hgext/largefiles/remotestore.py
author Mads Kiilerich <madski@unity3d.com>
Mon, 28 Jan 2013 15:19:44 +0100
branchstable
changeset 18484 d43823f928fe
parent 18482 6f219eb83435
child 18573 003730ca254d
permissions -rw-r--r--
largefiles: adapt remotestore._getfile to batched statlfile 9e1616307c4c introduced batching of statlfile, but not all codepaths got converted. _getfile gave _stat garbage and got garbage back. The garbage didn't match the expected error codes and was thus interpreted as success. It could thus end up trying to fetch a largefile that didn't exist. Instead we now pass _stat valid input and handle both correct and invalid output correctly. This makes the code work as intended ... but it would probably be better if it didn't abort on missing largefiles, just like it happened to do before.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     1
# Copyright 2010-2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     2
# Copyright 2010-2011 Unity Technologies
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     3
#
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     4
# This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     5
# GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     6
17425
e95ec38f86b0 fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents: 17127
diff changeset
     7
'''remote largefile store; the base class for wirestore'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     8
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     9
import urllib2
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    10
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    11
from mercurial import util
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    12
from mercurial.i18n import _
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    13
from mercurial.wireproto import remotebatch
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    14
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    15
import lfutil
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    16
import basestore
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    17
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    18
class remotestore(basestore.basestore):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15188
diff changeset
    19
    '''a largefile store accessed over a network'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    20
    def __init__(self, ui, repo, url):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    21
        super(remotestore, self).__init__(ui, repo, url)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    22
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    23
    def put(self, source, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    24
        if self.sendfile(source, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    25
            raise util.Abort(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    26
                _('remotestore: could not put %s to remote store %s')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    27
                % (source, self.url))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    28
        self.ui.debug(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    29
            _('remotestore: put %s to remote store %s') % (source, self.url))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    30
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    31
    def exists(self, hashes):
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    32
        return self._verify(hashes)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    33
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    34
    def sendfile(self, filename, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    35
        self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    36
        fd = None
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    37
        try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    38
            try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    39
                fd = lfutil.httpsendfile(self.ui, filename)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    40
            except IOError, e:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    41
                raise util.Abort(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    42
                    _('remotestore: could not open file %s: %s')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    43
                    % (filename, str(e)))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    44
            return self._put(hash, fd)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    45
        finally:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    46
            if fd:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    47
                fd.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    48
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    49
    def _getfile(self, tmpfile, filename, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    50
        # quit if the largefile isn't there
18484
d43823f928fe largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents: 18482
diff changeset
    51
        stat = self._stat([hash])[hash]
15253
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    52
        if stat == 1:
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    53
            raise util.Abort(_('remotestore: largefile %s is invalid') % hash)
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    54
        elif stat == 2:
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    55
            raise util.Abort(_('remotestore: largefile %s is missing') % hash)
18484
d43823f928fe largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents: 18482
diff changeset
    56
        elif stat != 0:
d43823f928fe largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents: 18482
diff changeset
    57
            raise RuntimeError('error getting file: unexpected response from '
d43823f928fe largefiles: adapt remotestore._getfile to batched statlfile
Mads Kiilerich <madski@unity3d.com>
parents: 18482
diff changeset
    58
                               'statlfile (%r)' % stat)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    59
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    60
        try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    61
            length, infile = self._get(hash)
15188
8e115063950d largefiles: don't break existing tests (syntax error, bad imports)
Greg Ward <greg@gerg.ca>
parents: 15168
diff changeset
    62
        except urllib2.HTTPError, e:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    63
            # 401s get converted to util.Aborts; everything else is fine being
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    64
            # turned into a StoreError
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    65
            raise basestore.StoreError(filename, hash, self.url, str(e))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    66
        except urllib2.URLError, e:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    67
            # This usually indicates a connection problem, so don't
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    68
            # keep trying with the other files... they will probably
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    69
            # all fail too.
15253
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    70
            raise util.Abort('%s: %s' % (self.url, e.reason))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    71
        except IOError, e:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    72
            raise basestore.StoreError(filename, hash, self.url, str(e))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    73
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    74
        # Mercurial does not close its SSH connections after writing a stream
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    75
        if length is not None:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    76
            infile = lfutil.limitreader(infile, length)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    77
        return lfutil.copyandhash(lfutil.blockstream(infile), tmpfile)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    78
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    79
    def _verify(self, hashes):
18481
ed647c59753b largefiles: let wirestore._stat return stats as expected by remotestore verify
Mads Kiilerich <madski@unity3d.com>
parents: 17425
diff changeset
    80
        return dict((h, s == 0) for (h, s) in self._stat(hashes).iteritems())
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    81
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    82
    def _verifyfile(self, cctx, cset, contents, standin, verified):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    83
        filename = lfutil.splitstandin(standin)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    84
        if not filename:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    85
            return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    86
        fctx = cctx[standin]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    87
        key = (filename, fctx.filenode())
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    88
        if key in verified:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    89
            return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    90
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    91
        verified.add(key)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    92
18482
6f219eb83435 largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents: 18481
diff changeset
    93
        expecthash = fctx.data()[0:40]
6f219eb83435 largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents: 18481
diff changeset
    94
        stat = self._stat([expecthash])[expecthash]
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    95
        if not stat:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    96
            return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    97
        elif stat == 1:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    98
            self.ui.warn(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    99
                _('changeset %s: %s: contents differ\n')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   100
                % (cset, filename))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   101
            return True # failed
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   102
        elif stat == 2:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   103
            self.ui.warn(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   104
                _('changeset %s: %s missing\n')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   105
                % (cset, filename))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   106
            return True # failed
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   107
        else:
15253
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
   108
            raise RuntimeError('verify failed: unexpected response from '
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
   109
                               'statlfile (%r)' % stat)
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
   110
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
   111
    def batch(self):
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
   112
        '''Support for remote batching.'''
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
   113
        return remotebatch(self)
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
   114