hgext/largefiles/remotestore.py
author Gregory Szorc <gregory.szorc@gmail.com>
Tue, 23 Jun 2015 22:20:08 -0700
changeset 25660 328739ea70c3
parent 25079 bee00e0c2e45
child 26587 56b2bcea2529
permissions -rw-r--r--
global: mass rewrite to use modern exception syntax Python 2.6 introduced the "except type as instance" syntax, replacing the "except type, instance" syntax that came before. Python 3 dropped support for the latter syntax. Since we no longer support Python 2.4 or 2.5, we have no need to continue supporting the "except type, instance". This patch mass rewrites the exception syntax to be Python 2.6+ and Python 3 compatible. This patch was produced by running `2to3 -f except -w -n .`.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     1
# Copyright 2010-2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     2
# Copyright 2010-2011 Unity Technologies
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     3
#
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     4
# This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     5
# GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     6
17425
e95ec38f86b0 fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents: 17127
diff changeset
     7
'''remote largefile store; the base class for wirestore'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     8
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     9
import urllib2
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    10
21084
70252bdfd39c largefiles: import whole modules instead of importing parts of them
Mads Kiilerich <madski@unity3d.com>
parents: 20688
diff changeset
    11
from mercurial import util, wireproto
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    12
from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    13
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    14
import lfutil
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    15
import basestore
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    16
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    17
class remotestore(basestore.basestore):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15188
diff changeset
    18
    '''a largefile store accessed over a network'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    19
    def __init__(self, ui, repo, url):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    20
        super(remotestore, self).__init__(ui, repo, url)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    21
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    22
    def put(self, source, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    23
        if self.sendfile(source, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    24
            raise util.Abort(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    25
                _('remotestore: could not put %s to remote store %s')
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    26
                % (source, util.hidepassword(self.url)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    27
        self.ui.debug(
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    28
            _('remotestore: put %s to remote store %s\n')
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    29
            % (source, util.hidepassword(self.url)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    30
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    31
    def exists(self, hashes):
20688
a61ed1c2d7a7 check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents: 19950
diff changeset
    32
        return dict((h, s == 0) for (h, s) in # dict-from-generator
a61ed1c2d7a7 check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents: 19950
diff changeset
    33
                    self._stat(hashes).iteritems())
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    34
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    35
    def sendfile(self, filename, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    36
        self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    37
        fd = None
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    38
        try:
25079
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
    39
            fd = lfutil.httpsendfile(self.ui, filename)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    40
            return self._put(hash, fd)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25079
diff changeset
    41
        except IOError as e:
25079
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
    42
            raise util.Abort(
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
    43
                _('remotestore: could not open file %s: %s')
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
    44
                % (filename, str(e)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    45
        finally:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    46
            if fd:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    47
                fd.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    48
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    49
    def _getfile(self, tmpfile, filename, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    50
        try:
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 19003
diff changeset
    51
            chunks = self._get(hash)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25079
diff changeset
    52
        except urllib2.HTTPError as e:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    53
            # 401s get converted to util.Aborts; everything else is fine being
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    54
            # turned into a StoreError
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    55
            raise basestore.StoreError(filename, hash, self.url, str(e))
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25079
diff changeset
    56
        except urllib2.URLError as e:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    57
            # This usually indicates a connection problem, so don't
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    58
            # keep trying with the other files... they will probably
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    59
            # all fail too.
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    60
            raise util.Abort('%s: %s' %
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    61
                             (util.hidepassword(self.url), e.reason))
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25079
diff changeset
    62
        except IOError as e:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    63
            raise basestore.StoreError(filename, hash, self.url, str(e))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    64
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 19003
diff changeset
    65
        return lfutil.copyandhash(chunks, tmpfile)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    66
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    67
    def _verifyfile(self, cctx, cset, contents, standin, verified):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    68
        filename = lfutil.splitstandin(standin)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    69
        if not filename:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    70
            return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    71
        fctx = cctx[standin]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    72
        key = (filename, fctx.filenode())
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    73
        if key in verified:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    74
            return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    75
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    76
        verified.add(key)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    77
18482
6f219eb83435 largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents: 18481
diff changeset
    78
        expecthash = fctx.data()[0:40]
6f219eb83435 largefiles: adapt verify to batched remote statlfile (issue3780)
Mads Kiilerich <madski@unity3d.com>
parents: 18481
diff changeset
    79
        stat = self._stat([expecthash])[expecthash]
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    80
        if not stat:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    81
            return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    82
        elif stat == 1:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    83
            self.ui.warn(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    84
                _('changeset %s: %s: contents differ\n')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    85
                % (cset, filename))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    86
            return True # failed
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    87
        elif stat == 2:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    88
            self.ui.warn(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    89
                _('changeset %s: %s missing\n')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    90
                % (cset, filename))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    91
            return True # failed
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    92
        else:
15253
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    93
            raise RuntimeError('verify failed: unexpected response from '
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    94
                               'statlfile (%r)' % stat)
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    95
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    96
    def batch(self):
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    97
        '''Support for remote batching.'''
21084
70252bdfd39c largefiles: import whole modules instead of importing parts of them
Mads Kiilerich <madski@unity3d.com>
parents: 20688
diff changeset
    98
        return wireproto.remotebatch(self)