hgext/largefiles/proto.py
author Martin Geisler <mg@aragost.com>
Thu, 23 Feb 2012 13:37:10 +0100
branchstable
changeset 16155 1b2b42e866be
parent 15778 f15c646bffc7
child 16247 d87d9d8a8e03
permissions -rw-r--r--
largefiles: respect store.createmode and avoid extra file copy Before, a tempfile was used to create a temp file was created with 600 permissions and the uploaded data was written into it. This file was then *copied* to .hg/largefiles/<hash>. We now simply use atomictempfile to write the data to a temp file with the right permissions and then rename that into place.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     1
# Copyright 2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     2
#
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     3
# This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     4
# GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     5
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     6
import os
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     7
import urllib2
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     8
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     9
from mercurial import error, httprepo, util, wireproto
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    10
from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    11
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    12
import lfutil
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    13
15255
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    14
LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    15
                           '\n\nPlease enable it in your Mercurial config '
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    16
                           'file.\n')
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    17
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    18
def putlfile(repo, proto, sha):
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    19
    '''Put a largefile into a repository's local store and into the
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    20
    user cache.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    21
    proto.redirect()
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    22
16155
1b2b42e866be largefiles: respect store.createmode and avoid extra file copy
Martin Geisler <mg@aragost.com>
parents: 15778
diff changeset
    23
    tmpfp = util.atomictempfile(lfutil.storepath(repo, sha),
1b2b42e866be largefiles: respect store.createmode and avoid extra file copy
Martin Geisler <mg@aragost.com>
parents: 15778
diff changeset
    24
                                createmode=repo.store.createmode)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    25
    try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    26
        try:
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    27
            proto.getfile(tmpfp)
16155
1b2b42e866be largefiles: respect store.createmode and avoid extra file copy
Martin Geisler <mg@aragost.com>
parents: 15778
diff changeset
    28
            tmpfp._fp.seek(0)
1b2b42e866be largefiles: respect store.createmode and avoid extra file copy
Martin Geisler <mg@aragost.com>
parents: 15778
diff changeset
    29
            if sha != lfutil.hexsha1(tmpfp._fp):
15778
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    30
                raise IOError(0, _('largefile contents do not match hash'))
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    31
            tmpfp.close()
16155
1b2b42e866be largefiles: respect store.createmode and avoid extra file copy
Martin Geisler <mg@aragost.com>
parents: 15778
diff changeset
    32
            lfutil.linktousercache(repo, sha)
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    33
        except IOError, e:
15778
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    34
            repo.ui.warn(_('largefiles: failed to put %s into store: %s') %
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    35
                         (sha, e.strerror))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    36
            return wireproto.pushres(1)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    37
    finally:
16155
1b2b42e866be largefiles: respect store.createmode and avoid extra file copy
Martin Geisler <mg@aragost.com>
parents: 15778
diff changeset
    38
        tmpfp.discard()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    39
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    40
    return wireproto.pushres(0)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    41
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    42
def getlfile(repo, proto, sha):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    43
    '''Retrieve a largefile from the repository-local cache or system
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    44
    cache.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    45
    filename = lfutil.findfile(repo, sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    46
    if not filename:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    47
        raise util.Abort(_('requested largefile %s not present in cache') % sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    48
    f = open(filename, 'rb')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    49
    length = os.fstat(f.fileno())[6]
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    50
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    51
    # Since we can't set an HTTP content-length header here, and
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    52
    # Mercurial core provides no way to give the length of a streamres
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    53
    # (and reading the entire file into RAM would be ill-advised), we
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    54
    # just send the length on the first line of the response, like the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    55
    # ssh proto does for string responses.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    56
    def generator():
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    57
        yield '%d\n' % length
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    58
        for chunk in f:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    59
            yield chunk
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    60
    return wireproto.streamres(generator())
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    61
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    62
def statlfile(repo, proto, sha):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    63
    '''Return '2\n' if the largefile is missing, '1\n' if it has a
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    64
    mismatched checksum, or '0\n' if it is in good condition'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    65
    filename = lfutil.findfile(repo, sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    66
    if not filename:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    67
        return '2\n'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    68
    fd = None
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    69
    try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    70
        fd = open(filename, 'rb')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    71
        return lfutil.hexsha1(fd) == sha and '0\n' or '1\n'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    72
    finally:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    73
        if fd:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    74
            fd.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    75
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    76
def wirereposetup(ui, repo):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    77
    class lfileswirerepository(repo.__class__):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    78
        def putlfile(self, sha, fd):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    79
            # unfortunately, httprepository._callpush tries to convert its
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    80
            # input file-like into a bundle before sending it, so we can't use
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    81
            # it ...
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    82
            if issubclass(self.__class__, httprepo.httprepository):
15778
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    83
                res = None
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    84
                try:
15778
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    85
                    res = self._call('putlfile', data=fd, sha=sha,
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    86
                        headers={'content-type':'application/mercurial-0.1'})
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    87
                    d, output = res.split('\n', 1)
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    88
                    for l in output.splitlines(True):
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    89
                        self.ui.warn(_('remote: '), l, '\n')
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    90
                    return int(d)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    91
                except (ValueError, urllib2.HTTPError):
15778
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
    92
                    self.ui.warn(_('unexpected putlfile response: %s') % res)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    93
                    return 1
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    94
            # ... but we can't use sshrepository._call because the data=
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    95
            # argument won't get sent, and _callpush does exactly what we want
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    96
            # in this case: send the data straight through
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    97
            else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    98
                try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    99
                    ret, output = self._callpush("putlfile", fd, sha=sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   100
                    if ret == "":
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   101
                        raise error.ResponseError(_('putlfile failed:'),
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   102
                                output)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   103
                    return int(ret)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   104
                except IOError:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   105
                    return 1
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   106
                except ValueError:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   107
                    raise error.ResponseError(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   108
                        _('putlfile failed (unexpected response):'), ret)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   109
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   110
        def getlfile(self, sha):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   111
            stream = self._callstream("getlfile", sha=sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   112
            length = stream.readline()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   113
            try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   114
                length = int(length)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   115
            except ValueError:
15170
c1a4a3220711 largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents: 15168
diff changeset
   116
                self._abort(error.ResponseError(_("unexpected response:"),
c1a4a3220711 largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents: 15168
diff changeset
   117
                                                length))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   118
            return (length, stream)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   119
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   120
        def statlfile(self, sha):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   121
            try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   122
                return int(self._call("statlfile", sha=sha))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   123
            except (ValueError, urllib2.HTTPError):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
   124
                # If the server returns anything but an integer followed by a
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   125
                # newline, newline, it's not speaking our language; if we get
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   126
                # an HTTP error, we can't be sure the largefile is present;
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
   127
                # either way, consider it missing.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   128
                return 2
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   129
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   130
    repo.__class__ = lfileswirerepository
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   131
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   132
# advertise the largefiles=serve capability
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   133
def capabilities(repo, proto):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   134
    return capabilities_orig(repo, proto) + ' largefiles=serve'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   135
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   136
# duplicate what Mercurial's new out-of-band errors mechanism does, because
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   137
# clients old and new alike both handle it well
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   138
def webproto_refuseclient(self, message):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   139
    self.req.header([('Content-Type', 'application/hg-error')])
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   140
    return message
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   141
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   142
def sshproto_refuseclient(self, message):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   143
    self.ui.write_err('%s\n-\n' % message)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   144
    self.fout.write('\n')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   145
    self.fout.flush()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   146
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   147
    return ''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   148
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   149
def heads(repo, proto):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   150
    if lfutil.islfilesrepo(repo):
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15170
diff changeset
   151
        return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   152
    return wireproto.heads(repo, proto)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   153
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   154
def sshrepo_callstream(self, cmd, **args):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   155
    if cmd == 'heads' and self.capable('largefiles'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   156
        cmd = 'lheads'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   157
    if cmd == 'batch' and self.capable('largefiles'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   158
        args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   159
    return ssh_oldcallstream(self, cmd, **args)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   160
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   161
def httprepo_callstream(self, cmd, **args):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   162
    if cmd == 'heads' and self.capable('largefiles'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   163
        cmd = 'lheads'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   164
    if cmd == 'batch' and self.capable('largefiles'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   165
        args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   166
    return http_oldcallstream(self, cmd, **args)