hgext/largefiles/proto.py
author Hao Lian <hao@fogcreek.com>
Thu, 20 Oct 2011 17:24:59 -0400
branchstable
changeset 15391 a5a6a9b7f3b9
parent 15317 41f371150ccb
child 15778 f15c646bffc7
permissions -rw-r--r--
largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp This is consistent with the rest of Mercurial's code, mirroring the try-finally-unlink structure elsewhere. Furthermore, it fixes the case where largefiles throws an IOError on Windows when the temporary file is opened a second time by copytocacheabsolute. This patch creates the temporary file in the repo's largefiles store rather than /tmp, which might be a different filesystem.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     1
# Copyright 2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     2
#
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     3
# This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     4
# GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     5
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     6
import os
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     7
import urllib2
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     8
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     9
from mercurial import error, httprepo, util, wireproto
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    10
from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    11
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    12
import lfutil
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    13
15255
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    14
LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    15
                           '\n\nPlease enable it in your Mercurial config '
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    16
                           'file.\n')
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    17
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    18
def putlfile(repo, proto, sha):
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    19
    '''Put a largefile into a repository's local store and into the
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    20
    user cache.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    21
    proto.redirect()
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    22
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    23
    fd, tmpname = lfutil.mkstemp(repo, prefix='hg-putlfile')
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    24
    tmpfp = os.fdopen(fd, 'wb+')
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    25
    try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    26
        try:
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    27
            proto.getfile(tmpfp)
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    28
            tmpfp.seek(0)
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    29
            if sha != lfutil.hexsha1(tmpfp):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    30
                return wireproto.pushres(1)
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    31
            tmpfp.close()
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    32
            lfutil.copytostoreabsolute(repo, tmpname, sha)
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    33
        except IOError, e:
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    34
            repo.ui.warn(_('largefiles: failed to put %s (%s) into store: %s') %
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    35
                         (sha, tmpname, e.strerror))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    36
            return wireproto.pushres(1)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    37
    finally:
15391
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    38
        tmpfp.close()
a5a6a9b7f3b9 largefiles: replace tempfile.NamedTemporaryFile with tempfile.mkstemp
Hao Lian <hao@fogcreek.com>
parents: 15317
diff changeset
    39
        os.unlink(tmpname)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    40
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    41
    return wireproto.pushres(0)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    42
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    43
def getlfile(repo, proto, sha):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    44
    '''Retrieve a largefile from the repository-local cache or system
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    45
    cache.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    46
    filename = lfutil.findfile(repo, sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    47
    if not filename:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    48
        raise util.Abort(_('requested largefile %s not present in cache') % sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    49
    f = open(filename, 'rb')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    50
    length = os.fstat(f.fileno())[6]
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    51
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    52
    # Since we can't set an HTTP content-length header here, and
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    53
    # Mercurial core provides no way to give the length of a streamres
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    54
    # (and reading the entire file into RAM would be ill-advised), we
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    55
    # just send the length on the first line of the response, like the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    56
    # ssh proto does for string responses.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    57
    def generator():
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    58
        yield '%d\n' % length
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    59
        for chunk in f:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    60
            yield chunk
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    61
    return wireproto.streamres(generator())
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    62
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    63
def statlfile(repo, proto, sha):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    64
    '''Return '2\n' if the largefile is missing, '1\n' if it has a
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    65
    mismatched checksum, or '0\n' if it is in good condition'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    66
    filename = lfutil.findfile(repo, sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    67
    if not filename:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    68
        return '2\n'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    69
    fd = None
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    70
    try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    71
        fd = open(filename, 'rb')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    72
        return lfutil.hexsha1(fd) == sha and '0\n' or '1\n'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    73
    finally:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    74
        if fd:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    75
            fd.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    76
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    77
def wirereposetup(ui, repo):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    78
    class lfileswirerepository(repo.__class__):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    79
        def putlfile(self, sha, fd):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    80
            # unfortunately, httprepository._callpush tries to convert its
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    81
            # input file-like into a bundle before sending it, so we can't use
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    82
            # it ...
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    83
            if issubclass(self.__class__, httprepo.httprepository):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    84
                try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    85
                    return int(self._call('putlfile', data=fd, sha=sha,
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    86
                        headers={'content-type':'application/mercurial-0.1'}))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    87
                except (ValueError, urllib2.HTTPError):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    88
                    return 1
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    89
            # ... but we can't use sshrepository._call because the data=
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    90
            # argument won't get sent, and _callpush does exactly what we want
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    91
            # in this case: send the data straight through
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    92
            else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    93
                try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    94
                    ret, output = self._callpush("putlfile", fd, sha=sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    95
                    if ret == "":
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    96
                        raise error.ResponseError(_('putlfile failed:'),
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    97
                                output)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    98
                    return int(ret)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    99
                except IOError:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   100
                    return 1
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   101
                except ValueError:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   102
                    raise error.ResponseError(
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   103
                        _('putlfile failed (unexpected response):'), ret)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   104
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   105
        def getlfile(self, sha):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   106
            stream = self._callstream("getlfile", sha=sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   107
            length = stream.readline()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   108
            try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   109
                length = int(length)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   110
            except ValueError:
15170
c1a4a3220711 largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents: 15168
diff changeset
   111
                self._abort(error.ResponseError(_("unexpected response:"),
c1a4a3220711 largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents: 15168
diff changeset
   112
                                                length))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   113
            return (length, stream)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   114
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   115
        def statlfile(self, sha):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   116
            try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   117
                return int(self._call("statlfile", sha=sha))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   118
            except (ValueError, urllib2.HTTPError):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
   119
                # If the server returns anything but an integer followed by a
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   120
                # newline, newline, it's not speaking our language; if we get
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   121
                # an HTTP error, we can't be sure the largefile is present;
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
   122
                # either way, consider it missing.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   123
                return 2
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   124
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   125
    repo.__class__ = lfileswirerepository
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   126
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   127
# advertise the largefiles=serve capability
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   128
def capabilities(repo, proto):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   129
    return capabilities_orig(repo, proto) + ' largefiles=serve'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   130
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   131
# duplicate what Mercurial's new out-of-band errors mechanism does, because
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   132
# clients old and new alike both handle it well
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   133
def webproto_refuseclient(self, message):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   134
    self.req.header([('Content-Type', 'application/hg-error')])
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   135
    return message
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   136
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   137
def sshproto_refuseclient(self, message):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   138
    self.ui.write_err('%s\n-\n' % message)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   139
    self.fout.write('\n')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   140
    self.fout.flush()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   141
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   142
    return ''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   143
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   144
def heads(repo, proto):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   145
    if lfutil.islfilesrepo(repo):
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15170
diff changeset
   146
        return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   147
    return wireproto.heads(repo, proto)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   148
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   149
def sshrepo_callstream(self, cmd, **args):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   150
    if cmd == 'heads' and self.capable('largefiles'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   151
        cmd = 'lheads'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   152
    if cmd == 'batch' and self.capable('largefiles'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   153
        args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   154
    return ssh_oldcallstream(self, cmd, **args)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   155
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   156
def httprepo_callstream(self, cmd, **args):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   157
    if cmd == 'heads' and self.capable('largefiles'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   158
        cmd = 'lheads'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   159
    if cmd == 'batch' and self.capable('largefiles'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   160
        args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   161
    return http_oldcallstream(self, cmd, **args)