hgext/largefiles/proto.py
author Martin von Zweigbergk <martinvonz@google.com>
Fri, 11 Oct 2019 22:02:44 -0700
changeset 43241 cca3c1755224
parent 43085 eef9a2d67051
child 43506 9f70512ae2cf
permissions -rw-r--r--
tests: show graph log before and after widening in more cases I'm about to make some changes to which revisions get sent during widening. Some more tests will make it clearer what changes. I've also switched to graph log so we easily notice if the graph shape changes. Differential Revision: https://phab.mercurial-scm.org/D7091
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     1
# Copyright 2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     2
#
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     3
# This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     4
# GNU General Public License version 2 or any later version.
29312
29139be0ccc7 py3: make largefiles/proto.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
     5
from __future__ import absolute_import
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     6
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     7
import os
19917
cff331cbb5ee largefiles: make the protocol hack for replacing heads with lheads more precise
Mads Kiilerich <madski@unity3d.com>
parents: 19009
diff changeset
     8
import re
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     9
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    10
from mercurial.i18n import _
43085
eef9a2d67051 py3: manually import pycompat.open into files that need it
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
    11
from mercurial.pycompat import open
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    12
29312
29139be0ccc7 py3: make largefiles/proto.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    13
from mercurial import (
29139be0ccc7 py3: make largefiles/proto.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    14
    error,
41062
0a7f582f6f1f largefiles: port wrapped functions to exthelper
Matt Harbison <matt_harbison@yahoo.com>
parents: 37614
diff changeset
    15
    exthelper,
29312
29139be0ccc7 py3: make largefiles/proto.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    16
    httppeer,
29139be0ccc7 py3: make largefiles/proto.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    17
    util,
36074
2f7290555c96 wireproto: introduce type for raw byte responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36070
diff changeset
    18
    wireprototypes,
37614
a81d02ea65db wireproto: move version 1 peer functionality to standalone module (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37484
diff changeset
    19
    wireprotov1peer,
41062
0a7f582f6f1f largefiles: port wrapped functions to exthelper
Matt Harbison <matt_harbison@yahoo.com>
parents: 37614
diff changeset
    20
    wireprotov1server,
29312
29139be0ccc7 py3: make largefiles/proto.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    21
)
29139be0ccc7 py3: make largefiles/proto.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    22
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    23
from . import lfutil
29312
29139be0ccc7 py3: make largefiles/proto.py use absolute_import
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    24
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28576
diff changeset
    25
urlerr = util.urlerr
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28576
diff changeset
    26
urlreq = util.urlreq
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28576
diff changeset
    27
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    28
LARGEFILES_REQUIRED_MSG = (
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    29
    b'\nThis repository uses the largefiles extension.'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    30
    b'\n\nPlease enable it in your Mercurial config '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    31
    b'file.\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    32
)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    33
41062
0a7f582f6f1f largefiles: port wrapped functions to exthelper
Matt Harbison <matt_harbison@yahoo.com>
parents: 37614
diff changeset
    34
eh = exthelper.exthelper()
0a7f582f6f1f largefiles: port wrapped functions to exthelper
Matt Harbison <matt_harbison@yahoo.com>
parents: 37614
diff changeset
    35
18922
d2c4d37f7db5 largefiles: quiet (and document) undefined name errors (issue3886)
Bryan O'Sullivan <bryano@fb.com>
parents: 18488
diff changeset
    36
# these will all be replaced by largefiles.uisetup
d2c4d37f7db5 largefiles: quiet (and document) undefined name errors (issue3886)
Bryan O'Sullivan <bryano@fb.com>
parents: 18488
diff changeset
    37
ssholdcallstream = None
d2c4d37f7db5 largefiles: quiet (and document) undefined name errors (issue3886)
Bryan O'Sullivan <bryano@fb.com>
parents: 18488
diff changeset
    38
httpoldcallstream = None
d2c4d37f7db5 largefiles: quiet (and document) undefined name errors (issue3886)
Bryan O'Sullivan <bryano@fb.com>
parents: 18488
diff changeset
    39
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    40
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    41
def putlfile(repo, proto, sha):
28576
33bd95443e7f largefiles: add some docstrings
Mads Kiilerich <madski@unity3d.com>
parents: 26825
diff changeset
    42
    '''Server command for putting a largefile into a repository's local store
33bd95443e7f largefiles: add some docstrings
Mads Kiilerich <madski@unity3d.com>
parents: 26825
diff changeset
    43
    and into the user cache.'''
36067
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    44
    with proto.mayberedirectstdio() as output:
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    45
        path = lfutil.storepath(repo, sha)
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    46
        util.makedirs(os.path.dirname(path))
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    47
        tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
16594
5516fdf3fe24 largefiles: in putlfile, ensure tempfile's directory exists prior to creation
hlian
parents: 16247
diff changeset
    48
36067
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    49
        try:
37414
2d965bfeb8f6 wireproto: allow direct stream processing for unbundle
Joerg Sonnenberger <joerg@bec.de>
parents: 37293
diff changeset
    50
            for p in proto.getpayload():
2d965bfeb8f6 wireproto: allow direct stream processing for unbundle
Joerg Sonnenberger <joerg@bec.de>
parents: 37293
diff changeset
    51
                tmpfp.write(p)
36067
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    52
            tmpfp._fp.seek(0)
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    53
            if sha != lfutil.hexsha1(tmpfp._fp):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    54
                raise IOError(0, _(b'largefile contents do not match hash'))
36067
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    55
            tmpfp.close()
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    56
            lfutil.linktousercache(repo, sha)
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    57
        except IOError as e:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    58
            repo.ui.warn(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    59
                _(b'largefiles: failed to put %s into store: %s\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    60
                % (sha, e.strerror)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    61
            )
37293
d5d665f6615a wireproto: stop aliasing wire protocol types (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36656
diff changeset
    62
            return wireprototypes.pushres(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    63
                1, output.getvalue() if output else b''
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    64
            )
36067
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    65
        finally:
caca3ac2ac04 wireproto: use maybecapturestdio() for push responses (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35750
diff changeset
    66
            tmpfp.discard()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    67
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    68
    return wireprototypes.pushres(0, output.getvalue() if output else b'')
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    69
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    70
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    71
def getlfile(repo, proto, sha):
28576
33bd95443e7f largefiles: add some docstrings
Mads Kiilerich <madski@unity3d.com>
parents: 26825
diff changeset
    72
    '''Server command for retrieving a largefile from the repository-local
33bd95443e7f largefiles: add some docstrings
Mads Kiilerich <madski@unity3d.com>
parents: 26825
diff changeset
    73
    cache or user cache.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    74
    filename = lfutil.findfile(repo, sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    75
    if not filename:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    76
        raise error.Abort(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    77
            _(b'requested largefile %s not present in cache') % sha
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    78
        )
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    79
    f = open(filename, b'rb')
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    80
    length = os.fstat(f.fileno())[6]
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    81
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    82
    # Since we can't set an HTTP content-length header here, and
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    83
    # Mercurial core provides no way to give the length of a streamres
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    84
    # (and reading the entire file into RAM would be ill-advised), we
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    85
    # just send the length on the first line of the response, like the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
    86
    # ssh proto does for string responses.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    87
    def generator():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    88
        yield b'%d\n' % length
19009
07e40d589b64 largefiles: use filechunkiter for iterating largefile when serving getlfile
Mads Kiilerich <madski@unity3d.com>
parents: 19006
diff changeset
    89
        for chunk in util.filechunkiter(f):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    90
            yield chunk
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    91
37293
d5d665f6615a wireproto: stop aliasing wire protocol types (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36656
diff changeset
    92
    return wireprototypes.streamreslegacy(gen=generator())
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    93
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
    94
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    95
def statlfile(repo, proto, sha):
28576
33bd95443e7f largefiles: add some docstrings
Mads Kiilerich <madski@unity3d.com>
parents: 26825
diff changeset
    96
    '''Server command for checking if a largefile is present - returns '2\n' if
33bd95443e7f largefiles: add some docstrings
Mads Kiilerich <madski@unity3d.com>
parents: 26825
diff changeset
    97
    the largefile is missing, '0\n' if it seems to be in good condition.
18488
a977b42df8b3 largefiles: don't verify largefile hashes on servers when processing statlfile
Mads Kiilerich <madski@unity3d.com>
parents: 18298
diff changeset
    98
a977b42df8b3 largefiles: don't verify largefile hashes on servers when processing statlfile
Mads Kiilerich <madski@unity3d.com>
parents: 18298
diff changeset
    99
    The value 1 is reserved for mismatched checksum, but that is too expensive
a977b42df8b3 largefiles: don't verify largefile hashes on servers when processing statlfile
Mads Kiilerich <madski@unity3d.com>
parents: 18298
diff changeset
   100
    to be verified on every stat and must be caught be running 'hg verify'
a977b42df8b3 largefiles: don't verify largefile hashes on servers when processing statlfile
Mads Kiilerich <madski@unity3d.com>
parents: 18298
diff changeset
   101
    server side.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   102
    filename = lfutil.findfile(repo, sha)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   103
    if not filename:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   104
        return wireprototypes.bytesresponse(b'2\n')
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   105
    return wireprototypes.bytesresponse(b'0\n')
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   106
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   107
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   108
def wirereposetup(ui, repo):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   109
    class lfileswirerepository(repo.__class__):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   110
        def putlfile(self, sha, fd):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   111
            # unfortunately, httprepository._callpush tries to convert its
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   112
            # input file-like into a bundle before sending it, so we can't use
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   113
            # it ...
17192
1ac628cd7113 peer: introduce real peer classes
Peter Arrenbrecht <peter.arrenbrecht@gmail.com>
parents: 17127
diff changeset
   114
            if issubclass(self.__class__, httppeer.httppeer):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   115
                res = self._call(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   116
                    b'putlfile',
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   117
                    data=fd,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   118
                    sha=sha,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   119
                    headers={r'content-type': r'application/mercurial-0.1'},
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   120
                )
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   121
                try:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   122
                    d, output = res.split(b'\n', 1)
15778
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
   123
                    for l in output.splitlines(True):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   124
                        self.ui.warn(_(b'remote: '), l)  # assume l ends with \n
15778
f15c646bffc7 largefiles: display remote errors from putlfile (issue3123) (issue3149)
Kevin Gessner <kevin@fogcreek.com>
parents: 15391
diff changeset
   125
                    return int(d)
26825
78539633acf3 largefiles: don't mute and obfuscate http errors when putlfile fails
Mads Kiilerich <madski@unity3d.com>
parents: 26587
diff changeset
   126
                except ValueError:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   127
                    self.ui.warn(_(b'unexpected putlfile response: %r\n') % res)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   128
                    return 1
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   129
            # ... but we can't use sshrepository._call because the data=
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   130
            # argument won't get sent, and _callpush does exactly what we want
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   131
            # in this case: send the data straight through
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   132
            else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   133
                try:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   134
                    ret, output = self._callpush(b"putlfile", fd, sha=sha)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   135
                    if ret == b"":
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   136
                        raise error.ResponseError(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   137
                            _(b'putlfile failed:'), output
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   138
                        )
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   139
                    return int(ret)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   140
                except IOError:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   141
                    return 1
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   142
                except ValueError:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   143
                    raise error.ResponseError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   144
                        _(b'putlfile failed (unexpected response):'), ret
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   145
                    )
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   146
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   147
        def getlfile(self, sha):
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 18922
diff changeset
   148
            """returns an iterable with the chunks of the file with sha sha"""
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   149
            stream = self._callstream(b"getlfile", sha=sha)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   150
            length = stream.readline()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   151
            try:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   152
                length = int(length)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   153
            except ValueError:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   154
                self._abort(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   155
                    error.ResponseError(_(b"unexpected response:"), length)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   156
                )
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 18922
diff changeset
   157
19005
1b84047e7d16 largefiles: drop limitreader, use filechunkiter limit
Mads Kiilerich <madski@unity3d.com>
parents: 19004
diff changeset
   158
            # SSH streams will block if reading more than length
30181
7356e6b1f5b8 util: increase filechunkiter size to 128k
Mads Kiilerich <madski@unity3d.com>
parents: 29312
diff changeset
   159
            for chunk in util.filechunkiter(stream, limit=length):
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 18922
diff changeset
   160
                yield chunk
19006
0b3b84222a2d largefiles: getlfile must hit end of HTTP chunked streams to reuse connections
Mads Kiilerich <madski@unity3d.com>
parents: 19005
diff changeset
   161
            # HTTP streams must hit the end to process the last empty
0b3b84222a2d largefiles: getlfile must hit end of HTTP chunked streams to reuse connections
Mads Kiilerich <madski@unity3d.com>
parents: 19005
diff changeset
   162
            # chunk of Chunked-Encoding so the connection can be reused.
0b3b84222a2d largefiles: getlfile must hit end of HTTP chunked streams to reuse connections
Mads Kiilerich <madski@unity3d.com>
parents: 19005
diff changeset
   163
            if issubclass(self.__class__, httppeer.httppeer):
0b3b84222a2d largefiles: getlfile must hit end of HTTP chunked streams to reuse connections
Mads Kiilerich <madski@unity3d.com>
parents: 19005
diff changeset
   164
                chunk = stream.read(1)
0b3b84222a2d largefiles: getlfile must hit end of HTTP chunked streams to reuse connections
Mads Kiilerich <madski@unity3d.com>
parents: 19005
diff changeset
   165
                if chunk:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   166
                    self._abort(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   167
                        error.ResponseError(_(b"unexpected response:"), chunk)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   168
                    )
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   169
37614
a81d02ea65db wireproto: move version 1 peer functionality to standalone module (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37484
diff changeset
   170
        @wireprotov1peer.batchable
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   171
        def statlfile(self, sha):
37614
a81d02ea65db wireproto: move version 1 peer functionality to standalone module (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37484
diff changeset
   172
            f = wireprotov1peer.future()
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   173
            result = {b'sha': sha}
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 16594
diff changeset
   174
            yield result, f
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   175
            try:
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 16594
diff changeset
   176
                yield int(f.value)
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28576
diff changeset
   177
            except (ValueError, urlerr.httperror):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
   178
                # If the server returns anything but an integer followed by a
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   179
                # newline, newline, it's not speaking our language; if we get
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   180
                # an HTTP error, we can't be sure the largefile is present;
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15224
diff changeset
   181
                # either way, consider it missing.
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 16594
diff changeset
   182
                yield 2
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   183
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   184
    repo.__class__ = lfileswirerepository
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   185
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   186
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   187
# advertise the largefiles=serve capability
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   188
@eh.wrapfunction(wireprotov1server, b'_capabilities')
35507
95a9be56c3bb largefiles: modernize how capabilities are added to the wire protocol
Matt Harbison <matt_harbison@yahoo.com>
parents: 35348
diff changeset
   189
def _capabilities(orig, repo, proto):
95a9be56c3bb largefiles: modernize how capabilities are added to the wire protocol
Matt Harbison <matt_harbison@yahoo.com>
parents: 35348
diff changeset
   190
    '''announce largefile server capability'''
95a9be56c3bb largefiles: modernize how capabilities are added to the wire protocol
Matt Harbison <matt_harbison@yahoo.com>
parents: 35348
diff changeset
   191
    caps = orig(repo, proto)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   192
    caps.append(b'largefiles=serve')
35507
95a9be56c3bb largefiles: modernize how capabilities are added to the wire protocol
Matt Harbison <matt_harbison@yahoo.com>
parents: 35348
diff changeset
   193
    return caps
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   194
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   195
37484
c22fd3c4c23e largefiles: wrap heads command handler more directly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37414
diff changeset
   196
def heads(orig, repo, proto):
28576
33bd95443e7f largefiles: add some docstrings
Mads Kiilerich <madski@unity3d.com>
parents: 26825
diff changeset
   197
    '''Wrap server command - largefile capable clients will know to call
33bd95443e7f largefiles: add some docstrings
Mads Kiilerich <madski@unity3d.com>
parents: 26825
diff changeset
   198
    lheads instead'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   199
    if lfutil.islfilesrepo(repo):
37293
d5d665f6615a wireproto: stop aliasing wire protocol types (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36656
diff changeset
   200
        return wireprototypes.ooberror(LARGEFILES_REQUIRED_MSG)
37484
c22fd3c4c23e largefiles: wrap heads command handler more directly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37414
diff changeset
   201
c22fd3c4c23e largefiles: wrap heads command handler more directly
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37414
diff changeset
   202
    return orig(repo, proto)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   203
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   204
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16155
diff changeset
   205
def sshrepocallstream(self, cmd, **args):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   206
    if cmd == b'heads' and self.capable(b'largefiles'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   207
        cmd = b'lheads'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   208
    if cmd == b'batch' and self.capable(b'largefiles'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   209
        args[r'cmds'] = args[r'cmds'].replace(b'heads ', b'lheads ')
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16155
diff changeset
   210
    return ssholdcallstream(self, cmd, **args)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   211
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   212
36312
3ac8b5c1c36c largefiles: mark headre as bytes regex
Augie Fackler <augie@google.com>
parents: 36074
diff changeset
   213
headsre = re.compile(br'(^|;)heads\b')
19917
cff331cbb5ee largefiles: make the protocol hack for replacing heads with lheads more precise
Mads Kiilerich <madski@unity3d.com>
parents: 19009
diff changeset
   214
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 41062
diff changeset
   215
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16155
diff changeset
   216
def httprepocallstream(self, cmd, **args):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   217
    if cmd == b'heads' and self.capable(b'largefiles'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   218
        cmd = b'lheads'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   219
    if cmd == b'batch' and self.capable(b'largefiles'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   220
        args[r'cmds'] = headsre.sub(b'lheads', args[r'cmds'])
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16155
diff changeset
   221
    return httpoldcallstream(self, cmd, **args)