hgext/largefiles/lfutil.py
author Mads Kiilerich <madski@unity3d.com>
Mon, 15 Apr 2013 23:43:50 +0200
changeset 19002 5083baa6cbf8
parent 19001 2a35296a6304
child 19005 1b84047e7d16
permissions -rw-r--r--
largefiles: remove blecch from lfutil.copyandhash - don't close the passed fd
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     1
# Copyright 2009-2010 Gregory P. Ward
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     2
# Copyright 2009-2010 Intelerad Medical Systems Incorporated
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     3
# Copyright 2010-2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     4
# Copyright 2010-2011 Unity Technologies
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     5
#
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     6
# This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     7
# GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     8
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     9
'''largefiles utility code: must not import other modules in this package.'''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    10
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    11
import os
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    12
import errno
15320
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
    13
import platform
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    14
import shutil
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    15
import stat
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    16
15226
2223ea21c98f largefiles: cleanup import, now that we can assume > 1.9 for bundled extension
Na'Tosha Bard <natosha@unity3d.com>
parents: 15224
diff changeset
    17
from mercurial import dirstate, httpconnection, match as match_, util, scmutil
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    18
from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    19
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    20
shortname = '.hglf'
18151
90ad387d9245 largefiles: use constant for '.hglf/'
Mads Kiilerich <madski@unity3d.com>
parents: 18150
diff changeset
    21
shortnameslash = shortname + '/'
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    22
longname = 'largefiles'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    23
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    24
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    25
# -- Private worker functions ------------------------------------------
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    26
15227
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    27
def getminsize(ui, assumelfiles, opt, default=10):
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    28
    lfsize = opt
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    29
    if not lfsize and assumelfiles:
15304
9aa9d4bb3d88 largefiles: rename config setting 'size' to 'minsize'
Greg Ward <greg@gerg.ca>
parents: 15255
diff changeset
    30
        lfsize = ui.config(longname, 'minsize', default=default)
15227
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    31
    if lfsize:
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    32
        try:
15228
ee625de3541e largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents: 15227
diff changeset
    33
            lfsize = float(lfsize)
15227
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    34
        except ValueError:
15228
ee625de3541e largefiles: allow minimum size to be a float
Greg Ward <greg@gerg.ca>
parents: 15227
diff changeset
    35
            raise util.Abort(_('largefiles: size must be number (not %s)\n')
15227
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    36
                             % lfsize)
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    37
    if lfsize is None:
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    38
        raise util.Abort(_('minimum size for largefiles must be specified'))
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    39
    return lfsize
a7686abf73a6 largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents: 15226
diff changeset
    40
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    41
def link(src, dest):
18998
d035c3902111 largefiles: refactoring - create destination dir in lfutil.link
Mads Kiilerich <madski@unity3d.com>
parents: 18980
diff changeset
    42
    util.makedirs(os.path.dirname(dest))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    43
    try:
15206
f85c76b16f27 largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents: 15188
diff changeset
    44
        util.oslink(src, dest)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    45
    except OSError:
15572
926bc23d0b6a largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents: 15571
diff changeset
    46
        # if hardlinks fail, fallback on atomic copy
926bc23d0b6a largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents: 15571
diff changeset
    47
        dst = util.atomictempfile(dest)
15699
84e55467093c largefiles: copy files in binary mode (issue3164)
Matt Mackall <mpm@selenic.com>
parents: 15658
diff changeset
    48
        for chunk in util.filechunkiter(open(src, 'rb')):
15572
926bc23d0b6a largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents: 15571
diff changeset
    49
            dst.write(chunk)
926bc23d0b6a largefiles: copy files into .hg/largefiles atomically
Martin Geisler <mg@aragost.com>
parents: 15571
diff changeset
    50
        dst.close()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    51
        os.chmod(dest, os.stat(src).st_mode)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    52
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
    53
def usercachepath(ui, hash):
15350
8b8dd13295db largefiles: use ui.configpath() where appropriate
Greg Ward <greg@gerg.ca>
parents: 15349
diff changeset
    54
    path = ui.configpath(longname, 'usercache', None)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    55
    if path:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    56
        path = os.path.join(path, hash)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    57
    else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    58
        if os.name == 'nt':
15255
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
    59
            appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA'))
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    60
            if appdata:
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    61
                path = os.path.join(appdata, longname, hash)
15320
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
    62
        elif platform.system() == 'Darwin':
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    63
            home = os.getenv('HOME')
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    64
            if home:
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    65
                path = os.path.join(home, 'Library', 'Caches',
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    66
                                    longname, hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    67
        elif os.name == 'posix':
15320
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
    68
            path = os.getenv('XDG_CACHE_HOME')
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
    69
            if path:
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
    70
                path = os.path.join(path, longname, hash)
681267a5f491 largefiles: use XDG and OS X-specific cache locations by default (issue3067)
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15319
diff changeset
    71
            else:
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    72
                home = os.getenv('HOME')
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    73
                if home:
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    74
                    path = os.path.join(home, '.cache', longname, hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    75
        else:
15253
67d010779907 largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents: 15252
diff changeset
    76
            raise util.Abort(_('unknown operating system: %s\n') % os.name)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    77
    return path
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    78
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
    79
def inusercache(ui, hash):
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    80
    path = usercachepath(ui, hash)
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
    81
    return path and os.path.exists(path)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    82
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    83
def findfile(repo, hash):
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
    84
    if instore(repo, hash):
16928
73b9286e667c largefiles: lowercase messages
Martin Geisler <mg@aragost.com>
parents: 16247
diff changeset
    85
        repo.ui.note(_('found %s in store\n') % hash)
15913
c35dcde25174 largefiles: refactor lfutil.findfiles to be more logical
Na'Tosha Bard <natosha@unity3d.com>
parents: 15796
diff changeset
    86
        return storepath(repo, hash)
15317
41f371150ccb largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15316
diff changeset
    87
    elif inusercache(repo.ui, hash):
16928
73b9286e667c largefiles: lowercase messages
Martin Geisler <mg@aragost.com>
parents: 16247
diff changeset
    88
        repo.ui.note(_('found %s in system cache\n') % hash)
15408
db8b0ee74025 largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents: 15392
diff changeset
    89
        path = storepath(repo, hash)
db8b0ee74025 largefiles: ensure destination directory exists before findfile links to there
Hao Lian <hao@fogcreek.com>
parents: 15392
diff changeset
    90
        link(usercachepath(repo.ui, hash), path)
15913
c35dcde25174 largefiles: refactor lfutil.findfiles to be more logical
Na'Tosha Bard <natosha@unity3d.com>
parents: 15796
diff changeset
    91
        return path
c35dcde25174 largefiles: refactor lfutil.findfiles to be more logical
Na'Tosha Bard <natosha@unity3d.com>
parents: 15796
diff changeset
    92
    return None
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    93
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
    94
class largefilesdirstate(dirstate.dirstate):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    95
    def __getitem__(self, key):
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
    96
        return super(largefilesdirstate, self).__getitem__(unixpath(key))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    97
    def normal(self, f):
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
    98
        return super(largefilesdirstate, self).normal(unixpath(f))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    99
    def remove(self, f):
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   100
        return super(largefilesdirstate, self).remove(unixpath(f))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   101
    def add(self, f):
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   102
        return super(largefilesdirstate, self).add(unixpath(f))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   103
    def drop(self, f):
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   104
        return super(largefilesdirstate, self).drop(unixpath(f))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   105
    def forget(self, f):
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   106
        return super(largefilesdirstate, self).forget(unixpath(f))
15793
3ef07ecdb0d5 largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents: 15700
diff changeset
   107
    def normallookup(self, f):
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   108
        return super(largefilesdirstate, self).normallookup(unixpath(f))
18148
bf6252d12c34 largefiles: simplify lfdirstate ignore handling - it is only for tracking .hglf
Mads Kiilerich <madski@unity3d.com>
parents: 18147
diff changeset
   109
    def _ignore(self):
bf6252d12c34 largefiles: simplify lfdirstate ignore handling - it is only for tracking .hglf
Mads Kiilerich <madski@unity3d.com>
parents: 18147
diff changeset
   110
        return False
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   111
17659
ae57920ac188 largefiles: enable islfilesrepo() prior to a commit (issue3541)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17270
diff changeset
   112
def openlfdirstate(ui, repo, create=True):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   113
    '''
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   114
    Return a dirstate object that tracks largefiles: i.e. its root is
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   115
    the repo root, but it is saved in .hg/largefiles/dirstate.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   116
    '''
18147
79f2493198e1 largefiles: rename 'admin' to more descriptive 'lfstoredir'
Mads Kiilerich <madski@unity3d.com>
parents: 18146
diff changeset
   117
    lfstoredir = repo.join(longname)
79f2493198e1 largefiles: rename 'admin' to more descriptive 'lfstoredir'
Mads Kiilerich <madski@unity3d.com>
parents: 18146
diff changeset
   118
    opener = scmutil.opener(lfstoredir)
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   119
    lfdirstate = largefilesdirstate(opener, ui, repo.root,
15349
63455eb771af largefiles: drop more unnecessary compatibility checks
Greg Ward <greg@gerg.ca>
parents: 15347
diff changeset
   120
                                     repo.dirstate._validate)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   121
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   122
    # If the largefiles dirstate does not exist, populate and create
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   123
    # it. This ensures that we create it on the first meaningful
15794
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   124
    # largefiles operation in a new clone.
18147
79f2493198e1 largefiles: rename 'admin' to more descriptive 'lfstoredir'
Mads Kiilerich <madski@unity3d.com>
parents: 18146
diff changeset
   125
    if create and not os.path.exists(os.path.join(lfstoredir, 'dirstate')):
79f2493198e1 largefiles: rename 'admin' to more descriptive 'lfstoredir'
Mads Kiilerich <madski@unity3d.com>
parents: 18146
diff changeset
   126
        util.makedirs(lfstoredir)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   127
        matcher = getstandinmatcher(repo)
18154
93c697d9c158 largefiles: remove trivial portability wrappers
Mads Kiilerich <madski@unity3d.com>
parents: 18153
diff changeset
   128
        for standin in repo.dirstate.walk(matcher, [], False, False):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   129
            lfile = splitstandin(standin)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   130
            hash = readstandin(repo, lfile)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   131
            lfdirstate.normallookup(lfile)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   132
            try:
15553
e89385e4ef8d largefiles: file storage should be relative to repo, not relative to cwd
Mads Kiilerich <mads@kiilerich.com>
parents: 15548
diff changeset
   133
                if hash == hashfile(repo.wjoin(lfile)):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   134
                    lfdirstate.normal(lfile)
15548
f76584098c88 largefiles: fix 'hg clone . ../foo' OSError abort
Martin Geisler <mg@lazybytes.net>
parents: 15408
diff changeset
   135
            except OSError, err:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   136
                if err.errno != errno.ENOENT:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   137
                    raise
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   138
    return lfdirstate
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   139
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   140
def lfdirstatestatus(lfdirstate, repo, rev):
15794
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   141
    match = match_.always(repo.root, repo.getcwd())
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   142
    s = lfdirstate.status(match, [], False, False, False)
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   143
    unsure, modified, added, removed, missing, unknown, ignored, clean = s
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   144
    for lfile in unsure:
18299
a7a88a458a4c largefiles: fix revert removing a largefile from a merge
Mads Kiilerich <madski@unity3d.com>
parents: 18154
diff changeset
   145
        try:
a7a88a458a4c largefiles: fix revert removing a largefile from a merge
Mads Kiilerich <madski@unity3d.com>
parents: 18154
diff changeset
   146
            fctx = repo[rev][standin(lfile)]
a7a88a458a4c largefiles: fix revert removing a largefile from a merge
Mads Kiilerich <madski@unity3d.com>
parents: 18154
diff changeset
   147
        except LookupError:
a7a88a458a4c largefiles: fix revert removing a largefile from a merge
Mads Kiilerich <madski@unity3d.com>
parents: 18154
diff changeset
   148
            fctx = None
a7a88a458a4c largefiles: fix revert removing a largefile from a merge
Mads Kiilerich <madski@unity3d.com>
parents: 18154
diff changeset
   149
        if not fctx or fctx.data().strip() != hashfile(repo.wjoin(lfile)):
15794
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   150
            modified.append(lfile)
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   151
        else:
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   152
            clean.append(lfile)
0d91211dd12f largefiles: fix inappropriate locking (issue3182)
Levi Bard <levi@unity3d.com>
parents: 15793
diff changeset
   153
            lfdirstate.normal(lfile)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   154
    return (modified, added, removed, missing, unknown, ignored, clean)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   155
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   156
def listlfiles(repo, rev=None, matcher=None):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   157
    '''return a list of largefiles in the working copy or the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   158
    specified changeset'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   159
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   160
    if matcher is None:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   161
        matcher = getstandinmatcher(repo)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   162
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   163
    # ignore unknown files in working directory
15255
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
   164
    return [splitstandin(f)
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
   165
            for f in repo[rev].walk(matcher)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   166
            if rev is not None or repo.dirstate[f] != '?']
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   167
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   168
def instore(repo, hash):
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   169
    return os.path.exists(storepath(repo, hash))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   170
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   171
def storepath(repo, hash):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   172
    return repo.join(os.path.join(longname, hash))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   173
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   174
def copyfromcache(repo, hash, filename):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   175
    '''Copy the specified largefile from the repo or system cache to
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   176
    filename in the repository. Return true on success or false if the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   177
    file was not found in either cache (which should not happened:
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   178
    this is meant to be called only after ensuring that the needed
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   179
    largefile exists in the cache).'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   180
    path = findfile(repo, hash)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   181
    if path is None:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   182
        return False
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   183
    util.makedirs(os.path.dirname(repo.wjoin(filename)))
15570
0f208626d503 largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents: 15553
diff changeset
   184
    # The write may fail before the file is fully written, but we
0f208626d503 largefiles: add comment about non-atomic working directory
Martin Geisler <mg@aragost.com>
parents: 15553
diff changeset
   185
    # don't use atomic writes in the working copy.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   186
    shutil.copy(path, repo.wjoin(filename))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   187
    return True
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   188
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   189
def copytostore(repo, rev, file, uploaded=False):
17877
92bbb21d4b13 largefiles: respect the rev when reading standins in copytostore() (issue3630)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17794
diff changeset
   190
    hash = readstandin(repo, file, rev)
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   191
    if instore(repo, hash):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   192
        return
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   193
    copytostoreabsolute(repo, repo.wjoin(file), hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   194
15796
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   195
def copyalltostore(repo, node):
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   196
    '''Copy all largefiles in a given revision to the store'''
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   197
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   198
    ctx = repo[node]
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   199
    for filename in ctx.files():
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   200
        if isstandin(filename) and filename in ctx.manifest():
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   201
            realfile = splitstandin(filename)
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   202
            copytostore(repo, ctx.node(), realfile)
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   203
3e5b6045ccfc largefiles: factor out a copyalltostore() function
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 15794
diff changeset
   204
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   205
def copytostoreabsolute(repo, file, hash):
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   206
    if inusercache(repo.ui, hash):
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   207
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
17878
d1d0140287b8 largefiles: don't copy largefiles from working dir to the store while converting
Matt Harbison <matt_harbison@yahoo.com>
parents: 17877
diff changeset
   208
    elif not getattr(repo, "_isconverting", False):
18998
d035c3902111 largefiles: refactoring - create destination dir in lfutil.link
Mads Kiilerich <madski@unity3d.com>
parents: 18980
diff changeset
   209
        util.makedirs(os.path.dirname(storepath(repo, hash)))
16153
05197f9fd1f3 largefiles: use repo.store.createmode for new files in .hg/largefiles
Martin Geisler <mg@aragost.com>
parents: 16103
diff changeset
   210
        dst = util.atomictempfile(storepath(repo, hash),
05197f9fd1f3 largefiles: use repo.store.createmode for new files in .hg/largefiles
Martin Geisler <mg@aragost.com>
parents: 16103
diff changeset
   211
                                  createmode=repo.store.createmode)
15699
84e55467093c largefiles: copy files in binary mode (issue3164)
Matt Mackall <mpm@selenic.com>
parents: 15658
diff changeset
   212
        for chunk in util.filechunkiter(open(file, 'rb')):
15571
809788118aa2 largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents: 15570
diff changeset
   213
            dst.write(chunk)
809788118aa2 largefiles: write .hg/largefiles/ files atomically
Martin Geisler <mg@aragost.com>
parents: 15570
diff changeset
   214
        dst.close()
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   215
        linktousercache(repo, hash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   216
15316
c65f5b6e26d4 largefiles: rename functions and methods to match desired behavior
Benjamin Pollack <benjamin@bitquabit.com>
parents: 15304
diff changeset
   217
def linktousercache(repo, hash):
15658
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
   218
    path = usercachepath(repo.ui, hash)
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
   219
    if path:
971c55ce03b8 largefiles: don't require a user cache (issue3088) (issue3155)
Kevin Gessner <kevin@fogcreek.com>
parents: 15572
diff changeset
   220
        link(storepath(repo, hash), path)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   221
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   222
def getstandinmatcher(repo, pats=[], opts={}):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   223
    '''Return a match object that applies pats to the standin directory'''
18150
14e31a631e41 largefiles: use plain wjoin instead of the complex pathto
Mads Kiilerich <madski@unity3d.com>
parents: 18148
diff changeset
   224
    standindir = repo.wjoin(shortname)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   225
    if pats:
18490
877f80599df0 largefiles: fix commit when using relative paths from subdirectory
Mads Kiilerich <madski@unity3d.com>
parents: 18300
diff changeset
   226
        pats = [os.path.join(standindir, pat) for pat in pats]
18724
894a5897a9dd largefiles: getstandinmatcher should not depend on existence of directories
Mads Kiilerich <madski@unity3d.com>
parents: 18490
diff changeset
   227
    else:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   228
        # no patterns: relative to repo root
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   229
        pats = [standindir]
18146
819c7e10d60d largefiles: simplify lfutil.getstandinmatcher by inlining getmatcher
Mads Kiilerich <madski@unity3d.com>
parents: 18143
diff changeset
   230
    # no warnings about missing files or directories
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
   231
    match = scmutil.match(repo[None], pats, opts)
18146
819c7e10d60d largefiles: simplify lfutil.getstandinmatcher by inlining getmatcher
Mads Kiilerich <madski@unity3d.com>
parents: 18143
diff changeset
   232
    match.bad = lambda f, msg: None
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   233
    return match
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   234
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   235
def composestandinmatcher(repo, rmatcher):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   236
    '''Return a matcher that accepts standins corresponding to the
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   237
    files accepted by rmatcher. Pass the list of files in the matcher
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   238
    as the paths specified by the user.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   239
    smatcher = getstandinmatcher(repo, rmatcher.files())
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   240
    isstandin = smatcher.matchfn
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   241
    def composedmatchfn(f):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   242
        return isstandin(f) and rmatcher.matchfn(splitstandin(f))
16247
d87d9d8a8e03 largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents: 16245
diff changeset
   243
    smatcher.matchfn = composedmatchfn
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   244
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   245
    return smatcher
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   246
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   247
def standin(filename):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   248
    '''Return the repo-relative path to the standin for the specified big
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   249
    file.'''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   250
    # Notes:
17425
e95ec38f86b0 fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents: 17270
diff changeset
   251
    # 1) Some callers want an absolute path, but for instance addlargefiles
18154
93c697d9c158 largefiles: remove trivial portability wrappers
Mads Kiilerich <madski@unity3d.com>
parents: 18153
diff changeset
   252
    #    needs it repo-relative so it can be passed to repo[None].add().  So
93c697d9c158 largefiles: remove trivial portability wrappers
Mads Kiilerich <madski@unity3d.com>
parents: 18153
diff changeset
   253
    #    leave it up to the caller to use repo.wjoin() to get an absolute path.
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   254
    # 2) Join with '/' because that's what dirstate always uses, even on
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   255
    #    Windows. Change existing separator to '/' first in case we are
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   256
    #    passed filenames from an external source (like the command line).
18151
90ad387d9245 largefiles: use constant for '.hglf/'
Mads Kiilerich <madski@unity3d.com>
parents: 18150
diff changeset
   257
    return shortnameslash + util.pconvert(filename)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   258
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   259
def isstandin(filename):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   260
    '''Return true if filename is a big file standin. filename must be
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   261
    in Mercurial's internal form (slash-separated).'''
18151
90ad387d9245 largefiles: use constant for '.hglf/'
Mads Kiilerich <madski@unity3d.com>
parents: 18150
diff changeset
   262
    return filename.startswith(shortnameslash)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   263
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   264
def splitstandin(filename):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   265
    # Split on / because that's what dirstate always uses, even on Windows.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   266
    # Change local separator to / first just in case we are passed filenames
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   267
    # from an external source (like the command line).
16066
6a42846cf769 i18n: use util.pconvert() instead of 'str.replace()' for problematic encoding
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15915
diff changeset
   268
    bits = util.pconvert(filename).split('/', 1)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   269
    if len(bits) == 2 and bits[0] == shortname:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   270
        return bits[1]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   271
    else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   272
        return None
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   273
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   274
def updatestandin(repo, standin):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   275
    file = repo.wjoin(splitstandin(standin))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   276
    if os.path.exists(file):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   277
        hash = hashfile(file)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   278
        executable = getexecutable(file)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   279
        writestandin(repo, standin, hash, executable)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   280
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   281
def readstandin(repo, filename, node=None):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   282
    '''read hex hash from standin for filename at given node, or working
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   283
    directory if no node is given'''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   284
    return repo[node][standin(filename)].data().strip()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   285
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   286
def writestandin(repo, standin, hash, executable):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   287
    '''write hash to <repo.root>/<standin>'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   288
    writehash(hash, repo.wjoin(standin), executable)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   289
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   290
def copyandhash(instream, outfile):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   291
    '''Read bytes from instream (iterable) and write them to outfile,
19002
5083baa6cbf8 largefiles: remove blecch from lfutil.copyandhash - don't close the passed fd
Mads Kiilerich <madski@unity3d.com>
parents: 19001
diff changeset
   292
    computing the SHA-1 hash of the data along the way. Return the hash.'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   293
    hasher = util.sha1('')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   294
    for data in instream:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   295
        hasher.update(data)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   296
        outfile.write(data)
18999
c1b5f9c4d989 largefiles: refactoring - return hex from _getfile and copyandhash
Mads Kiilerich <madski@unity3d.com>
parents: 18998
diff changeset
   297
    return hasher.hexdigest()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   298
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   299
def hashrepofile(repo, file):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   300
    return hashfile(repo.wjoin(file))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   301
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   302
def hashfile(file):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   303
    if not os.path.exists(file):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   304
        return ''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   305
    hasher = util.sha1('')
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   306
    fd = open(file, 'rb')
19001
2a35296a6304 largefiles: drop lfutil.blockstream - use filechunkiter like everybody else
Mads Kiilerich <madski@unity3d.com>
parents: 18999
diff changeset
   307
    for data in util.filechunkiter(fd, 128 * 1024):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   308
        hasher.update(data)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   309
    fd.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   310
    return hasher.hexdigest()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   311
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   312
class limitreader(object):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   313
    def __init__(self, f, limit):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   314
        self.f = f
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   315
        self.limit = limit
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   316
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   317
    def read(self, length):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   318
        if self.limit == 0:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   319
            return ''
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   320
        length = length > self.limit and self.limit or length
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   321
        self.limit -= length
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   322
        return self.f.read(length)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   323
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   324
    def close(self):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   325
        pass
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   326
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   327
def writehash(hash, filename, executable):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   328
    util.makedirs(os.path.dirname(filename))
15574
c9328c829cd9 largefiles: simplify lfutil.writehash
Martin Geisler <mg@aragost.com>
parents: 15572
diff changeset
   329
    util.writefile(filename, hash + '\n')
c9328c829cd9 largefiles: simplify lfutil.writehash
Martin Geisler <mg@aragost.com>
parents: 15572
diff changeset
   330
    os.chmod(filename, getmode(executable))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   331
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   332
def getexecutable(filename):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   333
    mode = os.stat(filename).st_mode
15255
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
   334
    return ((mode & stat.S_IXUSR) and
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
   335
            (mode & stat.S_IXGRP) and
7ab05d752405 largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents: 15253
diff changeset
   336
            (mode & stat.S_IXOTH))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   337
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   338
def getmode(executable):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   339
    if executable:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   340
        return 0755
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   341
    else:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   342
        return 0644
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   343
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   344
def urljoin(first, second, *arg):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   345
    def join(left, right):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   346
        if not left.endswith('/'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   347
            left += '/'
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   348
        if right.startswith('/'):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   349
            right = right[1:]
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   350
        return left + right
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   351
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   352
    url = join(first, second)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   353
    for a in arg:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   354
        url = join(url, a)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   355
    return url
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   356
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   357
def hexsha1(data):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   358
    """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   359
    object data"""
15347
799e56609ef6 largefiles: use util.sha1() instead of hashlib.sha1() everywhere
Thomas Arendsen Hein <thomas@intevation.de>
parents: 15333
diff changeset
   360
    h = util.sha1()
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   361
    for chunk in util.filechunkiter(data):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   362
        h.update(chunk)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   363
    return h.hexdigest()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   364
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   365
def httpsendfile(ui, filename):
15224
7c604d8c7e83 largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents: 15206
diff changeset
   366
    return httpconnection.httpsendfile(ui, filename, 'rb')
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   367
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   368
def unixpath(path):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15228
diff changeset
   369
    '''Return a version of path normalized for use with the lfdirstate.'''
16066
6a42846cf769 i18n: use util.pconvert() instead of 'str.replace()' for problematic encoding
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15915
diff changeset
   370
    return util.pconvert(os.path.normpath(path))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   371
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   372
def islfilesrepo(repo):
17659
ae57920ac188 largefiles: enable islfilesrepo() prior to a commit (issue3541)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17270
diff changeset
   373
    if ('largefiles' in repo.requirements and
18151
90ad387d9245 largefiles: use constant for '.hglf/'
Mads Kiilerich <madski@unity3d.com>
parents: 18150
diff changeset
   374
            util.any(shortnameslash in f[0] for f in repo.store.datafiles())):
17659
ae57920ac188 largefiles: enable islfilesrepo() prior to a commit (issue3541)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17270
diff changeset
   375
        return True
ae57920ac188 largefiles: enable islfilesrepo() prior to a commit (issue3541)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17270
diff changeset
   376
ae57920ac188 largefiles: enable islfilesrepo() prior to a commit (issue3541)
Matt Harbison <matt_harbison@yahoo.com>
parents: 17270
diff changeset
   377
    return util.any(openlfdirstate(repo.ui, repo, False))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   378
15333
f37b71fec602 largefiles: py2.4 doesn't have BaseException
Matt Mackall <mpm@selenic.com>
parents: 15320
diff changeset
   379
class storeprotonotcapable(Exception):
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   380
    def __init__(self, storetypes):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
   381
        self.storetypes = storetypes
16103
3e1efb458e8b largefiles: only cache largefiles in new heads
Na'Tosha Bard <natosha@unity3d.com>
parents: 16066
diff changeset
   382
16120
47ee41fcf42b largefiles: optimize update speed by only updating changed largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents: 16103
diff changeset
   383
def getstandinsstate(repo):
47ee41fcf42b largefiles: optimize update speed by only updating changed largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents: 16103
diff changeset
   384
    standins = []
47ee41fcf42b largefiles: optimize update speed by only updating changed largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents: 16103
diff changeset
   385
    matcher = getstandinmatcher(repo)
18154
93c697d9c158 largefiles: remove trivial portability wrappers
Mads Kiilerich <madski@unity3d.com>
parents: 18153
diff changeset
   386
    for standin in repo.dirstate.walk(matcher, [], False, False):
16120
47ee41fcf42b largefiles: optimize update speed by only updating changed largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents: 16103
diff changeset
   387
        lfile = splitstandin(standin)
18300
745bc16ccef2 largefiles: fix update from a merge with removed files
Mads Kiilerich <madski@unity3d.com>
parents: 18299
diff changeset
   388
        try:
745bc16ccef2 largefiles: fix update from a merge with removed files
Mads Kiilerich <madski@unity3d.com>
parents: 18299
diff changeset
   389
            hash = readstandin(repo, lfile)
745bc16ccef2 largefiles: fix update from a merge with removed files
Mads Kiilerich <madski@unity3d.com>
parents: 18299
diff changeset
   390
        except IOError:
745bc16ccef2 largefiles: fix update from a merge with removed files
Mads Kiilerich <madski@unity3d.com>
parents: 18299
diff changeset
   391
            hash = None
745bc16ccef2 largefiles: fix update from a merge with removed files
Mads Kiilerich <madski@unity3d.com>
parents: 18299
diff changeset
   392
        standins.append((lfile, hash))
16120
47ee41fcf42b largefiles: optimize update speed by only updating changed largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents: 16103
diff changeset
   393
    return standins
16245
a18ad914aa21 largefiles: move calculation of largefiles for updating to utility function
Na'Tosha Bard <natosha@unity3d.com>
parents: 16166
diff changeset
   394
a18ad914aa21 largefiles: move calculation of largefiles for updating to utility function
Na'Tosha Bard <natosha@unity3d.com>
parents: 16166
diff changeset
   395
def getlfilestoupdate(oldstandins, newstandins):
a18ad914aa21 largefiles: move calculation of largefiles for updating to utility function
Na'Tosha Bard <natosha@unity3d.com>
parents: 16166
diff changeset
   396
    changedstandins = set(oldstandins).symmetric_difference(set(newstandins))
a18ad914aa21 largefiles: move calculation of largefiles for updating to utility function
Na'Tosha Bard <natosha@unity3d.com>
parents: 16166
diff changeset
   397
    filelist = []
a18ad914aa21 largefiles: move calculation of largefiles for updating to utility function
Na'Tosha Bard <natosha@unity3d.com>
parents: 16166
diff changeset
   398
    for f in changedstandins:
a18ad914aa21 largefiles: move calculation of largefiles for updating to utility function
Na'Tosha Bard <natosha@unity3d.com>
parents: 16166
diff changeset
   399
        if f[0] not in filelist:
a18ad914aa21 largefiles: move calculation of largefiles for updating to utility function
Na'Tosha Bard <natosha@unity3d.com>
parents: 16166
diff changeset
   400
            filelist.append(f[0])
a18ad914aa21 largefiles: move calculation of largefiles for updating to utility function
Na'Tosha Bard <natosha@unity3d.com>
parents: 16166
diff changeset
   401
    return filelist