hgext/convert/git.py
author Mads Kiilerich <madski@unity3d.com>
Thu, 19 Mar 2015 17:40:19 +0100
changeset 24395 216fa1ba9993
parent 23206 a7f25a31e021
child 25698 307370c2dda2
permissions -rw-r--r--
convert: optimize convert of files that are unmodified from p2 in merges Conversion of a merge starts with p1 and re-adds the files that were changed in the merge or came unmodified from p2. Files that are unmodified from p1 will thus not be touched and take no time. Files that are unmodified from p2 would be retrieved and rehashed. They would end up getting the same hash as in p2 and end up reusing the filelog entry and look like the p1 case ... but it was slow. Instead, make getchanges also return 'files that are unmodified from p2' so the sink can reuse the existing p2 entry instead of calling getfile. Reuse of filelog entries can make a big difference when files are big and with long revlong chains so they take time to retrieve and hash, or when using an expensive custom getfile function (think http://mercurial.selenic.com/wiki/ConvertExtension#Customization with a code reformatter). This in combination with changes to reuse filectx entries in localrepo._filecommit make 'unchanged from p2' almost as fast as 'unchanged from p1'. This is so far only implemented for the combination of hg source and hg sink. This is a refactoring/optimization. It is covered by existing tests and show no changes - which is a good thing.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
8250
1b60efdb8bc5 convert: add copyright and license headers to back-ends
Martin Geisler <mg@lazybytes.net>
parents: 7875
diff changeset
     1
# git.py - git support for the convert extension
1b60efdb8bc5 convert: add copyright and license headers to back-ends
Martin Geisler <mg@lazybytes.net>
parents: 7875
diff changeset
     2
#
1b60efdb8bc5 convert: add copyright and license headers to back-ends
Martin Geisler <mg@lazybytes.net>
parents: 7875
diff changeset
     3
#  Copyright 2005-2009 Matt Mackall <mpm@selenic.com> and others
1b60efdb8bc5 convert: add copyright and license headers to back-ends
Martin Geisler <mg@lazybytes.net>
parents: 7875
diff changeset
     4
#
1b60efdb8bc5 convert: add copyright and license headers to back-ends
Martin Geisler <mg@lazybytes.net>
parents: 7875
diff changeset
     5
# This software may be used and distributed according to the terms of the
10263
25e572394f5c Update license to GPLv2+
Matt Mackall <mpm@selenic.com>
parents: 8456
diff changeset
     6
# GNU General Public License version 2 or any later version.
3821
158fce02dc40 Teach convert-repo to deal with mixed charsets in git
Matt Mackall <mpm@selenic.com>
parents: 2657
diff changeset
     7
4536
cc9b79216a76 Split convert extension into common and repository type modules
Brendan Cully <brendan@kublai.com>
parents: 4532
diff changeset
     8
import os
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
     9
import subprocess
17929
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    10
from mercurial import util, config
12144
be9c4131a8f4 clone, patch, convert: use hex(nullid) instead of '0'*40
Martin Geisler <mg@lazybytes.net>
parents: 11134
diff changeset
    11
from mercurial.node import hex, nullid
10939
9f6731b03906 convert: mark strings for translation
Martin Geisler <mg@lazybytes.net>
parents: 10938
diff changeset
    12
from mercurial.i18n import _
3938
0fab73b3f453 convert-repo: add some smarts
Matt Mackall <mpm@selenic.com>
parents: 3917
diff changeset
    13
5497
f0a3918abd42 convert: fail if an external required tool is not found
Patrick Mezard <pmezard@gmail.com>
parents: 5404
diff changeset
    14
from common import NoRepo, commit, converter_source, checktool
3954
9af4b853ed4d convert-repo: add CVS branch support
Matt Mackall <mpm@selenic.com>
parents: 3953
diff changeset
    15
17929
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    16
class submodule(object):
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    17
    def __init__(self, path, node, url):
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    18
        self.path = path
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    19
        self.node = node
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    20
        self.url = url
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    21
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    22
    def hgsub(self):
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    23
        return "%s = [git]%s" % (self.path, self.url)
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    24
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    25
    def hgsubstate(self):
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    26
        return "%s %s" % (self.node, self.path)
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
    27
4448
af013ae3ca10 use documented convert-repo interface
Daniel Holth <dholth@fastmail.fm>
parents: 4447
diff changeset
    28
class convert_git(converter_source):
5217
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    29
    # Windows does not support GIT_DIR= construct while other systems
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    30
    # cannot remove environment variable. Just assume none have
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    31
    # both issues.
14945
11aad09a6370 hgext: replace uses of hasattr with util.safehasattr
Augie Fackler <durin42@gmail.com>
parents: 14735
diff changeset
    32
    if util.safehasattr(os, 'unsetenv'):
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    33
        def gitopen(self, s, err=None):
5217
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    34
            prevgitdir = os.environ.get('GIT_DIR')
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    35
            os.environ['GIT_DIR'] = self.path
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    36
            try:
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    37
                if err == subprocess.PIPE:
13756
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
    38
                    (stdin, stdout, stderr) = util.popen3(s)
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
    39
                    return stdout
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    40
                elif err == subprocess.STDOUT:
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    41
                    return self.popen_with_stderr(s)
13756
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
    42
                else:
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
    43
                    return util.popen(s, 'rb')
5217
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    44
            finally:
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    45
                if prevgitdir is None:
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    46
                    del os.environ['GIT_DIR']
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    47
                else:
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    48
                    os.environ['GIT_DIR'] = prevgitdir
21630
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    49
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    50
        def gitpipe(self, s):
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    51
            prevgitdir = os.environ.get('GIT_DIR')
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    52
            os.environ['GIT_DIR'] = self.path
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    53
            try:
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    54
                return util.popen3(s)
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    55
            finally:
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    56
                if prevgitdir is None:
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    57
                    del os.environ['GIT_DIR']
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    58
                else:
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    59
                    os.environ['GIT_DIR'] = prevgitdir
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    60
5217
149742a628fd convert: fix issue702 about GIT_DIR= construct unsupported under Windows.
Patrick Mezard <pmezard@gmail.com>
parents: 5216
diff changeset
    61
    else:
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    62
        def gitopen(self, s, err=None):
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    63
            if err == subprocess.PIPE:
13756
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
    64
                (sin, so, se) = util.popen3('GIT_DIR=%s %s' % (self.path, s))
14177
fc004d16633f convert: fix error in git solaris code
timeless <timeless@mozdev.org>
parents: 13756
diff changeset
    65
                return so
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    66
            elif err == subprocess.STDOUT:
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    67
                    return self.popen_with_stderr(s)
13756
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
    68
            else:
14735
d297ee0b2d94 convert: fix git convert on solaris - it cannot remove environment variables
Mads Kiilerich <mads@kiilerich.com>
parents: 14177
diff changeset
    69
                return util.popen('GIT_DIR=%s %s' % (self.path, s), 'rb')
4767
2d0a823cbba5 convert: gitcmd wrapper for os.popen
Brendan Cully <brendan@kublai.com>
parents: 4760
diff changeset
    70
21630
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    71
        def gitpipe(self, s):
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    72
            return util.popen3('GIT_DIR=%s %s' % (self.path, s))
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
    73
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    74
    def popen_with_stderr(self, s):
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    75
        p = subprocess.Popen(s, shell=True, bufsize=-1,
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    76
                             close_fds=util.closefds,
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    77
                             stdin=subprocess.PIPE,
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    78
                             stdout=subprocess.PIPE,
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    79
                             stderr=subprocess.STDOUT,
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    80
                             universal_newlines=False,
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    81
                             env=None)
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    82
        return p.stdout
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
    83
10986
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
    84
    def gitread(self, s):
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
    85
        fh = self.gitopen(s)
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
    86
        data = fh.read()
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
    87
        return data, fh.close()
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
    88
4760
07efcce17d28 convert: add -r argument specifying latest revision to convert
Brendan Cully <brendan@kublai.com>
parents: 4759
diff changeset
    89
    def __init__(self, ui, path, rev=None):
4807
15a3cbfc6568 convert: call superclass init from engine init functions
Brendan Cully <brendan@kublai.com>
parents: 4768
diff changeset
    90
        super(convert_git, self).__init__(ui, path, rev=rev)
15a3cbfc6568 convert: call superclass init from engine init functions
Brendan Cully <brendan@kublai.com>
parents: 4768
diff changeset
    91
3938
0fab73b3f453 convert-repo: add some smarts
Matt Mackall <mpm@selenic.com>
parents: 3917
diff changeset
    92
        if os.path.isdir(path + "/.git"):
0fab73b3f453 convert-repo: add some smarts
Matt Mackall <mpm@selenic.com>
parents: 3917
diff changeset
    93
            path += "/.git"
4102
06d65498f73b convert-repo: use .git/objects/ rather than .git/HEAD to detect git repos
Matt Mackall <mpm@selenic.com>
parents: 4082
diff changeset
    94
        if not os.path.exists(path + "/objects"):
10939
9f6731b03906 convert: mark strings for translation
Martin Geisler <mg@lazybytes.net>
parents: 10938
diff changeset
    95
            raise NoRepo(_("%s does not look like a Git repository") % path)
5497
f0a3918abd42 convert: fail if an external required tool is not found
Patrick Mezard <pmezard@gmail.com>
parents: 5404
diff changeset
    96
22512
6b6da715cb96 convert: change default for git rename detection to 50%
Siddharth Agarwal <sid0@fb.com>
parents: 22511
diff changeset
    97
        # The default value (50) is based on the default for 'git diff'.
6b6da715cb96 convert: change default for git rename detection to 50%
Siddharth Agarwal <sid0@fb.com>
parents: 22511
diff changeset
    98
        similarity = ui.configint('convert', 'git.similarity', default=50)
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
    99
        if similarity < 0 or similarity > 100:
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   100
            raise util.Abort(_('similarity must be between 0 and 100'))
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   101
        if similarity > 0:
23206
a7f25a31e021 convert: use git diff-tree -Cn% instead of --find-copies=n% for older git
Thomas Arendsen Hein <thomas@intevation.de>
parents: 22512
diff changeset
   102
            self.simopt = '-C%d%%' % similarity
22471
cc5f94db672b convert: add support to find git copies from all files in the working copy
Siddharth Agarwal <sid0@fb.com>
parents: 22470
diff changeset
   103
            findcopiesharder = ui.configbool('convert', 'git.findcopiesharder',
cc5f94db672b convert: add support to find git copies from all files in the working copy
Siddharth Agarwal <sid0@fb.com>
parents: 22470
diff changeset
   104
                                             False)
cc5f94db672b convert: add support to find git copies from all files in the working copy
Siddharth Agarwal <sid0@fb.com>
parents: 22470
diff changeset
   105
            if findcopiesharder:
cc5f94db672b convert: add support to find git copies from all files in the working copy
Siddharth Agarwal <sid0@fb.com>
parents: 22470
diff changeset
   106
                self.simopt += ' --find-copies-harder'
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   107
        else:
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   108
            self.simopt = ''
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   109
6837
e30c56f337b1 convert: use git executable only, with subcommands
Dhruva Krishnamurthy <dhruvakm@gmail.com>
parents: 6001
diff changeset
   110
        checktool('git', 'git')
5497
f0a3918abd42 convert: fail if an external required tool is not found
Patrick Mezard <pmezard@gmail.com>
parents: 5404
diff changeset
   111
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   112
        self.path = path
17929
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   113
        self.submodules = []
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   114
21630
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   115
        self.catfilepipe = self.gitpipe('git cat-file --batch')
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   116
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   117
    def after(self):
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   118
        for f in self.catfilepipe:
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   119
            f.close()
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   120
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   121
    def getheads(self):
4768
f52bfe566583 convert: import all branches from git repositories
Brendan Cully <brendan@kublai.com>
parents: 4767
diff changeset
   122
        if not self.rev:
10986
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   123
            heads, ret = self.gitread('git rev-parse --branches --remotes')
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   124
            heads = heads.splitlines()
4768
f52bfe566583 convert: import all branches from git repositories
Brendan Cully <brendan@kublai.com>
parents: 4767
diff changeset
   125
        else:
10986
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   126
            heads, ret = self.gitread("git rev-parse --verify %s" % self.rev)
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   127
            heads = [heads[:-1]]
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   128
        if ret:
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   129
            raise util.Abort(_('cannot retrieve git heads'))
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   130
        return heads
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   131
692
695dd9a491da convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents: 450
diff changeset
   132
    def catfile(self, rev, type):
12144
be9c4131a8f4 clone, patch, convert: use hex(nullid) instead of '0'*40
Martin Geisler <mg@lazybytes.net>
parents: 11134
diff changeset
   133
        if rev == hex(nullid):
16687
e34106fa0dc3 cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents: 16683
diff changeset
   134
            raise IOError
21630
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   135
        self.catfilepipe[0].write(rev+'\n')
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   136
        self.catfilepipe[0].flush()
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   137
        info = self.catfilepipe[1].readline().split()
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   138
        if info[1] != type:
10986
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   139
            raise util.Abort(_('cannot read %r object at %s') % (type, rev))
21630
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   140
        size = int(info[2])
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   141
        data = self.catfilepipe[1].read(size)
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   142
        if len(data) < size:
21958
9663a7855798 convert: fix argument mismatch at formatting the abort message
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 21873
diff changeset
   143
            raise util.Abort(_('cannot read %r object at %s: unexpected size')
9663a7855798 convert: fix argument mismatch at formatting the abort message
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 21873
diff changeset
   144
                             % (type, rev))
21630
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   145
        # read the trailing newline
a204fd9b5ba9 convert: drastically speed up git conversions
David Schleimer <dschleimer@fb.com>
parents: 20373
diff changeset
   146
        self.catfilepipe[1].read(1)
10986
610f047326b9 convert/git: check status when reading the whole output
Patrick Mezard <pmezard@gmail.com>
parents: 10985
diff changeset
   147
        return data
692
695dd9a491da convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents: 450
diff changeset
   148
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   149
    def getfile(self, name, rev):
21868
3420346174b1 convert: detect removal of ".gitmodules" at git source revisions correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20373
diff changeset
   150
        if rev == hex(nullid):
22296
650b5b6e75ed convert: use None value for missing files instead of overloading IOError
Mads Kiilerich <madski@unity3d.com>
parents: 21958
diff changeset
   151
            return None, None
17929
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   152
        if name == '.hgsub':
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   153
            data = '\n'.join([m.hgsub() for m in self.submoditer()])
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   154
            mode = ''
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   155
        elif name == '.hgsubstate':
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   156
            data = '\n'.join([m.hgsubstate() for m in self.submoditer()])
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   157
            mode = ''
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   158
        else:
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   159
            data = self.catfile(rev, "blob")
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   160
            mode = self.modecache[(name, rev)]
11134
33010ff1fd6f convert: merge sources getmode() into getfile()
Patrick Mezard <pmezard@gmail.com>
parents: 10987
diff changeset
   161
        return data, mode
3956
558f52943cd2 convert-repo: add CVS exec bit support
Matt Mackall <mpm@selenic.com>
parents: 3954
diff changeset
   162
17929
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   163
    def submoditer(self):
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   164
        null = hex(nullid)
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   165
        for m in sorted(self.submodules, key=lambda p: p.path):
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   166
            if m.node != null:
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   167
                yield m
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   168
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   169
    def parsegitmodules(self, content):
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   170
        """Parse the formatted .gitmodules file, example file format:
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   171
        [submodule "sub"]\n
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   172
        \tpath = sub\n
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   173
        \turl = git://giturl\n
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   174
        """
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   175
        self.submodules = []
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   176
        c = config.config()
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   177
        # Each item in .gitmodules starts with \t that cant be parsed
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   178
        c.parse('.gitmodules', content.replace('\t',''))
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   179
        for sec in c.sections():
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   180
            s = c[sec]
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   181
            if 'url' in s and 'path' in s:
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   182
                self.submodules.append(submodule(s['path'], '', s['url']))
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   183
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   184
    def retrievegitmodules(self, version):
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   185
        modules, ret = self.gitread("git show %s:%s" % (version, '.gitmodules'))
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   186
        if ret:
17930
7788b5e7d9ef convert: fix a too-long line nag
Bryan O'Sullivan <bryano@fb.com>
parents: 17929
diff changeset
   187
            raise util.Abort(_('cannot read submodules config file in %s') %
7788b5e7d9ef convert: fix a too-long line nag
Bryan O'Sullivan <bryano@fb.com>
parents: 17929
diff changeset
   188
                             version)
17929
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   189
        self.parsegitmodules(modules)
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   190
        for m in self.submodules:
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   191
            node, ret = self.gitread("git rev-parse %s:%s" % (version, m.path))
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   192
            if ret:
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   193
                continue
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   194
            m.node = node.strip()
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   195
22300
35ab037de989 convert: introduce --full for converting all files
Mads Kiilerich <madski@unity3d.com>
parents: 22296
diff changeset
   196
    def getchanges(self, version, full):
35ab037de989 convert: introduce --full for converting all files
Mads Kiilerich <madski@unity3d.com>
parents: 22296
diff changeset
   197
        if full:
35ab037de989 convert: introduce --full for converting all files
Mads Kiilerich <madski@unity3d.com>
parents: 22296
diff changeset
   198
            raise util.Abort(_("convert from git do not support --full"))
3956
558f52943cd2 convert-repo: add CVS exec bit support
Matt Mackall <mpm@selenic.com>
parents: 3954
diff changeset
   199
        self.modecache = {}
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   200
        fh = self.gitopen("git diff-tree -z --root -m -r %s %s" % (
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   201
            self.simopt, version))
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   202
        changes = []
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   203
        copies = {}
8456
e9e2a2c9b294 convert: use set instead of dict
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8271
diff changeset
   204
        seen = set()
7242
d1dff8c492dd convert: fix non-ASCII filenames retrieval from git sources (issue 1360)
Patrick Mezard <pmezard@gmail.com>
parents: 7222
diff changeset
   205
        entry = None
22469
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   206
        subexists = [False]
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   207
        subdeleted = [False]
22467
333d654783ad convert: for git's getchanges, use explicit index for iteration
Siddharth Agarwal <sid0@fb.com>
parents: 22413
diff changeset
   208
        difftree = fh.read().split('\x00')
333d654783ad convert: for git's getchanges, use explicit index for iteration
Siddharth Agarwal <sid0@fb.com>
parents: 22413
diff changeset
   209
        lcount = len(difftree)
333d654783ad convert: for git's getchanges, use explicit index for iteration
Siddharth Agarwal <sid0@fb.com>
parents: 22413
diff changeset
   210
        i = 0
22469
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   211
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   212
        def add(entry, f, isdest):
22469
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   213
            seen.add(f)
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   214
            h = entry[3]
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   215
            p = (entry[1] == "100755")
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   216
            s = (entry[1] == "120000")
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   217
            renamesource = (not isdest and entry[4][0] == 'R')
22469
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   218
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   219
            if f == '.gitmodules':
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   220
                subexists[0] = True
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   221
                if entry[4] == 'D' or renamesource:
22469
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   222
                    subdeleted[0] = True
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   223
                    changes.append(('.hgsub', hex(nullid)))
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   224
                else:
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   225
                    changes.append(('.hgsub', ''))
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   226
            elif entry[1] == '160000' or entry[0] == ':160000':
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   227
                subexists[0] = True
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   228
            else:
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   229
                if renamesource:
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   230
                    h = hex(nullid)
22469
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   231
                self.modecache[(f, h)] = (p and "x") or (s and "l") or ""
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   232
                changes.append((f, h))
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   233
22467
333d654783ad convert: for git's getchanges, use explicit index for iteration
Siddharth Agarwal <sid0@fb.com>
parents: 22413
diff changeset
   234
        while i < lcount:
333d654783ad convert: for git's getchanges, use explicit index for iteration
Siddharth Agarwal <sid0@fb.com>
parents: 22413
diff changeset
   235
            l = difftree[i]
333d654783ad convert: for git's getchanges, use explicit index for iteration
Siddharth Agarwal <sid0@fb.com>
parents: 22413
diff changeset
   236
            i += 1
7242
d1dff8c492dd convert: fix non-ASCII filenames retrieval from git sources (issue 1360)
Patrick Mezard <pmezard@gmail.com>
parents: 7222
diff changeset
   237
            if not entry:
d1dff8c492dd convert: fix non-ASCII filenames retrieval from git sources (issue 1360)
Patrick Mezard <pmezard@gmail.com>
parents: 7222
diff changeset
   238
                if not l.startswith(':'):
d1dff8c492dd convert: fix non-ASCII filenames retrieval from git sources (issue 1360)
Patrick Mezard <pmezard@gmail.com>
parents: 7222
diff changeset
   239
                    continue
22468
5910184f1f7b convert: for git's getchanges, always split entry line into components
Siddharth Agarwal <sid0@fb.com>
parents: 22467
diff changeset
   240
                entry = l.split()
5335
88e931f74e8b convert_git: avoid returning two entries for the same file in getchanges
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5220
diff changeset
   241
                continue
7242
d1dff8c492dd convert: fix non-ASCII filenames retrieval from git sources (issue 1360)
Patrick Mezard <pmezard@gmail.com>
parents: 7222
diff changeset
   242
            f = l
d1dff8c492dd convert: fix non-ASCII filenames retrieval from git sources (issue 1360)
Patrick Mezard <pmezard@gmail.com>
parents: 7222
diff changeset
   243
            if f not in seen:
22470
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   244
                add(entry, f, False)
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   245
            # A file can be copied multiple times, or modified and copied
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   246
            # simultaneously. So f can be repeated even if fdest isn't.
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   247
            if entry[4][0] in 'RC':
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   248
                # rename or copy: next line is the destination
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   249
                fdest = difftree[i]
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   250
                i += 1
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   251
                if fdest not in seen:
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   252
                    add(entry, fdest, True)
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   253
                    # .gitmodules isn't imported at all, so it being copied to
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   254
                    # and fro doesn't really make sense
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   255
                    if f != '.gitmodules' and fdest != '.gitmodules':
8e0c4df28eec convert: add support to detect git renames and copies
Siddharth Agarwal <sid0@fb.com>
parents: 22469
diff changeset
   256
                        copies[fdest] = f
7242
d1dff8c492dd convert: fix non-ASCII filenames retrieval from git sources (issue 1360)
Patrick Mezard <pmezard@gmail.com>
parents: 7222
diff changeset
   257
            entry = None
10987
b3af02b1f19f convert/git: check status when reading output stream
Patrick Mezard <pmezard@gmail.com>
parents: 10986
diff changeset
   258
        if fh.close():
b3af02b1f19f convert/git: check status when reading output stream
Patrick Mezard <pmezard@gmail.com>
parents: 10986
diff changeset
   259
            raise util.Abort(_('cannot read changes in %s') % version)
17929
0eed66327ad4 convert: add support for converting git submodule (issue3528)
YaNan Xu <robot9@fb.com>
parents: 16689
diff changeset
   260
22469
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   261
        if subexists[0]:
15bc0431476b convert: for git, factor out code to add entries to a separate function
Siddharth Agarwal <sid0@fb.com>
parents: 22468
diff changeset
   262
            if subdeleted[0]:
21868
3420346174b1 convert: detect removal of ".gitmodules" at git source revisions correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20373
diff changeset
   263
                changes.append(('.hgsubstate', hex(nullid)))
3420346174b1 convert: detect removal of ".gitmodules" at git source revisions correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20373
diff changeset
   264
            else:
3420346174b1 convert: detect removal of ".gitmodules" at git source revisions correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20373
diff changeset
   265
                self.retrievegitmodules(version)
3420346174b1 convert: detect removal of ".gitmodules" at git source revisions correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20373
diff changeset
   266
                changes.append(('.hgsubstate', ''))
24395
216fa1ba9993 convert: optimize convert of files that are unmodified from p2 in merges
Mads Kiilerich <madski@unity3d.com>
parents: 23206
diff changeset
   267
        return (changes, copies, set())
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   268
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   269
    def getcommit(self, version):
692
695dd9a491da convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents: 450
diff changeset
   270
        c = self.catfile(version, "commit") # read the commit hash
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   271
        end = c.find("\n\n")
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10263
diff changeset
   272
        message = c[end + 2:]
4759
20ec5cc02f18 convert: ove recode method into converter_source
Brendan Cully <brendan@kublai.com>
parents: 4721
diff changeset
   273
        message = self.recode(message)
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   274
        l = c[:end].splitlines()
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   275
        parents = []
8271
e3d3dad805f9 Add committer tag only when needed in git conversion
Richard Quirk <richard.quirk@gmail.com>
parents: 8250
diff changeset
   276
        author = committer = None
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   277
        for e in l[1:]:
4532
c3a78a49d7f0 Some small cleanups for convert extension:
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4521
diff changeset
   278
            n, v = e.split(" ", 1)
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   279
            if n == "author":
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   280
                p = v.split()
1385
adb3de56635b convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents: 1335
diff changeset
   281
                tm, tz = p[-2:]
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   282
                author = " ".join(p[:-2])
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   283
                if author[0] == "<": author = author[1:-1]
4759
20ec5cc02f18 convert: ove recode method into converter_source
Brendan Cully <brendan@kublai.com>
parents: 4721
diff changeset
   284
                author = self.recode(author)
692
695dd9a491da convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents: 450
diff changeset
   285
            if n == "committer":
431
dfc44f3f587c convert-repo fixups
mpm@selenic.com
parents: 316
diff changeset
   286
                p = v.split()
1385
adb3de56635b convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents: 1335
diff changeset
   287
                tm, tz = p[-2:]
431
dfc44f3f587c convert-repo fixups
mpm@selenic.com
parents: 316
diff changeset
   288
                committer = " ".join(p[:-2])
dfc44f3f587c convert-repo fixups
mpm@selenic.com
parents: 316
diff changeset
   289
                if committer[0] == "<": committer = committer[1:-1]
4759
20ec5cc02f18 convert: ove recode method into converter_source
Brendan Cully <brendan@kublai.com>
parents: 4721
diff changeset
   290
                committer = self.recode(committer)
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10263
diff changeset
   291
            if n == "parent":
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10263
diff changeset
   292
                parents.append(v)
1385
adb3de56635b convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents: 1335
diff changeset
   293
8271
e3d3dad805f9 Add committer tag only when needed in git conversion
Richard Quirk <richard.quirk@gmail.com>
parents: 8250
diff changeset
   294
        if committer and committer != author:
e3d3dad805f9 Add committer tag only when needed in git conversion
Richard Quirk <richard.quirk@gmail.com>
parents: 8250
diff changeset
   295
            message += "\ncommitter: %s\n" % committer
1385
adb3de56635b convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents: 1335
diff changeset
   296
        tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
2093
5cc414722587 convert-repo: fix reversed time zone offset
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1715
diff changeset
   297
        tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
1385
adb3de56635b convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents: 1335
diff changeset
   298
        date = tm + " " + str(tz)
3954
9af4b853ed4d convert-repo: add CVS branch support
Matt Mackall <mpm@selenic.com>
parents: 3953
diff changeset
   299
4873
28b23b9073a8 convert: record the source revision in the changelog
Brendan Cully <brendan@kublai.com>
parents: 4810
diff changeset
   300
        c = commit(parents=parents, date=date, author=author, desc=message,
28b23b9073a8 convert: record the source revision in the changelog
Brendan Cully <brendan@kublai.com>
parents: 4810
diff changeset
   301
                   rev=version)
3954
9af4b853ed4d convert-repo: add CVS branch support
Matt Mackall <mpm@selenic.com>
parents: 3953
diff changeset
   302
        return c
316
c48d069163d6 Add new convert-repo script
mpm@selenic.com
parents:
diff changeset
   303
22413
3cb0318bb2dd convert: enable deterministic conversion progress bar for git
Augie Fackler <raf@durin42.com>
parents: 22300
diff changeset
   304
    def numcommits(self):
3cb0318bb2dd convert: enable deterministic conversion progress bar for git
Augie Fackler <raf@durin42.com>
parents: 22300
diff changeset
   305
        return len([None for _ in self.gitopen('git rev-list --all')])
3cb0318bb2dd convert: enable deterministic conversion progress bar for git
Augie Fackler <raf@durin42.com>
parents: 22300
diff changeset
   306
694
51eb248d3348 Teach convert-repo about tags
mpm@selenic.com
parents: 692
diff changeset
   307
    def gettags(self):
51eb248d3348 Teach convert-repo about tags
mpm@selenic.com
parents: 692
diff changeset
   308
        tags = {}
16259
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   309
        alltags = {}
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
   310
        fh = self.gitopen('git ls-remote --tags "%s"' % self.path,
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
   311
                          err=subprocess.STDOUT)
4062
516f883e3d79 convert-repo: handle packed git tags
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4047
diff changeset
   312
        prefix = 'refs/tags/'
16259
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   313
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   314
        # Build complete list of tags, both annotated and bare ones
4062
516f883e3d79 convert-repo: handle packed git tags
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4047
diff changeset
   315
        for line in fh:
516f883e3d79 convert-repo: handle packed git tags
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4047
diff changeset
   316
            line = line.strip()
18572
5fe58f9332a4 git convert: some versions of git use fatal: instead of error:
Augie Fackler <raf@durin42.com>
parents: 18570
diff changeset
   317
            if line.startswith("error:") or line.startswith("fatal:"):
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
   318
                raise util.Abort(_('cannot read tags from %s') % self.path)
4062
516f883e3d79 convert-repo: handle packed git tags
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4047
diff changeset
   319
            node, tag = line.split(None, 1)
516f883e3d79 convert-repo: handle packed git tags
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4047
diff changeset
   320
            if not tag.startswith(prefix):
516f883e3d79 convert-repo: handle packed git tags
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4047
diff changeset
   321
                continue
16259
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   322
            alltags[tag[len(prefix):]] = node
10987
b3af02b1f19f convert/git: check status when reading output stream
Patrick Mezard <pmezard@gmail.com>
parents: 10986
diff changeset
   323
        if fh.close():
b3af02b1f19f convert/git: check status when reading output stream
Patrick Mezard <pmezard@gmail.com>
parents: 10986
diff changeset
   324
            raise util.Abort(_('cannot read tags from %s') % self.path)
4062
516f883e3d79 convert-repo: handle packed git tags
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4047
diff changeset
   325
16259
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   326
        # Filter out tag objects for annotated tag refs
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   327
        for tag in alltags:
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   328
            if tag.endswith('^{}'):
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   329
                tags[tag[:-3]] = alltags[tag]
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   330
            else:
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   331
                if tag + '^{}' in alltags:
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   332
                    continue
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   333
                else:
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   334
                    tags[tag] = alltags[tag]
589aab2ca716 convert: support non annotated tags in git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 14945
diff changeset
   335
694
51eb248d3348 Teach convert-repo about tags
mpm@selenic.com
parents: 692
diff changeset
   336
        return tags
5380
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   337
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   338
    def getchangedfiles(self, version, i):
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   339
        changes = []
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   340
        if i is None:
10985
7fab6ae3f688 convert/git: rename gitcmd() into gitopen() for readability
Patrick Mezard <pmezard@gmail.com>
parents: 10939
diff changeset
   341
            fh = self.gitopen("git diff-tree --root -m -r %s" % version)
5380
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   342
            for l in fh:
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   343
                if "\t" not in l:
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   344
                    continue
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   345
                m, f = l[:-1].split("\t")
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   346
                changes.append(f)
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   347
        else:
16683
525fdb738975 cleanup: eradicate long lines
Brodie Rao <brodie@sf.io>
parents: 16295
diff changeset
   348
            fh = self.gitopen('git diff-tree --name-only --root -r %s '
525fdb738975 cleanup: eradicate long lines
Brodie Rao <brodie@sf.io>
parents: 16295
diff changeset
   349
                              '"%s^%s" --' % (version, version, i + 1))
5380
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   350
            changes = [f.rstrip('\n') for f in fh]
10987
b3af02b1f19f convert/git: check status when reading output stream
Patrick Mezard <pmezard@gmail.com>
parents: 10986
diff changeset
   351
        if fh.close():
b3af02b1f19f convert/git: check status when reading output stream
Patrick Mezard <pmezard@gmail.com>
parents: 10986
diff changeset
   352
            raise util.Abort(_('cannot read changes in %s') % version)
5380
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   353
a5a7f7fd5554 convert_git: add --filemap support
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 5336
diff changeset
   354
        return changes
13756
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   355
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   356
    def getbookmarks(self):
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   357
        bookmarks = {}
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   358
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   359
        # Interesting references in git are prefixed
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   360
        prefix = 'refs/heads/'
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   361
        prefixlen = len(prefix)
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   362
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   363
        # factor two commands
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   364
        gitcmd = { 'remote/': 'git ls-remote --heads origin',
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   365
                          '': 'git show-ref'}
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   366
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   367
        # Origin heads
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   368
        for reftype in gitcmd:
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   369
            try:
18570
dcf2d6fdf630 convert/git: catch errors from modern git-ls-remote (issue3428)
Ross Lagerwall <rosslagerwall@gmail.com>
parents: 17930
diff changeset
   370
                fh = self.gitopen(gitcmd[reftype], err=subprocess.PIPE)
13756
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   371
                for line in fh:
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   372
                    line = line.strip()
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   373
                    rev, name = line.split(None, 1)
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   374
                    if not name.startswith(prefix):
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   375
                        continue
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   376
                    name = '%s%s' % (reftype, name[prefixlen:])
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   377
                    bookmarks[name] = rev
16689
f366d4c2ff34 cleanup: replace naked excepts with except Exception: ...
Brodie Rao <brodie@sf.io>
parents: 16687
diff changeset
   378
            except Exception:
13756
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   379
                pass
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   380
6b7077df4aa5 convert: add bookmarks reading support to git backend
Edouard Gomez <ed.gomez@free.fr>
parents: 12144
diff changeset
   381
        return bookmarks
19121
478a04605ce1 splicemap: improve error handling when source is git (issue2084)
Ben Goswami <bengoswami@fb.com>
parents: 18572
diff changeset
   382
20373
e8203629371b convert: add mapname parameter to checkrevformat
Sean Farley <sean.michael.farley@gmail.com>
parents: 19121
diff changeset
   383
    def checkrevformat(self, revstr, mapname='splicemap'):
19121
478a04605ce1 splicemap: improve error handling when source is git (issue2084)
Ben Goswami <bengoswami@fb.com>
parents: 18572
diff changeset
   384
        """ git revision string is a 40 byte hex """
20373
e8203629371b convert: add mapname parameter to checkrevformat
Sean Farley <sean.michael.farley@gmail.com>
parents: 19121
diff changeset
   385
        self.checkhexformat(revstr, mapname)