convert: add commandline.xargs(), use it in svn_sink class
authorMaxim Dounin <mdounin@mdounin.ru>
Fri, 11 Jan 2008 06:07:43 +0300
changeset 5832 2192ed187319
parent 5831 4a40b7066525
child 5833 323b9c55b328
convert: add commandline.xargs(), use it in svn_sink class Introduce commandline.xargs() to limit argument list with respect to ARG_MAX bytes. If no ARG_MAX information available - use POSIX required minimum of 4096 bytes. Under Windows, while actual argument list length is limited to 32k, shells impose their own limits on command line length, down to 2047 bytes for cmd.exe under Windows NT/2k and about 2500 bytes for older 4nt.exe. See http://support.microsoft.com/kb/830473 for details about cmd.exe limitations. Since ARG_MAX is limit for argument list and environment, we reserve half of it and one byte for environment variables. This way with default ARG_MAX (4096 bytes) we get value 2047 bytes which is OK for Windows too.
hgext/convert/common.py
hgext/convert/subversion.py
--- a/hgext/convert/common.py	Fri Jan 11 15:01:29 2008 +0100
+++ b/hgext/convert/common.py	Fri Jan 11 06:07:43 2008 +0300
@@ -1,5 +1,6 @@
 # common code for the convert extension
 import base64, errno
+import os
 import cPickle as pickle
 from mercurial import util
 from mercurial.i18n import _
@@ -212,7 +213,7 @@
     def postrun(self):
         pass
 
-    def _run(self, cmd, *args, **kwargs):
+    def _cmdline(self, cmd, *args, **kwargs):
         cmdline = [self.command, cmd] + list(args)
         for k, v in kwargs.iteritems():
             if len(k) == 1:
@@ -230,7 +231,10 @@
         cmdline += ['<', util.nulldev]
         cmdline = ' '.join(cmdline)
         self.ui.debug(cmdline, '\n')
+        return cmdline
 
+    def _run(self, cmd, *args, **kwargs):
+        cmdline = self._cmdline(cmd, *args, **kwargs)
         self.prerun()
         try:
             return util.popen(cmdline)
@@ -256,6 +260,47 @@
         self.checkexit(status, output)
         return output
 
+    def getargmax(self):
+        if '_argmax' in self.__dict__:
+            return self._argmax
+
+        # POSIX requires at least 4096 bytes for ARG_MAX
+        self._argmax = 4096
+        try:
+            self._argmax = os.sysconf("SC_ARG_MAX")
+        except:
+            pass
+
+        # Windows shells impose their own limits on command line length,
+        # down to 2047 bytes for cmd.exe under Windows NT/2k and 2500 bytes
+        # for older 4nt.exe. See http://support.microsoft.com/kb/830473 for
+        # details about cmd.exe limitations.
+
+        # Since ARG_MAX is for command line _and_ environment, lower our limit
+        # (and make happy Windows shells while doing this).
+
+        self._argmax = self._argmax/2 - 1
+        return self._argmax
+
+    def limit_arglist(self, arglist, cmd, *args, **kwargs):
+        limit = self.getargmax() - len(self._cmdline(cmd, *args, **kwargs))
+        bytes = 0
+        fl = []
+        for fn in arglist:
+            b = len(fn) + 3
+            if bytes + b < limit or len(fl) == 0:
+                fl.append(fn)
+                bytes += b
+            else:
+                yield fl
+                fl = [fn]
+                bytes = b
+        if fl:
+            yield fl
+
+    def xargs(self, arglist, cmd, *args, **kwargs):
+        for l in self.limit_arglist(arglist, cmd, *args, **kwargs):
+            self.run0(cmd, *(list(args) + l), **kwargs)
 
 class mapfile(dict):
     def __init__(self, ui, path):
--- a/hgext/convert/subversion.py	Fri Jan 11 15:01:29 2008 +0100
+++ b/hgext/convert/subversion.py	Fri Jan 11 06:07:43 2008 +0300
@@ -707,27 +707,6 @@
 class svn_sink(converter_sink, commandline):
     commit_re = re.compile(r'Committed revision (\d+).', re.M)
 
-    # iterates sublist of given list for concatenated length is within limit
-    def limit_arglist(self, files):
-        if os.name != 'nt':
-            yield files
-            return
-        # When I tested on WinXP, limit = 2500 is NG, 2400 is OK
-        limit = 2000
-        bytes = 0
-        fl = []
-        for fn in files:
-            b = len(fn) + 1
-            if bytes + b < limit:
-                fl.append(fn)
-                bytes += b
-            else:
-                yield fl
-                fl = [fn]
-                bytes = b
-        if fl:
-            yield fl
-
     def prerun(self):
         if self.wc:
             os.chdir(self.wc)
@@ -866,14 +845,12 @@
                     if not os.path.exists(self.wjoin(d, '.svn', 'entries'))]
         if add_dirs:
             add_dirs.sort()
-            for fl in self.limit_arglist(add_dirs):
-                self.run('add', non_recursive=True, quiet=True, *fl)
+            self.xargs(add_dirs, 'add', non_recursive=True, quiet=True)
         return add_dirs
 
     def add_files(self, files):
         if files:
-            for fl in self.limit_arglist(files):
-                self.run('add', quiet=True, *fl)
+            self.xargs(files, 'add', quiet=True)
         return files
 
     def tidy_dirs(self, names):
@@ -907,18 +884,15 @@
                 self._copyfile(s, d)
             self.copies = []
         if self.delete:
-            for fl in self.limit_arglist(self.delete):
-                self.run0('delete', *fl)
+            self.xargs(self.delete, 'delete')
             self.delete = []
         entries.update(self.add_files(files.difference(entries)))
         entries.update(self.tidy_dirs(entries))
         if self.delexec:
-            for fl in self.limit_arglist(self.delexec):
-                self.run0('propdel', 'svn:executable', *fl)
+            self.xargs(self.delexec, 'propdel', 'svn:executable')
             self.delexec = []
         if self.setexec:
-            for fl in self.limit_arglist(self.setexec):
-                self.run0('propset', 'svn:executable', '*', *fl)
+            self.xargs(self.setexec, 'propset', 'svn:executable', '*')
             self.setexec = []
 
         fd, messagefile = tempfile.mkstemp(prefix='hg-convert-')