Merge crew and main.
--- a/mercurial/help/config.txt Sat Feb 09 22:54:34 2013 +0000
+++ b/mercurial/help/config.txt Sun Feb 10 04:04:22 2013 -0600
@@ -1463,3 +1463,15 @@
``templates``
Where to find the HTML templates. Default is install path.
+
+``worker``
+----------
+
+Parallel master/worker configuration. We currently perform working
+directory updates in parallel on Unix-like systems, which greatly
+helps performance.
+
+``numcpus``
+ Number of CPUs to use for parallel operations. Default is 4 or the
+ number of CPUs on the system, whichever is larger. A zero or
+ negative value is treated as ``use the default``.
--- a/mercurial/merge.py Sat Feb 09 22:54:34 2013 +0000
+++ b/mercurial/merge.py Sun Feb 10 04:04:22 2013 -0600
@@ -7,7 +7,7 @@
from node import nullid, nullrev, hex, bin
from i18n import _
-import error, util, filemerge, copies, subrepo
+import error, util, filemerge, copies, subrepo, worker
import errno, os, shutil
class mergestate(object):
@@ -342,6 +342,42 @@
def actionkey(a):
return a[1] == "r" and -1 or 0, a
+def getremove(repo, mctx, overwrite, args):
+ """apply usually-non-interactive updates to the working directory
+
+ mctx is the context to be merged into the working copy
+
+ yields tuples for progress updates
+ """
+ verbose = repo.ui.verbose
+ unlink = util.unlinkpath
+ wjoin = repo.wjoin
+ fctx = mctx.filectx
+ wwrite = repo.wwrite
+ audit = repo.wopener.audit
+ i = 0
+ for arg in args:
+ f = arg[0]
+ if arg[1] == 'r':
+ if verbose:
+ repo.ui.note(_("removing %s\n") % f)
+ audit(f)
+ try:
+ unlink(wjoin(f), ignoremissing=True)
+ except OSError, inst:
+ repo.ui.warn(_("update failed to remove %s: %s!\n") %
+ (f, inst.strerror))
+ else:
+ if verbose:
+ repo.ui.note(_("getting %s\n") % f)
+ wwrite(f, fctx(f).data(), arg[2][0])
+ if i == 100:
+ yield i, f
+ i = 0
+ i += 1
+ if i > 0:
+ yield i, f
+
def applyupdates(repo, actions, wctx, mctx, actx, overwrite):
"""apply the merge action list to the working directory
@@ -393,22 +429,34 @@
util.unlinkpath(repo.wjoin(f))
numupdates = len(actions)
+ workeractions = [a for a in actions if a[1] in 'gr']
+ updated = len([a for a in workeractions if a[1] == 'g'])
+ removed = len([a for a in workeractions if a[1] == 'r'])
+ actions = [a for a in actions if a[1] not in 'gr']
+
+ hgsub = [a[1] for a in workeractions if a[0] == '.hgsubstate']
+ if hgsub and hgsub[0] == 'r':
+ subrepo.submerge(repo, wctx, mctx, wctx, overwrite)
+
+ z = 0
+ prog = worker.worker(repo.ui, 0.001, getremove, (repo, mctx, overwrite),
+ workeractions)
+ for i, item in prog:
+ z += i
+ repo.ui.progress(_('updating'), z, item=item, total=numupdates,
+ unit=_('files'))
+
+ if hgsub and hgsub[0] == 'g':
+ subrepo.submerge(repo, wctx, mctx, wctx, overwrite)
+
+ _updating = _('updating')
+ _files = _('files')
+ progress = repo.ui.progress
+
for i, a in enumerate(actions):
f, m, args, msg = a
- repo.ui.progress(_('updating'), i + 1, item=f, total=numupdates,
- unit=_('files'))
- if m == "r": # remove
- repo.ui.note(_("removing %s\n") % f)
- audit(f)
- if f == '.hgsubstate': # subrepo states need updating
- subrepo.submerge(repo, wctx, mctx, wctx, overwrite)
- try:
- util.unlinkpath(repo.wjoin(f), ignoremissing=True)
- except OSError, inst:
- repo.ui.warn(_("update failed to remove %s: %s!\n") %
- (f, inst.strerror))
- removed += 1
- elif m == "m": # merge
+ progress(_updating, z + i + 1, item=f, total=numupdates, unit=_files)
+ if m == "m": # merge
if fd == '.hgsubstate': # subrepo states need updating
subrepo.submerge(repo, wctx, mctx, wctx.ancestor(mctx),
overwrite)
@@ -423,13 +471,6 @@
updated += 1
else:
merged += 1
- elif m == "g": # get
- flags, = args
- repo.ui.note(_("getting %s\n") % f)
- repo.wwrite(f, mctx.filectx(f).data(), flags)
- updated += 1
- if f == '.hgsubstate': # subrepo states need updating
- subrepo.submerge(repo, wctx, mctx, wctx, overwrite)
elif m == "d": # directory rename
f2, fd, flags = args
if f:
@@ -459,7 +500,7 @@
util.setflags(repo.wjoin(f), 'l' in flags, 'x' in flags)
updated += 1
ms.commit()
- repo.ui.progress(_('updating'), None, total=numupdates, unit=_('files'))
+ progress(_updating, None, total=numupdates, unit=_files)
return updated, merged, removed, unresolved
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/worker.py Sun Feb 10 04:04:22 2013 -0600
@@ -0,0 +1,123 @@
+# worker.py - master-slave parallelism support
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from i18n import _
+import os, signal, sys, util
+
+def countcpus():
+ '''try to count the number of CPUs on the system'''
+
+ # posix
+ try:
+ n = int(os.sysconf('SC_NPROCESSORS_ONLN'))
+ if n > 0:
+ return n
+ except (AttributeError, ValueError):
+ pass
+
+ # windows
+ try:
+ n = int(os.environ['NUMBER_OF_PROCESSORS'])
+ if n > 0:
+ return n
+ except (KeyError, ValueError):
+ pass
+
+ return 1
+
+def _numworkers(ui):
+ s = ui.config('worker', 'numcpus')
+ if s:
+ try:
+ n = int(s)
+ if n >= 1:
+ return n
+ except ValueError:
+ raise util.Abort(_('number of cpus must be an integer'))
+ return min(max(countcpus(), 4), 32)
+
+if os.name == 'posix':
+ _startupcost = 0.01
+else:
+ _startupcost = 1e30
+
+def worthwhile(ui, costperop, nops):
+ '''try to determine whether the benefit of multiple processes can
+ outweigh the cost of starting them'''
+ linear = costperop * nops
+ workers = _numworkers(ui)
+ benefit = linear - (_startupcost * workers + linear / workers)
+ return benefit >= 0.15
+
+def worker(ui, costperarg, func, staticargs, args):
+ '''run a function, possibly in parallel in multiple worker
+ processes.
+
+ returns a progress iterator
+
+ costperarg - cost of a single task
+
+ func - function to run
+
+ staticargs - arguments to pass to every invocation of the function
+
+ args - arguments to split into chunks, to pass to individual
+ workers
+ '''
+ if worthwhile(ui, costperarg, len(args)):
+ return _platformworker(ui, func, staticargs, args)
+ return func(*staticargs + (args,))
+
+def _posixworker(ui, func, staticargs, args):
+ rfd, wfd = os.pipe()
+ workers = _numworkers(ui)
+ for pargs in partition(args, workers):
+ pid = os.fork()
+ if pid == 0:
+ try:
+ os.close(rfd)
+ for i, item in func(*(staticargs + (pargs,))):
+ os.write(wfd, '%d %s\n' % (i, item))
+ os._exit(0)
+ except KeyboardInterrupt:
+ os._exit(255)
+ os.close(wfd)
+ fp = os.fdopen(rfd, 'rb', 0)
+ oldhandler = signal.getsignal(signal.SIGINT)
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ def cleanup():
+ # python 2.4 is too dumb for try/yield/finally
+ signal.signal(signal.SIGINT, oldhandler)
+ problems = 0
+ for i in xrange(workers):
+ problems |= os.wait()[1]
+ if problems:
+ sys.exit(1)
+ try:
+ for line in fp:
+ l = line.split(' ', 1)
+ yield int(l[0]), l[1][:-1]
+ except: # re-raises
+ cleanup()
+ raise
+ cleanup()
+
+if os.name != 'nt':
+ _platformworker = _posixworker
+
+def partition(lst, nslices):
+ '''partition a list into N slices of equal size'''
+ n = len(lst)
+ chunk, slop = n / nslices, n % nslices
+ end = 0
+ for i in xrange(nslices):
+ start = end
+ end = start + chunk
+ if slop:
+ end += 1
+ slop -= 1
+ yield lst[start:end]
--- a/tests/test-graft.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-graft.t Sun Feb 10 04:04:22 2013 -0600
@@ -150,8 +150,8 @@
branchmerge: True, force: True, partial: False
ancestor: 4c60f11aa304, local: 6b9e5368ca4e+, remote: 97f8bfe72746
e: remote is newer -> g
+ getting e
updating: e 1/1 files (100.00%)
- getting e
e
grafting revision 4
searching for copies back to rev 1
@@ -161,8 +161,8 @@
d: remote is newer -> g
e: versions differ -> m
preserving e for resolve of e
+ getting d
updating: d 1/2 files (50.00%)
- getting d
updating: e 2/2 files (100.00%)
picked tool 'internal:merge' for e (binary False symlink False)
merging e
--- a/tests/test-issue522.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-issue522.t Sun Feb 10 04:04:22 2013 -0600
@@ -32,8 +32,8 @@
branchmerge: True, force: False, partial: False
ancestor: bbd179dfa0a7, local: 71766447bdbb+, remote: 4d9e78aaceee
foo: remote is newer -> g
+ getting foo
updating: foo 1/1 files (100.00%)
- getting foo
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
(branch merge, don't forget to commit)
--- a/tests/test-issue672.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-issue672.t Sun Feb 10 04:04:22 2013 -0600
@@ -36,10 +36,9 @@
ancestor: 81f4b099af3d, local: c64f439569a9+, remote: c12dcd37c90a
1: other deleted -> r
1a: remote created -> g
- updating: 1 1/2 files (50.00%)
removing 1
+ getting 1a
updating: 1a 2/2 files (100.00%)
- getting 1a
1 files updated, 0 files merged, 1 files removed, 0 files unresolved
(branch merge, don't forget to commit)
--- a/tests/test-largefiles.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-largefiles.t Sun Feb 10 04:04:22 2013 -0600
@@ -1679,8 +1679,8 @@
branchmerge: False, force: False, partial: False
ancestor: 000000000000, local: 000000000000+, remote: cf03e5bb9936
.hglf/f1: remote created -> g
+ getting .hglf/f1
updating: .hglf/f1 1/1 files (100.00%)
- getting .hglf/f1
getting changed largefiles
using http://localhost:$HGPORT2/
sending capabilities command
--- a/tests/test-rename-dir-merge.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-rename-dir-merge.t Sun Feb 10 04:04:22 2013 -0600
@@ -44,16 +44,13 @@
a/c: remote renamed directory to b/c -> d
b/a: remote created -> g
b/b: remote created -> g
- updating: a/a 1/5 files (20.00%)
removing a/a
- updating: a/b 2/5 files (40.00%)
removing a/b
- updating: a/c 3/5 files (60.00%)
+ getting b/a
+ getting b/b
+ updating: b/b 4/5 files (80.00%)
+ updating: a/c 5/5 files (100.00%)
moving a/c to b/c
- updating: b/a 4/5 files (80.00%)
- getting b/a
- updating: b/b 5/5 files (100.00%)
- getting b/b
3 files updated, 0 files merged, 2 files removed, 0 files unresolved
(branch merge, don't forget to commit)
--- a/tests/test-rename-merge1.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-rename-merge1.t Sun Feb 10 04:04:22 2013 -0600
@@ -41,17 +41,17 @@
a2: divergent renames -> dr
b2: remote created -> g
removing a
- updating: a 1/3 files (33.33%)
+ getting b2
+ updating: b2 1/3 files (33.33%)
+ updating: a 2/3 files (66.67%)
picked tool 'internal:merge' for b (binary False symlink False)
merging a and b to b
my b@044f8520aeeb+ other b@85c198ef2f6c ancestor a@af1939970a1c
premerge successful
- updating: a2 2/3 files (66.67%)
+ updating: a2 3/3 files (100.00%)
note: possible conflict - a2 was renamed multiple times to:
c2
b2
- updating: b2 3/3 files (100.00%)
- getting b2
1 files updated, 1 files merged, 0 files removed, 0 files unresolved
(branch merge, don't forget to commit)
@@ -183,11 +183,11 @@
ancestor: 19d7f95df299, local: 0084274f6b67+, remote: 5d32493049f0
file: rename and delete -> rd
newfile: remote created -> g
- updating: file 1/2 files (50.00%)
+ getting newfile
+ updating: newfile 1/2 files (50.00%)
+ updating: file 2/2 files (100.00%)
note: possible conflict - file was deleted and renamed to:
newfile
- updating: newfile 2/2 files (100.00%)
- getting newfile
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
(branch merge, don't forget to commit)
$ hg status
--- a/tests/test-rename-merge2.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-rename-merge2.t Sun Feb 10 04:04:22 2013 -0600
@@ -126,8 +126,8 @@
preserving b for resolve of b
rev: versions differ -> m
preserving rev for resolve of rev
+ getting a
updating: a 1/3 files (33.33%)
- getting a
updating: b 2/3 files (66.67%)
picked tool 'python ../merge' for b (binary False symlink False)
merging b and a to b
@@ -231,8 +231,8 @@
b: remote created -> g
rev: versions differ -> m
preserving rev for resolve of rev
+ getting b
updating: b 1/2 files (50.00%)
- getting b
updating: rev 2/2 files (100.00%)
picked tool 'python ../merge' for rev (binary False symlink False)
merging rev
@@ -289,10 +289,9 @@
b: remote created -> g
rev: versions differ -> m
preserving rev for resolve of rev
- updating: a 1/3 files (33.33%)
removing a
+ getting b
updating: b 2/3 files (66.67%)
- getting b
updating: rev 3/3 files (100.00%)
picked tool 'python ../merge' for rev (binary False symlink False)
merging rev
@@ -380,12 +379,12 @@
c: remote created -> g
rev: versions differ -> m
preserving rev for resolve of rev
- updating: a 1/3 files (33.33%)
+ getting c
+ updating: c 1/3 files (33.33%)
+ updating: a 2/3 files (66.67%)
note: possible conflict - a was renamed multiple times to:
b
c
- updating: c 2/3 files (66.67%)
- getting c
updating: rev 3/3 files (100.00%)
picked tool 'python ../merge' for rev (binary False symlink False)
merging rev
@@ -439,8 +438,8 @@
preserving b for resolve of b
rev: versions differ -> m
preserving rev for resolve of rev
+ removing a
updating: a 1/3 files (33.33%)
- removing a
updating: b 2/3 files (66.67%)
picked tool 'python ../merge' for b (binary False symlink False)
merging b
@@ -469,8 +468,8 @@
preserving b for resolve of b
rev: versions differ -> m
preserving rev for resolve of rev
+ getting a
updating: a 1/3 files (33.33%)
- getting a
updating: b 2/3 files (66.67%)
picked tool 'python ../merge' for b (binary False symlink False)
merging b
@@ -500,8 +499,8 @@
preserving b for resolve of b
rev: versions differ -> m
preserving rev for resolve of rev
+ removing a
updating: a 1/3 files (33.33%)
- removing a
updating: b 2/3 files (66.67%)
picked tool 'python ../merge' for b (binary False symlink False)
merging b
@@ -530,8 +529,8 @@
preserving b for resolve of b
rev: versions differ -> m
preserving rev for resolve of rev
+ getting a
updating: a 1/3 files (33.33%)
- getting a
updating: b 2/3 files (66.67%)
picked tool 'python ../merge' for b (binary False symlink False)
merging b
@@ -591,8 +590,8 @@
preserving b for resolve of b
rev: versions differ -> m
preserving rev for resolve of rev
+ getting a
updating: a 1/3 files (33.33%)
- getting a
updating: b 2/3 files (66.67%)
picked tool 'python ../merge' for b (binary False symlink False)
merging b
@@ -731,13 +730,13 @@
c: remote created -> g
rev: versions differ -> m
preserving rev for resolve of rev
- updating: b 1/3 files (33.33%)
+ getting c
+ updating: c 1/3 files (33.33%)
+ updating: b 2/3 files (66.67%)
picked tool 'python ../merge' for b (binary False symlink False)
merging b and a to b
my b@02963e448370+ other a@2b958612230f ancestor a@924404dff337
premerge successful
- updating: c 2/3 files (66.67%)
- getting c
updating: rev 3/3 files (100.00%)
picked tool 'python ../merge' for rev (binary False symlink False)
merging rev
--- a/tests/test-subrepo.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-subrepo.t Sun Feb 10 04:04:22 2013 -0600
@@ -214,8 +214,8 @@
branchmerge: False, force: False, partial: False
ancestor: 60ca1237c194, local: 60ca1237c194+, remote: 6747d179aa9a
t: remote is newer -> g
+ getting t
updating: t 1/1 files (100.00%)
- getting t
0 files updated, 0 files merged, 0 files removed, 0 files unresolved
(branch merge, don't forget to commit)
$ hg debugsub
--- a/tests/test-up-local-change.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-up-local-change.t Sun Feb 10 04:04:22 2013 -0600
@@ -49,12 +49,12 @@
a: versions differ -> m
preserving a for resolve of a
b: remote created -> g
- updating: a 1/2 files (50.00%)
+ getting b
+ updating: b 1/2 files (50.00%)
+ updating: a 2/2 files (100.00%)
picked tool 'true' for a (binary False symlink False)
merging a
my a@c19d34741b0a+ other a@1e71731e6fbb ancestor a@c19d34741b0a
- updating: b 2/2 files (100.00%)
- getting b
1 files updated, 1 files merged, 0 files removed, 0 files unresolved
$ hg parents
changeset: 1:1e71731e6fbb
@@ -70,8 +70,8 @@
b: other deleted -> r
a: versions differ -> m
preserving a for resolve of a
+ removing b
updating: b 1/2 files (50.00%)
- removing b
updating: a 2/2 files (100.00%)
picked tool 'true' for a (binary False symlink False)
merging a
@@ -103,12 +103,12 @@
a: versions differ -> m
preserving a for resolve of a
b: remote created -> g
- updating: a 1/2 files (50.00%)
+ getting b
+ updating: b 1/2 files (50.00%)
+ updating: a 2/2 files (100.00%)
picked tool 'true' for a (binary False symlink False)
merging a
my a@c19d34741b0a+ other a@1e71731e6fbb ancestor a@c19d34741b0a
- updating: b 2/2 files (100.00%)
- getting b
1 files updated, 1 files merged, 0 files removed, 0 files unresolved
$ hg parents
changeset: 1:1e71731e6fbb
--- a/tests/test-update-reverse.t Sat Feb 09 22:54:34 2013 +0000
+++ b/tests/test-update-reverse.t Sun Feb 10 04:04:22 2013 -0600
@@ -71,12 +71,10 @@
side1: other deleted -> r
side2: other deleted -> r
main: remote created -> g
- updating: side1 1/3 files (33.33%)
removing side1
- updating: side2 2/3 files (66.67%)
removing side2
+ getting main
updating: main 3/3 files (100.00%)
- getting main
1 files updated, 0 files merged, 2 files removed, 0 files unresolved
$ ls