|
1 # Copyright 2009-2010 Gregory P. Ward |
|
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated |
|
3 # Copyright 2010-2011 Fog Creek Software |
|
4 # Copyright 2010-2011 Unity Technologies |
|
5 # |
|
6 # This software may be used and distributed according to the terms of the |
|
7 # GNU General Public License version 2 or any later version. |
|
8 |
|
9 '''High-level command functions: lfadd() et. al, plus the cmdtable.''' |
|
10 |
|
11 import os |
|
12 import shutil |
|
13 |
|
14 from mercurial import util, match as match_, hg, node, context, error |
|
15 from mercurial.i18n import _ |
|
16 |
|
17 import lfutil |
|
18 import basestore |
|
19 |
|
20 # -- Commands ---------------------------------------------------------- |
|
21 |
|
22 def lfconvert(ui, src, dest, *pats, **opts): |
|
23 '''Convert a normal repository to a largefiles repository |
|
24 |
|
25 Convert source repository creating an identical repository, except that all |
|
26 files that match the patterns given, or are over the given size will be |
|
27 added as largefiles. The size used to determine whether or not to track a |
|
28 file as a largefile is the size of the first version of the file. After |
|
29 running this command you will need to make sure that largefiles is enabled |
|
30 anywhere you intend to push the new repository.''' |
|
31 |
|
32 if opts['tonormal']: |
|
33 tolfile = False |
|
34 else: |
|
35 tolfile = True |
|
36 size = opts['size'] |
|
37 if not size: |
|
38 size = ui.config(lfutil.longname, 'size', default=None) |
|
39 try: |
|
40 size = int(size) |
|
41 except ValueError: |
|
42 raise util.Abort(_('largefiles.size must be integer, was %s\n') % \ |
|
43 size) |
|
44 except TypeError: |
|
45 raise util.Abort(_('size must be specified')) |
|
46 |
|
47 try: |
|
48 rsrc = hg.repository(ui, src) |
|
49 if not rsrc.local(): |
|
50 raise util.Abort(_('%s is not a local Mercurial repo') % src) |
|
51 except error.RepoError, err: |
|
52 ui.traceback() |
|
53 raise util.Abort(err.args[0]) |
|
54 if os.path.exists(dest): |
|
55 if not os.path.isdir(dest): |
|
56 raise util.Abort(_('destination %s already exists') % dest) |
|
57 elif os.listdir(dest): |
|
58 raise util.Abort(_('destination %s is not empty') % dest) |
|
59 try: |
|
60 ui.status(_('initializing destination %s\n') % dest) |
|
61 rdst = hg.repository(ui, dest, create=True) |
|
62 if not rdst.local(): |
|
63 raise util.Abort(_('%s is not a local Mercurial repo') % dest) |
|
64 except error.RepoError: |
|
65 ui.traceback() |
|
66 raise util.Abort(_('%s is not a repo') % dest) |
|
67 |
|
68 try: |
|
69 # Lock destination to prevent modification while it is converted to. |
|
70 # Don't need to lock src because we are just reading from its history |
|
71 # which can't change. |
|
72 dst_lock = rdst.lock() |
|
73 |
|
74 # Get a list of all changesets in the source. The easy way to do this |
|
75 # is to simply walk the changelog, using changelog.nodesbewteen(). |
|
76 # Take a look at mercurial/revlog.py:639 for more details. |
|
77 # Use a generator instead of a list to decrease memory usage |
|
78 ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None, |
|
79 rsrc.heads())[0]) |
|
80 revmap = {node.nullid: node.nullid} |
|
81 if tolfile: |
|
82 lfiles = set() |
|
83 normalfiles = set() |
|
84 if not pats: |
|
85 pats = ui.config(lfutil.longname, 'patterns', default=()) |
|
86 if pats: |
|
87 pats = pats.split(' ') |
|
88 if pats: |
|
89 matcher = match_.match(rsrc.root, '', list(pats)) |
|
90 else: |
|
91 matcher = None |
|
92 |
|
93 lfiletohash = {} |
|
94 for ctx in ctxs: |
|
95 ui.progress(_('converting revisions'), ctx.rev(), |
|
96 unit=_('revision'), total=rsrc['tip'].rev()) |
|
97 _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, |
|
98 lfiles, normalfiles, matcher, size, lfiletohash) |
|
99 ui.progress(_('converting revisions'), None) |
|
100 |
|
101 if os.path.exists(rdst.wjoin(lfutil.shortname)): |
|
102 shutil.rmtree(rdst.wjoin(lfutil.shortname)) |
|
103 |
|
104 for f in lfiletohash.keys(): |
|
105 if os.path.isfile(rdst.wjoin(f)): |
|
106 os.unlink(rdst.wjoin(f)) |
|
107 try: |
|
108 os.removedirs(os.path.dirname(rdst.wjoin(f))) |
|
109 except: |
|
110 pass |
|
111 |
|
112 else: |
|
113 for ctx in ctxs: |
|
114 ui.progress(_('converting revisions'), ctx.rev(), |
|
115 unit=_('revision'), total=rsrc['tip'].rev()) |
|
116 _addchangeset(ui, rsrc, rdst, ctx, revmap) |
|
117 |
|
118 ui.progress(_('converting revisions'), None) |
|
119 except: |
|
120 # we failed, remove the new directory |
|
121 shutil.rmtree(rdst.root) |
|
122 raise |
|
123 finally: |
|
124 dst_lock.release() |
|
125 |
|
126 def _addchangeset(ui, rsrc, rdst, ctx, revmap): |
|
127 # Convert src parents to dst parents |
|
128 parents = [] |
|
129 for p in ctx.parents(): |
|
130 parents.append(revmap[p.node()]) |
|
131 while len(parents) < 2: |
|
132 parents.append(node.nullid) |
|
133 |
|
134 # Generate list of changed files |
|
135 files = set(ctx.files()) |
|
136 if node.nullid not in parents: |
|
137 mc = ctx.manifest() |
|
138 mp1 = ctx.parents()[0].manifest() |
|
139 mp2 = ctx.parents()[1].manifest() |
|
140 files |= (set(mp1) | set(mp2)) - set(mc) |
|
141 for f in mc: |
|
142 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None): |
|
143 files.add(f) |
|
144 |
|
145 def getfilectx(repo, memctx, f): |
|
146 if lfutil.standin(f) in files: |
|
147 # if the file isn't in the manifest then it was removed |
|
148 # or renamed, raise IOError to indicate this |
|
149 try: |
|
150 fctx = ctx.filectx(lfutil.standin(f)) |
|
151 except error.LookupError: |
|
152 raise IOError() |
|
153 renamed = fctx.renamed() |
|
154 if renamed: |
|
155 renamed = lfutil.splitstandin(renamed[0]) |
|
156 |
|
157 hash = fctx.data().strip() |
|
158 path = lfutil.findfile(rsrc, hash) |
|
159 ### TODO: What if the file is not cached? |
|
160 data = '' |
|
161 fd = None |
|
162 try: |
|
163 fd = open(path, 'rb') |
|
164 data = fd.read() |
|
165 finally: |
|
166 if fd: fd.close() |
|
167 return context.memfilectx(f, data, 'l' in fctx.flags(), |
|
168 'x' in fctx.flags(), renamed) |
|
169 else: |
|
170 try: |
|
171 fctx = ctx.filectx(f) |
|
172 except error.LookupError: |
|
173 raise IOError() |
|
174 renamed = fctx.renamed() |
|
175 if renamed: |
|
176 renamed = renamed[0] |
|
177 data = fctx.data() |
|
178 if f == '.hgtags': |
|
179 newdata = [] |
|
180 for line in data.splitlines(): |
|
181 id, name = line.split(' ', 1) |
|
182 newdata.append('%s %s\n' % (node.hex(revmap[node.bin(id)]), |
|
183 name)) |
|
184 data = ''.join(newdata) |
|
185 return context.memfilectx(f, data, 'l' in fctx.flags(), |
|
186 'x' in fctx.flags(), renamed) |
|
187 |
|
188 dstfiles = [] |
|
189 for file in files: |
|
190 if lfutil.isstandin(file): |
|
191 dstfiles.append(lfutil.splitstandin(file)) |
|
192 else: |
|
193 dstfiles.append(file) |
|
194 # Commit |
|
195 mctx = context.memctx(rdst, parents, ctx.description(), dstfiles, |
|
196 getfilectx, ctx.user(), ctx.date(), ctx.extra()) |
|
197 ret = rdst.commitctx(mctx) |
|
198 rdst.dirstate.setparents(ret) |
|
199 revmap[ctx.node()] = rdst.changelog.tip() |
|
200 |
|
201 def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles, |
|
202 matcher, size, lfiletohash): |
|
203 # Convert src parents to dst parents |
|
204 parents = [] |
|
205 for p in ctx.parents(): |
|
206 parents.append(revmap[p.node()]) |
|
207 while len(parents) < 2: |
|
208 parents.append(node.nullid) |
|
209 |
|
210 # Generate list of changed files |
|
211 files = set(ctx.files()) |
|
212 if node.nullid not in parents: |
|
213 mc = ctx.manifest() |
|
214 mp1 = ctx.parents()[0].manifest() |
|
215 mp2 = ctx.parents()[1].manifest() |
|
216 files |= (set(mp1) | set(mp2)) - set(mc) |
|
217 for f in mc: |
|
218 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None): |
|
219 files.add(f) |
|
220 |
|
221 dstfiles = [] |
|
222 for f in files: |
|
223 if f not in lfiles and f not in normalfiles: |
|
224 islfile = _islfile(f, ctx, matcher, size) |
|
225 # If this file was renamed or copied then copy |
|
226 # the lfileness of its predecessor |
|
227 if f in ctx.manifest(): |
|
228 fctx = ctx.filectx(f) |
|
229 renamed = fctx.renamed() |
|
230 renamedlfile = renamed and renamed[0] in lfiles |
|
231 islfile |= renamedlfile |
|
232 if 'l' in fctx.flags(): |
|
233 if renamedlfile: |
|
234 raise util.Abort( |
|
235 _('Renamed/copied largefile %s becomes symlink') % f) |
|
236 islfile = False |
|
237 if islfile: |
|
238 lfiles.add(f) |
|
239 else: |
|
240 normalfiles.add(f) |
|
241 |
|
242 if f in lfiles: |
|
243 dstfiles.append(lfutil.standin(f)) |
|
244 # lfile in manifest if it has not been removed/renamed |
|
245 if f in ctx.manifest(): |
|
246 if 'l' in ctx.filectx(f).flags(): |
|
247 if renamed and renamed[0] in lfiles: |
|
248 raise util.Abort(_('largefile %s becomes symlink') % f) |
|
249 |
|
250 # lfile was modified, update standins |
|
251 fullpath = rdst.wjoin(f) |
|
252 lfutil.createdir(os.path.dirname(fullpath)) |
|
253 m = util.sha1('') |
|
254 m.update(ctx[f].data()) |
|
255 hash = m.hexdigest() |
|
256 if f not in lfiletohash or lfiletohash[f] != hash: |
|
257 try: |
|
258 fd = open(fullpath, 'wb') |
|
259 fd.write(ctx[f].data()) |
|
260 finally: |
|
261 if fd: |
|
262 fd.close() |
|
263 executable = 'x' in ctx[f].flags() |
|
264 os.chmod(fullpath, lfutil.getmode(executable)) |
|
265 lfutil.writestandin(rdst, lfutil.standin(f), hash, |
|
266 executable) |
|
267 lfiletohash[f] = hash |
|
268 else: |
|
269 # normal file |
|
270 dstfiles.append(f) |
|
271 |
|
272 def getfilectx(repo, memctx, f): |
|
273 if lfutil.isstandin(f): |
|
274 # if the file isn't in the manifest then it was removed |
|
275 # or renamed, raise IOError to indicate this |
|
276 srcfname = lfutil.splitstandin(f) |
|
277 try: |
|
278 fctx = ctx.filectx(srcfname) |
|
279 except error.LookupError: |
|
280 raise IOError() |
|
281 renamed = fctx.renamed() |
|
282 if renamed: |
|
283 # standin is always a lfile because lfileness |
|
284 # doesn't change after rename or copy |
|
285 renamed = lfutil.standin(renamed[0]) |
|
286 |
|
287 return context.memfilectx(f, lfiletohash[srcfname], 'l' in |
|
288 fctx.flags(), 'x' in fctx.flags(), renamed) |
|
289 else: |
|
290 try: |
|
291 fctx = ctx.filectx(f) |
|
292 except error.LookupError: |
|
293 raise IOError() |
|
294 renamed = fctx.renamed() |
|
295 if renamed: |
|
296 renamed = renamed[0] |
|
297 |
|
298 data = fctx.data() |
|
299 if f == '.hgtags': |
|
300 newdata = [] |
|
301 for line in data.splitlines(): |
|
302 id, name = line.split(' ', 1) |
|
303 newdata.append('%s %s\n' % (node.hex(revmap[node.bin(id)]), |
|
304 name)) |
|
305 data = ''.join(newdata) |
|
306 return context.memfilectx(f, data, 'l' in fctx.flags(), |
|
307 'x' in fctx.flags(), renamed) |
|
308 |
|
309 # Commit |
|
310 mctx = context.memctx(rdst, parents, ctx.description(), dstfiles, |
|
311 getfilectx, ctx.user(), ctx.date(), ctx.extra()) |
|
312 ret = rdst.commitctx(mctx) |
|
313 rdst.dirstate.setparents(ret) |
|
314 revmap[ctx.node()] = rdst.changelog.tip() |
|
315 |
|
316 def _islfile(file, ctx, matcher, size): |
|
317 ''' |
|
318 A file is a lfile if it matches a pattern or is over |
|
319 the given size. |
|
320 ''' |
|
321 # Never store hgtags or hgignore as lfiles |
|
322 if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs': |
|
323 return False |
|
324 if matcher and matcher(file): |
|
325 return True |
|
326 try: |
|
327 return ctx.filectx(file).size() >= size * 1024 * 1024 |
|
328 except error.LookupError: |
|
329 return False |
|
330 |
|
331 def uploadlfiles(ui, rsrc, rdst, files): |
|
332 '''upload largefiles to the central store''' |
|
333 |
|
334 # Don't upload locally. All largefiles are in the system wide cache |
|
335 # so the other repo can just get them from there. |
|
336 if not files or rdst.local(): |
|
337 return |
|
338 |
|
339 store = basestore._openstore(rsrc, rdst, put=True) |
|
340 |
|
341 at = 0 |
|
342 files = filter(lambda h: not store.exists(h), files) |
|
343 for hash in files: |
|
344 ui.progress(_('uploading largefiles'), at, unit='largefile', total=len(files)) |
|
345 source = lfutil.findfile(rsrc, hash) |
|
346 if not source: |
|
347 raise util.Abort(_('Missing largefile %s needs to be uploaded') % hash) |
|
348 # XXX check for errors here |
|
349 store.put(source, hash) |
|
350 at += 1 |
|
351 ui.progress('uploading largefiles', None) |
|
352 |
|
353 def verifylfiles(ui, repo, all=False, contents=False): |
|
354 '''Verify that every big file revision in the current changeset |
|
355 exists in the central store. With --contents, also verify that |
|
356 the contents of each big file revision are correct (SHA-1 hash |
|
357 matches the revision ID). With --all, check every changeset in |
|
358 this repository.''' |
|
359 if all: |
|
360 # Pass a list to the function rather than an iterator because we know a |
|
361 # list will work. |
|
362 revs = range(len(repo)) |
|
363 else: |
|
364 revs = ['.'] |
|
365 |
|
366 store = basestore._openstore(repo) |
|
367 return store.verify(revs, contents=contents) |
|
368 |
|
369 def cachelfiles(ui, repo, node): |
|
370 '''cachelfiles ensures that all largefiles needed by the specified revision |
|
371 are present in the repository's largefile cache. |
|
372 |
|
373 returns a tuple (cached, missing). cached is the list of files downloaded |
|
374 by this operation; missing is the list of files that were needed but could |
|
375 not be found.''' |
|
376 lfiles = lfutil.listlfiles(repo, node) |
|
377 toget = [] |
|
378 |
|
379 for lfile in lfiles: |
|
380 expectedhash = repo[node][lfutil.standin(lfile)].data().strip() |
|
381 # if it exists and its hash matches, it might have been locally |
|
382 # modified before updating and the user chose 'local'. in this case, |
|
383 # it will not be in any store, so don't look for it. |
|
384 if (not os.path.exists(repo.wjoin(lfile)) \ |
|
385 or expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and \ |
|
386 not lfutil.findfile(repo, expectedhash): |
|
387 toget.append((lfile, expectedhash)) |
|
388 |
|
389 if toget: |
|
390 store = basestore._openstore(repo) |
|
391 ret = store.get(toget) |
|
392 return ret |
|
393 |
|
394 return ([], []) |
|
395 |
|
396 def updatelfiles(ui, repo, filelist=None, printmessage=True): |
|
397 wlock = repo.wlock() |
|
398 try: |
|
399 lfdirstate = lfutil.openlfdirstate(ui, repo) |
|
400 lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) |
|
401 |
|
402 if filelist is not None: |
|
403 lfiles = [f for f in lfiles if f in filelist] |
|
404 |
|
405 printed = False |
|
406 if printmessage and lfiles: |
|
407 ui.status(_('getting changed largefiles\n')) |
|
408 printed = True |
|
409 cachelfiles(ui, repo, '.') |
|
410 |
|
411 updated, removed = 0, 0 |
|
412 for i in map(lambda f: _updatelfile(repo, lfdirstate, f), lfiles): |
|
413 # increment the appropriate counter according to _updatelfile's |
|
414 # return value |
|
415 updated += i > 0 and i or 0 |
|
416 removed -= i < 0 and i or 0 |
|
417 if printmessage and (removed or updated) and not printed: |
|
418 ui.status(_('getting changed largefiles\n')) |
|
419 printed = True |
|
420 |
|
421 lfdirstate.write() |
|
422 if printed and printmessage: |
|
423 ui.status(_('%d largefiles updated, %d removed\n') % (updated, |
|
424 removed)) |
|
425 finally: |
|
426 wlock.release() |
|
427 |
|
428 def _updatelfile(repo, lfdirstate, lfile): |
|
429 '''updates a single largefile and copies the state of its standin from |
|
430 the repository's dirstate to its state in the lfdirstate. |
|
431 |
|
432 returns 1 if the file was modified, -1 if the file was removed, 0 if the |
|
433 file was unchanged, and None if the needed largefile was missing from the |
|
434 cache.''' |
|
435 ret = 0 |
|
436 abslfile = repo.wjoin(lfile) |
|
437 absstandin = repo.wjoin(lfutil.standin(lfile)) |
|
438 if os.path.exists(absstandin): |
|
439 if os.path.exists(absstandin+'.orig'): |
|
440 shutil.copyfile(abslfile, abslfile+'.orig') |
|
441 expecthash = lfutil.readstandin(repo, lfile) |
|
442 if expecthash != '' and \ |
|
443 (not os.path.exists(abslfile) or \ |
|
444 expecthash != lfutil.hashfile(abslfile)): |
|
445 if not lfutil.copyfromcache(repo, expecthash, lfile): |
|
446 return None # don't try to set the mode or update the dirstate |
|
447 ret = 1 |
|
448 mode = os.stat(absstandin).st_mode |
|
449 if mode != os.stat(abslfile).st_mode: |
|
450 os.chmod(abslfile, mode) |
|
451 ret = 1 |
|
452 else: |
|
453 if os.path.exists(abslfile): |
|
454 os.unlink(abslfile) |
|
455 ret = -1 |
|
456 state = repo.dirstate[lfutil.standin(lfile)] |
|
457 if state == 'n': |
|
458 lfdirstate.normal(lfile) |
|
459 elif state == 'r': |
|
460 lfdirstate.remove(lfile) |
|
461 elif state == 'a': |
|
462 lfdirstate.add(lfile) |
|
463 elif state == '?': |
|
464 try: |
|
465 # Mercurial >= 1.9 |
|
466 lfdirstate.drop(lfile) |
|
467 except AttributeError: |
|
468 # Mercurial <= 1.8 |
|
469 lfdirstate.forget(lfile) |
|
470 return ret |
|
471 |
|
472 # -- hg commands declarations ------------------------------------------------ |
|
473 |
|
474 |
|
475 cmdtable = { |
|
476 'lfconvert': (lfconvert, |
|
477 [('s', 'size', 0, 'All files over this size (in megabytes) ' |
|
478 'will be considered largefiles. This can also be specified in ' |
|
479 'your hgrc as [largefiles].size.'), |
|
480 ('','tonormal',False, |
|
481 'Convert from a largefiles repo to a normal repo')], |
|
482 _('hg lfconvert SOURCE DEST [FILE ...]')), |
|
483 } |