37 # If this is a clone and it was requested to perform a "stream clone", |
37 # If this is a clone and it was requested to perform a "stream clone", |
38 # we obtain the raw files data from the remote then fall back to an |
38 # we obtain the raw files data from the remote then fall back to an |
39 # incremental pull. This is somewhat hacky and is not nearly robust enough |
39 # incremental pull. This is somewhat hacky and is not nearly robust enough |
40 # for long-term usage. |
40 # for long-term usage. |
41 if usingrawchangelogandmanifest: |
41 if usingrawchangelogandmanifest: |
42 with repo.transaction('clone'): |
42 with repo.transaction(b'clone'): |
43 _fetchrawstorefiles(repo, remote) |
43 _fetchrawstorefiles(repo, remote) |
44 repo.invalidate(clearfilecache=True) |
44 repo.invalidate(clearfilecache=True) |
45 |
45 |
46 tr = pullop.trmanager.transaction() |
46 tr = pullop.trmanager.transaction() |
47 |
47 |
49 # to exchange.pull() could be different. |
49 # to exchange.pull() could be different. |
50 narrowmatcher = narrowspec.match( |
50 narrowmatcher = narrowspec.match( |
51 repo.root, |
51 repo.root, |
52 # Empty maps to nevermatcher. So always |
52 # Empty maps to nevermatcher. So always |
53 # set includes if missing. |
53 # set includes if missing. |
54 pullop.includepats or {'path:.'}, |
54 pullop.includepats or {b'path:.'}, |
55 pullop.excludepats, |
55 pullop.excludepats, |
56 ) |
56 ) |
57 |
57 |
58 if pullop.includepats or pullop.excludepats: |
58 if pullop.includepats or pullop.excludepats: |
59 pathfilter = {} |
59 pathfilter = {} |
76 # New revisions are written to the changelog. But all other updates |
76 # New revisions are written to the changelog. But all other updates |
77 # are deferred. Do those now. |
77 # are deferred. Do those now. |
78 |
78 |
79 # Ensure all new changesets are draft by default. If the repo is |
79 # Ensure all new changesets are draft by default. If the repo is |
80 # publishing, the phase will be adjusted by the loop below. |
80 # publishing, the phase will be adjusted by the loop below. |
81 if csetres['added']: |
81 if csetres[b'added']: |
82 phases.registernew(repo, tr, phases.draft, csetres['added']) |
82 phases.registernew(repo, tr, phases.draft, csetres[b'added']) |
83 |
83 |
84 # And adjust the phase of all changesets accordingly. |
84 # And adjust the phase of all changesets accordingly. |
85 for phase in phases.phasenames: |
85 for phase in phases.phasenames: |
86 if phase == b'secret' or not csetres['nodesbyphase'][phase]: |
86 if phase == b'secret' or not csetres[b'nodesbyphase'][phase]: |
87 continue |
87 continue |
88 |
88 |
89 phases.advanceboundary( |
89 phases.advanceboundary( |
90 repo, |
90 repo, |
91 tr, |
91 tr, |
92 phases.phasenames.index(phase), |
92 phases.phasenames.index(phase), |
93 csetres['nodesbyphase'][phase], |
93 csetres[b'nodesbyphase'][phase], |
94 ) |
94 ) |
95 |
95 |
96 # Write bookmark updates. |
96 # Write bookmark updates. |
97 bookmarks.updatefromremote( |
97 bookmarks.updatefromremote( |
98 repo.ui, |
98 repo.ui, |
99 repo, |
99 repo, |
100 csetres['bookmarks'], |
100 csetres[b'bookmarks'], |
101 remote.url(), |
101 remote.url(), |
102 pullop.gettransaction, |
102 pullop.gettransaction, |
103 explicit=pullop.explicitbookmarks, |
103 explicit=pullop.explicitbookmarks, |
104 ) |
104 ) |
105 |
105 |
106 manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes']) |
106 manres = _fetchmanifests(repo, tr, remote, csetres[b'manifestnodes']) |
107 |
107 |
108 # We don't properly support shallow changeset and manifest yet. So we apply |
108 # We don't properly support shallow changeset and manifest yet. So we apply |
109 # depth limiting locally. |
109 # depth limiting locally. |
110 if pullop.depth: |
110 if pullop.depth: |
111 relevantcsetnodes = set() |
111 relevantcsetnodes = set() |
140 csetsforfiles.append(node) |
140 csetsforfiles.append(node) |
141 mnodesforfiles.append(mnode) |
141 mnodesforfiles.append(mnode) |
142 manifestlinkrevs[mnode] = rev |
142 manifestlinkrevs[mnode] = rev |
143 |
143 |
144 else: |
144 else: |
145 csetsforfiles = [n for n in csetres['added'] if csetrelevantfilter(n)] |
145 csetsforfiles = [n for n in csetres[b'added'] if csetrelevantfilter(n)] |
146 mnodesforfiles = manres['added'] |
146 mnodesforfiles = manres[b'added'] |
147 manifestlinkrevs = manres['linkrevs'] |
147 manifestlinkrevs = manres[b'linkrevs'] |
148 |
148 |
149 # Find all file nodes referenced by added manifests and fetch those |
149 # Find all file nodes referenced by added manifests and fetch those |
150 # revisions. |
150 # revisions. |
151 fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles) |
151 fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles) |
152 _fetchfilesfromcsets( |
152 _fetchfilesfromcsets( |
195 |
195 |
196 # First object is a summary of files data that follows. |
196 # First object is a summary of files data that follows. |
197 overall = next(objs) |
197 overall = next(objs) |
198 |
198 |
199 progress = repo.ui.makeprogress( |
199 progress = repo.ui.makeprogress( |
200 _('clone'), total=overall[b'totalsize'], unit=_('bytes') |
200 _(b'clone'), total=overall[b'totalsize'], unit=_(b'bytes') |
201 ) |
201 ) |
202 with progress: |
202 with progress: |
203 progress.update(0) |
203 progress.update(0) |
204 |
204 |
205 # Next are pairs of file metadata, data. |
205 # Next are pairs of file metadata, data. |
328 # we need to remain in the context manager in order to stream data. |
328 # we need to remain in the context manager in order to stream data. |
329 return _processchangesetdata(repo, tr, objs) |
329 return _processchangesetdata(repo, tr, objs) |
330 |
330 |
331 |
331 |
332 def _processchangesetdata(repo, tr, objs): |
332 def _processchangesetdata(repo, tr, objs): |
333 repo.hook('prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs)) |
333 repo.hook(b'prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs)) |
334 |
334 |
335 urepo = repo.unfiltered() |
335 urepo = repo.unfiltered() |
336 cl = urepo.changelog |
336 cl = urepo.changelog |
337 |
337 |
338 cl.delayupdate(tr) |
338 cl.delayupdate(tr) |
340 # The first emitted object is a header describing the data that |
340 # The first emitted object is a header describing the data that |
341 # follows. |
341 # follows. |
342 meta = next(objs) |
342 meta = next(objs) |
343 |
343 |
344 progress = repo.ui.makeprogress( |
344 progress = repo.ui.makeprogress( |
345 _('changesets'), unit=_('chunks'), total=meta.get(b'totalitems') |
345 _(b'changesets'), unit=_(b'chunks'), total=meta.get(b'totalitems') |
346 ) |
346 ) |
347 |
347 |
348 manifestnodes = {} |
348 manifestnodes = {} |
349 |
349 |
350 def linkrev(node): |
350 def linkrev(node): |
351 repo.ui.debug('add changeset %s\n' % short(node)) |
351 repo.ui.debug(b'add changeset %s\n' % short(node)) |
352 # Linkrev for changelog is always self. |
352 # Linkrev for changelog is always self. |
353 return len(cl) |
353 return len(cl) |
354 |
354 |
355 def onchangeset(cl, node): |
355 def onchangeset(cl, node): |
356 progress.increment() |
356 progress.increment() |
411 ) |
411 ) |
412 |
412 |
413 progress.complete() |
413 progress.complete() |
414 |
414 |
415 return { |
415 return { |
416 'added': added, |
416 b'added': added, |
417 'nodesbyphase': nodesbyphase, |
417 b'nodesbyphase': nodesbyphase, |
418 'bookmarks': remotebookmarks, |
418 b'bookmarks': remotebookmarks, |
419 'manifestnodes': manifestnodes, |
419 b'manifestnodes': manifestnodes, |
420 } |
420 } |
421 |
421 |
422 |
422 |
423 def _fetchmanifests(repo, tr, remote, manifestnodes): |
423 def _fetchmanifests(repo, tr, remote, manifestnodes): |
424 rootmanifest = repo.manifestlog.getstorage(b'') |
424 rootmanifest = repo.manifestlog.getstorage(b'') |
481 ) |
481 ) |
482 |
482 |
483 progress.increment() |
483 progress.increment() |
484 |
484 |
485 progress = repo.ui.makeprogress( |
485 progress = repo.ui.makeprogress( |
486 _('manifests'), unit=_('chunks'), total=len(fetchnodes) |
486 _(b'manifests'), unit=_(b'chunks'), total=len(fetchnodes) |
487 ) |
487 ) |
488 |
488 |
489 commandmeta = remote.apidescriptor[b'commands'][b'manifestdata'] |
489 commandmeta = remote.apidescriptor[b'commands'][b'manifestdata'] |
490 batchsize = commandmeta.get(b'recommendedbatchsize', 10000) |
490 batchsize = commandmeta.get(b'recommendedbatchsize', 10000) |
491 # TODO make size configurable on client? |
491 # TODO make size configurable on client? |
543 """ |
543 """ |
544 ml = repo.manifestlog |
544 ml = repo.manifestlog |
545 fnodes = collections.defaultdict(dict) |
545 fnodes = collections.defaultdict(dict) |
546 |
546 |
547 progress = repo.ui.makeprogress( |
547 progress = repo.ui.makeprogress( |
548 _('scanning manifests'), total=len(manifestnodes) |
548 _(b'scanning manifests'), total=len(manifestnodes) |
549 ) |
549 ) |
550 |
550 |
551 with progress: |
551 with progress: |
552 for manifestnode in manifestnodes: |
552 for manifestnode in manifestnodes: |
553 m = ml.get(b'', manifestnode) |
553 m = ml.get(b'', manifestnode) |