25 self.hash = hash |
25 self.hash = hash |
26 self.url = url |
26 self.url = url |
27 self.detail = detail |
27 self.detail = detail |
28 |
28 |
29 def longmessage(self): |
29 def longmessage(self): |
30 return _("error getting id %s from url %s for file %s: %s\n") % ( |
30 return _(b"error getting id %s from url %s for file %s: %s\n") % ( |
31 self.hash, |
31 self.hash, |
32 util.hidepassword(self.url), |
32 util.hidepassword(self.url), |
33 self.filename, |
33 self.filename, |
34 self.detail, |
34 self.detail, |
35 ) |
35 ) |
36 |
36 |
37 def __str__(self): |
37 def __str__(self): |
38 return "%s: %s" % (util.hidepassword(self.url), self.detail) |
38 return b"%s: %s" % (util.hidepassword(self.url), self.detail) |
39 |
39 |
40 |
40 |
41 class basestore(object): |
41 class basestore(object): |
42 def __init__(self, ui, repo, url): |
42 def __init__(self, ui, repo, url): |
43 self.ui = ui |
43 self.ui = ui |
44 self.repo = repo |
44 self.repo = repo |
45 self.url = url |
45 self.url = url |
46 |
46 |
47 def put(self, source, hash): |
47 def put(self, source, hash): |
48 '''Put source file into the store so it can be retrieved by hash.''' |
48 '''Put source file into the store so it can be retrieved by hash.''' |
49 raise NotImplementedError('abstract method') |
49 raise NotImplementedError(b'abstract method') |
50 |
50 |
51 def exists(self, hashes): |
51 def exists(self, hashes): |
52 '''Check to see if the store contains the given hashes. Given an |
52 '''Check to see if the store contains the given hashes. Given an |
53 iterable of hashes it returns a mapping from hash to bool.''' |
53 iterable of hashes it returns a mapping from hash to bool.''' |
54 raise NotImplementedError('abstract method') |
54 raise NotImplementedError(b'abstract method') |
55 |
55 |
56 def get(self, files): |
56 def get(self, files): |
57 '''Get the specified largefiles from the store and write to local |
57 '''Get the specified largefiles from the store and write to local |
58 files under repo.root. files is a list of (filename, hash) |
58 files under repo.root. files is a list of (filename, hash) |
59 tuples. Return (success, missing), lists of files successfully |
59 tuples. Return (success, missing), lists of files successfully |
67 ui = self.ui |
67 ui = self.ui |
68 |
68 |
69 at = 0 |
69 at = 0 |
70 available = self.exists(set(hash for (_filename, hash) in files)) |
70 available = self.exists(set(hash for (_filename, hash) in files)) |
71 with ui.makeprogress( |
71 with ui.makeprogress( |
72 _('getting largefiles'), unit=_('files'), total=len(files) |
72 _(b'getting largefiles'), unit=_(b'files'), total=len(files) |
73 ) as progress: |
73 ) as progress: |
74 for filename, hash in files: |
74 for filename, hash in files: |
75 progress.update(at) |
75 progress.update(at) |
76 at += 1 |
76 at += 1 |
77 ui.note(_('getting %s:%s\n') % (filename, hash)) |
77 ui.note(_(b'getting %s:%s\n') % (filename, hash)) |
78 |
78 |
79 if not available.get(hash): |
79 if not available.get(hash): |
80 ui.warn( |
80 ui.warn( |
81 _('%s: largefile %s not available from %s\n') |
81 _(b'%s: largefile %s not available from %s\n') |
82 % (filename, hash, util.hidepassword(self.url)) |
82 % (filename, hash, util.hidepassword(self.url)) |
83 ) |
83 ) |
84 missing.append(filename) |
84 missing.append(filename) |
85 continue |
85 continue |
86 |
86 |
94 def _gethash(self, filename, hash): |
94 def _gethash(self, filename, hash): |
95 """Get file with the provided hash and store it in the local repo's |
95 """Get file with the provided hash and store it in the local repo's |
96 store and in the usercache. |
96 store and in the usercache. |
97 filename is for informational messages only. |
97 filename is for informational messages only. |
98 """ |
98 """ |
99 util.makedirs(lfutil.storepath(self.repo, '')) |
99 util.makedirs(lfutil.storepath(self.repo, b'')) |
100 storefilename = lfutil.storepath(self.repo, hash) |
100 storefilename = lfutil.storepath(self.repo, hash) |
101 |
101 |
102 tmpname = storefilename + '.tmp' |
102 tmpname = storefilename + b'.tmp' |
103 with util.atomictempfile( |
103 with util.atomictempfile( |
104 tmpname, createmode=self.repo.store.createmode |
104 tmpname, createmode=self.repo.store.createmode |
105 ) as tmpfile: |
105 ) as tmpfile: |
106 try: |
106 try: |
107 gothash = self._getfile(tmpfile, filename, hash) |
107 gothash = self._getfile(tmpfile, filename, hash) |
108 except StoreError as err: |
108 except StoreError as err: |
109 self.ui.warn(err.longmessage()) |
109 self.ui.warn(err.longmessage()) |
110 gothash = "" |
110 gothash = b"" |
111 |
111 |
112 if gothash != hash: |
112 if gothash != hash: |
113 if gothash != "": |
113 if gothash != b"": |
114 self.ui.warn( |
114 self.ui.warn( |
115 _('%s: data corruption (expected %s, got %s)\n') |
115 _(b'%s: data corruption (expected %s, got %s)\n') |
116 % (filename, hash, gothash) |
116 % (filename, hash, gothash) |
117 ) |
117 ) |
118 util.unlink(tmpname) |
118 util.unlink(tmpname) |
119 return False |
119 return False |
120 |
120 |
126 '''Verify the existence (and, optionally, contents) of every big |
126 '''Verify the existence (and, optionally, contents) of every big |
127 file revision referenced by every changeset in revs. |
127 file revision referenced by every changeset in revs. |
128 Return 0 if all is well, non-zero on any errors.''' |
128 Return 0 if all is well, non-zero on any errors.''' |
129 |
129 |
130 self.ui.status( |
130 self.ui.status( |
131 _('searching %d changesets for largefiles\n') % len(revs) |
131 _(b'searching %d changesets for largefiles\n') % len(revs) |
132 ) |
132 ) |
133 verified = set() # set of (filename, filenode) tuples |
133 verified = set() # set of (filename, filenode) tuples |
134 filestocheck = [] # list of (cset, filename, expectedhash) |
134 filestocheck = [] # list of (cset, filename, expectedhash) |
135 for rev in revs: |
135 for rev in revs: |
136 cctx = self.repo[rev] |
136 cctx = self.repo[rev] |
137 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) |
137 cset = b"%d:%s" % (cctx.rev(), node.short(cctx.node())) |
138 |
138 |
139 for standin in cctx: |
139 for standin in cctx: |
140 filename = lfutil.splitstandin(standin) |
140 filename = lfutil.splitstandin(standin) |
141 if filename: |
141 if filename: |
142 fctx = cctx[standin] |
142 fctx = cctx[standin] |
150 |
150 |
151 numrevs = len(verified) |
151 numrevs = len(verified) |
152 numlfiles = len({fname for (fname, fnode) in verified}) |
152 numlfiles = len({fname for (fname, fnode) in verified}) |
153 if contents: |
153 if contents: |
154 self.ui.status( |
154 self.ui.status( |
155 _('verified contents of %d revisions of %d largefiles\n') |
155 _(b'verified contents of %d revisions of %d largefiles\n') |
156 % (numrevs, numlfiles) |
156 % (numrevs, numlfiles) |
157 ) |
157 ) |
158 else: |
158 else: |
159 self.ui.status( |
159 self.ui.status( |
160 _('verified existence of %d revisions of %d largefiles\n') |
160 _(b'verified existence of %d revisions of %d largefiles\n') |
161 % (numrevs, numlfiles) |
161 % (numrevs, numlfiles) |
162 ) |
162 ) |
163 return int(failed) |
163 return int(failed) |
164 |
164 |
165 def _getfile(self, tmpfile, filename, hash): |
165 def _getfile(self, tmpfile, filename, hash): |
166 '''Fetch one revision of one file from the store and write it |
166 '''Fetch one revision of one file from the store and write it |
167 to tmpfile. Compute the hash of the file on-the-fly as it |
167 to tmpfile. Compute the hash of the file on-the-fly as it |
168 downloads and return the hash. Close tmpfile. Raise |
168 downloads and return the hash. Close tmpfile. Raise |
169 StoreError if unable to download the file (e.g. it does not |
169 StoreError if unable to download the file (e.g. it does not |
170 exist in the store).''' |
170 exist in the store).''' |
171 raise NotImplementedError('abstract method') |
171 raise NotImplementedError(b'abstract method') |
172 |
172 |
173 def _verifyfiles(self, contents, filestocheck): |
173 def _verifyfiles(self, contents, filestocheck): |
174 '''Perform the actual verification of files in the store. |
174 '''Perform the actual verification of files in the store. |
175 'contents' controls verification of content hash. |
175 'contents' controls verification of content hash. |
176 'filestocheck' is list of files to check. |
176 'filestocheck' is list of files to check. |
177 Returns _true_ if any problems are found! |
177 Returns _true_ if any problems are found! |
178 ''' |
178 ''' |
179 raise NotImplementedError('abstract method') |
179 raise NotImplementedError(b'abstract method') |