mercurial: comparison hgext/clonebundles.py

equal deleted inserted replaced

-:5ae30ff79c76
+:971dc2369b04
 It is possible to set Mercurial to automatically re-generate clone bundles when
 new content is available.
 Mercurial will take care of the process asynchronously. The defined list of
-bundle type will be generated, uploaded, and advertised.
+bundle-type will be generated, uploaded, and advertised. Older bundles will get
+decommissioned as newer ones replace them.
 Bundles Generation:
 ...................
 The extension can generate multiple variants of the clone bundle. Each
 [clone-bundles]
 upload-command=sftp put $HGCB_BUNDLE_PATH \
 sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME
+If the file was already uploaded, the command must still succeed.
 After upload, the file should be available at an url defined by
 `clone-bundles.url-template`.
 [clone-bundles]
 url-template=https://bundles.host/cache/clone-bundles/{basename}
+Old bundles cleanup:
+....................
+When new bundles are generated, the older ones are no longer necessary and can
+be removed from storage. This is done through the `clone-bundles.delete-command`
+configuration. The command is given the url of the artifact to delete through
+the `$HGCB_BUNDLE_URL` environment variable.
+[clone-bundles]
+delete-command=sftp rm sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME
+If the file was already deleted, the command must still succeed.
 """
 import os
 import weakref
 configitem(b'clone-bundles', b'auto-generate.formats', default=list)
 configitem(b'clone-bundles', b'upload-command', default=None)
+configitem(b'clone-bundles', b'delete-command', default=None)
 configitem(b'clone-bundles', b'url-template', default=None)
 configitem(b'devel', b'debug.clonebundles', default=False)
 result = upload_bundle(repo, target)
 update_bundle_list(repo, new_bundles=[result])
 cleanup_tmp_bundle(repo, target)
+def find_outdated_bundles(repo, bundles):
+"""finds outdated bundles"""
+olds = []
+per_types = {}
+for b in bundles:
+if not b.valid_for(repo):
+olds.append(b)
+continue
+l = per_types.setdefault(b.bundle_type, [])
+l.append(b)
+for key in sorted(per_types):
+all = per_types[key]
+if len(all) > 1:
+all.sort(key=lambda b: b.revs, reverse=True)
+olds.extend(all[1:])
+return olds
+def collect_garbage(repo):
+"""finds outdated bundles and get them deleted"""
+with repo.clonebundles_lock():
+bundles = read_auto_gen(repo)
+olds = find_outdated_bundles(repo, bundles)
+for o in olds:
+delete_bundle(repo, o)
+update_bundle_list(repo, del_bundles=olds)
 def upload_bundle(repo, bundle):
 """upload the result of a GeneratingBundle and return a GeneratedBundle
 The upload is done using the `clone-bundles.upload-command`
 """
 .encode('utf8')
 )
 return bundle.uploaded(url, basename)
+def delete_bundle(repo, bundle):
+"""delete a bundle from storage"""
+assert bundle.ready
+msg = b'clone-bundles: deleting bundle %s\n'
+msg %= bundle.basename
+if repo.ui.configbool(b'devel', b'debug.clonebundles'):
+repo.ui.write(msg)
+else:
+repo.ui.debug(msg)
+cmd = repo.ui.config(b'clone-bundles', b'delete-command')
+variables = {
+b'HGCB_BUNDLE_URL': bundle.file_url,
+b'HGCB_BASENAME': bundle.basename,
+}
+env = procutil.shellenviron(environ=variables)
+ret = repo.ui.system(cmd, environ=env)
+if ret:
+raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
 def auto_bundle_needed_actions(repo, bundles, op_id):
 """find the list of bundles that need action
 returns a list of RequestedBundle objects that need to be generated and
 uploaded."""
 create_bundles = []
+delete_bundles = []
 repo = repo.filtered(b"immutable")
 targets = repo.ui.configlist(b'clone-bundles', b'auto-generate.formats')
 revs = len(repo.changelog)
 generic_data = {
 'revs': revs,
 for t in targets:
 data = generic_data.copy()
 data['bundle_type'] = t
 b = RequestedBundle(**data)
 create_bundles.append(b)
-return create_bundles
+delete_bundles.extend(find_outdated_bundles(repo, bundles))
+return create_bundles, delete_bundles
 def start_one_bundle(repo, bundle):
 """start the generation of a single bundle file
 def debugmakeclonebundles(ui, repo):
 """Internal command to auto-generate debug bundles"""
 requested_bundle = util.pickle.load(procutil.stdin)
 procutil.stdin.close()
+collect_garbage(repo)
 fname = requested_bundle.suggested_filename
 fpath = repo.vfs.makedirs(b'tmp-bundles')
 fpath = repo.vfs.join(b'tmp-bundles', fname)
 bundle = requested_bundle.generating(fpath)
 update_bundle_list(repo, new_bundles=[bundle])
 def autobundle(tr):
 repo = reporef()
 assert repo is not None
 bundles = read_auto_gen(repo)
-new = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr))
+new, __ = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr))
 for data in new:
 start_one_bundle(repo, data)
 return None
 return autobundle

changeset 50404	971dc2369b04
parent 50403	5ae30ff79c76
child 50405	5b70b9f5a2f9