Mercurial Mercurial > mercurial / changeset
summary | shortlog | changelog | graph | tags | bookmarks | branches | files | changeset | raw | help
Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
remotefilelog: replace repack lock to solve race condition
authorBoris Feld <boris.feld@octobus.net>
Wed, 11 Sep 2019 17:41:13 +0200
changeset 42943 5fadf6103790
parent 42942 adac17faa72e
child 42944 c2676b5a9f59
remotefilelog: replace repack lock to solve race condition 2c74337e6483 reduced the probability of race-conditions when starting background repack and prefetch and we saw the difference in our CI instance with all failures disappearing except one where one call to waitonrepack seems to returns too early. I'm not sure what exactly goes wrong but I realized that while the prefetch operation uses a standard Mercurial lock, the repack operation is using a custom lock based on `fcntl.flock` on available platforms. As `extutil.flock` fallback on traditional Mercurial locks on other platforms and the tests are stable on my laptop, our CI environment and GCC112, I'm sending this patch to standardize the behavior across environments. Differential Revision: https://phab.mercurial-scm.org/D6844
hgext/remotefilelog/debugcommands.py file | annotate | diff | comparison | revisions
hgext/remotefilelog/repack.py file | annotate | diff | comparison | revisions
tests/test-remotefilelog-bgprefetch.t file | annotate | diff | comparison | revisions
tests/test-remotefilelog-gc.t file | annotate | diff | comparison | revisions
tests/test-remotefilelog-repack-fast.t file | annotate | diff | comparison | revisions
tests/test-remotefilelog-repack.t file | annotate | diff | comparison | revisions 
--- a/hgext/remotefilelog/debugcommands.py	Tue Sep 17 18:36:30 2019 +0200
+++ b/hgext/remotefilelog/debugcommands.py	Wed Sep 11 17:41:13 2019 +0200
@@ -15,6 +15,7 @@
 from mercurial import (
     error,
     filelog,
+    lock as lockmod,
     node as nodemod,
     pycompat,
     revlog,
@@ -22,7 +23,6 @@
 from . import (
     constants,
     datapack,
-    extutil,
     fileserverclient,
     historypack,
     repack,
@@ -369,7 +369,7 @@
             short(p2node), short(linknode), copyfrom))
 
 def debugwaitonrepack(repo):
-    with extutil.flock(repack.repacklockvfs(repo).join('repacklock'), ''):
+    with lockmod.lock(repack.repacklockvfs(repo), "repacklock", timeout=-1):
         return
 
 def debugwaitonprefetch(repo):
--- a/hgext/remotefilelog/repack.py	Tue Sep 17 18:36:30 2019 +0200
+++ b/hgext/remotefilelog/repack.py	Wed Sep 11 17:41:13 2019 +0200
@@ -11,6 +11,7 @@
 from mercurial import (
     encoding,
     error,
+    lock as lockmod,
     mdiff,
     policy,
     pycompat,
@@ -23,7 +24,6 @@
     constants,
     contentstore,
     datapack,
-    extutil,
     historypack,
     metadatastore,
     shallowutil,
@@ -432,8 +432,8 @@
     def run(self, targetdata, targethistory):
         ledger = repackledger()
 
-        with extutil.flock(repacklockvfs(self.repo).join("repacklock"),
-                           _('repacking %s') % self.repo.origroot, timeout=0):
+        with lockmod.lock(repacklockvfs(self.repo), "repacklock", desc=None,
+                          timeout=0):
             self.repo.hook('prerepack')
 
             # Populate ledger from source
--- a/tests/test-remotefilelog-bgprefetch.t	Tue Sep 17 18:36:30 2019 +0200
+++ b/tests/test-remotefilelog-bgprefetch.t	Wed Sep 11 17:41:13 2019 +0200
@@ -117,7 +117,6 @@
   $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histpack
   $TESTTMP/hgcache/master/packs/8ce5ab3745465ab83bba30a7b9c295e0c8404652.dataidx
   $TESTTMP/hgcache/master/packs/8ce5ab3745465ab83bba30a7b9c295e0c8404652.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # background prefetch with repack on update when wcprevset configured
@@ -154,7 +153,6 @@
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that file 'w' was prefetched - it was not part of the update operation and therefore
@@ -207,7 +205,6 @@
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that file 'w' was prefetched - it was not part of the commit operation and therefore
@@ -246,7 +243,7 @@
   $ find $CACHEDIR -type f | sort
   $ hg rebase -s temporary -d foo
   rebasing 3:58147a5b5242 "b" (temporary tip)
-  saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg (glob)
+  saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg
   3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
   $ sleep 1
   $ hg debugwaitonprefetch >/dev/null 2>%1
@@ -301,7 +298,6 @@
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that files were prefetched
@@ -346,7 +342,6 @@
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.dataidx
   $TESTTMP/hgcache/master/packs/f4d50848e0b465e9bfd2875f213044c06cfd7407.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Ensure that files were prefetched
--- a/tests/test-remotefilelog-gc.t	Tue Sep 17 18:36:30 2019 +0200
+++ b/tests/test-remotefilelog-gc.t	Wed Sep 11 17:41:13 2019 +0200
@@ -93,7 +93,6 @@
   $TESTTMP/hgcache/master/packs/320dab99b7e3f60512b97f347689625263d22cf5.datapack
   $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histidx
   $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that warning is displayed when there are no valid repos in repofile
--- a/tests/test-remotefilelog-repack-fast.t	Tue Sep 17 18:36:30 2019 +0200
+++ b/tests/test-remotefilelog-repack-fast.t	Wed Sep 11 17:41:13 2019 +0200
@@ -56,7 +56,6 @@
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that the packs are readonly
@@ -65,7 +64,6 @@
   -r--r--r--     172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   -r--r--r--    1074 b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   -r--r--r--      72 b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  -rw-r--r--       0 repacklock
 
 # Test that the data in the new packs is accessible
   $ hg cat -r . x
@@ -89,7 +87,6 @@
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --traceback
@@ -99,7 +96,6 @@
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Verify all the file data is still available
@@ -119,7 +115,6 @@
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Run two repacks at once
@@ -143,7 +138,6 @@
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --background
@@ -155,7 +149,6 @@
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack
   $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
   $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test debug commands
@@ -232,7 +225,6 @@
   bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
   fb3aa57b22789ebcc45706c352e2d6af099c5816.dataidx
   fb3aa57b22789ebcc45706c352e2d6af099c5816.datapack
-  repacklock
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
--- a/tests/test-remotefilelog-repack.t	Tue Sep 17 18:36:30 2019 +0200
+++ b/tests/test-remotefilelog-repack.t	Wed Sep 11 17:41:13 2019 +0200
@@ -54,7 +54,6 @@
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test that the packs are readonly
@@ -63,7 +62,6 @@
   -r--r--r--     172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   -r--r--r--    1074 b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   -r--r--r--      72 b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  -rw-r--r--       0 repacklock
 
 # Test that the data in the new packs is accessible
   $ hg cat -r . x
@@ -87,7 +85,6 @@
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # First assert that with --packsonly, the loose object will be ignored:
@@ -100,7 +97,6 @@
   $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.dataidx
   $TESTTMP/hgcache/master/packs/b1e0cfc7f345e408a7825e3081501959488d59ce.datapack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --traceback
@@ -110,7 +106,6 @@
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Verify all the file data is still available
@@ -130,7 +125,6 @@
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Run two repacks at once
@@ -154,7 +148,6 @@
   $TESTTMP/hgcache/master/packs/78840d69389c7404327f7477e3931c89945c37d1.datapack
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
   $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
   $ hg repack --background
@@ -166,7 +159,6 @@
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack
   $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
   $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
-  $TESTTMP/hgcache/master/packs/repacklock
   $TESTTMP/hgcache/repos
 
 # Test debug commands
@@ -243,7 +235,6 @@
   bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
   fb3aa57b22789ebcc45706c352e2d6af099c5816.dataidx
   fb3aa57b22789ebcc45706c352e2d6af099c5816.datapack
-  repacklock
   $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
   
   x
mercurial