perf: clear vfs audit_cache before each run
When generating a stream clone, we spend a large amount of time auditing path.
Before this changes, the first run was warming the vfs cache for the other
runs, leading to a large runtime difference and a "faulty" reported timing for
the operation.
We now clear this important cache between run to get a more realistic timing.
Below are some example of median time change when clearing these cases. The
maximum time for a run did not changed significantly.
### data-env-vars.name = mozilla-central-2018-08-01-zstd-sparse-revlog
# benchmark.name = hg.perf.exchange.stream.generate
# bin-env-vars.hg.flavor = default
# bin-env-vars.hg.py-re2-module = default
# benchmark.variants.version = latest
no-clearing: 17.289905
cache-clearing: 21.587965 (+24.86%, +4.30)
## data-env-vars.name = mozilla-central-2024-03-22-zstd-sparse-revlog
no-clearing: 32.670748
cache-clearing: 40.467095 (+23.86%, +7.80)
## data-env-vars.name = mozilla-try-2019-02-18-zstd-sparse-revlog
no-clearing: 37.838858
cache-clearing: 46.072749 (+21.76%, +8.23)
## data-env-vars.name = mozilla-unified-2024-03-22-zstd-sparse-revlog
no-clearing: 32.969395
cache-clearing: 39.646209 (+20.25%, +6.68)
In addition, this significantly reduce the timing difference between the
performance command, from the perf extensions and a `real `hg bundle` call
producing a stream bundle. Some significant differences remain especially on
the "mozilla-try" repositories, but they are now smaller.
Note that some of that difference will actually not be
attributable to the stream generation (like maybe phases or branch map
computation).
Below are some benchmarks done on a currently draft changeset fixing some
unrelated slowness in `hg bundle` (34a78972af409d1ff37c29e60f6ca811ad1a457d)
### data-env-vars.name = mozilla-central-2018-08-01-zstd-sparse-revlog
# bin-env-vars.hg.flavor = default
# bin-env-vars.hg.py-re2-module = default
hg.perf.exchange.stream.generate: 21.587965
hg.command.bundle: 24.301799 (+12.57%, +2.71)
## data-env-vars.name = mozilla-central-2024-03-22-zstd-sparse-revlog
hg.perf.exchange.stream.generate: 40.467095
hg.command.bundle: 44.831317 (+10.78%, +4.36)
## data-env-vars.name = mozilla-unified-2024-03-22-zstd-sparse-revlog
hg.perf.exchange.stream.generate: 39.646209
hg.command.bundle: 45.395258 (+14.50%, +5.75)
## data-env-vars.name = mozilla-try-2019-02-18-zstd-sparse-revlog
hg.perf.exchange.stream.generate: 46.072749
hg.command.bundle: 55.882608 (+21.29%, +9.81)
## data-env-vars.name = mozilla-try-2023-03-22-zlib-general-delta
hg.perf.exchange.stream.generate: 334.716708
hg.command.bundle: 377.856767 (+12.89%, +43.14)
## data-env-vars.name = mozilla-try-2023-03-22-zstd-sparse-revlog
hg.perf.exchange.stream.generate: 302.972301
hg.command.bundle: 326.098755 (+7.63%, +23.13)
#require execbit
$ cat <<EOF >> $HGRCPATH
> [extensions]
> autodiff = $TESTDIR/autodiff.py
> [diff]
> nodates = 1
> EOF
$ hg init repo
$ cd repo
make a combination of new, changed and deleted file
$ echo regular > regular
$ echo rmregular > rmregular
$ "$PYTHON" -c "open('bintoregular', 'wb').write(b'\0')"
$ touch rmempty
$ echo exec > exec
$ chmod +x exec
$ echo rmexec > rmexec
$ chmod +x rmexec
$ echo setexec > setexec
$ echo unsetexec > unsetexec
$ chmod +x unsetexec
$ echo binary > binary
$ "$PYTHON" -c "open('rmbinary', 'wb').write(b'\0')"
$ hg ci -Am addfiles
adding binary
adding bintoregular
adding exec
adding regular
adding rmbinary
adding rmempty
adding rmexec
adding rmregular
adding setexec
adding unsetexec
$ echo regular >> regular
$ echo newregular >> newregular
$ rm rmempty
$ touch newempty
$ rm rmregular
$ echo exec >> exec
$ echo newexec > newexec
$ echo bintoregular > bintoregular
$ chmod +x newexec
$ rm rmexec
$ chmod +x setexec
$ chmod -x unsetexec
$ "$PYTHON" -c "open('binary', 'wb').write(b'\0\0')"
$ "$PYTHON" -c "open('newbinary', 'wb').write(b'\0')"
$ rm rmbinary
$ hg addremove -s 0
adding newbinary
adding newempty
adding newexec
adding newregular
removing rmbinary
removing rmempty
removing rmexec
removing rmregular
git=no: regular diff for all files
$ hg autodiff --git=no
diff -r a66d19b9302d binary
Binary file binary has changed
diff -r a66d19b9302d bintoregular
Binary file bintoregular has changed
diff -r a66d19b9302d exec
--- a/exec
+++ b/exec
@@ -1,1 +1,2 @@
exec
+exec
diff -r a66d19b9302d newbinary
Binary file newbinary has changed
diff -r a66d19b9302d newexec
--- /dev/null
+++ b/newexec
@@ -0,0 +1,1 @@
+newexec
diff -r a66d19b9302d newregular
--- /dev/null
+++ b/newregular
@@ -0,0 +1,1 @@
+newregular
diff -r a66d19b9302d regular
--- a/regular
+++ b/regular
@@ -1,1 +1,2 @@
regular
+regular
diff -r a66d19b9302d rmbinary
Binary file rmbinary has changed
diff -r a66d19b9302d rmexec
--- a/rmexec
+++ /dev/null
@@ -1,1 +0,0 @@
-rmexec
diff -r a66d19b9302d rmregular
--- a/rmregular
+++ /dev/null
@@ -1,1 +0,0 @@
-rmregular
git=yes: git diff for single regular file
$ hg autodiff --git=yes regular
diff --git a/regular b/regular
--- a/regular
+++ b/regular
@@ -1,1 +1,2 @@
regular
+regular
git=auto: regular diff for regular files and non-binary removals
$ hg autodiff --git=auto regular newregular rmregular rmexec
diff -r a66d19b9302d newregular
--- /dev/null
+++ b/newregular
@@ -0,0 +1,1 @@
+newregular
diff -r a66d19b9302d regular
--- a/regular
+++ b/regular
@@ -1,1 +1,2 @@
regular
+regular
diff -r a66d19b9302d rmexec
--- a/rmexec
+++ /dev/null
@@ -1,1 +0,0 @@
-rmexec
diff -r a66d19b9302d rmregular
--- a/rmregular
+++ /dev/null
@@ -1,1 +0,0 @@
-rmregular
$ for f in exec newexec setexec unsetexec binary newbinary newempty rmempty rmbinary bintoregular; do
> echo
> echo '% git=auto: git diff for' $f
> hg autodiff --git=auto $f
> done
% git=auto: git diff for exec
diff -r a66d19b9302d exec
--- a/exec
+++ b/exec
@@ -1,1 +1,2 @@
exec
+exec
% git=auto: git diff for newexec
diff --git a/newexec b/newexec
new file mode 100755
--- /dev/null
+++ b/newexec
@@ -0,0 +1,1 @@
+newexec
% git=auto: git diff for setexec
diff --git a/setexec b/setexec
old mode 100644
new mode 100755
% git=auto: git diff for unsetexec
diff --git a/unsetexec b/unsetexec
old mode 100755
new mode 100644
% git=auto: git diff for binary
diff --git a/binary b/binary
index a9128c283485202893f5af379dd9beccb6e79486..09f370e38f498a462e1ca0faa724559b6630c04f
GIT binary patch
literal 2
Jc${Nk0000200961
% git=auto: git diff for newbinary
diff --git a/newbinary b/newbinary
new file mode 100644
index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d
GIT binary patch
literal 1
Ic${MZ000310RR91
% git=auto: git diff for newempty
diff --git a/newempty b/newempty
new file mode 100644
% git=auto: git diff for rmempty
diff --git a/rmempty b/rmempty
deleted file mode 100644
% git=auto: git diff for rmbinary
diff --git a/rmbinary b/rmbinary
deleted file mode 100644
index f76dd238ade08917e6712764a16a22005a50573d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
% git=auto: git diff for bintoregular
diff --git a/bintoregular b/bintoregular
index f76dd238ade08917e6712764a16a22005a50573d..9c42f2b6427d8bf034b7bc23986152dc01bfd3ab
GIT binary patch
literal 13
Uc$`bh%qz(+N=+}#Ni5<5043uE82|tP
git=warn: regular diff with data loss warnings
$ hg autodiff --git=warn
diff -r a66d19b9302d binary
Binary file binary has changed
diff -r a66d19b9302d bintoregular
Binary file bintoregular has changed
diff -r a66d19b9302d exec
--- a/exec
+++ b/exec
@@ -1,1 +1,2 @@
exec
+exec
diff -r a66d19b9302d newbinary
Binary file newbinary has changed
diff -r a66d19b9302d newexec
--- /dev/null
+++ b/newexec
@@ -0,0 +1,1 @@
+newexec
diff -r a66d19b9302d newregular
--- /dev/null
+++ b/newregular
@@ -0,0 +1,1 @@
+newregular
diff -r a66d19b9302d regular
--- a/regular
+++ b/regular
@@ -1,1 +1,2 @@
regular
+regular
diff -r a66d19b9302d rmbinary
Binary file rmbinary has changed
diff -r a66d19b9302d rmexec
--- a/rmexec
+++ /dev/null
@@ -1,1 +0,0 @@
-rmexec
diff -r a66d19b9302d rmregular
--- a/rmregular
+++ /dev/null
@@ -1,1 +0,0 @@
-rmregular
data lost for: binary
data lost for: bintoregular
data lost for: newbinary
data lost for: newempty
data lost for: newexec
data lost for: rmbinary
data lost for: rmempty
data lost for: setexec
data lost for: unsetexec
git=abort: fail on execute bit change
$ hg autodiff --git=abort regular setexec
abort: losing data for setexec
[255]
git=abort: succeed on regular file
$ hg autodiff --git=abort regular
diff -r a66d19b9302d regular
--- a/regular
+++ b/regular
@@ -1,1 +1,2 @@
regular
+regular
$ cd ..