outgoing: rework the handling of the `missingroots` case to be faster
The previous implementation was slow, to the point it was taking a significant
amount of `hg bundle --type none-streamv2` call. We rework the code to compute
the same value much faster, making the operation disappear from the `hg bundle
--type none-streamv2` profile. Someone would remark that producing a streamclone
does not requires an `outgoing` object. However that is a matter for another
day. There is other user of `missingroots` (non stream `hg bundle` call for
example), and they will also benefit from this rework.
We implement an old TODO in the process, directly computing the missing and
common attribute as we have most element at hand already.
### benchmark.name = hg.command.bundle
# bin-env-vars.hg.flavor = default
# bin-env-vars.hg.py-re2-module = default
# benchmark.variants.revs = all
# benchmark.variants.type = none-streamv2
## data-env-vars.name = heptapod-public-2024-03-25-zstd-sparse-revlog
before: 7.750458
after: 6.665565 (-14.00%, -1.08)
## data-env-vars.name = mercurial-public-2024-03-22-zstd-sparse-revlog
before: 0.700229
after: 0.496050 (-29.16%, -0.20)
## data-env-vars.name = mozilla-try-2023-03-22-zstd-sparse-revlog
before: 346.508952
after: 316.749699 (-8.59%, -29.76)
## data-env-vars.name = pypy-2024-03-22-zstd-sparse-revlog
before: 3.401700
after: 2.915810 (-14.28%, -0.49)
## data-env-vars.name = tryton-public-2024-03-22-zstd-sparse-revlog
before: 1.870798
after: 1.461583 (-21.87%, -0.41)
note: this whole `missingroots` of outgoing has a limited number of callers and
could likely be replace by something simpler (like taking an explicit
"missing_revs" set for example). However this is a wider change and we focus on
a small impact, quick rework that does not change the API for now.
from mercurial import demandimport
demandimport.enable()
import os
import subprocess
import sys
import types
# Don't import pycompat because it has too many side-effects.
ispy311 = (sys.version_info.major, sys.version_info.minor) >= (3, 11)
# Only run if demandimport is allowed
if subprocess.call(
[os.environ['PYTHON'], '%s/hghave' % os.environ['TESTDIR'], 'demandimport']
):
sys.exit(80)
# We rely on assert, which gets optimized out.
if sys.flags.optimize:
sys.exit(80)
# The demand importer doesn't work on Python 3.5.
if sys.version_info[0:2] == (3, 5):
sys.exit(80)
from importlib.util import _LazyModule
try:
from importlib.util import _Module as moduletype
except ImportError:
moduletype = types.ModuleType
if os.name != 'nt':
try:
import distutils.msvc9compiler
print(
'distutils.msvc9compiler needs to be an immediate '
'importerror on non-windows platforms'
)
distutils.msvc9compiler
except ImportError:
pass
import re
rsub = re.sub
def f(obj):
l = repr(obj)
l = rsub("0x[0-9a-fA-F]+", "0x?", l)
l = rsub("from '.*'", "from '?'", l)
l = rsub("'<[a-z]*>'", "'<whatever>'", l)
return l
demandimport.disable()
os.environ['HGDEMANDIMPORT'] = 'disable'
# this enable call should not actually enable demandimport!
demandimport.enable()
from mercurial import node
# We use assert instead of a unittest test case because having imports inside
# functions changes behavior of the demand importer.
assert not isinstance(node, _LazyModule)
# now enable it for real
del os.environ['HGDEMANDIMPORT']
demandimport.enable()
# Test access to special attributes through demandmod proxy
assert 'mercurial.error' not in sys.modules
from mercurial import error as errorproxy
assert isinstance(errorproxy, _LazyModule)
assert f(errorproxy) == "<module 'mercurial.error' from '?'>", f(errorproxy)
doc = ' '.join(errorproxy.__doc__.split()[:3])
assert doc == 'Mercurial exceptions. This', doc
assert errorproxy.__name__ == 'mercurial.error', errorproxy.__name__
# __name__ must be accessible via __dict__ so the relative imports can be
# resolved
name = errorproxy.__dict__['__name__']
assert name == 'mercurial.error', name
assert not isinstance(errorproxy, _LazyModule)
assert f(errorproxy) == "<module 'mercurial.error' from '?'>", f(errorproxy)
import os
assert not isinstance(os, _LazyModule)
if ispy311:
assert f(os) == "<module 'os' (frozen)>", f(os)
else:
assert f(os) == "<module 'os' from '?'>", f(os)
assert f(os.system) == '<built-in function system>', f(os.system)
if ispy311:
assert f(os) == "<module 'os' (frozen)>", f(os)
else:
assert f(os) == "<module 'os' from '?'>", f(os)
assert 'mercurial.utils.procutil' not in sys.modules
from mercurial.utils import procutil
assert isinstance(procutil, _LazyModule)
assert f(procutil) == "<module 'mercurial.utils.procutil' from '?'>", f(
procutil
)
assert f(procutil.system) == '<function system at 0x?>', f(procutil.system)
assert procutil.__class__ == moduletype, procutil.__class__
assert f(procutil) == "<module 'mercurial.utils.procutil' from '?'>", f(
procutil
)
assert f(procutil.system) == '<function system at 0x?>', f(procutil.system)
assert 'mercurial.hgweb' not in sys.modules
from mercurial import hgweb
assert isinstance(hgweb, _LazyModule)
assert f(hgweb) == "<module 'mercurial.hgweb' from '?'>", f(hgweb)
assert isinstance(hgweb.hgweb_mod, _LazyModule)
assert f(hgweb.hgweb_mod) == "<module 'mercurial.hgweb.hgweb_mod' from '?'>", f(
hgweb.hgweb_mod
)
assert f(hgweb) == "<module 'mercurial.hgweb' from '?'>", f(hgweb)
import re as fred
assert not isinstance(fred, _LazyModule)
assert f(fred) == "<module 're' from '?'>"
import re as remod
assert not isinstance(remod, _LazyModule)
assert f(remod) == "<module 're' from '?'>"
import sys as re
assert not isinstance(re, _LazyModule)
assert f(re) == "<module 'sys' (built-in)>"
assert not isinstance(fred, _LazyModule)
assert f(fred) == "<module 're' from '?'>", f(fred)
assert f(fred.sub) == '<function sub at 0x?>', f(fred.sub)
assert not isinstance(fred, _LazyModule)
assert f(fred) == "<module 're' from '?'>", f(fred)
remod.escape # use remod
assert f(remod) == "<module 're' from '?'>", f(remod)
assert not isinstance(re, _LazyModule)
assert f(re) == "<module 'sys' (built-in)>"
assert f(type(re.stderr)) == "<class '_io.TextIOWrapper'>", f(type(re.stderr))
assert f(re) == "<module 'sys' (built-in)>"
assert 'telnetlib' not in sys.modules
import telnetlib
assert isinstance(telnetlib, _LazyModule)
assert f(telnetlib) == "<module 'telnetlib' from '?'>"
try:
from telnetlib import unknownattr
assert False, (
'no demandmod should be created for attribute of non-package '
'module:\ntelnetlib.unknownattr = %s' % f(unknownattr)
)
except ImportError as inst:
assert rsub(r"'", '', str(inst)).startswith(
'cannot import name unknownattr'
)
# Unlike the import statement, __import__() function should not raise
# ImportError even if fromlist has an unknown item
# (see Python/import.c:import_module_level() and ensure_fromlist())
assert 'ftplib' not in sys.modules
zipfileimp = __import__('ftplib', globals(), locals(), ['unknownattr'])
assert f(zipfileimp) == "<module 'ftplib' from '?'>", f(zipfileimp)
assert not hasattr(zipfileimp, 'unknownattr')
# test deactivation for issue6725
del sys.modules['telnetlib']
with demandimport.deactivated():
import telnetlib
assert telnetlib.__loader__ == telnetlib.__spec__.loader
assert telnetlib.__loader__.get_resource_reader