i18n/hggettext
author Pierre-Yves David <pierre-yves.david@octobus.net>
Tue, 09 Apr 2024 02:54:19 +0200
changeset 51586 1cef1412af3e
parent 48875 6000f5b25c9b
permissions -rwxr-xr-x
phases: rework the logic of _pushdiscoveryphase to bound complexity This rework the various graph traversal in _pushdiscoveryphase to keep the complexity in check. This is done though a couple of things: - first, limiting the space we have to explore, for example, if we are not in publishing push, we don't need to consider remote draft roots that are also draft locally, as there is nothing to be moved there. - avoid unbounded descendant computation, and use the faster "rev between" computation. This provide a massive boost to performance when exchanging with repository with a massive amount of draft, like mozilla-try: ### data-env-vars.name = mozilla-try-2023-03-22-zstd-sparse-revlog # benchmark.name = hg.command.push # bin-env-vars.hg.flavor = default # bin-env-vars.hg.py-re2-module = default # benchmark.variants.explicit-rev = all-out-heads # benchmark.variants.issue6528 = disabled # benchmark.variants.protocol = ssh # benchmark.variants.reuse-external-delta-parent = default ## benchmark.variants.revs = any-1-extra-rev before: 20.346590 seconds after: 11.232059 seconds (-38.15%, -7.48 seconds) ## benchmark.variants.revs = any-100-extra-rev before: 24.752051 seconds after: 15.367412 seconds (-37.91%, -9.38 seconds) After this changes, the push operation is still quite too slow. Some of this can be attributed to general phases slowness (reading all the roots from disk for example) and other know slowness (not using persistent-nodemap, branchmap, tags, etc. We are also working on them, but with this series, phase discovery during push no longer showing up in profile and this is a pretty nice and bit low-hanging fruit out of the way. ### (same case as the above) # benchmark.variants.revs = any-1-extra-rev pre-%ln-change: 44.235070 this-changeset: 11.232059 seconds (-74.61%, -33.00 seconds) # benchmark.variants.revs = any-100-extra-rev pre-%ln-change: 49.234697 this-changeset: 15.367412 seconds (-68.79%, -33.87 seconds) Note that with this change, the `hg push` performance is now much closer to the `hg pull` performance, even it still lagging behind a bit. (and the overall performance are still too slow). ### data-env-vars.name = mozilla-try-2023-03-22-ds2-pnm # benchmark.variants.explicit-rev = all-out-heads # benchmark.variants.issue6528 = disabled # benchmark.variants.protocol = ssh # benchmark.variants.pulled-delta-reuse-policy = default # bin-env-vars.hg.flavor = rust ## benchmark.variants.revs = any-1-extra-rev hg.command.pull: 6.517450 hg.command.push: 11.219888 ## benchmark.variants.revs = any-100-extra-rev hg.command.pull: 10.160991 hg.command.push: 14.251107 ### data-env-vars.name = mozilla-try-2023-03-22-zstd-sparse-revlog # bin-env-vars.hg.py-re2-module = default # benchmark.variants.explicit-rev = all-out-heads # benchmark.variants.issue6528 = disabled # benchmark.variants.protocol = ssh # benchmark.variants.pulled-delta-reuse-policy = default ## bin-env-vars.hg.flavor = default ## benchmark.variants.revs = any-1-extra-rev hg.command.pull: 8.577772 hg.command.push: 11.232059 ## bin-env-vars.hg.flavor = default ## benchmark.variants.revs = any-100-extra-rev hg.command.pull: 13.152976 hg.command.push: 15.367412 ## bin-env-vars.hg.flavor = rust ## benchmark.variants.revs = any-1-extra-rev hg.command.pull: 8.731982 hg.command.push: 11.178751 ## bin-env-vars.hg.flavor = rust ## benchmark.variants.revs = any-100-extra-rev hg.command.pull: 13.184236 hg.command.push: 15.620843

#!/usr/bin/env python3
#
# hggettext - carefully extract docstrings for Mercurial
#
# Copyright 2009 Olivia Mackall <olivia@selenic.com> and others
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

# The normalize function is taken from pygettext which is distributed
# with Python under the Python License, which is GPL compatible.

"""Extract docstrings from Mercurial commands.

Compared to pygettext, this script knows about the cmdtable and table
dictionaries used by Mercurial, and will only extract docstrings from
functions mentioned therein.

Use xgettext like normal to extract strings marked as translatable and
join the message cataloges to get the final catalog.
"""


import inspect
import os
import re
import sys


def escape(s):
    # The order is important, the backslash must be escaped first
    # since the other replacements introduce new backslashes
    # themselves.
    s = s.replace('\\', '\\\\')
    s = s.replace('\n', '\\n')
    s = s.replace('\r', '\\r')
    s = s.replace('\t', '\\t')
    s = s.replace('"', '\\"')
    return s


def normalize(s):
    # This converts the various Python string types into a format that
    # is appropriate for .po files, namely much closer to C style.
    lines = s.split('\n')
    if len(lines) == 1:
        s = '"' + escape(s) + '"'
    else:
        if not lines[-1]:
            del lines[-1]
            lines[-1] = lines[-1] + '\n'
        lines = map(escape, lines)
        lineterm = '\\n"\n"'
        s = '""\n"' + lineterm.join(lines) + '"'
    return s


def poentry(path, lineno, s):
    return (
        '#: %s:%d\n' % (path, lineno)
        + 'msgid %s\n' % normalize(s)
        + 'msgstr ""\n'
    )


doctestre = re.compile(r'^ +>>> ', re.MULTILINE)


def offset(src, doc, name, lineno, default):
    """Compute offset or issue a warning on stdout."""
    # remove doctest part, in order to avoid backslash mismatching
    m = doctestre.search(doc)
    if m:
        doc = doc[: m.start()]

    # Backslashes in doc appear doubled in src.
    end = src.find(doc.replace('\\', '\\\\'))
    if end == -1:
        # This can happen if the docstring contains unnecessary escape
        # sequences such as \" in a triple-quoted string. The problem
        # is that \" is turned into " and so doc wont appear in src.
        sys.stderr.write(
            "%s:%d:warning:"
            " unknown docstr offset, assuming %d lines\n"
            % (name, lineno, default)
        )
        return default
    else:
        return src.count('\n', 0, end)


def importpath(path):
    """Import a path like foo/bar/baz.py and return the baz module."""
    if path.endswith('.py'):
        path = path[:-3]
    if path.endswith('/__init__'):
        path = path[:-9]
    path = path.replace('/', '.')
    mod = __import__(path)
    for comp in path.split('.')[1:]:
        mod = getattr(mod, comp)
    return mod


def docstrings(path):
    """Extract docstrings from path.

    This respects the Mercurial cmdtable/table convention and will
    only extract docstrings from functions mentioned in these tables.
    """
    mod = importpath(path)
    if not path.startswith('mercurial/') and mod.__doc__:
        with open(path) as fobj:
            src = fobj.read()
        lineno = 1 + offset(src, mod.__doc__, path, 1, 7)
        print(poentry(path, lineno, mod.__doc__))

    functions = list(getattr(mod, 'i18nfunctions', []))
    functions = [(f, True) for f in functions]

    cmdtable = getattr(mod, 'cmdtable', {})
    if not cmdtable:
        # Maybe we are processing mercurial.commands?
        cmdtable = getattr(mod, 'table', {})
    functions.extend((c[0], False) for c in cmdtable.values())

    for func, rstrip in functions:
        if func.__doc__:
            docobj = func  # this might be a proxy to provide formatted doc
            func = getattr(func, '_origfunc', func)
            funcmod = inspect.getmodule(func)
            extra = ''
            if funcmod.__package__ == funcmod.__name__:
                extra = '/__init__'
            actualpath = '%s%s.py' % (funcmod.__name__.replace('.', '/'), extra)

            src = inspect.getsource(func)
            lineno = inspect.getsourcelines(func)[1]
            doc = docobj.__doc__
            origdoc = getattr(docobj, '_origdoc', '')
            if rstrip:
                doc = doc.rstrip()
                origdoc = origdoc.rstrip()
            if origdoc:
                lineno += offset(src, origdoc, actualpath, lineno, 1)
            else:
                lineno += offset(src, doc, actualpath, lineno, 1)
            print(poentry(actualpath, lineno, doc))


def rawtext(path):
    with open(path) as f:
        src = f.read()
    print(poentry(path, 1, src))


if __name__ == "__main__":
    # It is very important that we import the Mercurial modules from
    # the source tree where hggettext is executed. Otherwise we might
    # accidentally import and extract strings from a Mercurial
    # installation mentioned in PYTHONPATH.
    sys.path.insert(0, os.getcwd())
    from mercurial import demandimport

    demandimport.enable()
    for path in sys.argv[1:]:
        if path.endswith('.txt'):
            rawtext(path)
        else:
            docstrings(path)