codemod: register core configitems using a script This is done by a script [2] using RedBaron [1], a tool designed for doing code refactoring. All "default" values are decided by the script and are strongly consistent with the existing code. There are 2 changes done manually to fix tests: [warn] mercurial/exchange.py: experimental.bundle2-output-capture: default needs manual removal [warn] mercurial/localrepo.py: experimental.hook-track-tags: default needs manual removal Since RedBaron is not confident about how to indent things [2]. [1]: https://github.com/PyCQA/redbaron [2]: https://github.com/PyCQA/redbaron/issues/100 [3]: #!/usr/bin/env python # codemod_configitems.py - codemod tool to fill configitems # # Copyright 2017 Facebook, Inc. # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. from __future__ import absolute_import, print_function import os import sys import redbaron def readpath(path): with open(path) as f: return f.read() def writepath(path, content): with open(path, 'w') as f: f.write(content) _configmethods = {'config', 'configbool', 'configint', 'configbytes', 'configlist', 'configdate'} def extractstring(rnode): """get the string from a RedBaron string or call_argument node""" while rnode.type != 'string': rnode = rnode.value return rnode.value[1:-1] # unquote, "'str'" -> "str" def uiconfigitems(red): """match *.ui.config* pattern, yield (node, method, args, section, name)""" for node in red.find_all('atomtrailers'): entry = None try: obj = node[-3].value method = node[-2].value args = node[-1] section = args[0].value name = args[1].value if (obj in ('ui', 'self') and method in _configmethods and section.type == 'string' and name.type == 'string'): entry = (node, method, args, extractstring(section), extractstring(name)) except Exception: pass else: if entry: yield entry def coreconfigitems(red): """match coreconfigitem(...) pattern, yield (node, args, section, name)""" for node in red.find_all('atomtrailers'): entry = None try: args = node[1] section = args[0].value name = args[1].value if (node[0].value == 'coreconfigitem' and section.type == 'string' and name.type == 'string'): entry = (node, args, extractstring(section), extractstring(name)) except Exception: pass else: if entry: yield entry def registercoreconfig(cfgred, section, name, defaultrepr): """insert coreconfigitem to cfgred AST section and name are plain string, defaultrepr is a string """ # find a place to insert the "coreconfigitem" item entries = list(coreconfigitems(cfgred)) for node, args, nodesection, nodename in reversed(entries): if (nodesection, nodename) < (section, name): # insert after this entry node.insert_after( 'coreconfigitem(%r, %r,\n' ' default=%s,\n' ')' % (section, name, defaultrepr)) return def main(argv): if not argv: print('Usage: codemod_configitems.py FILES\n' 'For example, FILES could be "{hgext,mercurial}/*/**.py"') dirname = os.path.dirname reporoot = dirname(dirname(dirname(os.path.abspath(__file__)))) # register configitems to this destination cfgpath = os.path.join(reporoot, 'mercurial', 'configitems.py') cfgred = redbaron.RedBaron(readpath(cfgpath)) # state about what to do registered = set((s, n) for n, a, s, n in coreconfigitems(cfgred)) toregister = {} # {(section, name): defaultrepr} coreconfigs = set() # {(section, name)}, whether it's used in core # first loop: scan all files before taking any action for i, path in enumerate(argv): print('(%d/%d) scanning %s' % (i + 1, len(argv), path)) iscore = ('mercurial' in path) and ('hgext' not in path) red = redbaron.RedBaron(readpath(path)) # find all repo.ui.config* and ui.config* calls, and collect their # section, name and default value information. for node, method, args, section, name in uiconfigitems(red): if section == 'web': # [web] section has some weirdness, ignore them for now continue defaultrepr = None key = (section, name) if len(args) == 2: if key in registered: continue if method == 'configlist': defaultrepr = 'list' elif method == 'configbool': defaultrepr = 'False' else: defaultrepr = 'None' elif len(args) >= 3 and (args[2].target is None or args[2].target.value == 'default'): # try to understand the "default" value dnode = args[2].value if dnode.type == 'name': if dnode.value in {'None', 'True', 'False'}: defaultrepr = dnode.value elif dnode.type == 'string': defaultrepr = repr(dnode.value[1:-1]) elif dnode.type in ('int', 'float'): defaultrepr = dnode.value # inconsistent default if key in toregister and toregister[key] != defaultrepr: defaultrepr = None # interesting to rewrite if key not in registered: if defaultrepr is None: print('[note] %s: %s.%s: unsupported default' % (path, section, name)) registered.add(key) # skip checking it again else: toregister[key] = defaultrepr if iscore: coreconfigs.add(key) # second loop: rewrite files given "toregister" result for path in argv: # reconstruct redbaron - trade CPU for memory red = redbaron.RedBaron(readpath(path)) changed = False for node, method, args, section, name in uiconfigitems(red): key = (section, name) defaultrepr = toregister.get(key) if defaultrepr is None or key not in coreconfigs: continue if len(args) >= 3 and (args[2].target is None or args[2].target.value == 'default'): try: del args[2] changed = True except Exception: # redbaron fails to do the rewrite due to indentation # see https://github.com/PyCQA/redbaron/issues/100 print('[warn] %s: %s.%s: default needs manual removal' % (path, section, name)) if key not in registered: print('registering %s.%s' % (section, name)) registercoreconfig(cfgred, section, name, defaultrepr) registered.add(key) if changed: print('updating %s' % path) writepath(path, red.dumps()) if toregister: print('updating configitems.py') writepath(cfgpath, cfgred.dumps()) if __name__ == "__main__": sys.exit(main(sys.argv[1:]))

     1

# profiling.py - profiling functions

     2

#

     3

# Copyright 2016 Gregory Szorc <gregory.szorc@gmail.com>

     4

#

     5

# This software may be used and distributed according to the terms of the

     6

# GNU General Public License version 2 or any later version.

     7

     8

from __future__ import absolute_import, print_function

     9

    10

import contextlib

    11

    12

from .i18n import _

    13

from . import (

    14

    encoding,

    15

    error,

    16

    extensions,

    17

    util,

    18

)

    19

    20

def _loadprofiler(ui, profiler):

    21

    """load profiler extension. return profile method, or None on failure"""

    22

    extname = profiler

    23

    extensions.loadall(ui, whitelist=[extname])

    24

    try:

    25

        mod = extensions.find(extname)

    26

    except KeyError:

    27

        return None

    28

    else:

    29

        return getattr(mod, 'profile', None)

    30

    31

@contextlib.contextmanager

    32

def lsprofile(ui, fp):

    33

    format = ui.config('profiling', 'format')

    34

    field = ui.config('profiling', 'sort')

    35

    limit = ui.configint('profiling', 'limit')

    36

    climit = ui.configint('profiling', 'nested')

    37

    38

    if format not in ['text', 'kcachegrind']:

    39

        ui.warn(_("unrecognized profiling format '%s'"

    40

                    " - Ignored\n") % format)

    41

        format = 'text'

    42

    43

    try:

    44

        from . import lsprof

    45

    except ImportError:

    46

        raise error.Abort(_(

    47

            'lsprof not available - install from '

    48

            'http://codespeak.net/svn/user/arigo/hack/misc/lsprof/'))

    49

    p = lsprof.Profiler()

    50

    p.enable(subcalls=True)

    51

    try:

    52

        yield

    53

    finally:

    54

        p.disable()

    55

    56

        if format == 'kcachegrind':

    57

            from . import lsprofcalltree

    58

            calltree = lsprofcalltree.KCacheGrind(p)

    59

            calltree.output(fp)

    60

        else:

    61

            # format == 'text'

    62

            stats = lsprof.Stats(p.getstats())

    63

            stats.sort(field)

    64

            stats.pprint(limit=limit, file=fp, climit=climit)

    65

    66

@contextlib.contextmanager

    67

def flameprofile(ui, fp):

    68

    try:

    69

        from flamegraph import flamegraph

    70

    except ImportError:

    71

        raise error.Abort(_(

    72

            'flamegraph not available - install from '

    73

            'https://github.com/evanhempel/python-flamegraph'))

    74

    # developer config: profiling.freq

    75

    freq = ui.configint('profiling', 'freq')

    76

    filter_ = None

    77

    collapse_recursion = True

    78

    thread = flamegraph.ProfileThread(fp, 1.0 / freq,

    79

                                      filter_, collapse_recursion)

    80

    start_time = util.timer()

    81

    try:

    82

        thread.start()

    83

        yield

    84

    finally:

    85

        thread.stop()

    86

        thread.join()

    87

        print('Collected %d stack frames (%d unique) in %2.2f seconds.' % (

    88

            util.timer() - start_time, thread.num_frames(),

    89

            thread.num_frames(unique=True)))

    90

    91

@contextlib.contextmanager

    92

def statprofile(ui, fp):

    93

    from . import statprof

    94

    95

    freq = ui.configint('profiling', 'freq')

    96

    if freq > 0:

    97

        # Cannot reset when profiler is already active. So silently no-op.

    98

        if statprof.state.profile_level == 0:

    99

            statprof.reset(freq)

   100

    else:

   101

        ui.warn(_("invalid sampling frequency '%s' - ignoring\n") % freq)

   102

   103

    statprof.start(mechanism='thread')

   104

   105

    try:

   106

        yield

   107

    finally:

   108

        data = statprof.stop()

   109

   110

        profformat = ui.config('profiling', 'statformat')

   111

   112

        formats = {

   113

            'byline': statprof.DisplayFormats.ByLine,

   114

            'bymethod': statprof.DisplayFormats.ByMethod,

   115

            'hotpath': statprof.DisplayFormats.Hotpath,

   116

            'json': statprof.DisplayFormats.Json,

   117

            'chrome': statprof.DisplayFormats.Chrome,

   118

        }

   119

   120

        if profformat in formats:

   121

            displayformat = formats[profformat]

   122

        else:

   123

            ui.warn(_('unknown profiler output format: %s\n') % profformat)

   124

            displayformat = statprof.DisplayFormats.Hotpath

   125

   126

        kwargs = {}

   127

   128

        def fraction(s):

   129

            if isinstance(s, (float, int)):

   130

                return float(s)

   131

            if s.endswith('%'):

   132

                v = float(s[:-1]) / 100

   133

            else:

   134

                v = float(s)

   135

            if 0 <= v <= 1:

   136

                return v

   137

            raise ValueError(s)

   138

   139

        if profformat == 'chrome':

   140

            showmin = ui.configwith(fraction, 'profiling', 'showmin', 0.005)

   141

            showmax = ui.configwith(fraction, 'profiling', 'showmax', 0.999)

   142

            kwargs.update(minthreshold=showmin, maxthreshold=showmax)

   143

        elif profformat == 'hotpath':

   144

            # inconsistent config: profiling.showmin

   145

            limit = ui.configwith(fraction, 'profiling', 'showmin', 0.05)

   146

            kwargs['limit'] = limit

   147

   148

        statprof.display(fp, data=data, format=displayformat, **kwargs)

   149

   150

class profile(object):

   151

    """Start profiling.

   152

   153

    Profiling is active when the context manager is active. When the context

   154

    manager exits, profiling results will be written to the configured output.

   155

    """

   156

    def __init__(self, ui, enabled=True):

   157

        self._ui = ui

   158

        self._output = None

   159

        self._fp = None

   160

        self._fpdoclose = True

   161

        self._profiler = None

   162

        self._enabled = enabled

   163

        self._entered = False

   164

        self._started = False

   165

   166

    def __enter__(self):

   167

        self._entered = True

   168

        if self._enabled:

   169

            self.start()

   170

        return self

   171

   172

    def start(self):

   173

        """Start profiling.

   174

   175

        The profiling will stop at the context exit.

   176

   177

        If the profiler was already started, this has no effect."""

   178

        if not self._entered:

   179

            raise error.ProgrammingError()

   180

        if self._started:

   181

            return

   182

        self._started = True

   183

        profiler = encoding.environ.get('HGPROF')

   184

        proffn = None

   185

        if profiler is None:

   186

            profiler = self._ui.config('profiling', 'type', default='stat')

   187

        if profiler not in ('ls', 'stat', 'flame'):

   188

            # try load profiler from extension with the same name

   189

            proffn = _loadprofiler(self._ui, profiler)

   190

            if proffn is None:

   191

                self._ui.warn(_("unrecognized profiler '%s' - ignored\n")

   192

                              % profiler)

   193

                profiler = 'stat'

   194

   195

        self._output = self._ui.config('profiling', 'output')

   196

   197

        try:

   198

            if self._output == 'blackbox':

   199

                self._fp = util.stringio()

   200

            elif self._output:

   201

                path = self._ui.expandpath(self._output)

   202

                self._fp = open(path, 'wb')

   203

            else:

   204

                self._fpdoclose = False

   205

                self._fp = self._ui.ferr

   206

   207

            if proffn is not None:

   208

                pass

   209

            elif profiler == 'ls':

   210

                proffn = lsprofile

   211

            elif profiler == 'flame':

   212

                proffn = flameprofile

   213

            else:

   214

                proffn = statprofile

   215

   216

            self._profiler = proffn(self._ui, self._fp)

   217

            self._profiler.__enter__()

   218

        except: # re-raises

   219

            self._closefp()

   220

            raise

   221

   222

    def __exit__(self, exception_type, exception_value, traceback):

   223

        propagate = None

   224

        if self._profiler is not None:

   225

            propagate = self._profiler.__exit__(exception_type, exception_value,

   226

                                                traceback)

   227

            if self._output == 'blackbox':

   228

                val = 'Profile:\n%s' % self._fp.getvalue()

   229

                # ui.log treats the input as a format string,

   230

                # so we need to escape any % signs.

   231

                val = val.replace('%', '%%')

   232

                self._ui.log('profile', val)

   233

        self._closefp()

   234

        return propagate

   235

   236

    def _closefp(self):

   237

        if self._fpdoclose and self._fp is not None:

   238

            self._fp.close()