# testparseutil.py - utilities to parse test script for check tools

#

#  Copyright 2018 FUJIWARA Katsunori <foozy@lares.dti.ne.jp> and others

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

from __future__ import absolute_import, print_function

import abc

import re

import sys

####################

# for Python3 compatibility (almost comes from mercurial/pycompat.py)

ispy3 = (sys.version_info[0] >= 3)

def identity(a):

    return a

def _rapply(f, xs):

    if xs is None:

        # assume None means non-value of optional data

        return xs

    if isinstance(xs, (list, set, tuple)):

        return type(xs)(_rapply(f, x) for x in xs)

    if isinstance(xs, dict):

        return type(xs)((_rapply(f, k), _rapply(f, v)) for k, v in xs.items())

    return f(xs)

def rapply(f, xs):

    if f is identity:

        # fast path mainly for py2

        return xs

    return _rapply(f, xs)

if ispy3:

    import builtins

    # TODO: .buffer might not exist if std streams were replaced; we'll need

    # a silly wrapper to make a bytes stream backed by a unicode one.

    stdin = sys.stdin.buffer

    stdout = sys.stdout.buffer

    stderr = sys.stderr.buffer

    def bytestr(s):

        # tiny version of pycompat.bytestr

        return s.encode('latin1')

    def sysstr(s):

        if isinstance(s, builtins.str):

            return s

        return s.decode(u'latin-1')

    def opentext(f):

        return open(f, 'rb')

else:

    stdin = sys.stdin

    stdout = sys.stdout

    stderr = sys.stderr

    bytestr = str

    sysstr = identity

    opentext = open

def b2s(x):

    # convert BYTES elements in "x" to SYSSTR recursively

    return rapply(sysstr, x)

def writeout(data):

    # write "data" in BYTES into stdout

    stdout.write(data)

def writeerr(data):

    # write "data" in BYTES into stderr

    stderr.write(data)

####################

class embeddedmatcher(object):

    """Base class to detect embedded code fragments in *.t test script

    """

    __metaclass__ = abc.ABCMeta

    def __init__(self, desc):

        self.desc = desc

    @abc.abstractmethod

    def startsat(self, line):

        """Examine whether embedded code starts at line

        This can return arbitrary object, and it is used as 'ctx' for

        subsequent method invocations.

        """

    @abc.abstractmethod

    def endsat(self, ctx, line):

        """Examine whether embedded code ends at line"""

    @abc.abstractmethod

    def isinside(self, ctx, line):

        """Examine whether line is inside embedded code, if not yet endsat

        """

    @abc.abstractmethod

    def ignores(self, ctx):

        """Examine whether detected embedded code should be ignored"""

    @abc.abstractmethod

    def filename(self, ctx):

        """Return filename of embedded code

        If filename isn't specified for embedded code explicitly, this

        returns None.

        """

    @abc.abstractmethod

    def codeatstart(self, ctx, line):

        """Return actual code at the start line of embedded code

        This might return None, if the start line doesn't contain

        actual code.

        """

    @abc.abstractmethod

    def codeatend(self, ctx, line):

        """Return actual code at the end line of embedded code

        This might return None, if the end line doesn't contain actual

        code.

        """

    @abc.abstractmethod

    def codeinside(self, ctx, line):

        """Return actual code at line inside embedded code"""

def embedded(basefile, lines, errors, matchers):

    """pick embedded code fragments up from given lines

    This is common parsing logic, which examines specified matchers on

    given lines.

    :basefile: a name of a file, from which lines to be parsed come.

    :lines: to be parsed (might be a value returned by "open(basefile)")

    :errors: an array, into which messages for detected error are stored

    :matchers: an array of embeddedmatcher objects

    This function yields '(filename, starts, ends, code)' tuple.

    :filename: a name of embedded code, if it is explicitly specified

               (e.g.  "foobar" of "cat >> foobar <<EOF").

               Otherwise, this is None

    :starts: line number (1-origin), at which embedded code starts (inclusive)

    :ends: line number (1-origin), at which embedded code ends (exclusive)

    :code: extracted embedded code, which is single-stringified

    >>> class ambigmatcher(object):

    ...     # mock matcher class to examine implementation of

    ...     # "ambiguous matching" corner case

    ...     def __init__(self, desc, matchfunc):

    ...         self.desc = desc

    ...         self.matchfunc = matchfunc

    ...     def startsat(self, line):

    ...         return self.matchfunc(line)

    >>> ambig1 = ambigmatcher(b'ambiguous #1',

    ...                       lambda l: l.startswith(b'  $ cat '))

    >>> ambig2 = ambigmatcher(b'ambiguous #2',

    ...                       lambda l: l.endswith(b'<< EOF\\n'))

    >>> lines = [b'  $ cat > foo.py << EOF\\n']

    >>> errors = []

    >>> matchers = [ambig1, ambig2]

    >>> list(t for t in embedded(b'<dummy>', lines, errors, matchers))

    []

    >>> b2s(errors)

    ['<dummy>:1: ambiguous line for "ambiguous #1", "ambiguous #2"']

    """

    matcher = None

    ctx = filename = code = startline = None # for pyflakes

    for lineno, line in enumerate(lines, 1):

        if not line.endswith(b'\n'):

            line += b'\n' # to normalize EOF line

        if matcher: # now, inside embedded code

            if matcher.endsat(ctx, line):

                codeatend = matcher.codeatend(ctx, line)

                if codeatend is not None:

                    code.append(codeatend)

                if not matcher.ignores(ctx):

                    yield (filename, startline, lineno, b''.join(code))

                matcher = None

                # DO NOT "continue", because line might start next fragment

            elif not matcher.isinside(ctx, line):

                # this is an error of basefile

                # (if matchers are implemented correctly)

                errors.append(b'%s:%d: unexpected line for "%s"'

                              % (basefile, lineno, matcher.desc))

                # stop extracting embedded code by current 'matcher',

                # because appearance of unexpected line might mean

                # that expected end-of-embedded-code line might never

                # appear

                matcher = None

                # DO NOT "continue", because line might start next fragment

            else:

                code.append(matcher.codeinside(ctx, line))

                continue

        # examine whether current line starts embedded code or not

        assert not matcher

        matched = []

        for m in matchers:

            ctx = m.startsat(line)

            if ctx:

                matched.append((m, ctx))

        if matched:

            if len(matched) > 1:

                # this is an error of matchers, maybe

                errors.append(b'%s:%d: ambiguous line for %s' %

                              (basefile, lineno,

                               b', '.join([b'"%s"' % m.desc

                                           for m, c in matched])))

                # omit extracting embedded code, because choosing

                # arbitrary matcher from matched ones might fail to

                # detect the end of embedded code as expected.

                continue

            matcher, ctx = matched[0]

            filename = matcher.filename(ctx)

            code = []

            codeatstart = matcher.codeatstart(ctx, line)

            if codeatstart is not None:

                code.append(codeatstart)

                startline = lineno

            else:

                startline = lineno + 1

    if matcher:

        # examine whether EOF ends embedded code, because embedded

        # code isn't yet ended explicitly

        if matcher.endsat(ctx, b'\n'):

            codeatend = matcher.codeatend(ctx, b'\n')

            if codeatend is not None:

                code.append(codeatend)

            if not matcher.ignores(ctx):

                yield (filename, startline, lineno + 1, b''.join(code))

        else:

            # this is an error of basefile

            # (if matchers are implemented correctly)

            errors.append(b'%s:%d: unexpected end of file for "%s"'

                          % (basefile, lineno, matcher.desc))

# heredoc limit mark to ignore embedded code at check-code.py or so

heredocignorelimit = b'NO_CHECK_EOF'

# the pattern to match against cases below, and to return a limit mark

# string as 'lname' group

#

# - << LIMITMARK

# - << "LIMITMARK"

# - << 'LIMITMARK'

heredoclimitpat = br'\s*<<\s*(?P<lquote>["\']?)(?P<limit>\w+)(?P=lquote)'

class fileheredocmatcher(embeddedmatcher):

    """Detect "cat > FILE << LIMIT" style embedded code

    >>> matcher = fileheredocmatcher(b'heredoc .py file', br'[^<]+\.py')

    >>> b2s(matcher.startsat(b'  $ cat > file.py << EOF\\n'))

    ('file.py', '  > EOF\\n')

    >>> b2s(matcher.startsat(b'  $ cat   >>file.py   <<EOF\\n'))

    ('file.py', '  > EOF\\n')

    >>> b2s(matcher.startsat(b'  $ cat>  \\x27any file.py\\x27<<  "EOF"\\n'))

    ('any file.py', '  > EOF\\n')

    >>> b2s(matcher.startsat(b"  $ cat > file.py << 'ANYLIMIT'\\n"))

    ('file.py', '  > ANYLIMIT\\n')

    >>> b2s(matcher.startsat(b'  $ cat<<ANYLIMIT>"file.py"\\n'))

    ('file.py', '  > ANYLIMIT\\n')

    >>> start = b'  $ cat > file.py << EOF\\n'

    >>> ctx = matcher.startsat(start)

    >>> matcher.codeatstart(ctx, start)

    >>> b2s(matcher.filename(ctx))

    'file.py'

    >>> matcher.ignores(ctx)

    False

    >>> inside = b'  > foo = 1\\n'

    >>> matcher.endsat(ctx, inside)

    False

    >>> matcher.isinside(ctx, inside)

    True

    >>> b2s(matcher.codeinside(ctx, inside))

    'foo = 1\\n'

    >>> end = b'  > EOF\\n'

    >>> matcher.endsat(ctx, end)

    True

    >>> matcher.codeatend(ctx, end)

    >>> matcher.endsat(ctx, b'  > EOFEOF\\n')

    False

    >>> ctx = matcher.startsat(b'  $ cat > file.py << NO_CHECK_EOF\\n')

    >>> matcher.ignores(ctx)

    True

    """

    _prefix = b'  > '

    def __init__(self, desc, namepat):

        super(fileheredocmatcher, self).__init__(desc)

        # build the pattern to match against cases below (and ">>"

        # variants), and to return a target filename string as 'name'

        # group

        #

        # - > NAMEPAT

        # - > "NAMEPAT"

        # - > 'NAMEPAT'

        namepat = (br'\s*>>?\s*(?P<nquote>["\']?)(?P<name>%s)(?P=nquote)'

                   % namepat)

        self._fileres = [

            # "cat > NAME << LIMIT" case

            re.compile(br'  \$ \s*cat' + namepat + heredoclimitpat),

            # "cat << LIMIT > NAME" case

            re.compile(br'  \$ \s*cat' + heredoclimitpat + namepat),

        ]

    def startsat(self, line):

        # ctx is (filename, END-LINE-OF-EMBEDDED-CODE) tuple

        for filere in self._fileres:

            matched = filere.match(line)

            if matched:

                return (matched.group('name'),

                        b'  > %s\n' % matched.group('limit'))

    def endsat(self, ctx, line):

        return ctx[1] == line

    def isinside(self, ctx, line):

        return line.startswith(self._prefix)

    def ignores(self, ctx):

        return b'  > %s\n' % heredocignorelimit == ctx[1]

    def filename(self, ctx):

        return ctx[0]

    def codeatstart(self, ctx, line):

        return None # no embedded code at start line

    def codeatend(self, ctx, line):

        return None # no embedded code at end line

    def codeinside(self, ctx, line):

        return line[len(self._prefix):] # strip prefix

####

# for embedded python script

class pydoctestmatcher(embeddedmatcher):

    """Detect ">>> code" style embedded python code

    >>> matcher = pydoctestmatcher()

    >>> startline = b'  >>> foo = 1\\n'

    >>> matcher.startsat(startline)

    True

    >>> matcher.startsat(b'  ... foo = 1\\n')

    False

    >>> ctx = matcher.startsat(startline)

    >>> matcher.filename(ctx)

    >>> matcher.ignores(ctx)

    False

    >>> b2s(matcher.codeatstart(ctx, startline))

    'foo = 1\\n'

    >>> inside = b'  >>> foo = 1\\n'

    >>> matcher.endsat(ctx, inside)

    False

    >>> matcher.isinside(ctx, inside)

    True

    >>> b2s(matcher.codeinside(ctx, inside))

    'foo = 1\\n'

    >>> inside = b'  ... foo = 1\\n'

    >>> matcher.endsat(ctx, inside)

    False

    >>> matcher.isinside(ctx, inside)

    True

    >>> b2s(matcher.codeinside(ctx, inside))

    'foo = 1\\n'

    >>> inside = b'  expected output\\n'

    >>> matcher.endsat(ctx, inside)

    False

    >>> matcher.isinside(ctx, inside)

    True

    >>> b2s(matcher.codeinside(ctx, inside))

    '\\n'

    >>> inside = b'  \\n'

    >>> matcher.endsat(ctx, inside)

    False

    >>> matcher.isinside(ctx, inside)

    True

    >>> b2s(matcher.codeinside(ctx, inside))

    '\\n'

    >>> end = b'  $ foo bar\\n'

    >>> matcher.endsat(ctx, end)

    True

    >>> matcher.codeatend(ctx, end)

    >>> end = b'\\n'

    >>> matcher.endsat(ctx, end)

    True

    >>> matcher.codeatend(ctx, end)

    """

    _prefix = b'  >>> '

    _prefixre = re.compile(br'  (>>>|\.\.\.) ')

    # If a line matches against not _prefixre but _outputre, that line

    # is "an expected output line" (= not a part of code fragment).

    #

    # Strictly speaking, a line matching against "(#if|#else|#endif)"

    # is also treated similarly in "inline python code" semantics by

    # run-tests.py. But "directive line inside inline python code"

    # should be rejected by Mercurial reviewers. Therefore, this

    # regexp does not matche against such directive lines.

    _outputre = re.compile(br'  $|  [^$]')

    def __init__(self):

        super(pydoctestmatcher, self).__init__(b"doctest style python code")

    def startsat(self, line):

        # ctx is "True"

        return line.startswith(self._prefix)

    def endsat(self, ctx, line):

        return not (self._prefixre.match(line) or self._outputre.match(line))

    def isinside(self, ctx, line):

        return True # always true, if not yet ended

    def ignores(self, ctx):

        return False # should be checked always

    def filename(self, ctx):

        return None # no filename

    def codeatstart(self, ctx, line):