# phabricator.py - simple Phabricator integration

#

# Copyright 2017 Facebook, Inc.

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

"""simple Phabricator integration (EXPERIMENTAL)

This extension provides a ``phabsend`` command which sends a stack of

changesets to Phabricator, and a ``phabread`` command which prints a stack of

revisions in a format suitable for :hg:`import`, and a ``phabupdate`` command

to update statuses in batch.

By default, Phabricator requires ``Test Plan`` which might prevent some

changeset from being sent. The requirement could be disabled by changing

``differential.require-test-plan-field`` config server side.

Config::

    [phabricator]

    # Phabricator URL

    url = https://phab.example.com/

    # Repo callsign. If a repo has a URL https://$HOST/diffusion/FOO, then its

    # callsign is "FOO".

    callsign = FOO

    # curl command to use. If not set (default), use builtin HTTP library to

    # communicate. If set, use the specified curl command. This could be useful

    # if you need to specify advanced options that is not easily supported by

    # the internal library.

    curlcmd = curl --connect-timeout 2 --retry 3 --silent

    [auth]

    example.schemes = https

    example.prefix = phab.example.com

    # API token. Get it from https://$HOST/conduit/login/

    example.phabtoken = cli-xxxxxxxxxxxxxxxxxxxxxxxxxxxx

"""

from __future__ import absolute_import

import contextlib

import itertools

import json

import operator

import re

from mercurial.node import bin, nullid

from mercurial.i18n import _

from mercurial import (

    cmdutil,

    context,

    encoding,

    error,

    httpconnection as httpconnectionmod,

    mdiff,

    obsutil,

    parser,

    patch,

    phases,

    pycompat,

    registrar,

    scmutil,

    smartset,

    tags,

    templatefilters,

    templateutil,

    url as urlmod,

    util,

)

from mercurial.utils import (

    procutil,

    stringutil,

)

# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for

# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should

# be specifying the version(s) of Mercurial they are tested with, or

# leave the attribute unspecified.

testedwith = 'ships-with-hg-core'

cmdtable = {}

command = registrar.command(cmdtable)

configtable = {}

configitem = registrar.configitem(configtable)

# developer config: phabricator.batchsize

configitem(b'phabricator', b'batchsize',

    default=12,

)

configitem(b'phabricator', b'callsign',

    default=None,

)

configitem(b'phabricator', b'curlcmd',

    default=None,

)

# developer config: phabricator.repophid

configitem(b'phabricator', b'repophid',

    default=None,

)

configitem(b'phabricator', b'url',

    default=None,

)

configitem(b'phabsend', b'confirm',

    default=False,

)

colortable = {

    b'phabricator.action.created': b'green',

    b'phabricator.action.skipped': b'magenta',

    b'phabricator.action.updated': b'magenta',

    b'phabricator.desc': b'',

    b'phabricator.drev': b'bold',

    b'phabricator.node': b'',

}

_VCR_FLAGS = [

    (b'', b'test-vcr', b'',

     _(b'Path to a vcr file. If nonexistent, will record a new vcr transcript'

       b', otherwise will mock all http requests using the specified vcr file.'

       b' (ADVANCED)'

     )),

]

def vcrcommand(name, flags, spec, helpcategory=None, optionalrepo=False):

    fullflags = flags + _VCR_FLAGS

    def hgmatcher(r1, r2):

        if r1.uri != r2.uri or r1.method != r2.method:

            return False

        r1params = r1.body.split(b'&')

        r2params = r2.body.split(b'&')

        return set(r1params) == set(r2params)

    def sanitiserequest(request):

        request.body = re.sub(

            r'cli-[a-z0-9]+',

            r'cli-hahayouwish',

            request.body

        )

        return request

    def sanitiseresponse(response):

        if r'set-cookie' in response[r'headers']:

            del response[r'headers'][r'set-cookie']

        return response

    def decorate(fn):

        def inner(*args, **kwargs):

            cassette = pycompat.fsdecode(kwargs.pop(r'test_vcr', None))

            if cassette:

                import hgdemandimport

                with hgdemandimport.deactivated():

                    import vcr as vcrmod

                    import vcr.stubs as stubs

                    vcr = vcrmod.VCR(

                        serializer=r'json',

                        before_record_request=sanitiserequest,

                        before_record_response=sanitiseresponse,

                        custom_patches=[

                            (urlmod, r'httpconnection',

                             stubs.VCRHTTPConnection),

                            (urlmod, r'httpsconnection',

                             stubs.VCRHTTPSConnection),

                        ])

                    vcr.register_matcher(r'hgmatcher', hgmatcher)

                    with vcr.use_cassette(cassette, match_on=[r'hgmatcher']):

                        return fn(*args, **kwargs)

            return fn(*args, **kwargs)

        inner.__name__ = fn.__name__

        inner.__doc__ = fn.__doc__

        return command(name, fullflags, spec, helpcategory=helpcategory,

                       optionalrepo=optionalrepo)(inner)

    return decorate

def urlencodenested(params):

    """like urlencode, but works with nested parameters.

    For example, if params is {'a': ['b', 'c'], 'd': {'e': 'f'}}, it will be

    flattened to {'a[0]': 'b', 'a[1]': 'c', 'd[e]': 'f'} and then passed to

    urlencode. Note: the encoding is consistent with PHP's http_build_query.

    """

    flatparams = util.sortdict()

    def process(prefix, obj):

        if isinstance(obj, bool):

            obj = {True: b'true', False: b'false'}[obj]  # Python -> PHP form

        lister = lambda l: [(b'%d' % k, v) for k, v in enumerate(l)]

        items = {list: lister, dict: lambda x: x.items()}.get(type(obj))

        if items is None:

            flatparams[prefix] = obj

        else:

            for k, v in items(obj):

                if prefix:

                    process(b'%s[%s]' % (prefix, k), v)

                else:

                    process(k, v)

    process(b'', params)

    return util.urlreq.urlencode(flatparams)

def readurltoken(ui):

    """return conduit url, token and make sure they exist

    Currently read from [auth] config section. In the future, it might

    make sense to read from .arcconfig and .arcrc as well.

    """

    url = ui.config(b'phabricator', b'url')

    if not url:

        raise error.Abort(_(b'config %s.%s is required')

                          % (b'phabricator', b'url'))

    res = httpconnectionmod.readauthforuri(ui, url, util.url(url).user)

    token = None

    if res:

        group, auth = res

        ui.debug(b"using auth.%s.* for authentication\n" % group)

        token = auth.get(b'phabtoken')

    if not token:

        raise error.Abort(_(b'Can\'t find conduit token associated to %s')

                            % (url,))

    return url, token

def callconduit(ui, name, params):

    """call Conduit API, params is a dict. return json.loads result, or None"""

    host, token = readurltoken(ui)

    url, authinfo = util.url(b'/'.join([host, b'api', name])).authinfo()

    ui.debug(b'Conduit Call: %s %s\n' % (url, pycompat.byterepr(params)))

    params = params.copy()

    params[b'api.token'] = token

    data = urlencodenested(params)

    curlcmd = ui.config(b'phabricator', b'curlcmd')

    if curlcmd:

        sin, sout = procutil.popen2(b'%s -d @- %s'

                                    % (curlcmd, procutil.shellquote(url)))

        sin.write(data)

        sin.close()

        body = sout.read()

    else:

        urlopener = urlmod.opener(ui, authinfo)

        request = util.urlreq.request(pycompat.strurl(url), data=data)

        with contextlib.closing(urlopener.open(request)) as rsp:

            body = rsp.read()

    ui.debug(b'Conduit Response: %s\n' % body)

    parsed = pycompat.rapply(

        lambda x: encoding.unitolocal(x) if isinstance(x, pycompat.unicode)

        else x,

        json.loads(body)

    )

    if parsed.get(b'error_code'):

        msg = (_(b'Conduit Error (%s): %s')

               % (parsed[b'error_code'], parsed[b'error_info']))

        raise error.Abort(msg)

    return parsed[b'result']

@vcrcommand(b'debugcallconduit', [], _(b'METHOD'), optionalrepo=True)

def debugcallconduit(ui, repo, name):

    """call Conduit API

    Call parameters are read from stdin as a JSON blob. Result will be written

    to stdout as a JSON blob.

    """

    # json.loads only accepts bytes from 3.6+

    rawparams = encoding.unifromlocal(ui.fin.read())

    # json.loads only returns unicode strings

    params = pycompat.rapply(lambda x:

        encoding.unitolocal(x) if isinstance(x, pycompat.unicode) else x,

        json.loads(rawparams)

    )

    # json.dumps only accepts unicode strings

    result = pycompat.rapply(lambda x:

        encoding.unifromlocal(x) if isinstance(x, bytes) else x,

        callconduit(ui, name, params)

    )

    s = json.dumps(result, sort_keys=True, indent=2, separators=(u',', u': '))

    ui.write(b'%s\n' % encoding.unitolocal(s))

def getrepophid(repo):

    """given callsign, return repository PHID or None"""

    # developer config: phabricator.repophid

    repophid = repo.ui.config(b'phabricator', b'repophid')

    if repophid:

        return repophid

    callsign = repo.ui.config(b'phabricator', b'callsign')

    if not callsign:

        return None

    query = callconduit(repo.ui, b'diffusion.repository.search',

                        {b'constraints': {b'callsigns': [callsign]}})

    if len(query[b'data']) == 0:

        return None

    repophid = query[b'data'][0][b'phid']

    repo.ui.setconfig(b'phabricator', b'repophid', repophid)

    return repophid

_differentialrevisiontagre = re.compile(br'\AD([1-9][0-9]*)\Z')

_differentialrevisiondescre = re.compile(

    br'^Differential Revision:\s*(?P<url>(?:.*)D(?P<id>[1-9][0-9]*))$', re.M)

def getoldnodedrevmap(repo, nodelist):

    """find previous nodes that has been sent to Phabricator

    return {node: (oldnode, Differential diff, Differential Revision ID)}

    for node in nodelist with known previous sent versions, or associated

    Differential Revision IDs. ``oldnode`` and ``Differential diff`` could

    be ``None``.

    Examines commit messages like "Differential Revision:" to get the

    association information.

    If such commit message line is not found, examines all precursors and their

    tags. Tags with format like "D1234" are considered a match and the node

    with that tag, and the number after "D" (ex. 1234) will be returned.

    The ``old node``, if not None, is guaranteed to be the last diff of

    corresponding Differential Revision, and exist in the repo.

    """

    unfi = repo.unfiltered()

    nodemap = unfi.changelog.nodemap

    result = {} # {node: (oldnode?, lastdiff?, drev)}

    toconfirm = {} # {node: (force, {precnode}, drev)}

    for node in nodelist:

        ctx = unfi[node]

        # For tags like "D123", put them into "toconfirm" to verify later

        precnodes = list(obsutil.allpredecessors(unfi.obsstore, [node]))

        for n in precnodes:

            if n in nodemap:

                for tag in unfi.nodetags(n):

                    m = _differentialrevisiontagre.match(tag)

                    if m:

                        toconfirm[node] = (0, set(precnodes), int(m.group(1)))

                        continue

        # Check commit message

        m = _differentialrevisiondescre.search(ctx.description())

        if m:

            toconfirm[node] = (1, set(precnodes), int(m.group(r'id')))

    # Double check if tags are genuine by collecting all old nodes from

    # Phabricator, and expect precursors overlap with it.

    if toconfirm:

        drevs = [drev for force, precs, drev in toconfirm.values()]

        alldiffs = callconduit(unfi.ui, b'differential.querydiffs',

                               {b'revisionIDs': drevs})

        getnode = lambda d: bin(

            getdiffmeta(d).get(b'node', b'')) or None

        for newnode, (force, precset, drev) in toconfirm.items():

            diffs = [d for d in alldiffs.values()

                     if int(d[b'revisionID']) == drev]

            # "precursors" as known by Phabricator

            phprecset = set(getnode(d) for d in diffs)

            # Ignore if precursors (Phabricator and local repo) do not overlap,

            # and force is not set (when commit message says nothing)

            if not force and not bool(phprecset & precset):

                tagname = b'D%d' % drev

                tags.tag(repo, tagname, nullid, message=None, user=None,

                         date=None, local=True)

                unfi.ui.warn(_(b'D%s: local tag removed - does not match '

                               b'Differential history\n') % drev)

                continue

            # Find the last node using Phabricator metadata, and make sure it

            # exists in the repo

            oldnode = lastdiff = None

            if diffs:

                lastdiff = max(diffs, key=lambda d: int(d[b'id']))

                oldnode = getnode(lastdiff)

                if oldnode and oldnode not in nodemap: