# phabricator.py - simple Phabricator integration

#

# Copyright 2017 Facebook, Inc.

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

"""simple Phabricator integration (EXPERIMENTAL)

This extension provides a ``phabsend`` command which sends a stack of

changesets to Phabricator, and a ``phabread`` command which prints a stack of

revisions in a format suitable for :hg:`import`, and a ``phabupdate`` command

to update statuses in batch.

By default, Phabricator requires ``Test Plan`` which might prevent some

changeset from being sent. The requirement could be disabled by changing

``differential.require-test-plan-field`` config server side.

Config::

    [phabricator]

    # Phabricator URL

    url = https://phab.example.com/

    # Repo callsign. If a repo has a URL https://$HOST/diffusion/FOO, then its

    # callsign is "FOO".

    callsign = FOO

    # curl command to use. If not set (default), use builtin HTTP library to

    # communicate. If set, use the specified curl command. This could be useful

    # if you need to specify advanced options that is not easily supported by

    # the internal library.

    curlcmd = curl --connect-timeout 2 --retry 3 --silent

    [auth]

    example.schemes = https

    example.prefix = phab.example.com

    # API token. Get it from https://$HOST/conduit/login/

    example.phabtoken = cli-xxxxxxxxxxxxxxxxxxxxxxxxxxxx

"""

from __future__ import absolute_import

import contextlib

import itertools

import json

import operator

import re

from mercurial.node import bin, nullid

from mercurial.i18n import _

from mercurial import (

    cmdutil,

    context,

    encoding,

    error,

    httpconnection as httpconnectionmod,

    mdiff,

    obsutil,

    parser,

    patch,

    phases,

    registrar,

    scmutil,

    smartset,

    tags,

    templateutil,

    url as urlmod,

    util,

)

from mercurial.utils import (

    procutil,

    stringutil,

)

# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for

# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should

# be specifying the version(s) of Mercurial they are tested with, or

# leave the attribute unspecified.

testedwith = 'ships-with-hg-core'

cmdtable = {}

command = registrar.command(cmdtable)

configtable = {}

configitem = registrar.configitem(configtable)

# developer config: phabricator.batchsize

configitem(b'phabricator', b'batchsize',

    default=12,

)

configitem(b'phabricator', b'callsign',

    default=None,

)

configitem(b'phabricator', b'curlcmd',

    default=None,

)

# developer config: phabricator.repophid

configitem(b'phabricator', b'repophid',

    default=None,

)

configitem(b'phabricator', b'url',

    default=None,

)

configitem(b'phabsend', b'confirm',

    default=False,

)

colortable = {

    b'phabricator.action.created': b'green',

    b'phabricator.action.skipped': b'magenta',

    b'phabricator.action.updated': b'magenta',

    b'phabricator.desc': b'',

    b'phabricator.drev': b'bold',

    b'phabricator.node': b'',

}

_VCR_FLAGS = [

    (b'', b'test-vcr', b'',

     _(b'Path to a vcr file. If nonexistent, will record a new vcr transcript'

       b', otherwise will mock all http requests using the specified vcr file.'

       b' (ADVANCED)'

     )),

]

def vcrcommand(name, flags, spec, helpcategory=None):

    fullflags = flags + _VCR_FLAGS

    def decorate(fn):

        def inner(*args, **kwargs):

            cassette = kwargs.pop(r'test_vcr', None)

            if cassette:

                import hgdemandimport

                with hgdemandimport.deactivated():

                    import vcr as vcrmod

                    import vcr.stubs as stubs

                    vcr = vcrmod.VCR(

                        serializer=r'json',

                        custom_patches=[

                            (urlmod, 'httpconnection', stubs.VCRHTTPConnection),

                            (urlmod, 'httpsconnection',

                             stubs.VCRHTTPSConnection),

                        ])

                    with vcr.use_cassette(cassette):

                        return fn(*args, **kwargs)

            return fn(*args, **kwargs)

        inner.__name__ = fn.__name__

        inner.__doc__ = fn.__doc__

        return command(name, fullflags, spec, helpcategory=helpcategory)(inner)

    return decorate

def urlencodenested(params):

    """like urlencode, but works with nested parameters.

    For example, if params is {'a': ['b', 'c'], 'd': {'e': 'f'}}, it will be

    flattened to {'a[0]': 'b', 'a[1]': 'c', 'd[e]': 'f'} and then passed to

    urlencode. Note: the encoding is consistent with PHP's http_build_query.

    """

    flatparams = util.sortdict()

    def process(prefix, obj):

        if isinstance(obj, bool):

            obj = {True: b'true', False: b'false'}[obj]  # Python -> PHP form

        items = {list: enumerate, dict: lambda x: x.items()}.get(type(obj))

        if items is None:

            flatparams[prefix] = obj

        else:

            for k, v in items(obj):

                if prefix:

                    process(b'%s[%s]' % (prefix, k), v)

                else:

                    process(k, v)

    process(b'', params)

    return util.urlreq.urlencode(flatparams)

def readurltoken(repo):

    """return conduit url, token and make sure they exist

    Currently read from [auth] config section. In the future, it might

    make sense to read from .arcconfig and .arcrc as well.

    """

    url = repo.ui.config(b'phabricator', b'url')

    if not url:

        raise error.Abort(_(b'config %s.%s is required')

                          % (b'phabricator', b'url'))

    res = httpconnectionmod.readauthforuri(repo.ui, url, util.url(url).user)

    token = None

    if res:

        group, auth = res

        repo.ui.debug(b"using auth.%s.* for authentication\n" % group)

        token = auth.get(b'phabtoken')

    if not token:

        raise error.Abort(_(b'Can\'t find conduit token associated to %s')

                            % (url,))

    return url, token

def callconduit(repo, name, params):

    """call Conduit API, params is a dict. return json.loads result, or None"""

    host, token = readurltoken(repo)

    url, authinfo = util.url(b'/'.join([host, b'api', name])).authinfo()

    repo.ui.debug(b'Conduit Call: %s %s\n' % (url, params))

    params = params.copy()

    params[b'api.token'] = token

    data = urlencodenested(params)

    curlcmd = repo.ui.config(b'phabricator', b'curlcmd')

    if curlcmd:

        sin, sout = procutil.popen2(b'%s -d @- %s'

                                    % (curlcmd, procutil.shellquote(url)))

        sin.write(data)

        sin.close()

        body = sout.read()

    else:

        urlopener = urlmod.opener(repo.ui, authinfo)

        request = util.urlreq.request(url, data=data)

        with contextlib.closing(urlopener.open(request)) as rsp:

            body = rsp.read()

    repo.ui.debug(b'Conduit Response: %s\n' % body)

    parsed = json.loads(body)

    if parsed.get(r'error_code'):

        msg = (_(b'Conduit Error (%s): %s')

               % (parsed[r'error_code'], parsed[r'error_info']))

        raise error.Abort(msg)

    return parsed[r'result']

@vcrcommand(b'debugcallconduit', [], _(b'METHOD'))

def debugcallconduit(ui, repo, name):

    """call Conduit API

    Call parameters are read from stdin as a JSON blob. Result will be written

    to stdout as a JSON blob.

    """

    params = json.loads(ui.fin.read())

    result = callconduit(repo, name, params)

    s = json.dumps(result, sort_keys=True, indent=2, separators=(b',', b': '))

    ui.write(b'%s\n' % s)

def getrepophid(repo):

    """given callsign, return repository PHID or None"""

    # developer config: phabricator.repophid

    repophid = repo.ui.config(b'phabricator', b'repophid')

    if repophid:

        return repophid

    callsign = repo.ui.config(b'phabricator', b'callsign')

    if not callsign:

        return None

    query = callconduit(repo, b'diffusion.repository.search',

                        {b'constraints': {b'callsigns': [callsign]}})

    if len(query[r'data']) == 0:

        return None

    repophid = encoding.strtolocal(query[r'data'][0][r'phid'])

    repo.ui.setconfig(b'phabricator', b'repophid', repophid)

    return repophid

_differentialrevisiondescre = re.compile(

def getoldnodedrevmap(repo, nodelist):

    """find previous nodes that has been sent to Phabricator

    return {node: (oldnode, Differential diff, Differential Revision ID)}

    for node in nodelist with known previous sent versions, or associated

    Differential Revision IDs. ``oldnode`` and ``Differential diff`` could

    be ``None``.

    Examines commit messages like "Differential Revision:" to get the

    association information.

    If such commit message line is not found, examines all precursors and their

    tags. Tags with format like "D1234" are considered a match and the node

    with that tag, and the number after "D" (ex. 1234) will be returned.

    The ``old node``, if not None, is guaranteed to be the last diff of

    corresponding Differential Revision, and exist in the repo.

    """

    unfi = repo.unfiltered()

    nodemap = unfi.changelog.nodemap

    result = {} # {node: (oldnode?, lastdiff?, drev)}

    toconfirm = {} # {node: (force, {precnode}, drev)}

    for node in nodelist:

        ctx = unfi[node]

        # For tags like "D123", put them into "toconfirm" to verify later

        precnodes = list(obsutil.allpredecessors(unfi.obsstore, [node]))

        for n in precnodes:

            if n in nodemap:

                for tag in unfi.nodetags(n):

                    m = _differentialrevisiontagre.match(tag)

                    if m:

                        toconfirm[node] = (0, set(precnodes), int(m.group(1)))

                        continue

        # Check commit message

        m = _differentialrevisiondescre.search(ctx.description())

        if m:

            toconfirm[node] = (1, set(precnodes), int(m.group(b'id')))

    # Double check if tags are genuine by collecting all old nodes from

    # Phabricator, and expect precursors overlap with it.

    if toconfirm:

        drevs = [drev for force, precs, drev in toconfirm.values()]

        alldiffs = callconduit(unfi, b'differential.querydiffs',

                               {b'revisionIDs': drevs})

        getnode = lambda d: bin(encoding.unitolocal(

            getdiffmeta(d).get(r'node', b''))) or None

        for newnode, (force, precset, drev) in toconfirm.items():

            diffs = [d for d in alldiffs.values()

                     if int(d[r'revisionID']) == drev]

            # "precursors" as known by Phabricator

            phprecset = set(getnode(d) for d in diffs)

            # Ignore if precursors (Phabricator and local repo) do not overlap,

            # and force is not set (when commit message says nothing)

            if not force and not bool(phprecset & precset):

                tagname = b'D%d' % drev

                tags.tag(repo, tagname, nullid, message=None, user=None,

                         date=None, local=True)

                unfi.ui.warn(_(b'D%s: local tag removed - does not match '

                               b'Differential history\n') % drev)

                continue

            # Find the last node using Phabricator metadata, and make sure it

            # exists in the repo

            oldnode = lastdiff = None

            if diffs:

                lastdiff = max(diffs, key=lambda d: int(d[r'id']))

                oldnode = getnode(lastdiff)

                if oldnode and oldnode not in nodemap:

                    oldnode = None

            result[newnode] = (oldnode, lastdiff, drev)

    return result

def getdiff(ctx, diffopts):

    """plain-text diff without header (user, commit message, etc)"""

    output = util.stringio()

    for chunk, _label in patch.diffui(ctx.repo(), ctx.p1().node(), ctx.node(),

                                      None, opts=diffopts):

        output.write(chunk)

    return output.getvalue()

def creatediff(ctx):

    """create a Differential Diff"""

    repo = ctx.repo()

    repophid = getrepophid(repo)

    # Create a "Differential Diff" via "differential.createrawdiff" API

    params = {b'diff': getdiff(ctx, mdiff.diffopts(git=True, context=32767))}

    if repophid:

        params[b'repositoryPHID'] = repophid

    diff = callconduit(repo, b'differential.createrawdiff', params)

    if not diff:

        raise error.Abort(_(b'cannot create diff for %s') % ctx)

    return diff

def writediffproperties(ctx, diff):

    """write metadata to diff so patches could be applied losslessly"""

    params = {

        b'diff_id': diff[r'id'],

        b'name': b'hg:meta',

        b'data': json.dumps({

            b'user': ctx.user(),

            b'date': b'%d %d' % ctx.date(),

            b'node': ctx.hex(),

            b'parent': ctx.p1().hex(),

        }),

    }

    callconduit(ctx.repo(), b'differential.setdiffproperty', params)

    params = {

        b'diff_id': diff[r'id'],

        b'name': b'local:commits',