# url.py - HTTP handling for mercurial

#

# Copyright 2005, 2006, 2007, 2008 Matt Mackall <mpm@selenic.com>

# Copyright 2006, 2007 Alexis S. L. Carvalho <alexis@cecm.usp.br>

# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

import urllib, urllib2, urlparse, httplib, os, re, socket, cStringIO

import __builtin__

from i18n import _

import keepalive, util

def _urlunparse(scheme, netloc, path, params, query, fragment, url):

    '''Handle cases where urlunparse(urlparse(x://)) doesn't preserve the "//"'''

    result = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))

    if (scheme and

        result.startswith(scheme + ':') and

        not result.startswith(scheme + '://') and

        url.startswith(scheme + '://')

       ):

        result = scheme + '://' + result[len(scheme + ':'):]

    return result

def hidepassword(url):

    '''hide user credential in a url string'''

    scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)

    netloc = re.sub('([^:]*):([^@]*)@(.*)', r'\1:***@\3', netloc)

    return _urlunparse(scheme, netloc, path, params, query, fragment, url)

def removeauth(url):

    '''remove all authentication information from a url string'''

    scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)

    netloc = netloc[netloc.find('@')+1:]

    return _urlunparse(scheme, netloc, path, params, query, fragment, url)

def netlocsplit(netloc):

    '''split [user[:passwd]@]host[:port] into 4-tuple.'''

    a = netloc.find('@')

    if a == -1:

        user, passwd = None, None

    else:

        userpass, netloc = netloc[:a], netloc[a + 1:]

        c = userpass.find(':')

        if c == -1:

            user, passwd = urllib.unquote(userpass), None

        else:

            user = urllib.unquote(userpass[:c])

            passwd = urllib.unquote(userpass[c + 1:])

    c = netloc.find(':')

    if c == -1:

        host, port = netloc, None

    else:

        host, port = netloc[:c], netloc[c + 1:]

    return host, port, user, passwd

def netlocunsplit(host, port, user=None, passwd=None):

    '''turn host, port, user, passwd into [user[:passwd]@]host[:port].'''

    if port:

        hostport = host + ':' + port

    else:

        hostport = host

    if user:

        quote = lambda s: urllib.quote(s, safe='')

        if passwd:

            userpass = quote(user) + ':' + quote(passwd)

        else:

            userpass = quote(user)

        return userpass + '@' + hostport

    return hostport

_safe = ('abcdefghijklmnopqrstuvwxyz'

         'ABCDEFGHIJKLMNOPQRSTUVWXYZ'

         '0123456789' '_.-/')

_safeset = None

_hex = None

def quotepath(path):

    '''quote the path part of a URL

    This is similar to urllib.quote, but it also tries to avoid

    quoting things twice (inspired by wget):

    >>> quotepath('abc def')

    'abc%20def'

    >>> quotepath('abc%20def')

    'abc%20def'

    >>> quotepath('abc%20 def')

    'abc%20%20def'

    >>> quotepath('abc def%20')

    'abc%20def%20'

    >>> quotepath('abc def%2')

    'abc%20def%252'

    >>> quotepath('abc def%')

    'abc%20def%25'

    '''

    global _safeset, _hex

    if _safeset is None:

        _safeset = set(_safe)

        _hex = set('abcdefABCDEF0123456789')

    l = list(path)

    for i in xrange(len(l)):

        c = l[i]

        if (c == '%' and i + 2 < len(l) and

            l[i + 1] in _hex and l[i + 2] in _hex):

            pass

        elif c not in _safeset:

            l[i] = '%%%02X' % ord(c)

    return ''.join(l)

class passwordmgr(urllib2.HTTPPasswordMgrWithDefaultRealm):

    def __init__(self, ui):

        urllib2.HTTPPasswordMgrWithDefaultRealm.__init__(self)

        self.ui = ui

    def find_user_password(self, realm, authuri):

        authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm.find_user_password(

            self, realm, authuri)

        user, passwd = authinfo

        if user and passwd:

            self._writedebug(user, passwd)

            return (user, passwd)

        if not user:

            auth = self.readauthtoken(authuri)

            if auth:

                user, passwd = auth.get('username'), auth.get('password')

        if not user or not passwd:

            if not self.ui.interactive():

                raise util.Abort(_('http authorization required'))

            self.ui.write(_("http authorization required\n"))

            self.ui.status(_("realm: %s\n") % realm)

            if user:

                self.ui.status(_("user: %s\n") % user)

            else:

                user = self.ui.prompt(_("user:"), default=None)

            if not passwd:

                passwd = self.ui.getpass()

        self.add_password(realm, authuri, user, passwd)

        self._writedebug(user, passwd)

        return (user, passwd)

    def _writedebug(self, user, passwd):

        msg = _('http auth: user %s, password %s\n')

        self.ui.debug(msg % (user, passwd and '*' * len(passwd) or 'not set'))

    def readauthtoken(self, uri):

        # Read configuration

        config = dict()

        for key, val in self.ui.configitems('auth'):

            if '.' not in key:

                self.ui.warn(_("ignoring invalid [auth] key '%s'\n") % key)

                continue

            group, setting = key.split('.', 1)

            gdict = config.setdefault(group, dict())

            if setting in ('username', 'cert', 'key'):

                val = util.expandpath(val)

            gdict[setting] = val

        # Find the best match

        scheme, hostpath = uri.split('://', 1)

        bestlen = 0

        bestauth = None

        for auth in config.itervalues():

            prefix = auth.get('prefix')

            if not prefix:

                continue

            p = prefix.split('://', 1)

            if len(p) > 1:

                schemes, prefix = [p[0]], p[1]

            else:

                schemes = (auth.get('schemes') or 'https').split()

            if (prefix == '*' or hostpath.startswith(prefix)) and \

                len(prefix) > bestlen and scheme in schemes:

                bestlen = len(prefix)

                bestauth = auth

        return bestauth

class proxyhandler(urllib2.ProxyHandler):

    def __init__(self, ui):

        proxyurl = ui.config("http_proxy", "host") or os.getenv('http_proxy')

        # XXX proxyauthinfo = None

        if proxyurl:

            # proxy can be proper url or host[:port]

            if not (proxyurl.startswith('http:') or

                    proxyurl.startswith('https:')):

                proxyurl = 'http://' + proxyurl + '/'

            snpqf = urlparse.urlsplit(proxyurl)

            proxyscheme, proxynetloc, proxypath, proxyquery, proxyfrag = snpqf

            hpup = netlocsplit(proxynetloc)

            proxyhost, proxyport, proxyuser, proxypasswd = hpup

            if not proxyuser:

                proxyuser = ui.config("http_proxy", "user")

                proxypasswd = ui.config("http_proxy", "passwd")

            # see if we should use a proxy for this url

            no_list = ["localhost", "127.0.0.1"]

            no_list.extend([p.lower() for

                            p in ui.configlist("http_proxy", "no")])

            no_list.extend([p.strip().lower() for

                            p in os.getenv("no_proxy", '').split(',')

                            if p.strip()])

            # "http_proxy.always" config is for running tests on localhost

            if ui.configbool("http_proxy", "always"):

                self.no_list = []

            else:

                self.no_list = no_list

            proxyurl = urlparse.urlunsplit((

                proxyscheme, netlocunsplit(proxyhost, proxyport,

                                                proxyuser, proxypasswd or ''),

                proxypath, proxyquery, proxyfrag))

            proxies = {'http': proxyurl, 'https': proxyurl}

            ui.debug('proxying through http://%s:%s\n' %

                      (proxyhost, proxyport))

        else:

            proxies = {}

        # urllib2 takes proxy values from the environment and those

        # will take precedence if found, so drop them

        for env in ["HTTP_PROXY", "http_proxy", "no_proxy"]:

            try:

                if env in os.environ:

                    del os.environ[env]

            except OSError:

                pass

        urllib2.ProxyHandler.__init__(self, proxies)

        self.ui = ui

    def proxy_open(self, req, proxy, type_):

        host = req.get_host().split(':')[0]

        if host in self.no_list:

            return None

        # work around a bug in Python < 2.4.2

        # (it leaves a "\n" at the end of Proxy-authorization headers)

        baseclass = req.__class__

        class _request(baseclass):

            def add_header(self, key, val):

                if key.lower() == 'proxy-authorization':

                    val = val.strip()

                return baseclass.add_header(self, key, val)

        req.__class__ = _request

        return urllib2.ProxyHandler.proxy_open(self, req, proxy, type_)

class httpsendfile(object):

    """This is a wrapper around the objects returned by python's "open".

    Its purpose is to send file-like objects via HTTP and, to do so, it

    defines a __len__ attribute to feed the Content-Length header.

    """

    def __init__(self, *args, **kwargs):

        # We can't just "self._data = open(*args, **kwargs)" here because there

        # is an "open" function defined in this module that shadows the global

        # one

        self._data = __builtin__.open(*args, **kwargs)

        self.read = self._data.read

        self.seek = self._data.seek

        self.close = self._data.close

        self.write = self._data.write

    def __len__(self):

        return os.fstat(self._data.fileno()).st_size

def _gen_sendfile(connection):

    def _sendfile(self, data):

        # send a file

        if isinstance(data, httpsendfile):

            # if auth required, some data sent twice, so rewind here

            data.seek(0)

            for chunk in util.filechunkiter(data):

                connection.send(self, chunk)

        else:

            connection.send(self, data)

    return _sendfile

has_https = hasattr(urllib2, 'HTTPSHandler')

if has_https:

    try:

        # avoid using deprecated/broken FakeSocket in python 2.6

        import ssl

        _ssl_wrap_socket = ssl.wrap_socket

        CERT_REQUIRED = ssl.CERT_REQUIRED

    except ImportError:

        CERT_REQUIRED = 2

        def _ssl_wrap_socket(sock, key_file, cert_file,

                             cert_reqs=CERT_REQUIRED, ca_certs=None):

            if ca_certs:

                raise util.Abort(_(

                    'certificate checking requires Python 2.6'))

            ssl = socket.ssl(sock, key_file, cert_file)

            return httplib.FakeSocket(sock, ssl)

    try:

        _create_connection = socket.create_connection

    except AttributeError:

        _GLOBAL_DEFAULT_TIMEOUT = object()

        def _create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,

                               source_address=None):

            # lifted from Python 2.6

            msg = "getaddrinfo returns an empty list"

            host, port = address

            for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):

                af, socktype, proto, canonname, sa = res

                sock = None

                try:

                    sock = socket.socket(af, socktype, proto)

                    if timeout is not _GLOBAL_DEFAULT_TIMEOUT:

                        sock.settimeout(timeout)

                    if source_address:

                        sock.bind(source_address)

                    sock.connect(sa)

                    return sock

                except socket.error, msg:

                    if sock is not None:

                        sock.close()

            raise socket.error, msg

class httpconnection(keepalive.HTTPConnection):

    # must be able to send big bundle as stream.

    send = _gen_sendfile(keepalive.HTTPConnection)

    def connect(self):

        if has_https and self.realhostport: # use CONNECT proxy

            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

            self.sock.connect((self.host, self.port))

            if _generic_proxytunnel(self):

                # we do not support client x509 certificates

                self.sock = _ssl_wrap_socket(self.sock, None, None)

        else:

            keepalive.HTTPConnection.connect(self)

    def getresponse(self):

        proxyres = getattr(self, 'proxyres', None)

        if proxyres:

            if proxyres.will_close:

                self.close()

            self.proxyres = None

            return proxyres

        return keepalive.HTTPConnection.getresponse(self)

# general transaction handler to support different ways to handle

# HTTPS proxying before and after Python 2.6.3.

def _generic_start_transaction(handler, h, req):

    if hasattr(req, '_tunnel_host') and req._tunnel_host:

        tunnel_host = req._tunnel_host

        if tunnel_host[:7] not in ['http://', 'https:/']:

            tunnel_host = 'https://' + tunnel_host

        new_tunnel = True

    else:

        tunnel_host = req.get_selector()

        new_tunnel = False

    if new_tunnel or tunnel_host == req.get_full_url(): # has proxy

        urlparts = urlparse.urlparse(tunnel_host)

        if new_tunnel or urlparts[0] == 'https': # only use CONNECT for HTTPS

            realhostport = urlparts[1]

            if realhostport[-1] == ']' or ':' not in realhostport: