"""Python wrapper for Google web APIs

This module allows you to access Google's web APIs through SOAP,

to do things like search Google and get the results programmatically.

Described here:

  http://www.google.com/apis/

You need a Google-provided license key to use these services.

Follow the link above to get one.  These functions will look in

several places (in this order) for the license key:

- the "license_key" argument of each function

- the module-level LICENSE_KEY variable (call setLicense once to set it)

- an environment variable called GOOGLE_LICENSE_KEY

- a file called ".googlekey" in the current directory

- a file called "googlekey.txt" in the current directory

- a file called ".googlekey" in your home directory

- a file called "googlekey.txt" in your home directory

- a file called ".googlekey" in the same directory as google.py

- a file called "googlekey.txt" in the same directory as google.py

Sample usage:

>>> import google

>>> google.setLicense('...') # must get your own key!

>>> data = google.doGoogleSearch('python')

>>> data.meta.searchTime

0.043221000000000002

>>> data.results[0].URL

'http://www.python.org/'

>>> data.results[0].title

'<b>Python</b> Language Website'

See documentation of SearchResultsMetaData and SearchResult classes

for other available attributes.

"""

__author__ = "Mark Pilgrim (f8dy@diveintomark.org)"

__version__ = "0.5.2"

__cvsversion__ = "$Revision: 1.1.1.1 $"[11:-2]

__date__ = "$Date: 2005/09/29 21:38:49 $"[7:-2]

__copyright__ = "Copyright (c) 2002 Mark Pilgrim"

__license__ = "Python"

__credits__ = """David Ascher, for the install script

Erik Max Francis, for the command line interface

Michael Twomey, for HTTP proxy support"""

import SOAP

import os, sys, getopt

LICENSE_KEY = None

HTTP_PROXY = None

# don't touch the rest of these constants

class NoLicenseKey(Exception): pass

_url = 'http://api.google.com/search/beta2'

_namespace = 'urn:GoogleSearch'

_false = SOAP.booleanType(0)

_true = SOAP.booleanType(1)

_googlefile1 = ".googlekey"

_googlefile2 = "googlekey.txt"

_licenseLocations = (

    (lambda key: key, 'passed to the function in license_key variable'),

    (lambda key: LICENSE_KEY, 'module-level LICENSE_KEY variable (call setLicense to set it)'),

    (lambda key: os.environ.get('GOOGLE_LICENSE_KEY', None), 'an environment variable called GOOGLE_LICENSE_KEY'),

    (lambda key: _contentsOf(os.getcwd(), _googlefile1), '%s in the current directory' % _googlefile1),

    (lambda key: _contentsOf(os.getcwd(), _googlefile2), '%s in the current directory' % _googlefile2),

    (lambda key: _contentsOf(os.environ.get('HOME', ''), _googlefile1), '%s in your home directory' % _googlefile1),

    (lambda key: _contentsOf(os.environ.get('HOME', ''), _googlefile2), '%s in your home directory' % _googlefile2),

    (lambda key: _contentsOf(_getScriptDir(), _googlefile1), '%s in the google.py directory' % _googlefile1),

    (lambda key: _contentsOf(_getScriptDir(), _googlefile2), '%s in the google.py directory' % _googlefile2)

    )

## administrative functions

def version():

    print """PyGoogle %(__version__)s

%(__copyright__)s

released %(__date__)s

Thanks to:

%(__credits__)s""" % globals()

def usage():

    program = os.path.basename(sys.argv[0])

    print """Usage: %(program)s [options] [querytype] query

options:

  -k, --key= <license key> Google license key (see important note below)

  -1, -l, --lucky          show only first hit

  -m, --meta               show meta information

  -r, --reverse            show results in reverse order

  -x, --proxy= <url>       use HTTP proxy

  -h, --help               print this help

  -v, --version            print version and copyright information

  -t, --test               run test queries

querytype:

  -s, --search= <query>    search (default)

  -c, --cache= <url>       retrieve cached page

  -p, --spelling= <word>   check spelling

IMPORTANT NOTE: all Google functions require a valid license key;

visit http://www.google.com/apis/ to get one.  %(program)s will look in

these places (in order) and use the first license key it finds:

  * the key specified on the command line""" % vars()

    for get, location in _licenseLocations[2:]:

        print "  *", location

## utility functions

def setLicense(license_key):

    """set license key"""

    global LICENSE_KEY

    LICENSE_KEY = license_key

def getLicense(license_key = None):

    """get license key

    license key can come from any number of locations;

    see module docs for search order"""

    for get, location in _licenseLocations:

        rc = get(license_key)

        if rc: return rc

    usage()

    raise NoLicenseKey, 'get a license key at http://www.google.com/apis/'

def setProxy(http_proxy):

    """set HTTP proxy"""

    global HTTP_PROXY

    HTTP_PROXY = http_proxy

def getProxy(http_proxy = None):

    """get HTTP proxy"""

    return http_proxy or HTTP_PROXY

def _contentsOf(dirname, filename):

    filename = os.path.join(dirname, filename)

    if not os.path.exists(filename): return None

    fsock = open(filename)

    contents = fsock.read()

    fsock.close()

    return contents

def _getScriptDir():

    if __name__ == '__main__':

        return os.path.abspath(os.path.dirname(sys.argv[0]))

    else:

        return os.path.abspath(os.path.dirname(sys.modules[__name__].__file__))

def _marshalBoolean(value):

    if value:

        return _true

    else:

        return _false

## output formatters

def makeFormatter(outputFormat):

    classname = "%sOutputFormatter" % outputFormat.capitalize()

    return globals()[classname]()

def output(results, params):

    formatter = makeFormatter(params.get("outputFormat", "text"))

    outputmethod = getattr(formatter, params["func"])

    outputmethod(results, params)

class OutputFormatter:

    def boil(self, data):

        if type(data) == type(u""):

            return data.encode("ISO-8859-1", "replace")

        else:

            return data

class TextOutputFormatter(OutputFormatter):

    def common(self, data, params):

        if params.get("showMeta", 0):

            meta = data.meta

            for category in meta.directoryCategories:

                print "directoryCategory: %s" % self.boil(category["fullViewableName"])

            for attr in [node for node in dir(meta) if node <> "directoryCategories" and node[:2] <> '__']:

                print "%s:" % attr, self.boil(getattr(meta, attr))

    def doGoogleSearch(self, data, params):

        results = data.results

        if params.get("feelingLucky", 0):

            results = results[:1]

        if params.get("reverseOrder", 0):

            results.reverse()

        for result in results:

            for attr in dir(result):

                if attr == "directoryCategory":

                    print "directoryCategory:", self.boil(result.directoryCategory["fullViewableName"])

                elif attr[:2] <> '__':

                    print "%s:" % attr, self.boil(getattr(result, attr))

            print

        self.common(data, params)

    def doGetCachedPage(self, data, params):

        print data

        self.common(data, params)

    doSpellingSuggestion = doGetCachedPage

## search results classes

class _SearchBase:

    def __init__(self, params):

        for k, v in params.items():

            if isinstance(v, SOAP.structType):

                v = v._asdict

            try:

                if isinstance(v[0], SOAP.structType):

                    v = [node._asdict for node in v]

            except:

                pass

            self.__dict__[str(k)] = v

class SearchResultsMetaData(_SearchBase):

    """metadata of search query results

    Available attributes:

    documentFiltering - flag indicates whether duplicate page filtering was perfomed in this search

    searchComments - human-readable informational message (example: "'the' is a very common word

        and was not included in your search")

    estimatedTotalResultsCount - estimated total number of results for this query

    estimateIsExact - flag indicates whether estimatedTotalResultsCount is an exact value

    searchQuery - search string that initiated this search

    startIndex - index of first result returned (zero-based)

    endIndex - index of last result returned (zero-based)

    searchTips - human-readable informational message on how to use Google bette

    directoryCategories - list of dictionaries like this:

        {'fullViewableName': Open Directory category,

         'specialEncoding': encoding scheme of this directory category}

    searchTime - total search time, in seconds

    """    

    pass

class SearchResult(_SearchBase):

    """search result

    Available attributes:

    URL - URL

    title - title (HTML)

    snippet - snippet showing query context (HTML)

    cachedSize - size of cached version of this result, (KB)

    relatedInformationPresent - flag indicates that the "related:" keyword is supported for this URL

    hostName: When filtering occurs, a maximum of two results from any given host is returned.

        When this occurs, the second resultElement that comes from that host contains

        the host name in this parameter.

    directoryCategory: dictionary like this:

        {'fullViewableName': Open Directory category,

         'specialEncoding': encoding scheme of this directory category}

    directoryTitle: Open Directory title of this result (or blank)

    summary - Open Directory summary for this result (or blank)

    """

    pass

class SearchReturnValue:

    """complete search results for a single query

    Available attributes:

    meta - SearchResultsMetaData

    results - list of SearchResult

    """

    def __init__(self, metadata, results):

        self.meta = metadata

        self.results = results

## main functions

def doGoogleSearch(q, start=0, maxResults=10, filter=1, restrict='',

                   safeSearch=0, language='', inputencoding='', outputencoding='',

                   license_key = None, http_proxy = None):

    """search Google

    You need a license key to call this function; see

    http://www.google.com/apis/ to get one.  Then you can either pass it to

    this function every time, or set it globally; see the module docs for details.

    Parameters:

    q - search string.  Anything you could type at google.com, you can pass here.

        See http://www.google.com/help/features.html for examples of advanced features.

    start (optional) - zero-based index of first desired result (for paging through

        multiple pages of results)

    maxResults (optional) - maximum number of results, currently capped at 10

    filter (optional) - set to 1 to filter out similar results, set to 0 to see everything

    restrict (optional) - restrict results by country or topic.  Examples:

        Ukraine - search only sites located in Ukraine

        linux - search Linux sites only

        mac - search Mac sites only

        bsd - search FreeBSD sites only

        See the APIs_reference.html file in the SDK (http://www.google.com/apis/download.html)

        for more advanced examples and a full list of country codes and topics.

    safeSearch (optional) - set to 1 to filter results with SafeSearch (no adult material)

    language (optional) - restricts search to documents in one or more languages.  Example:

        lang_en - only return pages in English

        lang_fr - only return pages in French

        See the APIs_reference.html file in the SDK (http://www.google.com/apis/download.html)

        for more advanced examples and a full list of language codes.

    inputencoding (optional) - sets the character encoding of q parameter

    outputencoding (optional) - sets the character encoding of the returned results

        See the APIs_reference.html file in the SDK (http://www.google.com/apis/download.html)

        for a full list of encodings.

    http_proxy (optional) - address of HTTP proxy to use for sending and receiving SOAP messages

    Returns: SearchReturnValue

    .meta - SearchMetaData

    .results - list of SearchResult

    See documentation of these individual classes for list of available attributes

    """

    http_proxy = getProxy(http_proxy)

    remoteserver = SOAP.SOAPProxy(_url, namespace=_namespace, http_proxy=http_proxy)

    license_key = getLicense(license_key)

    filter = _marshalBoolean(filter)

    safeSearch = _marshalBoolean(safeSearch)

    data = remoteserver.doGoogleSearch(license_key, q, start, maxResults, filter, restrict,

                                       safeSearch, language, inputencoding, outputencoding)

    metadata = data._asdict

    del metadata["resultElements"]

    metadata = SearchResultsMetaData(metadata)

    results = [SearchResult(node._asdict) for node in data.resultElements]

    return SearchReturnValue(metadata, results)

def doGetCachedPage(url, license_key = None, http_proxy = None):

    """get page from Google cache

    You need a license key to call this function; see

    http://www.google.com/apis/ to get one.  Then you can either pass it to

    this function every time, or set it globally; see the module docs for details.

    Parameters:

    url - address of page to get

    license_key (optional) - Google license key

    http_proxy (optional) - address of HTTP proxy to use for sending and receiving SOAP messages

    Returns: string, text of cached page    

    """

    http_proxy = getProxy(http_proxy)

    remoteserver = SOAP.SOAPProxy(_url, namespace=_namespace, http_proxy=http_proxy)

    license_key = getLicense(license_key)

    return remoteserver.doGetCachedPage(license_key, url)

def doSpellingSuggestion(phrase, license_key = None, http_proxy = None):

    """get spelling suggestions from Google

    You need a license key to call this function; see

    http://www.google.com/apis/ to get one.  Then you can either pass it to

    this function every time, or set it globally; see the module docs for details.

    Parameters:

    phrase - word or phrase to spell-check

    http_proxy (optional) - address of HTTP proxy to use for sending and receiving SOAP messages

    Returns: text of suggested replacement, or None

    """

    http_proxy = getProxy(http_proxy)

    remoteserver = SOAP.SOAPProxy(_url, namespace=_namespace, http_proxy=http_proxy)

    license_key = getLicense(license_key)

    return remoteserver.doSpellingSuggestion(license_key, phrase)

## functional test suite (see googletest.py for unit test suite)

def test():

    try:

        getLicense(None)

    except NoLicenseKey:

        return

    print "Searching for Python at google.com..."

    data = doGoogleSearch("Python")

    output(data, {"func": "doGoogleSearch"})

    print "\nSearching for 5 _French_ pages about Python, encoded in ISO-8859-1..."

    data = doGoogleSearch("Python", language='lang_fr', outputencoding='ISO-8859-1', maxResults=5)

    output(data, {"func": "doGoogleSearch"})

    phrase = "Pyhton programming languager"

    print "\nTesting spelling suggetions for '%s'..." % phrase

    data = doSpellingSuggestion(phrase)

    output(data, {"func": "doSpellingSuggestion"})

## main driver for command-line use

def main(argv):

    if not argv:

        usage()

        return

    q = None

    func = None

    http_proxy = None

    license_key = None

    feelingLucky = 0

    showMeta = 0

    reverseOrder = 0

    runTest = 0

    outputFormat = "text"

    try:

        opts, args = getopt.getopt(argv, "s:c:p:k:lmrx:hvt1",

            ["search=", "cache=", "spelling=", "key=", "lucky", "meta", "reverse", "proxy=", "help", "version", "test"])

    except getopt.GetoptError:

        usage()

        sys.exit(2)

    for opt, arg in opts:

        if opt in ("-s", "--search"):

            q = arg

            func = "doGoogleSearch"

        elif opt in ("-c", "--cache"):

            q = arg

            func = "doGetCachedPage"

        elif opt in ("-p", "--spelling"):

            q = arg

            func = "doSpellingSuggestion"

        elif opt in ("-k", "--key"):

            license_key = arg

        elif opt in ("-l", "-1", "--lucky"):

            feelingLucky = 1

        elif opt in ("-m", "--meta"):

            showMeta = 1

        elif opt in ("-r", "--reverse"):

            reverseOrder = 1

        elif opt in ("-x", "--proxy"):

            http_proxy = arg

        elif opt in ("-h", "--help"):

            usage()

        elif opt in ("-v", "--version"):

            version()

        elif opt in ("-t", "--test"):

            runTest = 1

    if runTest:

        setLicense(license_key)

        setProxy(http_proxy)