# __init__.py - remotefilelog extension

#

# Copyright 2013 Facebook, Inc.

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

"""remotefilelog causes Mercurial to lazilly fetch file contents (EXPERIMENTAL)

Configs:

    ``packs.maxchainlen`` specifies the maximum delta chain length in pack files

    ``packs.maxpacksize`` specifies the maximum pack file size

    ``packs.maxpackfilecount`` specifies the maximum number of packs in the

      shared cache (trees only for now)

    ``remotefilelog.backgroundprefetch`` runs prefetch in background when True

    ``remotefilelog.bgprefetchrevs`` specifies revisions to fetch on commit and

      update, and on other commands that use them. Different from pullprefetch.

    ``remotefilelog.gcrepack`` does garbage collection during repack when True

    ``remotefilelog.nodettl`` specifies maximum TTL of a node in seconds before

      it is garbage collected

    ``remotefilelog.repackonhggc`` runs repack on hg gc when True

    ``remotefilelog.prefetchdays`` specifies the maximum age of a commit in

      days after which it is no longer prefetched.

    ``remotefilelog.prefetchdelay`` specifies delay between background

      prefetches in seconds after operations that change the working copy parent

    ``remotefilelog.data.gencountlimit`` constraints the minimum number of data

      pack files required to be considered part of a generation. In particular,

      minimum number of packs files > gencountlimit.

    ``remotefilelog.data.generations`` list for specifying the lower bound of

      each generation of the data pack files. For example, list ['100MB','1MB']

      or ['1MB', '100MB'] will lead to three generations: [0, 1MB), [

      1MB, 100MB) and [100MB, infinity).

    ``remotefilelog.data.maxrepackpacks`` the maximum number of pack files to

      include in an incremental data repack.

    ``remotefilelog.data.repackmaxpacksize`` the maximum size of a pack file for

      it to be considered for an incremental data repack.

    ``remotefilelog.data.repacksizelimit`` the maximum total size of pack files

      to include in an incremental data repack.

    ``remotefilelog.history.gencountlimit`` constraints the minimum number of

      history pack files required to be considered part of a generation. In

      particular, minimum number of packs files > gencountlimit.

    ``remotefilelog.history.generations`` list for specifying the lower bound of

      each generation of the historhy pack files. For example, list [

      '100MB', '1MB'] or ['1MB', '100MB'] will lead to three generations: [

      0, 1MB), [1MB, 100MB) and [100MB, infinity).

    ``remotefilelog.history.maxrepackpacks`` the maximum number of pack files to

      include in an incremental history repack.

    ``remotefilelog.history.repackmaxpacksize`` the maximum size of a pack file

      for it to be considered for an incremental history repack.

    ``remotefilelog.history.repacksizelimit`` the maximum total size of pack

      files to include in an incremental history repack.

    ``remotefilelog.backgroundrepack`` automatically consolidate packs in the

      background

    ``remotefilelog.cachepath`` path to cache

    ``remotefilelog.cachegroup`` if set, make cache directory sgid to this

      group

    ``remotefilelog.cacheprocess`` binary to invoke for fetching file data

    ``remotefilelog.debug`` turn on remotefilelog-specific debug output

    ``remotefilelog.excludepattern`` pattern of files to exclude from pulls

    ``remotefilelog.includepattern``pattern of files to include in pulls

    ``remotefilelog.fetchwarning``: message to print when too many

      single-file fetches occur

    ``remotefilelog.getfilesstep`` number of files to request in a single RPC

    ``remotefilelog.getfilestype`` if set to 'threaded' use threads to fetch

      files, otherwise use optimistic fetching

    ``remotefilelog.pullprefetch`` revset for selecting files that should be

      eagerly downloaded rather than lazily

    ``remotefilelog.reponame`` name of the repo. If set, used to partition

      data from other repos in a shared store.

    ``remotefilelog.server`` if true, enable server-side functionality

    ``remotefilelog.servercachepath`` path for caching blobs on the server

    ``remotefilelog.serverexpiration`` number of days to keep cached server

      blobs

    ``remotefilelog.validatecache`` if set, check cache entries for corruption

      before returning blobs

    ``remotefilelog.validatecachelog`` if set, check cache entries for

      corruption before returning metadata

"""

from __future__ import absolute_import

import os

import time

import traceback

from mercurial.node import hex

from mercurial.i18n import _

from mercurial import (

    changegroup,

    changelog,

    cmdutil,

    commands,

    configitems,

    context,

    copies,

    debugcommands as hgdebugcommands,

    dispatch,

    error,

    exchange,

    extensions,

    hg,

    localrepo,

    match,

    merge,

    node as nodemod,

    patch,

    registrar,

    repair,

    repoview,

    revset,

    scmutil,

    smartset,

    templatekw,

    util,

)

from . import (

    debugcommands,

    fileserverclient,

    remotefilectx,

    remotefilelog,

    remotefilelogserver,

    repack as repackmod,

    shallowbundle,

    shallowrepo,

    shallowstore,

    shallowutil,

    shallowverifier,

)

# ensures debug commands are registered

hgdebugcommands.command

try:

    from mercurial import streamclone

    streamclone._walkstreamfiles

    hasstreamclone = True

except Exception:

    hasstreamclone = False

cmdtable = {}

command = registrar.command(cmdtable)

configtable = {}

configitem = registrar.configitem(configtable)

configitem('remotefilelog', 'debug', default=False)

configitem('remotefilelog', 'reponame', default='')

configitem('remotefilelog', 'cachepath', default=None)

configitem('remotefilelog', 'cachegroup', default=None)

configitem('remotefilelog', 'cacheprocess', default=None)

configitem('remotefilelog', 'cacheprocess.includepath', default=None)

configitem("remotefilelog", "cachelimit", default="1000 GB")

configitem('remotefilelog', 'fallbackpath', default=configitems.dynamicdefault,

           alias=[('remotefilelog', 'fallbackrepo')])

configitem('remotefilelog', 'validatecachelog', default=None)

configitem('remotefilelog', 'validatecache', default='on')

configitem('remotefilelog', 'server', default=None)

configitem('remotefilelog', 'servercachepath', default=None)

configitem("remotefilelog", "serverexpiration", default=30)

configitem('remotefilelog', 'backgroundrepack', default=False)

configitem('remotefilelog', 'bgprefetchrevs', default=None)

configitem('remotefilelog', 'pullprefetch', default=None)

configitem('remotefilelog', 'backgroundprefetch', default=False)

configitem('remotefilelog', 'prefetchdelay', default=120)

configitem('remotefilelog', 'prefetchdays', default=14)

configitem('remotefilelog', 'getfilesstep', default=10000)

configitem('remotefilelog', 'getfilestype', default='optimistic')

configitem('remotefilelog', 'batchsize', configitems.dynamicdefault)

configitem('remotefilelog', 'fetchwarning', default='')

configitem('remotefilelog', 'includepattern', default=None)

configitem('remotefilelog', 'excludepattern', default=None)

configitem('remotefilelog', 'gcrepack', default=False)

configitem('remotefilelog', 'repackonhggc', default=False)

configitem('remotefilelog', 'datapackversion', default=0)

configitem('repack', 'chainorphansbysize', default=True)

configitem('packs', 'maxpacksize', default=0)

configitem('packs', 'maxchainlen', default=1000)

configitem('remotefilelog', 'historypackv1', default=False)

#  default TTL limit is 30 days

_defaultlimit = 60 * 60 * 24 * 30

configitem('remotefilelog', 'nodettl', default=_defaultlimit)

configitem('remotefilelog', 'data.gencountlimit', default=2),

configitem('remotefilelog', 'data.generations',

           default=['1GB', '100MB', '1MB'])

configitem('remotefilelog', 'data.maxrepackpacks', default=50)

configitem('remotefilelog', 'data.repackmaxpacksize', default='4GB')

configitem('remotefilelog', 'data.repacksizelimit', default='100MB')

configitem('remotefilelog', 'history.gencountlimit', default=2),

configitem('remotefilelog', 'history.generations', default=['100MB'])

configitem('remotefilelog', 'history.maxrepackpacks', default=50)

configitem('remotefilelog', 'history.repackmaxpacksize', default='400MB')

configitem('remotefilelog', 'history.repacksizelimit', default='100MB')

# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for

# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should

# be specifying the version(s) of Mercurial they are tested with, or

# leave the attribute unspecified.

testedwith = 'ships-with-hg-core'

repoclass = localrepo.localrepository

repoclass._basesupported.add(shallowrepo.requirement)

def uisetup(ui):

    """Wraps user facing Mercurial commands to swap them out with shallow

    versions.

    """

    hg.wirepeersetupfuncs.append(fileserverclient.peersetup)

    entry = extensions.wrapcommand(commands.table, 'clone', cloneshallow)

    entry[1].append(('', 'shallow', None,

                     _("create a shallow clone which uses remote file "

                       "history")))

    extensions.wrapcommand(commands.table, 'debugindex',

        debugcommands.debugindex)

    extensions.wrapcommand(commands.table, 'debugindexdot',

        debugcommands.debugindexdot)

    extensions.wrapcommand(commands.table, 'log', log)

    extensions.wrapcommand(commands.table, 'pull', pull)

    # Prevent 'hg manifest --all'

    def _manifest(orig, ui, repo, *args, **opts):

        if shallowrepo.requirement in repo.requirements and opts.get('all'):

            raise error.Abort(_("--all is not supported in a shallow repo"))

        return orig(ui, repo, *args, **opts)

    extensions.wrapcommand(commands.table, "manifest", _manifest)

    # Wrap remotefilelog with lfs code

    def _lfsloaded(loaded=False):

        lfsmod = None

        try:

            lfsmod = extensions.find('lfs')

        except KeyError:

            pass

        if lfsmod:

            lfsmod.wrapfilelog(remotefilelog.remotefilelog)

            fileserverclient._lfsmod = lfsmod

    extensions.afterloaded('lfs', _lfsloaded)

    # debugdata needs remotefilelog.len to work

    extensions.wrapcommand(commands.table, 'debugdata', debugdatashallow)

def cloneshallow(orig, ui, repo, *args, **opts):

    if opts.get('shallow'):

        repos = []

        def pull_shallow(orig, self, *args, **kwargs):

            if shallowrepo.requirement not in self.requirements:

                repos.append(self.unfiltered())

                # set up the client hooks so the post-clone update works

                setupclient(self.ui, self.unfiltered())

                # setupclient fixed the class on the repo itself

                # but we also need to fix it on the repoview

                if isinstance(self, repoview.repoview):

                    self.__class__.__bases__ = (self.__class__.__bases__[0],

                                                self.unfiltered().__class__)

                self.requirements.add(shallowrepo.requirement)

                self._writerequirements()

                # Since setupclient hadn't been called, exchange.pull was not

                # wrapped. So we need to manually invoke our version of it.

                return exchangepull(orig, self, *args, **kwargs)

            else:

                return orig(self, *args, **kwargs)

        extensions.wrapfunction(exchange, 'pull', pull_shallow)

        # Wrap the stream logic to add requirements and to pass include/exclude

        # patterns around.

        def setup_streamout(repo, remote):

            # Replace remote.stream_out with a version that sends file

            # patterns.

            def stream_out_shallow(orig):

                caps = remote.capabilities()

                    opts = {}

                    if repo.includepattern:

                        opts['includepattern'] = '\0'.join(repo.includepattern)

                    if repo.excludepattern:

                        opts['excludepattern'] = '\0'.join(repo.excludepattern)

                    return remote._callstream('stream_out_shallow', **opts)

                else:

                    return orig()

            extensions.wrapfunction(remote, 'stream_out', stream_out_shallow)

        if hasstreamclone:

            def stream_wrap(orig, op):

                setup_streamout(op.repo, op.remote)

                return orig(op)

            extensions.wrapfunction(

                streamclone, 'maybeperformlegacystreamclone', stream_wrap)

            def canperformstreamclone(orig, pullop, bundle2=False):

                # remotefilelog is currently incompatible with the

                # bundle2 flavor of streamclones, so force us to use

                # v1 instead.

                if 'v2' in pullop.remotebundle2caps.get('stream', []):

                    pullop.remotebundle2caps['stream'] = [

                        c for c in pullop.remotebundle2caps['stream']

                        if c != 'v2']

                if bundle2:

                    return False, None

                supported, requirements = orig(pullop, bundle2=bundle2)

                if requirements is not None:

                    requirements.add(shallowrepo.requirement)

                return supported, requirements

            extensions.wrapfunction(

                streamclone, 'canperformstreamclone', canperformstreamclone)

        else:

            def stream_in_shallow(orig, repo, remote, requirements):

                setup_streamout(repo, remote)

                requirements.add(shallowrepo.requirement)

                return orig(repo, remote, requirements)

            extensions.wrapfunction(

                localrepo.localrepository, 'stream_in', stream_in_shallow)

    try:

        orig(ui, repo, *args, **opts)

    finally:

        if opts.get('shallow'):

            for r in repos:

                if util.safehasattr(r, 'fileservice'):

                    r.fileservice.close()

def debugdatashallow(orig, *args, **kwds):

    oldlen = remotefilelog.remotefilelog.__len__

    try:

        remotefilelog.remotefilelog.__len__ = lambda x: 1

        return orig(*args, **kwds)

    finally:

        remotefilelog.remotefilelog.__len__ = oldlen

def reposetup(ui, repo):

    if not isinstance(repo, localrepo.localrepository):

        return

    # put here intentionally bc doesnt work in uisetup

    ui.setconfig('hooks', 'update.prefetch', wcpprefetch)

    ui.setconfig('hooks', 'commit.prefetch', wcpprefetch)

    isserverenabled = ui.configbool('remotefilelog', 'server')

    isshallowclient = shallowrepo.requirement in repo.requirements

    if isserverenabled and isshallowclient:

        raise RuntimeError("Cannot be both a server and shallow client.")

    if isshallowclient:

        setupclient(ui, repo)

    if isserverenabled:

        remotefilelogserver.setupserver(ui, repo)

def setupclient(ui, repo):

    if not isinstance(repo, localrepo.localrepository):

        return

    # Even clients get the server setup since they need to have the

    # wireprotocol endpoints registered.

    remotefilelogserver.onetimesetup(ui)

    onetimeclientsetup(ui)

    shallowrepo.wraprepo(repo)

    repo.store = shallowstore.wrapstore(repo.store)

clientonetime = False

def onetimeclientsetup(ui):

    global clientonetime

    if clientonetime:

        return

    clientonetime = True

    changegroup.cgpacker = shallowbundle.shallowcg1packer

    extensions.wrapfunction(changegroup, '_addchangegroupfiles',

                            shallowbundle.addchangegroupfiles)

    extensions.wrapfunction(

        changegroup, 'makechangegroup', shallowbundle.makechangegroup)

    def storewrapper(orig, requirements, path, vfstype):

        s = orig(requirements, path, vfstype)

        if shallowrepo.requirement in requirements:

            s = shallowstore.wrapstore(s)

        return s

    extensions.wrapfunction(localrepo, 'makestore', storewrapper)

    extensions.wrapfunction(exchange, 'pull', exchangepull)

    # prefetch files before update

    def applyupdates(orig, repo, actions, wctx, mctx, overwrite, labels=None):

        if shallowrepo.requirement in repo.requirements:

            manifest = mctx.manifest()

            files = []

            for f, args, msg in actions['g']:

                files.append((f, hex(manifest[f])))

            # batch fetch the needed files from the server

            repo.fileservice.prefetch(files)

        return orig(repo, actions, wctx, mctx, overwrite, labels=labels)

    extensions.wrapfunction(merge, 'applyupdates', applyupdates)