# __init__.py - remotefilelog extension

#

# Copyright 2013 Facebook, Inc.

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

"""remotefilelog causes Mercurial to lazilly fetch file contents (EXPERIMENTAL)

This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY

GUARANTEES. This means that repositories created with this extension may

only be usable with the exact version of this extension/Mercurial that was

used. The extension attempts to enforce this in order to prevent repository

corruption.

remotefilelog works by fetching file contents lazily and storing them

in a cache on the client rather than in revlogs. This allows enormous

histories to be transferred only partially, making them easier to

operate on.

Configs:

    ``packs.maxchainlen`` specifies the maximum delta chain length in pack files

    ``packs.maxpacksize`` specifies the maximum pack file size

    ``packs.maxpackfilecount`` specifies the maximum number of packs in the

      shared cache (trees only for now)

    ``remotefilelog.backgroundprefetch`` runs prefetch in background when True

    ``remotefilelog.bgprefetchrevs`` specifies revisions to fetch on commit and

      update, and on other commands that use them. Different from pullprefetch.

    ``remotefilelog.gcrepack`` does garbage collection during repack when True

    ``remotefilelog.nodettl`` specifies maximum TTL of a node in seconds before

      it is garbage collected

    ``remotefilelog.repackonhggc`` runs repack on hg gc when True

    ``remotefilelog.prefetchdays`` specifies the maximum age of a commit in

      days after which it is no longer prefetched.

    ``remotefilelog.prefetchdelay`` specifies delay between background

      prefetches in seconds after operations that change the working copy parent

    ``remotefilelog.data.gencountlimit`` constraints the minimum number of data

      pack files required to be considered part of a generation. In particular,

      minimum number of packs files > gencountlimit.

    ``remotefilelog.data.generations`` list for specifying the lower bound of

      each generation of the data pack files. For example, list ['100MB','1MB']

      or ['1MB', '100MB'] will lead to three generations: [0, 1MB), [

      1MB, 100MB) and [100MB, infinity).

    ``remotefilelog.data.maxrepackpacks`` the maximum number of pack files to

      include in an incremental data repack.

    ``remotefilelog.data.repackmaxpacksize`` the maximum size of a pack file for

      it to be considered for an incremental data repack.

    ``remotefilelog.data.repacksizelimit`` the maximum total size of pack files

      to include in an incremental data repack.

    ``remotefilelog.history.gencountlimit`` constraints the minimum number of

      history pack files required to be considered part of a generation. In

      particular, minimum number of packs files > gencountlimit.

    ``remotefilelog.history.generations`` list for specifying the lower bound of

      each generation of the history pack files. For example, list [

      '100MB', '1MB'] or ['1MB', '100MB'] will lead to three generations: [

      0, 1MB), [1MB, 100MB) and [100MB, infinity).

    ``remotefilelog.history.maxrepackpacks`` the maximum number of pack files to

      include in an incremental history repack.

    ``remotefilelog.history.repackmaxpacksize`` the maximum size of a pack file

      for it to be considered for an incremental history repack.

    ``remotefilelog.history.repacksizelimit`` the maximum total size of pack

      files to include in an incremental history repack.

    ``remotefilelog.backgroundrepack`` automatically consolidate packs in the

      background

    ``remotefilelog.cachepath`` path to cache

    ``remotefilelog.cachegroup`` if set, make cache directory sgid to this

      group

    ``remotefilelog.cacheprocess`` binary to invoke for fetching file data

    ``remotefilelog.debug`` turn on remotefilelog-specific debug output

    ``remotefilelog.excludepattern`` pattern of files to exclude from pulls

    ``remotefilelog.includepattern`` pattern of files to include in pulls

    ``remotefilelog.fetchwarning``: message to print when too many

      single-file fetches occur

    ``remotefilelog.getfilesstep`` number of files to request in a single RPC

    ``remotefilelog.getfilestype`` if set to 'threaded' use threads to fetch

      files, otherwise use optimistic fetching

    ``remotefilelog.pullprefetch`` revset for selecting files that should be

      eagerly downloaded rather than lazily

    ``remotefilelog.reponame`` name of the repo. If set, used to partition

      data from other repos in a shared store.

    ``remotefilelog.server`` if true, enable server-side functionality

    ``remotefilelog.servercachepath`` path for caching blobs on the server

    ``remotefilelog.serverexpiration`` number of days to keep cached server

      blobs

    ``remotefilelog.validatecache`` if set, check cache entries for corruption

      before returning blobs

    ``remotefilelog.validatecachelog`` if set, check cache entries for

      corruption before returning metadata

"""

from __future__ import absolute_import

import os

import time

import traceback

from mercurial.node import hex

from mercurial.i18n import _

from mercurial import (

    changegroup,

    changelog,

    cmdutil,

    commands,

    configitems,

    context,

    copies,

    debugcommands as hgdebugcommands,

    dispatch,

    error,

    exchange,

    extensions,

    hg,

    localrepo,

    match,

    merge,

    node as nodemod,

    patch,

    pycompat,

    registrar,

    repair,

    repoview,

    revset,

    scmutil,

    smartset,

    streamclone,

    util,

)

from . import (

    constants,

    debugcommands,

    fileserverclient,

    remotefilectx,

    remotefilelog,

    remotefilelogserver,

    repack as repackmod,

    shallowbundle,

    shallowrepo,

    shallowstore,

    shallowutil,

    shallowverifier,

)

# ensures debug commands are registered

hgdebugcommands.command

cmdtable = {}

command = registrar.command(cmdtable)

configtable = {}

configitem = registrar.configitem(configtable)

configitem('remotefilelog', 'debug', default=False)

configitem('remotefilelog', 'reponame', default='')

configitem('remotefilelog', 'cachepath', default=None)

configitem('remotefilelog', 'cachegroup', default=None)

configitem('remotefilelog', 'cacheprocess', default=None)

configitem('remotefilelog', 'cacheprocess.includepath', default=None)

configitem("remotefilelog", "cachelimit", default="1000 GB")

configitem('remotefilelog', 'fallbackpath', default=configitems.dynamicdefault,

           alias=[('remotefilelog', 'fallbackrepo')])

configitem('remotefilelog', 'validatecachelog', default=None)

configitem('remotefilelog', 'validatecache', default='on')

configitem('remotefilelog', 'server', default=None)

configitem('remotefilelog', 'servercachepath', default=None)

configitem("remotefilelog", "serverexpiration", default=30)

configitem('remotefilelog', 'backgroundrepack', default=False)

configitem('remotefilelog', 'bgprefetchrevs', default=None)

configitem('remotefilelog', 'pullprefetch', default=None)

configitem('remotefilelog', 'backgroundprefetch', default=False)

configitem('remotefilelog', 'prefetchdelay', default=120)

configitem('remotefilelog', 'prefetchdays', default=14)

configitem('remotefilelog', 'getfilesstep', default=10000)

configitem('remotefilelog', 'getfilestype', default='optimistic')

configitem('remotefilelog', 'batchsize', configitems.dynamicdefault)

configitem('remotefilelog', 'fetchwarning', default='')

configitem('remotefilelog', 'includepattern', default=None)

configitem('remotefilelog', 'excludepattern', default=None)

configitem('remotefilelog', 'gcrepack', default=False)

configitem('remotefilelog', 'repackonhggc', default=False)

configitem('repack', 'chainorphansbysize', default=True, experimental=True)

configitem('packs', 'maxpacksize', default=0)

configitem('packs', 'maxchainlen', default=1000)

configitem('devel', 'remotefilelog.ensurestart', default=False)

#  default TTL limit is 30 days

_defaultlimit = 60 * 60 * 24 * 30

configitem('remotefilelog', 'nodettl', default=_defaultlimit)

configitem('remotefilelog', 'data.gencountlimit', default=2),

configitem('remotefilelog', 'data.generations',

           default=['1GB', '100MB', '1MB'])

configitem('remotefilelog', 'data.maxrepackpacks', default=50)

configitem('remotefilelog', 'data.repackmaxpacksize', default='4GB')

configitem('remotefilelog', 'data.repacksizelimit', default='100MB')

configitem('remotefilelog', 'history.gencountlimit', default=2),

configitem('remotefilelog', 'history.generations', default=['100MB'])

configitem('remotefilelog', 'history.maxrepackpacks', default=50)

configitem('remotefilelog', 'history.repackmaxpacksize', default='400MB')

configitem('remotefilelog', 'history.repacksizelimit', default='100MB')

# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for

# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should

# be specifying the version(s) of Mercurial they are tested with, or

# leave the attribute unspecified.

testedwith = 'ships-with-hg-core'

repoclass = localrepo.localrepository

repoclass._basesupported.add(constants.SHALLOWREPO_REQUIREMENT)

isenabled = shallowutil.isenabled

def uisetup(ui):

    """Wraps user facing Mercurial commands to swap them out with shallow

    versions.

    """

    hg.wirepeersetupfuncs.append(fileserverclient.peersetup)

    entry = extensions.wrapcommand(commands.table, 'clone', cloneshallow)

    entry[1].append(('', 'shallow', None,

                     _("create a shallow clone which uses remote file "

                       "history")))

    extensions.wrapcommand(commands.table, 'debugindex',

        debugcommands.debugindex)

    extensions.wrapcommand(commands.table, 'debugindexdot',

        debugcommands.debugindexdot)

    extensions.wrapcommand(commands.table, 'log', log)

    extensions.wrapcommand(commands.table, 'pull', pull)

    # Prevent 'hg manifest --all'

    def _manifest(orig, ui, repo, *args, **opts):

        if (isenabled(repo) and opts.get(r'all')):

            raise error.Abort(_("--all is not supported in a shallow repo"))

        return orig(ui, repo, *args, **opts)

    extensions.wrapcommand(commands.table, "manifest", _manifest)

    # Wrap remotefilelog with lfs code

    def _lfsloaded(loaded=False):

        lfsmod = None

        try:

            lfsmod = extensions.find('lfs')

        except KeyError:

            pass

        if lfsmod:

            lfsmod.wrapfilelog(remotefilelog.remotefilelog)

            fileserverclient._lfsmod = lfsmod

    extensions.afterloaded('lfs', _lfsloaded)

    # debugdata needs remotefilelog.len to work

    extensions.wrapcommand(commands.table, 'debugdata', debugdatashallow)

    changegroup.cgpacker = shallowbundle.shallowcg1packer

    extensions.wrapfunction(changegroup, '_addchangegroupfiles',

                            shallowbundle.addchangegroupfiles)

    extensions.wrapfunction(

        changegroup, 'makechangegroup', shallowbundle.makechangegroup)

    extensions.wrapfunction(localrepo, 'makestore', storewrapper)

    extensions.wrapfunction(exchange, 'pull', exchangepull)

    extensions.wrapfunction(merge, 'applyupdates', applyupdates)

    extensions.wrapfunction(merge, '_checkunknownfiles', checkunknownfiles)

    extensions.wrapfunction(context.workingctx, '_checklookup', checklookup)

    extensions.wrapfunction(scmutil, '_findrenames', findrenames)

    extensions.wrapfunction(copies, '_computeforwardmissing',

                            computeforwardmissing)

    extensions.wrapfunction(dispatch, 'runcommand', runcommand)

    extensions.wrapfunction(repair, '_collectbrokencsets', _collectbrokencsets)

    extensions.wrapfunction(context.changectx, 'filectx', filectx)

    extensions.wrapfunction(context.workingctx, 'filectx', workingfilectx)

    extensions.wrapfunction(patch, 'trydiff', trydiff)

    extensions.wrapfunction(hg, 'verify', _verify)

    scmutil.fileprefetchhooks.add('remotefilelog', _fileprefetchhook)

    # disappointing hacks below

    extensions.wrapfunction(scmutil, 'getrenamedfn', getrenamedfn)

    extensions.wrapfunction(revset, 'filelog', filelogrevset)

    revset.symbols['filelog'] = revset.filelog

    extensions.wrapfunction(cmdutil, 'walkfilerevs', walkfilerevs)

def cloneshallow(orig, ui, repo, *args, **opts):

    if opts.get(r'shallow'):

        repos = []

        def pull_shallow(orig, self, *args, **kwargs):

            if not isenabled(self):

                repos.append(self.unfiltered())

                # set up the client hooks so the post-clone update works

                setupclient(self.ui, self.unfiltered())

                # setupclient fixed the class on the repo itself

                # but we also need to fix it on the repoview

                if isinstance(self, repoview.repoview):

                    self.__class__.__bases__ = (self.__class__.__bases__[0],

                                                self.unfiltered().__class__)

                self.requirements.add(constants.SHALLOWREPO_REQUIREMENT)

                self._writerequirements()

                # Since setupclient hadn't been called, exchange.pull was not

                # wrapped. So we need to manually invoke our version of it.

                return exchangepull(orig, self, *args, **kwargs)

            else:

                return orig(self, *args, **kwargs)

        extensions.wrapfunction(exchange, 'pull', pull_shallow)

        # Wrap the stream logic to add requirements and to pass include/exclude

        # patterns around.

        def setup_streamout(repo, remote):

            # Replace remote.stream_out with a version that sends file

            # patterns.

            def stream_out_shallow(orig):

                caps = remote.capabilities()

                if constants.NETWORK_CAP_LEGACY_SSH_GETFILES in caps:

                    opts = {}

                    if repo.includepattern:

                        opts[r'includepattern'] = '\0'.join(repo.includepattern)

                    if repo.excludepattern:

                        opts[r'excludepattern'] = '\0'.join(repo.excludepattern)

                    return remote._callstream('stream_out_shallow', **opts)

                else:

                    return orig()

            extensions.wrapfunction(remote, 'stream_out', stream_out_shallow)

        def stream_wrap(orig, op):

            setup_streamout(op.repo, op.remote)

            return orig(op)

        extensions.wrapfunction(

            streamclone, 'maybeperformlegacystreamclone', stream_wrap)

        def canperformstreamclone(orig, pullop, bundle2=False):

            # remotefilelog is currently incompatible with the

            # bundle2 flavor of streamclones, so force us to use

            # v1 instead.

            if 'v2' in pullop.remotebundle2caps.get('stream', []):

                pullop.remotebundle2caps['stream'] = [

                    c for c in pullop.remotebundle2caps['stream']

                    if c != 'v2']

            if bundle2:

                return False, None

            supported, requirements = orig(pullop, bundle2=bundle2)

            if requirements is not None:

                requirements.add(constants.SHALLOWREPO_REQUIREMENT)

            return supported, requirements

        extensions.wrapfunction(

            streamclone, 'canperformstreamclone', canperformstreamclone)

    try:

        orig(ui, repo, *args, **opts)

    finally:

        if opts.get(r'shallow'):

            for r in repos:

                if util.safehasattr(r, 'fileservice'):

                    r.fileservice.close()

def debugdatashallow(orig, *args, **kwds):

    oldlen = remotefilelog.remotefilelog.__len__

    try:

        remotefilelog.remotefilelog.__len__ = lambda x: 1

        return orig(*args, **kwds)

    finally:

        remotefilelog.remotefilelog.__len__ = oldlen