from __future__ import absolute_import

import os

import time

from mercurial.i18n import _

from mercurial.node import (

    nullid,

    short,

)

from mercurial import (

    encoding,

    error,

    mdiff,

    policy,

    pycompat,

    scmutil,

    util,

    vfs,

)

from mercurial.utils import procutil

from . import (

    constants,

    contentstore,

    datapack,

    extutil,

    historypack,

    metadatastore,

    shallowutil,

)

osutil = policy.importmod(r'osutil')

class RepackAlreadyRunning(error.Abort):

    pass

if util.safehasattr(util, '_hgexecutable'):

    # Before 5be286db

    _hgexecutable = util.hgexecutable

else:

    from mercurial.utils import procutil

    _hgexecutable = procutil.hgexecutable

def backgroundrepack(repo, incremental=True, packsonly=False):

    cmd = [_hgexecutable(), '-R', repo.origroot, 'repack']

    msg = _("(running background repack)\n")

    if incremental:

        cmd.append('--incremental')

        msg = _("(running background incremental repack)\n")

    if packsonly:

        cmd.append('--packsonly')

    repo.ui.warn(msg)

def fullrepack(repo, options=None):

    """If ``packsonly`` is True, stores creating only loose objects are skipped.

    """

    if util.safehasattr(repo, 'shareddatastores'):

        datasource = contentstore.unioncontentstore(

            *repo.shareddatastores)

        historysource = metadatastore.unionmetadatastore(

            *repo.sharedhistorystores,

            allowincomplete=True)

        packpath = shallowutil.getcachepackpath(

            repo,

            constants.FILEPACK_CATEGORY)

        _runrepack(repo, datasource, historysource, packpath,

                   constants.FILEPACK_CATEGORY, options=options)

    if util.safehasattr(repo.manifestlog, 'datastore'):

        localdata, shareddata = _getmanifeststores(repo)

        lpackpath, ldstores, lhstores = localdata

        spackpath, sdstores, shstores = shareddata

        # Repack the shared manifest store

        datasource = contentstore.unioncontentstore(*sdstores)

        historysource = metadatastore.unionmetadatastore(

                        *shstores,

                        allowincomplete=True)

        _runrepack(repo, datasource, historysource, spackpath,

                   constants.TREEPACK_CATEGORY, options=options)

        # Repack the local manifest store

        datasource = contentstore.unioncontentstore(

                        *ldstores,

                        allowincomplete=True)

        historysource = metadatastore.unionmetadatastore(

                        *lhstores,

                        allowincomplete=True)

        _runrepack(repo, datasource, historysource, lpackpath,

                   constants.TREEPACK_CATEGORY, options=options)

def incrementalrepack(repo, options=None):

    """This repacks the repo by looking at the distribution of pack files in the

    repo and performing the most minimal repack to keep the repo in good shape.

    """

    if util.safehasattr(repo, 'shareddatastores'):

        packpath = shallowutil.getcachepackpath(

            repo,

            constants.FILEPACK_CATEGORY)

        _incrementalrepack(repo,

                           repo.shareddatastores,

                           repo.sharedhistorystores,

                           packpath,

                           constants.FILEPACK_CATEGORY,

                           options=options)

    if util.safehasattr(repo.manifestlog, 'datastore'):

        localdata, shareddata = _getmanifeststores(repo)

        lpackpath, ldstores, lhstores = localdata

        spackpath, sdstores, shstores = shareddata

        # Repack the shared manifest store

        _incrementalrepack(repo,

                           sdstores,

                           shstores,

                           spackpath,

                           constants.TREEPACK_CATEGORY,

                           options=options)

        # Repack the local manifest store

        _incrementalrepack(repo,

                           ldstores,

                           lhstores,

                           lpackpath,

                           constants.TREEPACK_CATEGORY,

                           allowincompletedata=True,

                           options=options)

def _getmanifeststores(repo):

    shareddatastores = repo.manifestlog.shareddatastores

    localdatastores = repo.manifestlog.localdatastores

    sharedhistorystores = repo.manifestlog.sharedhistorystores

    localhistorystores = repo.manifestlog.localhistorystores

    sharedpackpath = shallowutil.getcachepackpath(repo,

                                            constants.TREEPACK_CATEGORY)

    localpackpath = shallowutil.getlocalpackpath(repo.svfs.vfs.base,

                                            constants.TREEPACK_CATEGORY)

    return ((localpackpath, localdatastores, localhistorystores),

            (sharedpackpath, shareddatastores, sharedhistorystores))

def _topacks(packpath, files, constructor):

    paths = list(os.path.join(packpath, p) for p in files)

    packs = list(constructor(p) for p in paths)

    return packs

def _deletebigpacks(repo, folder, files):

    """Deletes packfiles that are bigger than ``packs.maxpacksize``.

    Returns ``files` with the removed files omitted."""

    maxsize = repo.ui.configbytes("packs", "maxpacksize")

    if maxsize <= 0:

        return files

    # This only considers datapacks today, but we could broaden it to include

    # historypacks.

    VALIDEXTS = [".datapack", ".dataidx"]

    # Either an oversize index or datapack will trigger cleanup of the whole

    # pack:

    oversized = set([os.path.splitext(path)[0] for path, ftype, stat in files

        if (stat.st_size > maxsize and (os.path.splitext(path)[1]

                                        in VALIDEXTS))])

    for rootfname in oversized:

        rootpath = os.path.join(folder, rootfname)

        for ext in VALIDEXTS:

            path = rootpath + ext

            repo.ui.debug('removing oversize packfile %s (%s)\n' %

                          (path, util.bytecount(os.stat(path).st_size)))

            os.unlink(path)

    return [row for row in files if os.path.basename(row[0]) not in oversized]

def _incrementalrepack(repo, datastore, historystore, packpath, category,

        allowincompletedata=False, options=None):

    shallowutil.mkstickygroupdir(repo.ui, packpath)

    files = osutil.listdir(packpath, stat=True)

    files = _deletebigpacks(repo, packpath, files)

    datapacks = _topacks(packpath,

        _computeincrementaldatapack(repo.ui, files),

        datapack.datapack)

    datapacks.extend(s for s in datastore

                     if not isinstance(s, datapack.datapackstore))

    historypacks = _topacks(packpath,

        _computeincrementalhistorypack(repo.ui, files),

        historypack.historypack)

    historypacks.extend(s for s in historystore

                        if not isinstance(s, historypack.historypackstore))

    # ``allhistory{files,packs}`` contains all known history packs, even ones we

    # don't plan to repack. They are used during the datapack repack to ensure

    # good ordering of nodes.

    allhistoryfiles = _allpackfileswithsuffix(files, historypack.PACKSUFFIX,

                            historypack.INDEXSUFFIX)

    allhistorypacks = _topacks(packpath,

        (f for f, mode, stat in allhistoryfiles),

        historypack.historypack)

    allhistorypacks.extend(s for s in historystore

                        if not isinstance(s, historypack.historypackstore))

    _runrepack(repo,

               contentstore.unioncontentstore(

                   *datapacks,

                   allowincomplete=allowincompletedata),

               metadatastore.unionmetadatastore(

                   *historypacks,

                   allowincomplete=True),

               packpath, category,

               fullhistory=metadatastore.unionmetadatastore(

                   *allhistorypacks,

                   allowincomplete=True),

                options=options)

def _computeincrementaldatapack(ui, files):

    opts = {

        'gencountlimit' : ui.configint(

            'remotefilelog', 'data.gencountlimit'),

        'generations' : ui.configlist(

            'remotefilelog', 'data.generations'),

        'maxrepackpacks' : ui.configint(

            'remotefilelog', 'data.maxrepackpacks'),

        'repackmaxpacksize' : ui.configbytes(

            'remotefilelog', 'data.repackmaxpacksize'),

        'repacksizelimit' : ui.configbytes(

            'remotefilelog', 'data.repacksizelimit'),

    }

    packfiles = _allpackfileswithsuffix(

        files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX)

    return _computeincrementalpack(packfiles, opts)

def _computeincrementalhistorypack(ui, files):

    opts = {

        'gencountlimit' : ui.configint(

            'remotefilelog', 'history.gencountlimit'),

        'generations' : ui.configlist(

            'remotefilelog', 'history.generations', ['100MB']),

        'maxrepackpacks' : ui.configint(

            'remotefilelog', 'history.maxrepackpacks'),

        'repackmaxpacksize' : ui.configbytes(

            'remotefilelog', 'history.repackmaxpacksize', '400MB'),

        'repacksizelimit' : ui.configbytes(

            'remotefilelog', 'history.repacksizelimit'),

    }

    packfiles = _allpackfileswithsuffix(

        files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX)

    return _computeincrementalpack(packfiles, opts)

def _allpackfileswithsuffix(files, packsuffix, indexsuffix):

    result = []

    fileset = set(fn for fn, mode, stat in files)

    for filename, mode, stat in files:

        if not filename.endswith(packsuffix):

            continue

        prefix = filename[:-len(packsuffix)]

        # Don't process a pack if it doesn't have an index.

        if (prefix + indexsuffix) not in fileset:

            continue

        result.append((prefix, mode, stat))

    return result

def _computeincrementalpack(files, opts):

    """Given a set of pack files along with the configuration options, this

    function computes the list of files that should be packed as part of an

    incremental repack.

    It tries to strike a balance between keeping incremental repacks cheap (i.e.

    packing small things when possible, and rolling the packs up to the big ones

    over time).

    """

    limits = list(sorted((util.sizetoint(s) for s in opts['generations']),

                                reverse=True))

    limits.append(0)

    # Group the packs by generation (i.e. by size)

    generations = []

    for i in pycompat.xrange(len(limits)):

        generations.append([])

    sizes = {}

    for prefix, mode, stat in files:

        size = stat.st_size

        if size > opts['repackmaxpacksize']:

            continue

        sizes[prefix] = size

        for i, limit in enumerate(limits):

            if size > limit:

                generations[i].append(prefix)

                break

    # Steps for picking what packs to repack:

    # 1. Pick the largest generation with > gencountlimit pack files.

    # 2. Take the smallest three packs.

    # 3. While total-size-of-packs < repacksizelimit: add another pack

    # Find the largest generation with more than gencountlimit packs

    genpacks = []

    for i, limit in enumerate(limits):

        if len(generations[i]) > opts['gencountlimit']:

            # Sort to be smallest last, for easy popping later

            genpacks.extend(sorted(generations[i], reverse=True,

                                   key=lambda x: sizes[x]))

            break

    # Take as many packs from the generation as we can

    chosenpacks = genpacks[-3:]

    genpacks = genpacks[:-3]

    repacksize = sum(sizes[n] for n in chosenpacks)

    while (repacksize < opts['repacksizelimit'] and genpacks and

           len(chosenpacks) < opts['maxrepackpacks']):

        chosenpacks.append(genpacks.pop())

        repacksize += sizes[chosenpacks[-1]]

    return chosenpacks

def _runrepack(repo, data, history, packpath, category, fullhistory=None,

               options=None):

    shallowutil.mkstickygroupdir(repo.ui, packpath)

    def isold(repo, filename, node):

        """Check if the file node is older than a limit.

        Unless a limit is specified in the config the default limit is taken.

        """

        filectx = repo.filectx(filename, fileid=node)

        filetime = repo[filectx.linkrev()].date()

        ttl = repo.ui.configint('remotefilelog', 'nodettl')

        limit = time.time() - ttl

        return filetime[0] < limit

    garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack')

    if not fullhistory:

        fullhistory = history

    packer = repacker(repo, data, history, fullhistory, category,

                      gc=garbagecollect, isold=isold, options=options)

    # internal config: remotefilelog.datapackversion

    dv = repo.ui.configint('remotefilelog', 'datapackversion', 0)

    with datapack.mutabledatapack(repo.ui, packpath, version=dv) as dpack:

        with historypack.mutablehistorypack(repo.ui, packpath) as hpack:

            try:

                packer.run(dpack, hpack)

            except error.LockHeld:

                raise RepackAlreadyRunning(_("skipping repack - another repack "

                                             "is already running"))

def keepset(repo, keyfn, lastkeepkeys=None):

    """Computes a keepset which is not garbage collected.

    'keyfn' is a function that maps filename, node to a unique key.

    'lastkeepkeys' is an optional argument and if provided the keepset

    function updates lastkeepkeys with more keys and returns the result.

    """

    if not lastkeepkeys:

        keepkeys = set()

    else:

        keepkeys = lastkeepkeys

    # We want to keep:

    # 1. Working copy parent

    # 2. Draft commits

    # 3. Parents of draft commits

    # 4. Pullprefetch and bgprefetchrevs revsets if specified

    revs = ['.', 'draft()', 'parents(draft())']

    prefetchrevs = repo.ui.config('remotefilelog', 'pullprefetch', None)

    if prefetchrevs:

        revs.append('(%s)' % prefetchrevs)

    prefetchrevs = repo.ui.config('remotefilelog', 'bgprefetchrevs', None)

    if prefetchrevs:

        revs.append('(%s)' % prefetchrevs)

    revs = '+'.join(revs)

    revs = ['sort((%s), "topo")' % revs]

    keep = scmutil.revrange(repo, revs)

    processed = set()

    lastmanifest = None

    # process the commits in toposorted order starting from the oldest

    for r in reversed(keep._list):

        if repo[r].p1().rev() in processed:

            # if the direct parent has already been processed

            # then we only need to process the delta

            m = repo[r].manifestctx().readdelta()

        else:

            # otherwise take the manifest and diff it

            # with the previous manifest if one exists

            if lastmanifest:

                m = repo[r].manifest().diff(lastmanifest)

            else: