from __future__ import absolute_import

import os

import time

from mercurial.i18n import _

from mercurial.node import (

    nullid,

    short,

)

from mercurial import (

    encoding,

    error,

    lock as lockmod,

    mdiff,

    policy,

    pycompat,

    scmutil,

    util,

    vfs,

)

from mercurial.utils import procutil

from . import (

    constants,

    contentstore,

    datapack,

    historypack,

    metadatastore,

    shallowutil,

)

osutil = policy.importmod('osutil')

class RepackAlreadyRunning(error.Abort):

    pass

def backgroundrepack(repo, incremental=True, packsonly=False):

    cmd = [procutil.hgexecutable(), b'-R', repo.origroot, b'repack']

    msg = _(b"(running background repack)\n")

    if incremental:

        cmd.append(b'--incremental')

        msg = _(b"(running background incremental repack)\n")

    if packsonly:

        cmd.append(b'--packsonly')

    repo.ui.warn(msg)

    # We know this command will find a binary, so don't block on it starting.

    kwargs = {}

    if repo.ui.configbool(b'devel', b'remotefilelog.bg-wait'):

        kwargs['record_wait'] = repo.ui.atexit

    procutil.runbgcommand(cmd, encoding.environ, ensurestart=False, **kwargs)

def fullrepack(repo, options=None):

    if util.safehasattr(repo, 'shareddatastores'):

        datasource = contentstore.unioncontentstore(*repo.shareddatastores)

        historysource = metadatastore.unionmetadatastore(

            *repo.sharedhistorystores, allowincomplete=True

        )

        packpath = shallowutil.getcachepackpath(

            repo, constants.FILEPACK_CATEGORY

        )

        _runrepack(

            repo,

            datasource,

            historysource,

            packpath,

            constants.FILEPACK_CATEGORY,

            options=options,

        )

    if util.safehasattr(repo.manifestlog, 'datastore'):

        localdata, shareddata = _getmanifeststores(repo)

        lpackpath, ldstores, lhstores = localdata

        spackpath, sdstores, shstores = shareddata

        # Repack the shared manifest store

        datasource = contentstore.unioncontentstore(*sdstores)

        historysource = metadatastore.unionmetadatastore(

            *shstores, allowincomplete=True

        )

        _runrepack(

            repo,

            datasource,

            historysource,

            spackpath,

            constants.TREEPACK_CATEGORY,

            options=options,

        )

        # Repack the local manifest store

        datasource = contentstore.unioncontentstore(

            *ldstores, allowincomplete=True

        )

        historysource = metadatastore.unionmetadatastore(

            *lhstores, allowincomplete=True

        )

        _runrepack(

            repo,

            datasource,

            historysource,

            lpackpath,

            constants.TREEPACK_CATEGORY,

            options=options,

        )

def incrementalrepack(repo, options=None):

    """This repacks the repo by looking at the distribution of pack files in the

    repo and performing the most minimal repack to keep the repo in good shape.

    """

    if util.safehasattr(repo, 'shareddatastores'):

        packpath = shallowutil.getcachepackpath(

            repo, constants.FILEPACK_CATEGORY

        )

        _incrementalrepack(

            repo,

            repo.shareddatastores,

            repo.sharedhistorystores,

            packpath,

            constants.FILEPACK_CATEGORY,

            options=options,

        )

    if util.safehasattr(repo.manifestlog, 'datastore'):

        localdata, shareddata = _getmanifeststores(repo)

        lpackpath, ldstores, lhstores = localdata

        spackpath, sdstores, shstores = shareddata

        # Repack the shared manifest store

        _incrementalrepack(

            repo,

            sdstores,

            shstores,

            spackpath,

            constants.TREEPACK_CATEGORY,

            options=options,

        )

        # Repack the local manifest store

        _incrementalrepack(

            repo,

            ldstores,

            lhstores,

            lpackpath,

            constants.TREEPACK_CATEGORY,

            allowincompletedata=True,

            options=options,

        )

def _getmanifeststores(repo):

    shareddatastores = repo.manifestlog.shareddatastores

    localdatastores = repo.manifestlog.localdatastores

    sharedhistorystores = repo.manifestlog.sharedhistorystores

    localhistorystores = repo.manifestlog.localhistorystores

    sharedpackpath = shallowutil.getcachepackpath(

        repo, constants.TREEPACK_CATEGORY

    )

    localpackpath = shallowutil.getlocalpackpath(

        repo.svfs.vfs.base, constants.TREEPACK_CATEGORY

    )

    return (

        (localpackpath, localdatastores, localhistorystores),

        (sharedpackpath, shareddatastores, sharedhistorystores),

    )

def _topacks(packpath, files, constructor):

    paths = list(os.path.join(packpath, p) for p in files)

    packs = list(constructor(p) for p in paths)

    return packs

def _deletebigpacks(repo, folder, files):

    """Deletes packfiles that are bigger than ``packs.maxpacksize``.

    Returns ``files` with the removed files omitted."""

    maxsize = repo.ui.configbytes(b"packs", b"maxpacksize")

    if maxsize <= 0:

        return files

    # This only considers datapacks today, but we could broaden it to include

    # historypacks.

    VALIDEXTS = [b".datapack", b".dataidx"]

    # Either an oversize index or datapack will trigger cleanup of the whole

    # pack:

    oversized = {

        os.path.splitext(path)[0]

        for path, ftype, stat in files

        if (stat.st_size > maxsize and (os.path.splitext(path)[1] in VALIDEXTS))

    }

    for rootfname in oversized:

        rootpath = os.path.join(folder, rootfname)

        for ext in VALIDEXTS:

            path = rootpath + ext

            repo.ui.debug(

                b'removing oversize packfile %s (%s)\n'

                % (path, util.bytecount(os.stat(path).st_size))

            )

            os.unlink(path)

    return [row for row in files if os.path.basename(row[0]) not in oversized]

def _incrementalrepack(

    repo,

    datastore,

    historystore,

    packpath,

    category,

    allowincompletedata=False,

    options=None,

):

    shallowutil.mkstickygroupdir(repo.ui, packpath)

    files = osutil.listdir(packpath, stat=True)

    files = _deletebigpacks(repo, packpath, files)

    datapacks = _topacks(

        packpath, _computeincrementaldatapack(repo.ui, files), datapack.datapack

    )

    datapacks.extend(

        s for s in datastore if not isinstance(s, datapack.datapackstore)

    )

    historypacks = _topacks(

        packpath,

        _computeincrementalhistorypack(repo.ui, files),

        historypack.historypack,

    )

    historypacks.extend(

        s

        for s in historystore

        if not isinstance(s, historypack.historypackstore)

    )

    # ``allhistory{files,packs}`` contains all known history packs, even ones we

    # don't plan to repack. They are used during the datapack repack to ensure

    # good ordering of nodes.

    allhistoryfiles = _allpackfileswithsuffix(

        files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX

    )

    allhistorypacks = _topacks(

        packpath,

        (f for f, mode, stat in allhistoryfiles),

        historypack.historypack,

    )

    allhistorypacks.extend(

        s

        for s in historystore

        if not isinstance(s, historypack.historypackstore)

    )

    _runrepack(

        repo,

        contentstore.unioncontentstore(

            *datapacks, allowincomplete=allowincompletedata

        ),

        metadatastore.unionmetadatastore(*historypacks, allowincomplete=True),

        packpath,

        category,

        fullhistory=metadatastore.unionmetadatastore(

            *allhistorypacks, allowincomplete=True

        ),

        options=options,

    )

def _computeincrementaldatapack(ui, files):

    opts = {

        b'gencountlimit': ui.configint(b'remotefilelog', b'data.gencountlimit'),

        b'generations': ui.configlist(b'remotefilelog', b'data.generations'),

        b'maxrepackpacks': ui.configint(

            b'remotefilelog', b'data.maxrepackpacks'

        ),

        b'repackmaxpacksize': ui.configbytes(

            b'remotefilelog', b'data.repackmaxpacksize'

        ),

        b'repacksizelimit': ui.configbytes(

            b'remotefilelog', b'data.repacksizelimit'

        ),

    }

    packfiles = _allpackfileswithsuffix(

        files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX

    )

    return _computeincrementalpack(packfiles, opts)

def _computeincrementalhistorypack(ui, files):

    opts = {

        b'gencountlimit': ui.configint(

            b'remotefilelog', b'history.gencountlimit'

        ),

        b'generations': ui.configlist(

            b'remotefilelog', b'history.generations', [b'100MB']

        ),

        b'maxrepackpacks': ui.configint(

            b'remotefilelog', b'history.maxrepackpacks'

        ),

        b'repackmaxpacksize': ui.configbytes(

            b'remotefilelog', b'history.repackmaxpacksize', b'400MB'

        ),

        b'repacksizelimit': ui.configbytes(

            b'remotefilelog', b'history.repacksizelimit'

        ),

    }

    packfiles = _allpackfileswithsuffix(

        files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX

    )

    return _computeincrementalpack(packfiles, opts)

def _allpackfileswithsuffix(files, packsuffix, indexsuffix):

    result = []

    fileset = {fn for fn, mode, stat in files}

    for filename, mode, stat in files:

        if not filename.endswith(packsuffix):

            continue

        prefix = filename[: -len(packsuffix)]

        # Don't process a pack if it doesn't have an index.

        if (prefix + indexsuffix) not in fileset:

            continue

        result.append((prefix, mode, stat))

    return result

def _computeincrementalpack(files, opts):

    """Given a set of pack files along with the configuration options, this

    function computes the list of files that should be packed as part of an

    incremental repack.

    It tries to strike a balance between keeping incremental repacks cheap (i.e.

    packing small things when possible, and rolling the packs up to the big ones

    over time).

    """

    limits = list(

        sorted((util.sizetoint(s) for s in opts[b'generations']), reverse=True)

    )

    limits.append(0)

    # Group the packs by generation (i.e. by size)

    generations = []

    for i in pycompat.xrange(len(limits)):

        generations.append([])

    sizes = {}

    for prefix, mode, stat in files:

        size = stat.st_size

        if size > opts[b'repackmaxpacksize']:

            continue

        sizes[prefix] = size

        for i, limit in enumerate(limits):

            if size > limit:

                generations[i].append(prefix)

                break

    # Steps for picking what packs to repack:

    # 1. Pick the largest generation with > gencountlimit pack files.

    # 2. Take the smallest three packs.

    # 3. While total-size-of-packs < repacksizelimit: add another pack

    # Find the largest generation with more than gencountlimit packs

    genpacks = []

    for i, limit in enumerate(limits):

        if len(generations[i]) > opts[b'gencountlimit']:

            # Sort to be smallest last, for easy popping later

            genpacks.extend(

                sorted(generations[i], reverse=True, key=lambda x: sizes[x])

            )

            break

    # Take as many packs from the generation as we can

    chosenpacks = genpacks[-3:]

    genpacks = genpacks[:-3]

    repacksize = sum(sizes[n] for n in chosenpacks)

    while (

        repacksize < opts[b'repacksizelimit']

        and genpacks

        and len(chosenpacks) < opts[b'maxrepackpacks']

    ):

        chosenpacks.append(genpacks.pop())

        repacksize += sizes[chosenpacks[-1]]

    return chosenpacks

def _runrepack(