from __future__ import absolute_import

import os

import time

from mercurial.i18n import _

from mercurial.node import (

    nullid,

    short,

)

from mercurial import (

    encoding,

    error,

    lock as lockmod,

    mdiff,

    policy,

    pycompat,

    scmutil,

    util,

    vfs,

)

from mercurial.utils import procutil

from . import (

    constants,

    contentstore,

    datapack,

    historypack,

    metadatastore,

    shallowutil,

)

osutil = policy.importmod(r'osutil')

class RepackAlreadyRunning(error.Abort):

    pass

def backgroundrepack(

    repo, incremental=True, packsonly=False, ensurestart=False

):

    cmd = [procutil.hgexecutable(), b'-R', repo.origroot, b'repack']

    msg = _(b"(running background repack)\n")

    if incremental:

        cmd.append(b'--incremental')

        msg = _(b"(running background incremental repack)\n")

    if packsonly:

        cmd.append(b'--packsonly')

    repo.ui.warn(msg)

    # We know this command will find a binary, so don't block on it starting.

    procutil.runbgcommand(cmd, encoding.environ, ensurestart=ensurestart)

def fullrepack(repo, options=None):

    """If ``packsonly`` is True, stores creating only loose objects are skipped.

    """

        datasource = contentstore.unioncontentstore(*repo.shareddatastores)

        historysource = metadatastore.unionmetadatastore(

            *repo.sharedhistorystores, allowincomplete=True

        )

        packpath = shallowutil.getcachepackpath(

            repo, constants.FILEPACK_CATEGORY

        )

        _runrepack(

            repo,

            datasource,

            historysource,

            packpath,

            constants.FILEPACK_CATEGORY,

            options=options,

        )

        localdata, shareddata = _getmanifeststores(repo)

        lpackpath, ldstores, lhstores = localdata

        spackpath, sdstores, shstores = shareddata

        # Repack the shared manifest store

        datasource = contentstore.unioncontentstore(*sdstores)

        historysource = metadatastore.unionmetadatastore(

            *shstores, allowincomplete=True

        )

        _runrepack(

            repo,

            datasource,

            historysource,

            spackpath,

            constants.TREEPACK_CATEGORY,

            options=options,

        )

        # Repack the local manifest store

        datasource = contentstore.unioncontentstore(

            *ldstores, allowincomplete=True

        )

        historysource = metadatastore.unionmetadatastore(

            *lhstores, allowincomplete=True

        )

        _runrepack(

            repo,

            datasource,

            historysource,

            lpackpath,

            constants.TREEPACK_CATEGORY,

            options=options,

        )

def incrementalrepack(repo, options=None):

    """This repacks the repo by looking at the distribution of pack files in the

    repo and performing the most minimal repack to keep the repo in good shape.

    """

        packpath = shallowutil.getcachepackpath(

            repo, constants.FILEPACK_CATEGORY

        )

        _incrementalrepack(

            repo,

            repo.shareddatastores,

            repo.sharedhistorystores,

            packpath,

            constants.FILEPACK_CATEGORY,

            options=options,

        )

        localdata, shareddata = _getmanifeststores(repo)

        lpackpath, ldstores, lhstores = localdata

        spackpath, sdstores, shstores = shareddata

        # Repack the shared manifest store

        _incrementalrepack(

            repo,

            sdstores,

            shstores,

            spackpath,

            constants.TREEPACK_CATEGORY,

            options=options,

        )

        # Repack the local manifest store

        _incrementalrepack(

            repo,

            ldstores,

            lhstores,

            lpackpath,

            constants.TREEPACK_CATEGORY,

            allowincompletedata=True,

            options=options,

        )

def _getmanifeststores(repo):

    shareddatastores = repo.manifestlog.shareddatastores

    localdatastores = repo.manifestlog.localdatastores

    sharedhistorystores = repo.manifestlog.sharedhistorystores

    localhistorystores = repo.manifestlog.localhistorystores

    sharedpackpath = shallowutil.getcachepackpath(

        repo, constants.TREEPACK_CATEGORY

    )

    localpackpath = shallowutil.getlocalpackpath(

        repo.svfs.vfs.base, constants.TREEPACK_CATEGORY

    )

    return (

        (localpackpath, localdatastores, localhistorystores),

        (sharedpackpath, shareddatastores, sharedhistorystores),

    )

def _topacks(packpath, files, constructor):

    paths = list(os.path.join(packpath, p) for p in files)

    packs = list(constructor(p) for p in paths)

    return packs

def _deletebigpacks(repo, folder, files):

    """Deletes packfiles that are bigger than ``packs.maxpacksize``.

    Returns ``files` with the removed files omitted."""

    maxsize = repo.ui.configbytes(b"packs", b"maxpacksize")

    if maxsize <= 0:

        return files

    # This only considers datapacks today, but we could broaden it to include

    # historypacks.

    VALIDEXTS = [b".datapack", b".dataidx"]

    # Either an oversize index or datapack will trigger cleanup of the whole

    # pack:

    oversized = {

        os.path.splitext(path)[0]

        for path, ftype, stat in files

        if (stat.st_size > maxsize and (os.path.splitext(path)[1] in VALIDEXTS))

    }

    for rootfname in oversized:

        rootpath = os.path.join(folder, rootfname)

        for ext in VALIDEXTS:

            path = rootpath + ext

            repo.ui.debug(

                b'removing oversize packfile %s (%s)\n'

                % (path, util.bytecount(os.stat(path).st_size))

            )

            os.unlink(path)

    return [row for row in files if os.path.basename(row[0]) not in oversized]

def _incrementalrepack(

    repo,

    datastore,

    historystore,

    packpath,

    category,

    allowincompletedata=False,

    options=None,

):

    shallowutil.mkstickygroupdir(repo.ui, packpath)

    files = osutil.listdir(packpath, stat=True)

    files = _deletebigpacks(repo, packpath, files)

    datapacks = _topacks(

        packpath, _computeincrementaldatapack(repo.ui, files), datapack.datapack

    )

    datapacks.extend(

        s for s in datastore if not isinstance(s, datapack.datapackstore)

    )

    historypacks = _topacks(

        packpath,

        _computeincrementalhistorypack(repo.ui, files),

        historypack.historypack,

    )

    historypacks.extend(

        s

        for s in historystore

        if not isinstance(s, historypack.historypackstore)

    )

    # ``allhistory{files,packs}`` contains all known history packs, even ones we

    # don't plan to repack. They are used during the datapack repack to ensure

    # good ordering of nodes.

    allhistoryfiles = _allpackfileswithsuffix(

        files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX

    )

    allhistorypacks = _topacks(

        packpath,

        (f for f, mode, stat in allhistoryfiles),

        historypack.historypack,

    )

    allhistorypacks.extend(

        s

        for s in historystore

        if not isinstance(s, historypack.historypackstore)

    )

    _runrepack(

        repo,

        contentstore.unioncontentstore(

            *datapacks, allowincomplete=allowincompletedata

        ),

        metadatastore.unionmetadatastore(*historypacks, allowincomplete=True),

        packpath,

        category,

        fullhistory=metadatastore.unionmetadatastore(

            *allhistorypacks, allowincomplete=True

        ),

        options=options,

    )

def _computeincrementaldatapack(ui, files):

    opts = {

        b'gencountlimit': ui.configint(b'remotefilelog', b'data.gencountlimit'),

        b'generations': ui.configlist(b'remotefilelog', b'data.generations'),

        b'maxrepackpacks': ui.configint(

            b'remotefilelog', b'data.maxrepackpacks'

        ),

        b'repackmaxpacksize': ui.configbytes(

            b'remotefilelog', b'data.repackmaxpacksize'

        ),

        b'repacksizelimit': ui.configbytes(

            b'remotefilelog', b'data.repacksizelimit'

        ),

    }

    packfiles = _allpackfileswithsuffix(

        files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX

    )

    return _computeincrementalpack(packfiles, opts)

def _computeincrementalhistorypack(ui, files):

    opts = {

        b'gencountlimit': ui.configint(

            b'remotefilelog', b'history.gencountlimit'

        ),

        b'generations': ui.configlist(

            b'remotefilelog', b'history.generations', [b'100MB']

        ),

        b'maxrepackpacks': ui.configint(

            b'remotefilelog', b'history.maxrepackpacks'

        ),

        b'repackmaxpacksize': ui.configbytes(

            b'remotefilelog', b'history.repackmaxpacksize', b'400MB'

        ),

        b'repacksizelimit': ui.configbytes(

            b'remotefilelog', b'history.repacksizelimit'

        ),

    }

    packfiles = _allpackfileswithsuffix(

        files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX

    )

    return _computeincrementalpack(packfiles, opts)

def _allpackfileswithsuffix(files, packsuffix, indexsuffix):

    result = []

    fileset = set(fn for fn, mode, stat in files)

    for filename, mode, stat in files:

        if not filename.endswith(packsuffix):

            continue

        prefix = filename[: -len(packsuffix)]

        # Don't process a pack if it doesn't have an index.

        if (prefix + indexsuffix) not in fileset:

            continue

        result.append((prefix, mode, stat))

    return result

def _computeincrementalpack(files, opts):

    """Given a set of pack files along with the configuration options, this

    function computes the list of files that should be packed as part of an

    incremental repack.

    It tries to strike a balance between keeping incremental repacks cheap (i.e.

    packing small things when possible, and rolling the packs up to the big ones

    over time).

    """

    limits = list(

        sorted((util.sizetoint(s) for s in opts[b'generations']), reverse=True)

    )

    limits.append(0)

    # Group the packs by generation (i.e. by size)

    generations = []

    for i in pycompat.xrange(len(limits)):

        generations.append([])

    sizes = {}

    for prefix, mode, stat in files:

        size = stat.st_size

        if size > opts[b'repackmaxpacksize']:

            continue

        sizes[prefix] = size

        for i, limit in enumerate(limits):

            if size > limit:

                generations[i].append(prefix)

                break

    # Steps for picking what packs to repack:

    # 1. Pick the largest generation with > gencountlimit pack files.

    # 2. Take the smallest three packs.

    # 3. While total-size-of-packs < repacksizelimit: add another pack

    # Find the largest generation with more than gencountlimit packs

    genpacks = []

    for i, limit in enumerate(limits):

        if len(generations[i]) > opts[b'gencountlimit']:

            # Sort to be smallest last, for easy popping later

            genpacks.extend(

                sorted(generations[i], reverse=True, key=lambda x: sizes[x])

            )

            break

    # Take as many packs from the generation as we can

    chosenpacks = genpacks[-3:]

    genpacks = genpacks[:-3]

    repacksize = sum(sizes[n] for n in chosenpacks)

    while (

        repacksize < opts[b'repacksizelimit']

        and genpacks

        and len(chosenpacks) < opts[b'maxrepackpacks']

    ):

        chosenpacks.append(genpacks.pop())

        repacksize += sizes[chosenpacks[-1]]

    return chosenpacks

def _runrepack(

    repo, data, history, packpath, category, fullhistory=None, options=None

):