1 # Copyright 2017 Facebook, Inc.

     2 #

     3 # This software may be used and distributed according to the terms of the

     4 # GNU General Public License version 2 or any later version.

     5 """

     6     [infinitepushbackup]

     7     # Whether to enable automatic backups. If this option is True then a backup

     8     # process will be started after every mercurial command that modifies the

     9     # repo, for example, commit, amend, histedit, rebase etc.

    10     autobackup = False

    11 

    12     # path to the directory where pushback logs should be stored

    13     logdir = path/to/dir

    14 

    15     # Backup at most maxheadstobackup heads, other heads are ignored.

    16     # Negative number means backup everything.

    17     maxheadstobackup = -1

    18 

    19     # Nodes that should not be backed up. Ancestors of these nodes won't be

    20     # backed up either

    21     dontbackupnodes = []

    22 

    23     # Special option that may be used to trigger re-backuping. For example,

    24     # if there was a bug in infinitepush backups, then changing the value of

    25     # this option will force all clients to make a "clean" backup

    26     backupgeneration = 0

    27 

    28     # Hostname value to use. If not specified then socket.gethostname() will

    29     # be used

    30     hostname = ''

    31 

    32     # Enable reporting of infinitepush backup status as a summary at the end

    33     # of smartlog.

    34     enablestatus = False

    35 

    36     # Whether or not to save information about the latest successful backup.

    37     # This information includes the local revision number and unix timestamp

    38     # of the last time we successfully made a backup.

    39     savelatestbackupinfo = False

    40 """

    41 

    42 from __future__ import absolute_import

    43 

    44 import collections

    45 import errno

    46 import json

    47 import os

    48 import re

    49 import socket

    50 import stat

    51 import subprocess

    52 import time

    53 

    54 from mercurial.node import (

    55     bin,

    56     hex,

    57     nullrev,

    58     short,

    59 )

    60 

    61 from mercurial.i18n import _

    62 

    63 from mercurial import (

    64     bundle2,

    65     changegroup,

    66     commands,

    67     discovery,

    68     dispatch,

    69     encoding,

    70     error,

    71     extensions,

    72     hg,

    73     localrepo,

    74     lock as lockmod,

    75     phases,

    76     policy,

    77     registrar,

    78     scmutil,

    79     util,

    80 )

    81 

    82 from . import bundleparts

    83 

    84 getscratchbookmarkspart = bundleparts.getscratchbookmarkspart

    85 getscratchbranchparts = bundleparts.getscratchbranchparts

    86 

    87 from hgext3rd import shareutil

    88 

    89 osutil = policy.importmod(r'osutil')

    90 

    91 cmdtable = {}

    92 command = registrar.command(cmdtable)

    93 revsetpredicate = registrar.revsetpredicate()

    94 templatekeyword = registrar.templatekeyword()

    95 

    96 backupbookmarktuple = collections.namedtuple('backupbookmarktuple',

    97                                  ['hostname', 'reporoot', 'localbookmark'])

    98 

    99 class backupstate(object):

   100     def __init__(self):

   101         self.heads = set()

   102         self.localbookmarks = {}

   103 

   104     def empty(self):

   105         return not self.heads and not self.localbookmarks

   106 

   107 class WrongPermissionsException(Exception):

   108     def __init__(self, logdir):

   109         self.logdir = logdir

   110 

   111 restoreoptions = [

   112      ('', 'reporoot', '', 'root of the repo to restore'),

   113      ('', 'user', '', 'user who ran the backup'),

   114      ('', 'hostname', '', 'hostname of the repo to restore'),

   115 ]

   116 

   117 _backuplockname = 'infinitepushbackup.lock'

   118 

   119 def extsetup(ui):

   120     if ui.configbool('infinitepushbackup', 'autobackup', False):

   121         extensions.wrapfunction(dispatch, 'runcommand',

   122                                 _autobackupruncommandwrapper)

   123         extensions.wrapfunction(localrepo.localrepository, 'transaction',

   124                                 _transaction)

   125 

   126 @command('pushbackup',

   127          [('', 'background', None, 'run backup in background')])

   128 def backup(ui, repo, dest=None, **opts):

   129     """

   130     Pushes commits, bookmarks and heads to infinitepush.

   131     New non-extinct commits are saved since the last `hg pushbackup`

   132     or since 0 revision if this backup is the first.

   133     Local bookmarks are saved remotely as:

   134         infinitepush/backups/USERNAME/HOST/REPOROOT/bookmarks/LOCAL_BOOKMARK

   135     Local heads are saved remotely as:

   136         infinitepush/backups/USERNAME/HOST/REPOROOT/heads/HEAD_HASH

   137     """

   138 

   139     if opts.get('background'):

   140         _dobackgroundbackup(ui, repo, dest)

   141         return 0

   142 

   143     try:

   144         # Wait at most 30 seconds, because that's the average backup time

   145         timeout = 30

   146         srcrepo = shareutil.getsrcrepo(repo)

   147         with lockmod.lock(srcrepo.vfs, _backuplockname, timeout=timeout):

   148             return _dobackup(ui, repo, dest, **opts)

   149     except error.LockHeld as e:

   150         if e.errno == errno.ETIMEDOUT:

   151             ui.warn(_('timeout waiting on backup lock\n'))

   152             return 0

   153         else:

   154             raise

   155 

   156 @command('pullbackup', restoreoptions)

   157 def restore(ui, repo, dest=None, **opts):

   158     """

   159     Pulls commits from infinitepush that were previously saved with

   160     `hg pushbackup`.

   161     If user has only one backup for the `dest` repo then it will be restored.

   162     But user may have backed up many local repos that points to `dest` repo.

   163     These local repos may reside on different hosts or in different

   164     repo roots. It makes restore ambiguous; `--reporoot` and `--hostname`

   165     options are used to disambiguate.

   166     """

   167 

   168     other = _getremote(repo, ui, dest, **opts)

   169 

   170     sourcereporoot = opts.get('reporoot')

   171     sourcehostname = opts.get('hostname')

   172     namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))

   173     allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,

   174                                            sourcehostname, namingmgr)

   175     if len(allbackupstates) == 0:

   176         ui.warn(_('no backups found!'))

   177         return 1

   178     _checkbackupstates(allbackupstates)

   179 

   180     __, backupstate = allbackupstates.popitem()

   181     pullcmd, pullopts = _getcommandandoptions('^pull')

   182     # pull backuped heads and nodes that are pointed by bookmarks

   183     pullopts['rev'] = list(backupstate.heads |

   184                            set(backupstate.localbookmarks.values()))

   185     if dest:

   186         pullopts['source'] = dest

   187     result = pullcmd(ui, repo, **pullopts)

   188 

   189     with repo.wlock(), repo.lock(), repo.transaction('bookmark') as tr:

   190         changes = []

   191         for book, hexnode in backupstate.localbookmarks.iteritems():

   192             if hexnode in repo:

   193                 changes.append((book, bin(hexnode)))

   194             else:

   195                 ui.warn(_('%s not found, not creating %s bookmark') %

   196                         (hexnode, book))

   197         repo._bookmarks.applychanges(repo, tr, changes)

   198 

   199     # manually write local backup state and flag to not autobackup

   200     # just after we restored, which would be pointless

   201     _writelocalbackupstate(repo.vfs,

   202                            list(backupstate.heads),

   203                            backupstate.localbookmarks)

   204     repo.ignoreautobackup = True

   205 

   206     return result

   207 

   208 @command('getavailablebackups',

   209     [('', 'user', '', _('username, defaults to current user')),

   210      ('', 'json', None, _('print available backups in json format'))])

   211 def getavailablebackups(ui, repo, dest=None, **opts):

   212     other = _getremote(repo, ui, dest, **opts)

   213 

   214     sourcereporoot = opts.get('reporoot')

   215     sourcehostname = opts.get('hostname')

   216 

   217     namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))

   218     allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,

   219                                            sourcehostname, namingmgr)

   220 

   221     if opts.get('json'):

   222         jsondict = collections.defaultdict(list)

   223         for hostname, reporoot in allbackupstates.keys():

   224             jsondict[hostname].append(reporoot)

   225             # make sure the output is sorted. That's not an efficient way to

   226             # keep list sorted but we don't have that many backups.

   227             jsondict[hostname].sort()

   228         ui.write('%s\n' % json.dumps(jsondict))

   229     else:

   230         if not allbackupstates:

   231             ui.write(_('no backups available for %s\n') % namingmgr.username)

   232 

   233         ui.write(_('user %s has %d available backups:\n') %

   234                  (namingmgr.username, len(allbackupstates)))

   235 

   236         for hostname, reporoot in sorted(allbackupstates.keys()):

   237             ui.write(_('%s on %s\n') % (reporoot, hostname))

   238 

   239 @command('debugcheckbackup',

   240          [('', 'all', None, _('check all backups that user have')),

   241          ] + restoreoptions)

   242 def checkbackup(ui, repo, dest=None, **opts):

   243     """

   244     Checks that all the nodes that backup needs are available in bundlestore

   245     This command can check either specific backup (see restoreoptions) or all

   246     backups for the user

   247     """

   248 

   249     sourcereporoot = opts.get('reporoot')

   250     sourcehostname = opts.get('hostname')

   251 

   252     other = _getremote(repo, ui, dest, **opts)

   253     namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))

   254     allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,

   255                                            sourcehostname, namingmgr)

   256     if not opts.get('all'):

   257         _checkbackupstates(allbackupstates)

   258 

   259     ret = 0

   260     while allbackupstates:

   261         key, bkpstate = allbackupstates.popitem()

   262         ui.status(_('checking %s on %s\n') % (key[1], key[0]))

   263         if not _dobackupcheck(bkpstate, ui, repo, dest, **opts):

   264             ret = 255

   265     return ret

   266 

   267 @command('debugwaitbackup', [('', 'timeout', '', 'timeout value')])

   268 def waitbackup(ui, repo, timeout):

   269     try:

   270         if timeout:

   271             timeout = int(timeout)

   272         else:

   273             timeout = -1

   274     except ValueError:

   275         raise error.Abort('timeout should be integer')

   276 

   277     try:

   278         repo = shareutil.getsrcrepo(repo)

   279         with lockmod.lock(repo.vfs, _backuplockname, timeout=timeout):

   280             pass

   281     except error.LockHeld as e:

   282         if e.errno == errno.ETIMEDOUT:

   283             raise error.Abort(_('timeout while waiting for backup'))

   284         raise

   285 

   286 @command('isbackedup',

   287      [('r', 'rev', [], _('show the specified revision or revset'), _('REV'))])

   288 def isbackedup(ui, repo, **opts):

   289     """checks if commit was backed up to infinitepush

   290 

   291     If no revision are specified then it checks working copy parent

   292     """

   293 

   294     revs = opts.get('rev')

   295     if not revs:

   296         revs = ['.']

   297     bkpstate = _readlocalbackupstate(ui, repo)

   298     unfi = repo.unfiltered()

   299     backeduprevs = unfi.revs('draft() and ::%ls', bkpstate.heads)

   300     for r in scmutil.revrange(unfi, revs):

   301         ui.write(_(unfi[r].hex() + ' '))

   302         ui.write(_('backed up' if r in backeduprevs else 'not backed up'))

   303         ui.write(_('\n'))

   304 

   305 @revsetpredicate('backedup')

   306 def backedup(repo, subset, x):

   307     """Draft changesets that have been backed up by infinitepush"""

   308     unfi = repo.unfiltered()

   309     bkpstate = _readlocalbackupstate(repo.ui, repo)

   310     return subset & unfi.revs('draft() and ::%ls and not hidden()',

   311                               bkpstate.heads)

   312 

   313 @revsetpredicate('notbackedup')

   314 def notbackedup(repo, subset, x):

   315     """Changesets that have not yet been backed up by infinitepush"""

   316     bkpstate = _readlocalbackupstate(repo.ui, repo)

   317     bkpheads = set(bkpstate.heads)

   318     candidates = set(_backupheads(repo.ui, repo))

   319     notbackeduprevs = set()

   320     # Find all revisions that are ancestors of the expected backup heads,

   321     # stopping when we reach either a public commit or a known backup head.

   322     while candidates:

   323         candidate = candidates.pop()

   324         if candidate not in bkpheads:

   325             ctx = repo[candidate]

   326             rev = ctx.rev()

   327             if rev not in notbackeduprevs and ctx.phase() != phases.public:

   328                 # This rev may not have been backed up.  Record it, and add its

   329                 # parents as candidates.

   330                 notbackeduprevs.add(rev)

   331                 candidates.update([p.hex() for p in ctx.parents()])

   332     if notbackeduprevs:

   333         # Some revisions in this set may actually have been backed up by

   334         # virtue of being an ancestor of a different backup head, which may

   335         # have been hidden since the backup was made.  Find these and remove

   336         # them from the set.

   337         unfi = repo.unfiltered()

   338         candidates = bkpheads

   339         while candidates:

   340             candidate = candidates.pop()

   341             if candidate in unfi:

   342                 ctx = unfi[candidate]

   343                 if ctx.phase() != phases.public:

   344                     notbackeduprevs.discard(ctx.rev())

   345                     candidates.update([p.hex() for p in ctx.parents()])

   346     return subset & notbackeduprevs

   347 

   348 @templatekeyword('backingup')

   349 def backingup(repo, ctx, **args):

   350     """Whether infinitepush is currently backing up commits."""

   351     # If the backup lock exists then a backup should be in progress.

   352     srcrepo = shareutil.getsrcrepo(repo)

   353     return srcrepo.vfs.lexists(_backuplockname)

   354 

   355 def smartlogsummary(ui, repo):

   356     if not ui.configbool('infinitepushbackup', 'enablestatus'):

   357         return

   358 

   359     # Don't output the summary if a backup is currently in progress.

   360     srcrepo = shareutil.getsrcrepo(repo)

   361     if srcrepo.vfs.lexists(_backuplockname):

   362         return

   363 

   364     unbackeduprevs = repo.revs('notbackedup()')

   365 

   366     # Count the number of changesets that haven't been backed up for 10 minutes.

   367     # If there is only one, also print out its hash.

   368     backuptime = time.time() - 10 * 60  # 10 minutes ago

   369     count = 0

   370     singleunbackeduprev = None

   371     for rev in unbackeduprevs:

   372         if repo[rev].date()[0] <= backuptime:

   373             singleunbackeduprev = rev

   374             count += 1

   375     if count > 0:

   376         if count > 1:

   377             ui.warn(_('note: %d changesets are not backed up.\n') % count)

   378         else:

   379             ui.warn(_('note: changeset %s is not backed up.\n') %

   380                     short(repo[singleunbackeduprev].node()))

   381         ui.warn(_('Run `hg pushbackup` to perform a backup.  If this fails,\n'

   382                   'please report to the Source Control @ FB group.\n'))

   383 

   384 def _autobackupruncommandwrapper(orig, lui, repo, cmd, fullargs, *args):

   385     '''

   386     If this wrapper is enabled then auto backup is started after every command

   387     that modifies a repository.

   388     Since we don't want to start auto backup after read-only commands,

   389     then this wrapper checks if this command opened at least one transaction.

   390     If yes then background backup will be started.

   391     '''

   392 

   393     # For chg, do not wrap the "serve" runcommand call

   394     if 'CHGINTERNALMARK' in encoding.environ:

   395         return orig(lui, repo, cmd, fullargs, *args)

   396 

   397     try:

   398         return orig(lui, repo, cmd, fullargs, *args)

   399     finally:

   400         if getattr(repo, 'txnwasopened', False) \

   401                 and not getattr(repo, 'ignoreautobackup', False):

   402             lui.debug("starting infinitepush autobackup in the background\n")

   403             _dobackgroundbackup(lui, repo)

   404 

   405 def _transaction(orig, self, *args, **kwargs):

   406     ''' Wrapper that records if a transaction was opened.

   407 

   408     If a transaction was opened then we want to start background backup process.

   409     This hook records the fact that transaction was opened.

   410     '''

   411     self.txnwasopened = True

   412     return orig(self, *args, **kwargs)

   413 

   414 def _backupheads(ui, repo):

   415     """Returns the set of heads that should be backed up in this repo."""

   416     maxheadstobackup = ui.configint('infinitepushbackup',

   417                                     'maxheadstobackup', -1)

   418 

   419     revset = 'heads(draft()) & not obsolete()'

   420 

   421     backupheads = [ctx.hex() for ctx in repo.set(revset)]

   422     if maxheadstobackup > 0:

   423         backupheads = backupheads[-maxheadstobackup:]

   424     elif maxheadstobackup == 0:

   425         backupheads = []

   426     return set(backupheads)

   427 

   428 def _dobackup(ui, repo, dest, **opts):

   429     ui.status(_('starting backup %s\n') % time.strftime('%H:%M:%S %d %b %Y %Z'))

   430     start = time.time()

   431     # to handle multiple working copies correctly

   432     repo = shareutil.getsrcrepo(repo)

   433     currentbkpgenerationvalue = _readbackupgenerationfile(repo.vfs)

   434     newbkpgenerationvalue = ui.configint('infinitepushbackup',

   435                                          'backupgeneration', 0)

   436     if currentbkpgenerationvalue != newbkpgenerationvalue:

   437         # Unlinking local backup state will trigger re-backuping

   438         _deletebackupstate(repo)

   439         _writebackupgenerationfile(repo.vfs, newbkpgenerationvalue)

   440     bkpstate = _readlocalbackupstate(ui, repo)

   441 

   442     # this variable stores the local store info (tip numeric revision and date)

   443     # which we use to quickly tell if our backup is stale

   444     afterbackupinfo = _getlocalinfo(repo)

   445 

   446     # This variable will store what heads will be saved in backup state file

   447     # if backup finishes successfully

   448     afterbackupheads = _backupheads(ui, repo)

   449     other = _getremote(repo, ui, dest, **opts)

   450     outgoing, badhexnodes = _getrevstobackup(repo, ui, other,

   451                                              afterbackupheads - bkpstate.heads)

   452     # If remotefilelog extension is enabled then there can be nodes that we

   453     # can't backup. In this case let's remove them from afterbackupheads

   454     afterbackupheads.difference_update(badhexnodes)

   455 

   456     # As afterbackupheads this variable stores what heads will be saved in

   457     # backup state file if backup finishes successfully

   458     afterbackuplocalbooks = _getlocalbookmarks(repo)

   459     afterbackuplocalbooks = _filterbookmarks(

   460         afterbackuplocalbooks, repo, afterbackupheads)

   461 

   462     newheads = afterbackupheads - bkpstate.heads

   463     removedheads = bkpstate.heads - afterbackupheads

   464     newbookmarks = _dictdiff(afterbackuplocalbooks, bkpstate.localbookmarks)

   465     removedbookmarks = _dictdiff(bkpstate.localbookmarks, afterbackuplocalbooks)

   466 

   467     namingmgr = BackupBookmarkNamingManager(ui, repo)

   468     bookmarkstobackup = _getbookmarkstobackup(

   469         repo, newbookmarks, removedbookmarks,

   470         newheads, removedheads, namingmgr)

   471 

   472     # Special case if backup state is empty. Clean all backup bookmarks from the

   473     # server.

   474     if bkpstate.empty():

   475         bookmarkstobackup[namingmgr.getbackupheadprefix()] = ''

   476         bookmarkstobackup[namingmgr.getbackupbookmarkprefix()] = ''

   477 

   478     # Wrap deltaparent function to make sure that bundle takes less space

   479     # See _deltaparent comments for details

   480     extensions.wrapfunction(changegroup.cg2packer, 'deltaparent', _deltaparent)

   481     try:

   482         bundler = _createbundler(ui, repo, other)

   483         bundler.addparam("infinitepush", "True")

   484         backup = False

   485         if outgoing and outgoing.missing:

   486             backup = True

   487             parts = getscratchbranchparts(repo, other, outgoing,

   488                                           confignonforwardmove=False,

   489                                           ui=ui, bookmark=None,

   490                                           create=False)

   491             for part in parts:

   492                 bundler.addpart(part)

   493 

   494         if bookmarkstobackup:

   495             backup = True

   496             bundler.addpart(getscratchbookmarkspart(other, bookmarkstobackup))

   497 

   498         if backup:

   499             _sendbundle(bundler, other)

   500             _writelocalbackupstate(repo.vfs, afterbackupheads,

   501                                    afterbackuplocalbooks)

   502             if ui.config('infinitepushbackup', 'savelatestbackupinfo'):

   503                 _writelocalbackupinfo(repo.vfs, **afterbackupinfo)

   504         else:

   505             ui.status(_('nothing to backup\n'))

   506     finally:

   507         # cleanup ensures that all pipes are flushed

   508         cleanup = getattr(other, '_cleanup', None) or getattr(other, 'cleanup')

   509         try:

   510             cleanup()

   511         except Exception:

   512             ui.warn(_('remote connection cleanup failed\n'))

   513         ui.status(_('finished in %f seconds\n') % (time.time() - start))

   514         extensions.unwrapfunction(changegroup.cg2packer, 'deltaparent',

   515                                   _deltaparent)

   516     return 0

   517 

   518 def _dobackgroundbackup(ui, repo, dest=None):

   519     background_cmd = ['hg', 'pushbackup']

   520     if dest:

   521         background_cmd.append(dest)

   522     logfile = None

   523     logdir = ui.config('infinitepushbackup', 'logdir')

   524     if logdir:

   525         # make newly created files and dirs non-writable

   526         oldumask = os.umask(0o022)

   527         try:

   528             try:

   529                 username = util.shortuser(ui.username())

   530             except Exception:

   531                 username = 'unknown'

   532 

   533             if not _checkcommonlogdir(logdir):

   534                 raise WrongPermissionsException(logdir)

   535 

   536             userlogdir = os.path.join(logdir, username)

   537             util.makedirs(userlogdir)

   538 

   539             if not _checkuserlogdir(userlogdir):

   540                 raise WrongPermissionsException(userlogdir)

   541 

   542             reporoot = repo.origroot

   543             reponame = os.path.basename(reporoot)

   544             _removeoldlogfiles(userlogdir, reponame)

   545             logfile = _getlogfilename(logdir, username, reponame)

   546         except (OSError, IOError) as e:

   547             ui.debug('infinitepush backup log is disabled: %s\n' % e)

   548         except WrongPermissionsException as e:

   549             ui.debug(('%s directory has incorrect permission, ' +

   550                      'infinitepush backup logging will be disabled\n') %

   551                      e.logdir)

   552         finally:

   553             os.umask(oldumask)

   554 

   555     if not logfile:

   556         logfile = os.devnull

   557 

   558     with open(logfile, 'a') as f:

   559         subprocess.Popen(background_cmd, shell=False, stdout=f,

   560                          stderr=subprocess.STDOUT)

   561 

   562 def _dobackupcheck(bkpstate, ui, repo, dest, **opts):

   563     remotehexnodes = sorted(

   564         set(bkpstate.heads).union(bkpstate.localbookmarks.values()))

   565     if not remotehexnodes:

   566         return True

   567     other = _getremote(repo, ui, dest, **opts)

   568     batch = other.iterbatch()

   569     for hexnode in remotehexnodes:

   570         batch.lookup(hexnode)

   571     batch.submit()

   572     lookupresults = batch.results()

   573     i = 0

   574     try:

   575         for i, r in enumerate(lookupresults):

   576             # iterate over results to make it throw if revision

   577             # was not found

   578             pass

   579         return True

   580     except error.RepoError:

   581         ui.warn(_('unknown revision %r\n') % remotehexnodes[i])

   582         return False

   583 

   584 _backuplatestinfofile = 'infinitepushlatestbackupinfo'

   585 _backupstatefile = 'infinitepushbackupstate'

   586 _backupgenerationfile = 'infinitepushbackupgeneration'

   587 

   588 # Common helper functions

   589 def _getlocalinfo(repo):

   590     localinfo = {}

   591     localinfo['rev'] = repo[repo.changelog.tip()].rev()

   592     localinfo['time'] = int(time.time())

   593     return localinfo

   594 

   595 def _getlocalbookmarks(repo):

   596     localbookmarks = {}

   597     for bookmark, node in repo._bookmarks.iteritems():

   598         hexnode = hex(node)

   599         localbookmarks[bookmark] = hexnode

   600     return localbookmarks

   601 

   602 def _filterbookmarks(localbookmarks, repo, headstobackup):

   603     '''Filters out some bookmarks from being backed up

   604 

   605     Filters out bookmarks that do not point to ancestors of headstobackup or

   606     public commits

   607     '''

   608 

   609     headrevstobackup = [repo[hexhead].rev() for hexhead in headstobackup]

   610     ancestors = repo.changelog.ancestors(headrevstobackup, inclusive=True)

   611     filteredbooks = {}

   612     for bookmark, hexnode in localbookmarks.iteritems():

   613         if (repo[hexnode].rev() in ancestors or

   614                 repo[hexnode].phase() == phases.public):

   615             filteredbooks[bookmark] = hexnode

   616     return filteredbooks

   617 

   618 def _downloadbackupstate(ui, other, sourcereporoot, sourcehostname, namingmgr):

   619     pattern = namingmgr.getcommonuserprefix()

   620     fetchedbookmarks = other.listkeyspatterns('bookmarks', patterns=[pattern])

   621     allbackupstates = collections.defaultdict(backupstate)

   622     for book, hexnode in fetchedbookmarks.iteritems():

   623         parsed = _parsebackupbookmark(book, namingmgr)

   624         if parsed:

   625             if sourcereporoot and sourcereporoot != parsed.reporoot:

   626                 continue

   627             if sourcehostname and sourcehostname != parsed.hostname:

   628                 continue

   629             key = (parsed.hostname, parsed.reporoot)

   630             if parsed.localbookmark:

   631                 bookname = parsed.localbookmark

   632                 allbackupstates[key].localbookmarks[bookname] = hexnode

   633             else:

   634                 allbackupstates[key].heads.add(hexnode)

   635         else:

   636             ui.warn(_('wrong format of backup bookmark: %s') % book)

   637 

   638     return allbackupstates

   639 

   640 def _checkbackupstates(allbackupstates):

   641     if len(allbackupstates) == 0:

   642         raise error.Abort('no backups found!')

   643 

   644     hostnames = set(key[0] for key in allbackupstates.iterkeys())

   645     reporoots = set(key[1] for key in allbackupstates.iterkeys())

   646 

   647     if len(hostnames) > 1:

   648         raise error.Abort(

   649             _('ambiguous hostname to restore: %s') % sorted(hostnames),

   650             hint=_('set --hostname to disambiguate'))

   651 

   652     if len(reporoots) > 1:

   653         raise error.Abort(

   654             _('ambiguous repo root to restore: %s') % sorted(reporoots),

   655             hint=_('set --reporoot to disambiguate'))

   656 

   657 class BackupBookmarkNamingManager(object):

   658     def __init__(self, ui, repo, username=None):

   659         self.ui = ui

   660         self.repo = repo

   661         if not username:

   662             username = util.shortuser(ui.username())

   663         self.username = username

   664 

   665         self.hostname = self.ui.config('infinitepushbackup', 'hostname')

   666         if not self.hostname:

   667             self.hostname = socket.gethostname()

   668 

   669     def getcommonuserprefix(self):

   670         return '/'.join((self._getcommonuserprefix(), '*'))

   671 

   672     def getcommonprefix(self):

   673         return '/'.join((self._getcommonprefix(), '*'))

   674 

   675     def getbackupbookmarkprefix(self):

   676         return '/'.join((self._getbackupbookmarkprefix(), '*'))

   677 

   678     def getbackupbookmarkname(self, bookmark):

   679         bookmark = _escapebookmark(bookmark)

   680         return '/'.join((self._getbackupbookmarkprefix(), bookmark))

   681 

   682     def getbackupheadprefix(self):

   683         return '/'.join((self._getbackupheadprefix(), '*'))

   684 

   685     def getbackupheadname(self, hexhead):

   686         return '/'.join((self._getbackupheadprefix(), hexhead))

   687 

   688     def _getbackupbookmarkprefix(self):

   689         return '/'.join((self._getcommonprefix(), 'bookmarks'))

   690 

   691     def _getbackupheadprefix(self):

   692         return '/'.join((self._getcommonprefix(), 'heads'))

   693 

   694     def _getcommonuserprefix(self):

   695         return '/'.join(('infinitepush', 'backups', self.username))

   696 

   697     def _getcommonprefix(self):

   698         reporoot = self.repo.origroot

   699 

   700         result = '/'.join((self._getcommonuserprefix(), self.hostname))

   701         if not reporoot.startswith('/'):

   702             result += '/'

   703         result += reporoot

   704         if result.endswith('/'):

   705             result = result[:-1]

   706         return result

   707 

   708 def _escapebookmark(bookmark):

   709     '''

   710     If `bookmark` contains "bookmarks" as a substring then replace it with

   711     "bookmarksbookmarks". This will make parsing remote bookmark name

   712     unambigious.

   713     '''

   714 

   715     bookmark = encoding.fromlocal(bookmark)

   716     return bookmark.replace('bookmarks', 'bookmarksbookmarks')

   717 

   718 def _unescapebookmark(bookmark):

   719     bookmark = encoding.tolocal(bookmark)

   720     return bookmark.replace('bookmarksbookmarks', 'bookmarks')

   721 

   722 def _getremote(repo, ui, dest, **opts):

   723     path = ui.paths.getpath(dest, default=('infinitepush', 'default'))

   724     if not path:

   725         raise error.Abort(_('default repository not configured!'),

   726                          hint=_("see 'hg help config.paths'"))

   727     dest = path.pushloc or path.loc

   728     return hg.peer(repo, opts, dest)

   729 

   730 def _getcommandandoptions(command):

   731     cmd = commands.table[command][0]

   732     opts = dict(opt[1:3] for opt in commands.table[command][1])

   733     return cmd, opts

   734 

   735 # Backup helper functions

   736 

   737 def _deltaparent(orig, self, revlog, rev, p1, p2, prev):

   738     # This version of deltaparent prefers p1 over prev to use less space

   739     dp = revlog.deltaparent(rev)

   740     if dp == nullrev and not revlog.storedeltachains:

   741         # send full snapshot only if revlog configured to do so

   742         return nullrev

   743     return p1

   744 

   745 def _getbookmarkstobackup(repo, newbookmarks, removedbookmarks,

   746                           newheads, removedheads, namingmgr):

   747     bookmarkstobackup = {}

   748 

   749     for bookmark, hexnode in removedbookmarks.items():

   750         backupbookmark = namingmgr.getbackupbookmarkname(bookmark)

   751         bookmarkstobackup[backupbookmark] = ''

   752 

   753     for bookmark, hexnode in newbookmarks.items():

   754         backupbookmark = namingmgr.getbackupbookmarkname(bookmark)

   755         bookmarkstobackup[backupbookmark] = hexnode

   756 

   757     for hexhead in removedheads:

   758         headbookmarksname = namingmgr.getbackupheadname(hexhead)

   759         bookmarkstobackup[headbookmarksname] = ''

   760 

   761     for hexhead in newheads:

   762         headbookmarksname = namingmgr.getbackupheadname(hexhead)

   763         bookmarkstobackup[headbookmarksname] = hexhead

   764 

   765     return bookmarkstobackup

   766 

   767 def _createbundler(ui, repo, other):

   768     bundler = bundle2.bundle20(ui, bundle2.bundle2caps(other))

   769     # Disallow pushback because we want to avoid taking repo locks.

   770     # And we don't need pushback anyway

   771     capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo,

   772                                                       allowpushback=False))

   773     bundler.newpart('replycaps', data=capsblob)

   774     return bundler

   775 

   776 def _sendbundle(bundler, other):

   777     stream = util.chunkbuffer(bundler.getchunks())

   778     try:

   779         other.unbundle(stream, ['force'], other.url())

   780     except error.BundleValueError as exc:

   781         raise error.Abort(_('missing support for %s') % exc)

   782 

   783 def findcommonoutgoing(repo, ui, other, heads):

   784     if heads:

   785         # Avoid using remotenames fastheaddiscovery heuristic. It uses

   786         # remotenames file to quickly find commonoutgoing set, but it can

   787         # result in sending public commits to infinitepush servers.

   788         # For example:

   789         #

   790         #        o draft

   791         #       /

   792         #      o C1

   793         #      |

   794         #     ...

   795         #      |

   796         #      o remote/master

   797         #

   798         # pushbackup in that case results in sending to the infinitepush server

   799         # all public commits from 'remote/master' to C1. It increases size of

   800         # the bundle + it may result in storing data about public commits

   801         # in infinitepush table.

   802 

   803         with ui.configoverride({("remotenames", "fastheaddiscovery"): False}):

   804             nodes = map(repo.changelog.node, heads)

   805             return discovery.findcommonoutgoing(repo, other, onlyheads=nodes)

   806     else:

   807         return None

   808 

   809 def _getrevstobackup(repo, ui, other, headstobackup):

   810     # In rare cases it's possible to have a local node without filelogs.

   811     # This is possible if remotefilelog is enabled and if the node was

   812     # stripped server-side. We want to filter out these bad nodes and all

   813     # of their descendants.

   814     badnodes = ui.configlist('infinitepushbackup', 'dontbackupnodes', [])

   815     badnodes = [node for node in badnodes if node in repo]

   816     badrevs = [repo[node].rev() for node in badnodes]

   817     badnodesdescendants = repo.set('%ld::', badrevs) if badrevs else set()

   818     badnodesdescendants = set(ctx.hex() for ctx in badnodesdescendants)

   819     filteredheads = filter(lambda head: head in badnodesdescendants,

   820                            headstobackup)

   821 

   822     if filteredheads:

   823         ui.warn(_('filtering nodes: %s\n') % filteredheads)

   824         ui.log('infinitepushbackup', 'corrupted nodes found',

   825                infinitepushbackupcorruptednodes='failure')

   826     headstobackup = filter(lambda head: head not in badnodesdescendants,

   827                            headstobackup)

   828 

   829     revs = list(repo[hexnode].rev() for hexnode in headstobackup)

   830     outgoing = findcommonoutgoing(repo, ui, other, revs)

   831     nodeslimit = 1000

   832     if outgoing and len(outgoing.missing) > nodeslimit:

   833         # trying to push too many nodes usually means that there is a bug

   834         # somewhere. Let's be safe and avoid pushing too many nodes at once

   835         raise error.Abort('trying to back up too many nodes: %d' %

   836                           (len(outgoing.missing),))

   837     return outgoing, set(filteredheads)

   838 

   839 def _localbackupstateexists(repo):

   840     return repo.vfs.exists(_backupstatefile)

   841 

   842 def _deletebackupstate(repo):

   843     return repo.vfs.tryunlink(_backupstatefile)

   844 

   845 def _readlocalbackupstate(ui, repo):

   846     repo = shareutil.getsrcrepo(repo)

   847     if not _localbackupstateexists(repo):

   848         return backupstate()

   849 

   850     with repo.vfs(_backupstatefile) as f:

   851         try:

   852             state = json.loads(f.read())

   853             if (not isinstance(state['bookmarks'], dict) or

   854                     not isinstance(state['heads'], list)):

   855                 raise ValueError('bad types of bookmarks or heads')

   856 

   857             result = backupstate()

   858             result.heads = set(map(str, state['heads']))

   859             result.localbookmarks = state['bookmarks']

   860             return result

   861         except (ValueError, KeyError, TypeError) as e:

   862             ui.warn(_('corrupt file: %s (%s)\n') % (_backupstatefile, e))

   863             return backupstate()

   864     return backupstate()

   865 

   866 def _writelocalbackupstate(vfs, heads, bookmarks):

   867     with vfs(_backupstatefile, 'w') as f:

   868         f.write(json.dumps({'heads': list(heads), 'bookmarks': bookmarks}))

   869 

   870 def _readbackupgenerationfile(vfs):

   871     try:

   872         with vfs(_backupgenerationfile) as f:

   873             return int(f.read())

   874     except (IOError, OSError, ValueError):

   875         return 0

   876 

   877 def _writebackupgenerationfile(vfs, backupgenerationvalue):

   878     with vfs(_backupgenerationfile, 'w', atomictemp=True) as f:

   879         f.write(str(backupgenerationvalue))

   880 

   881 def _writelocalbackupinfo(vfs, rev, time):

   882     with vfs(_backuplatestinfofile, 'w', atomictemp=True) as f:

   883         f.write(('backuprevision=%d\nbackuptime=%d\n') % (rev, time))

   884 

   885 # Restore helper functions

   886 def _parsebackupbookmark(backupbookmark, namingmgr):

   887     '''Parses backup bookmark and returns info about it

   888 

   889     Backup bookmark may represent either a local bookmark or a head.

   890     Returns None if backup bookmark has wrong format or tuple.

   891     First entry is a hostname where this bookmark came from.

   892     Second entry is a root of the repo where this bookmark came from.

   893     Third entry in a tuple is local bookmark if backup bookmark

   894     represents a local bookmark and None otherwise.

   895     '''

   896 

   897     backupbookmarkprefix = namingmgr._getcommonuserprefix()

   898     commonre = '^{0}/([-\w.]+)(/.*)'.format(re.escape(backupbookmarkprefix))

   899     bookmarkre = commonre + '/bookmarks/(.*)$'

   900     headsre = commonre + '/heads/[a-f0-9]{40}$'

   901 

   902     match = re.search(bookmarkre, backupbookmark)

   903     if not match:

   904         match = re.search(headsre, backupbookmark)

   905         if not match:

   906             return None

   907         # It's a local head not a local bookmark.

   908         # That's why localbookmark is None

   909         return backupbookmarktuple(hostname=match.group(1),

   910                                    reporoot=match.group(2),

   911                                    localbookmark=None)

   912 

   913     return backupbookmarktuple(hostname=match.group(1),

   914                                reporoot=match.group(2),

   915                                localbookmark=_unescapebookmark(match.group(3)))

   916 

   917 _timeformat = '%Y%m%d'

   918 

   919 def _getlogfilename(logdir, username, reponame):

   920     '''Returns name of the log file for particular user and repo

   921 

   922     Different users have different directories inside logdir. Log filename

   923     consists of reponame (basename of repo path) and current day

   924     (see _timeformat). That means that two different repos with the same name

   925     can share the same log file. This is not a big problem so we ignore it.

   926     '''

   927 

   928     currentday = time.strftime(_timeformat)

   929     return os.path.join(logdir, username, reponame + currentday)

   930 

   931 def _removeoldlogfiles(userlogdir, reponame):

   932     existinglogfiles = []

   933     for entry in osutil.listdir(userlogdir):

   934         filename = entry[0]

   935         fullpath = os.path.join(userlogdir, filename)

   936         if filename.startswith(reponame) and os.path.isfile(fullpath):

   937             try:

   938                 time.strptime(filename[len(reponame):], _timeformat)

   939             except ValueError:

   940                 continue

   941             existinglogfiles.append(filename)

   942 

   943     # _timeformat gives us a property that if we sort log file names in

   944     # descending order then newer files are going to be in the beginning

   945     existinglogfiles = sorted(existinglogfiles, reverse=True)

   946     # Delete logs that are older than 5 days

   947     maxlogfilenumber = 5

   948     if len(existinglogfiles) > maxlogfilenumber:

   949         for filename in existinglogfiles[maxlogfilenumber:]:

   950             os.unlink(os.path.join(userlogdir, filename))

   951 

   952 def _checkcommonlogdir(logdir):

   953     '''Checks permissions of the log directory

   954 

   955     We want log directory to actually be a directory, have restricting

   956     deletion flag set (sticky bit)

   957     '''

   958 

   959     try:

   960         st = os.stat(logdir)

   961         return stat.S_ISDIR(st.st_mode) and st.st_mode & stat.S_ISVTX

   962     except OSError:

   963         # is raised by os.stat()

   964         return False

   965 

   966 def _checkuserlogdir(userlogdir):

   967     '''Checks permissions of the user log directory

   968 

   969     We want user log directory to be writable only by the user who created it

   970     and be owned by `username`

   971     '''

   972 

   973     try:

   974         st = os.stat(userlogdir)

   975         # Check that `userlogdir` is owned by `username`

   976         if os.getuid() != st.st_uid:

   977             return False

   978         return ((st.st_mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)) ==

   979                 stat.S_IWUSR)

   980     except OSError:

   981         # is raised by os.stat()

   982         return False

   983 

   984 def _dictdiff(first, second):

   985     '''Returns new dict that contains items from the first dict that are missing

   986     from the second dict.

   987     '''

   988     result = {}

   989     for book, hexnode in first.items():

   990         if second.get(book) != hexnode:

   991             result[book] = hexnode

   992     return result