"Fossies" - the Fresh Open Source Software Archive

Member "backintime-1.2.0/common/snapshots.py" (27 Apr 2019, 99997 Bytes) of package /linux/privat/backintime-1.2.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "snapshots.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 1.1.24_vs_1.2.0.

    1 #   Back In Time
    2 #   Copyright (C) 2008-2019 Oprea Dan, Bart de Koning, Richard Bailey, Germar Reitze, Taylor Raack
    3 #
    4 #   This program is free software; you can redistribute it and/or modify
    5 #   it under the terms of the GNU General Public License as published by
    6 #   the Free Software Foundation; either version 2 of the License, or
    7 #   (at your option) any later version.
    8 #
    9 #   This program is distributed in the hope that it will be useful,
   10 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
   11 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   12 #   GNU General Public License for more details.
   13 #
   14 #   You should have received a copy of the GNU General Public License along
   15 #   with this program; if not, write to the Free Software Foundation, Inc.,
   16 #   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   17 
   18 
   19 import json
   20 import os
   21 import stat
   22 import datetime
   23 import gettext
   24 import bz2
   25 import pwd
   26 import grp
   27 import subprocess
   28 import shutil
   29 import time
   30 import re
   31 import fcntl
   32 from tempfile import TemporaryDirectory
   33 
   34 import config
   35 import configfile
   36 import logger
   37 import tools
   38 import encfstools
   39 import mount
   40 import progress
   41 import bcolors
   42 import snapshotlog
   43 from applicationinstance import ApplicationInstance
   44 from exceptions import MountException, LastSnapshotSymlink
   45 
   46 _=gettext.gettext
   47 
   48 
   49 class Snapshots:
   50     """
   51     Collection of take-snapshot and restore commands.
   52 
   53     Args:
   54         cfg (config.Config): current config
   55     """
   56     SNAPSHOT_VERSION = 3
   57     GLOBAL_FLOCK = '/tmp/backintime.lock'
   58 
   59     def __init__(self, cfg = None):
   60         self.config = cfg
   61         if self.config is None:
   62             self.config = config.Config()
   63         self.snapshotLog = snapshotlog.SnapshotLog(self.config)
   64 
   65         self.clearIdCache()
   66         self.clearNameCache()
   67 
   68         #rsync --info=progress2 output
   69         #search for:     517.38K  26%   14.46MB/s    0:02:36
   70         #or:             497.84M   4% -449.39kB/s   ??:??:??
   71         #but filter out: 517.38K  26%   14.46MB/s    0:00:53 (xfr#53, to-chk=169/452)
   72         #                because this shows current run time
   73         self.reRsyncProgress = re.compile(r'.*?'                            #trash at start
   74                                           r'(\d*[,\.]?\d+[KkMGT]?)\s+'      #bytes sent
   75                                           r'(\d*)%\s+'                      #percent done
   76                                           r'(-?\d*[,\.]?\d*[KkMGT]?B/s)\s+' #speed
   77                                           r'([\d\?]+:[\d\?]{2}:[\d\?]{2})'  #estimated time of arrival
   78                                           r'(.*$)')                         #trash at the end
   79 
   80         self.lastBusyCheck = datetime.datetime(1,1,1)
   81         self.flock = None
   82         self.restorePermissionFailed = False
   83 
   84     #TODO: make own class for takeSnapshotMessage
   85     def clearTakeSnapshotMessage(self):
   86         files = (self.config.takeSnapshotMessageFile(), \
   87                  self.config.takeSnapshotProgressFile())
   88         for f in files:
   89             if os.path.exists(f):
   90                 os.remove(f)
   91 
   92     #TODO: make own class for takeSnapshotMessage
   93     def takeSnapshotMessage(self):
   94         wait = datetime.datetime.now() - datetime.timedelta(seconds = 5)
   95         if self.lastBusyCheck < wait:
   96             self.lastBusyCheck = datetime.datetime.now()
   97             if not self.busy():
   98                 self.clearTakeSnapshotMessage()
   99                 return None
  100 
  101         if not os.path.exists(self.config.takeSnapshotMessageFile()):
  102             return None
  103         try:
  104             with open(self.config.takeSnapshotMessageFile(), 'rt') as f:
  105                 items = f.read().split('\n')
  106         except Exception as e:
  107             logger.debug('Failed to get takeSnapshot message from %s: %s'
  108                          %(self.config.takeSnapshotMessageFile(), str(e)),
  109                          self)
  110             return None
  111 
  112         if len(items) < 2:
  113             return None
  114 
  115         mid = 0
  116         try:
  117             mid = int(items[0])
  118         except Exception as e:
  119             logger.debug('Failed extract message ID from %s: %s'
  120                          %(items[0], str(e)),
  121                          self)
  122 
  123         del items[0]
  124         message = '\n'.join(items)
  125 
  126         return(mid, message)
  127 
  128     #TODO: make own class for takeSnapshotMessage
  129     def setTakeSnapshotMessage(self, type_id, message, timeout = -1):
  130         data = str(type_id) + '\n' + message
  131 
  132         try:
  133             with open(self.config.takeSnapshotMessageFile(), 'wt') as f:
  134                 f.write(data)
  135         except Exception as e:
  136             logger.debug('Failed to set takeSnapshot message to %s: %s'
  137                          %(self.config.takeSnapshotMessageFile(), str(e)),
  138                          self)
  139 
  140         if 1 == type_id:
  141             self.snapshotLog.append('[E] ' + message, 1)
  142         else:
  143             self.snapshotLog.append('[I] '  + message, 3)
  144 
  145         try:
  146             profile_id =self.config.currentProfile()
  147             profile_name = self.config.profileName(profile_id)
  148             self.config.PLUGIN_MANAGER.message(profile_id, profile_name, type_id, message, timeout)
  149         except Exception as e:
  150             logger.debug('Failed to send message to plugins: %s'
  151                          %str(e),
  152                          self)
  153 
  154     def busy(self):
  155         instance = ApplicationInstance(self.config.takeSnapshotInstanceFile(), False)
  156         return instance.busy()
  157 
  158     def pid(self):
  159         instance = ApplicationInstance(self.config.takeSnapshotInstanceFile(), False)
  160         return instance.readPidFile()[0]
  161 
  162     def clearNameCache(self):
  163         """
  164         Reset the cache for user and group names.
  165         """
  166         self.userCache = {}
  167         self.groupCache = {}
  168 
  169     def clearIdCache(self):
  170         """
  171         Reset the cache for UIDs and GIDs.
  172         """
  173         self.uidCache = {}
  174         self.gidCache = {}
  175 
  176     def uid(self, name, callback = None, backup = None):
  177         """
  178         Get the User identifier (UID) for the user in ``name``.
  179         name->uid will be cached to speed up subsequent requests.
  180 
  181         Args:
  182             name (:py:class:`str`, :py:class:`bytes`):
  183                                 username to search for
  184             callback (method):  callable which will handle a given message
  185             backup (int):       UID wich will be used if the username is unknown
  186                                 on this machine
  187 
  188         Returns:
  189             int:                UID of the user in name or -1 if not found
  190         """
  191         if isinstance(name, bytes):
  192             name = name.decode()
  193 
  194         if name in self.uidCache:
  195             return self.uidCache[name]
  196         else:
  197             uid = -1
  198             try:
  199                 uid = pwd.getpwnam(name).pw_uid
  200             except Exception as e:
  201                 if backup:
  202                     uid = backup
  203                     msg = "UID for '%s' is not available on this system. Using UID %s from snapshot." %(name, backup)
  204                     logger.info(msg, self)
  205                     if callback is not None:
  206                         callback(msg)
  207                 else:
  208                     self.restorePermissionFailed = True
  209                     msg = 'Failed to get UID for %s: %s' %(name, str(e))
  210                     logger.error(msg, self)
  211                     if callback:
  212                         callback(msg)
  213 
  214             self.uidCache[name] = uid
  215             return uid
  216 
  217     def gid(self, name, callback = None, backup = None):
  218         """
  219         Get the Group identifier (GID) for the group in ``name``.
  220         name->gid will be cached to speed up subsequent requests.
  221 
  222         Args:
  223             name (:py:class:`str`, :py:class:`bytes`):
  224                                 groupname to search for
  225             callback (method):  callable which will handle a given message
  226             backup (int):       GID wich will be used if the groupname is unknown
  227                                 on this machine
  228 
  229         Returns:
  230             int:                GID of the group in name or -1 if not found
  231         """
  232         if isinstance(name, bytes):
  233             name = name.decode()
  234 
  235         if name in self.gidCache:
  236             return self.gidCache[name]
  237         else:
  238             gid = -1
  239             try:
  240                 gid = grp.getgrnam(name).gr_gid
  241             except Exception as e:
  242                 if backup is not None:
  243                     gid = backup
  244                     msg = "GID for '%s' is not available on this system. Using GID %s from snapshot." %(name, backup)
  245                     logger.info(msg, self)
  246                     if callback:
  247                         callback(msg)
  248                 else:
  249                     self.restorePermissionFailed = True
  250                     msg = 'Failed to get GID for %s: %s' %(name, str(e))
  251                     logger.error(msg, self)
  252                     if callback:
  253                         callback(msg)
  254 
  255             self.gidCache[name] = gid
  256             return gid
  257 
  258     def userName(self, uid):
  259         """
  260         Get the username for the given uid.
  261         uid->name will be cached to speed up subsequent requests.
  262 
  263         Args:
  264             uid (int):  User identifier (UID) to search for
  265 
  266         Returns:
  267             str:        name of the user with UID uid or '-' if not found
  268         """
  269         if uid in self.userCache:
  270             return self.userCache[uid]
  271         else:
  272             name = '-'
  273             try:
  274                 name = pwd.getpwuid(uid).pw_name
  275             except Exception as e:
  276                 logger.debug('Failed to get user name for UID %s: %s'
  277                              %(uid, str(e)),
  278                              self)
  279 
  280             self.userCache[uid] = name
  281             return name
  282 
  283     def groupName(self, gid):
  284         """
  285         Get the groupname for the given gid.
  286         gid->name will be cached to speed up subsequent requests.
  287 
  288         Args:
  289             gid (int):  Group identifier (GID) to search for
  290 
  291         Returns:
  292             str:        name of the Group with GID gid or '.' if not found
  293         """
  294         if gid in self.groupCache:
  295             return self.groupCache[gid]
  296         else:
  297             name = '-'
  298             try:
  299                 name = grp.getgrgid(gid).gr_name
  300             except Exception as e:
  301                 logger.debug('Failed to get group name for GID %s: %s'
  302                              %(gid, str(e)),
  303                              self)
  304 
  305             self.groupCache[gid] = name
  306             return name
  307 
  308     def restoreCallback(self, callback, ok, msg):
  309         """
  310         Format messages thrown by restore depending on whether they where
  311         successful or failed.
  312 
  313         Args:
  314             callback (method):  callable instance which will handle the message
  315             ok (bool):          ``True`` if the logged action was successful
  316                                 or ``False`` if it failed
  317             msg (str):          message that should be send to callback
  318         """
  319         if not callback is None:
  320             if not ok:
  321                 msg = msg + " : " + _("FAILED")
  322                 self.restorePermissionFailed = True
  323             callback(msg)
  324 
  325     def restorePermission(self, key_path, path, fileInfoDict, callback = None):
  326         """
  327         Restore permissions (owner, group and mode). If permissions are
  328         already identical with the new ones just skip. Otherwise try to
  329         'chown' to new owner and new group. If that fails (most probably because
  330         we are not running as root and normal user has no rights to change
  331         ownership of files) try to at least 'chgrp' to the new group. Finally
  332         'chmod' the new mode.
  333 
  334         Args:
  335             key_path (bytes):       original path during backup.
  336                                     Same as in fileInfoDict.
  337             path (bytes):           current path of file that should be changed.
  338             fileInfoDict (FileInfoDict):    FileInfoDict
  339         """
  340         assert isinstance(key_path, bytes), 'key_path is not bytes type: %s' % key_path
  341         assert isinstance(path, bytes), 'path is not bytes type: %s' % path
  342         assert isinstance(fileInfoDict, FileInfoDict), 'fileInfoDict is not FileInfoDict type: %s' % fileInfoDict
  343         if key_path not in fileInfoDict or not os.path.exists(path):
  344             return
  345         info = fileInfoDict[key_path]
  346 
  347         #restore uid/gid
  348         uid = self.uid(info[1], callback)
  349         gid = self.gid(info[2], callback)
  350 
  351         #current file stats
  352         st = os.stat(path)
  353 
  354         # logger.debug('%(path)s: uid %(target_uid)s/%(cur_uid)s, gid %(target_gid)s/%(cur_gid)s, mod %(target_mod)s/%(cur_mod)s'
  355         #              %{'path': path.decode(),
  356         #                'target_uid': uid,
  357         #                'cur_uid': st.st_uid,
  358         #                'target_gid': gid,
  359         #                'cur_gid': st.st_gid,
  360         #                'target_mod': info[0],
  361         #                'cur_mod': st.st_mode
  362         #                })
  363 
  364         if uid != -1 or gid != -1:
  365             ok = False
  366             if uid != st.st_uid:
  367                 try:
  368                     os.chown(path, uid, gid)
  369                     ok = True
  370                 except:
  371                     pass
  372                 self.restoreCallback(callback, ok, "chown %s %s : %s" % (path.decode(errors = 'ignore'), uid, gid))
  373                 st = os.stat(path)
  374 
  375             #if restore uid/gid failed try to restore at least gid
  376             if not ok and gid != st.st_gid:
  377                 try:
  378                     os.chown(path, -1, gid)
  379                     ok = True
  380                 except:
  381                     pass
  382                 self.restoreCallback(callback, ok, "chgrp %s %s" % (path.decode(errors = 'ignore'), gid))
  383                 st = os.stat(path)
  384 
  385         #restore perms
  386         ok = False
  387         if info[0] != st.st_mode:
  388             try:
  389                 os.chmod(path, info[0])
  390                 ok = True
  391             except:
  392                 pass
  393             self.restoreCallback(callback, ok, "chmod %s %04o" % (path.decode(errors = 'ignore'), info[0]))
  394 
  395     def restore(self,
  396                 sid,
  397                 paths,
  398                 callback = None,
  399                 restore_to = '',
  400                 delete = False,
  401                 backup = True,
  402                 only_new = False):
  403         """
  404         Restore one or more files from snapshot ``sid`` to either original
  405         or a different destination. Restore is done with rsync. If available
  406         permissions will be restored from ``fileinfo.bz2``.
  407 
  408         Args:
  409             sid (SID):                  snapshot from whom to restore
  410             paths (:py:class:`list`, :py:class:`tuple` or :py:class:`str`):
  411                                         single path (str) or multiple
  412                                         paths (list, tuple) that should be
  413                                         restored. For every path this will run
  414                                         a new rsync process. Permissions will be
  415                                         restored for all paths in one run
  416             callback (method):          callable instance which will handle
  417                                         messages
  418             restore_to (str):           full path to restore to. If empty
  419                                         restore to original destiantion
  420             delete (bool):              delete newer files which are not in the
  421                                         snapshot
  422             backup (bool):              create backup files (\*.backup.YYYYMMDD)
  423                                         before changing or deleting local files.
  424             only_new (bool):            Only restore files which does not exist
  425                                         or are newer than those in destination.
  426                                         Using "rsync --update" option.
  427         """
  428         instance = ApplicationInstance(self.config.restoreInstanceFile(), False, flock = True)
  429         if instance.check():
  430             instance.startApplication()
  431         else:
  432             logger.warning('Restore is already running', self)
  433             return
  434 
  435         if restore_to.endswith('/'):
  436             restore_to = restore_to[: -1]
  437 
  438         if not isinstance(paths, (list, tuple)):
  439             paths = (paths,)
  440 
  441         logger.info("Restore: %s to: %s"
  442                     %(', '.join(paths), restore_to),
  443                     self)
  444 
  445         info = sid.info
  446 
  447         cmd_prefix = tools.rsyncPrefix(self.config, no_perms = False, use_mode = ['ssh'])
  448         cmd_prefix.extend(('-R', '-v'))
  449         if backup:
  450             cmd_prefix.extend(('--backup', '--suffix=%s' %self.backupSuffix()))
  451         if delete:
  452             cmd_prefix.append('--delete')
  453             cmd_prefix.append('--filter=protect %s' % self.config.snapshotsPath())
  454             cmd_prefix.append('--filter=protect %s' % self.config._LOCAL_DATA_FOLDER)
  455             cmd_prefix.append('--filter=protect %s' % self.config._MOUNT_ROOT)
  456         if only_new:
  457             cmd_prefix.append('--update')
  458 
  459         restored_paths = []
  460         for path in paths:
  461             tools.makeDirs(os.path.dirname(path))
  462             src_path = path
  463             src_delta = 0
  464             src_base = sid.pathBackup(use_mode = ['ssh'])
  465             if not src_base.endswith(os.sep):
  466                 src_base += os.sep
  467             cmd = cmd_prefix[:]
  468             if restore_to:
  469                 items = os.path.split(src_path)
  470                 aux = items[0].lstrip(os.sep)
  471                 #bugfix: restore system root ended in <src_base>//.<src_path>
  472                 if aux:
  473                     src_base = os.path.join(src_base, aux) + '/'
  474                 src_path = '/' + items[1]
  475                 if items[0] == '/':
  476                     src_delta = 0
  477                 else:
  478                     src_delta = len(items[0])
  479 
  480             cmd.append(self.rsyncRemotePath('%s.%s' %(src_base, src_path), use_mode = ['ssh']))
  481             cmd.append('%s/' %restore_to)
  482             proc = tools.Execute(cmd,
  483                                  callback = callback,
  484                                  filters = (self.filterRsyncProgress,),
  485                                  parent = self)
  486             self.restoreCallback(callback, True, proc.printable_cmd)
  487             proc.run()
  488             self.restoreCallback(callback, True, ' ')
  489             restored_paths.append((path, src_delta))
  490         try:
  491             os.remove(self.config.takeSnapshotProgressFile())
  492         except Exception as e:
  493             logger.debug('Failed to remove snapshot progress file %s: %s'
  494                          %(self.config.takeSnapshotProgressFile(), str(e)),
  495                          self)
  496 
  497         #restore permissions
  498         logger.info('Restore permissions', self)
  499         self.restoreCallback(callback, True, ' ')
  500         self.restoreCallback(callback, True, _("Restore permissions:"))
  501         self.restorePermissionFailed = False
  502         fileInfoDict = sid.fileInfo
  503 
  504         #cache uids/gids
  505         for uid, name in info.listValue('user', ('int:uid', 'str:name')):
  506             self.uid(name.encode(), callback = callback, backup = uid)
  507         for gid, name in info.listValue('group', ('int:gid', 'str:name')):
  508             self.gid(name.encode(), callback = callback, backup = gid)
  509 
  510         if fileInfoDict:
  511             all_dirs = [] #restore dir permissions after all files are done
  512             for path, src_delta in restored_paths:
  513                 #explore items
  514                 snapshot_path_to = sid.pathBackup(path).rstrip('/')
  515                 root_snapshot_path_to = sid.pathBackup().rstrip('/')
  516                 #use bytes instead of string from here
  517                 if isinstance(path, str):
  518                     path = path.encode()
  519                 if isinstance(restore_to, str):
  520                     restore_to = restore_to.encode()
  521 
  522                 if not restore_to:
  523                     path_items = path.strip(b'/').split(b'/')
  524                     curr_path = b'/'
  525                     for path_item in path_items:
  526                         curr_path = os.path.join(curr_path, path_item)
  527                         if curr_path not in all_dirs:
  528                             all_dirs.append(curr_path)
  529                 else:
  530                     if path not in all_dirs:
  531                         all_dirs.append(path)
  532 
  533                 if os.path.isdir(snapshot_path_to) and not os.path.islink(snapshot_path_to):
  534                     head = len(root_snapshot_path_to.encode())
  535                     for explore_path, dirs, files in os.walk(snapshot_path_to.encode()):
  536                         for item in dirs:
  537                             item_path = os.path.join(explore_path, item)[head:]
  538                             if item_path not in all_dirs:
  539                                 all_dirs.append(item_path)
  540 
  541                         for item in files:
  542                             item_path = os.path.join(explore_path, item)[head:]
  543                             real_path = restore_to + item_path[src_delta:]
  544                             self.restorePermission(item_path, real_path, fileInfoDict, callback)
  545 
  546             all_dirs.reverse()
  547             for item_path in all_dirs:
  548                 real_path = restore_to + item_path[src_delta:]
  549                 self.restorePermission(item_path, real_path, fileInfoDict, callback)
  550 
  551             self.restoreCallback(callback, True, '')
  552             if self.restorePermissionFailed:
  553                 status = _('FAILED')
  554             else:
  555                 status = _('Done')
  556             self.restoreCallback(callback, True, _("Restore permissions:") + ' ' + status)
  557 
  558         instance.exitApplication()
  559 
  560     def backupSuffix(self):
  561         """
  562         Get suffix for backup files.
  563 
  564         Returns:
  565             str:    backup suffix in form of '.backup.YYYYMMDD'
  566         """
  567         return '.backup.' + datetime.date.today().strftime('%Y%m%d')
  568 
  569     def remove(self, sid):
  570         """
  571         Remove snapshot ``sid``.
  572 
  573         Args:
  574             sid (SID):              snapshot to remove
  575         """
  576         if isinstance(sid, RootSnapshot):
  577             return
  578         rsync = tools.rsyncRemove(self.config)
  579         with TemporaryDirectory() as d:
  580             rsync.append(d + os.sep)
  581             rsync.append(self.rsyncRemotePath(sid.path(use_mode = ['ssh', 'ssh_encfs'])))
  582             tools.Execute(rsync).run()
  583             shutil.rmtree(sid.path())
  584 
  585     def backup(self, force = False):
  586         """
  587         Wrapper for :py:func:`takeSnapshot` which will prepair and clean up
  588         things for the main :py:func:`takeSnapshot` method. This will check
  589         that no other snapshots are running at the same time, there is nothing
  590         prohibing a new snapshot (e.g. on battery) and the profile is configured
  591         correctly. This will also mount and unmount remote destinations.
  592 
  593         Args:
  594             force (bool):   force taking a new snapshot even if the profile is
  595                             not scheduled or the machine is running on battery
  596 
  597         Returns:
  598             bool:           ``True`` if there was an error
  599         """
  600         ret_val, ret_error = False, True
  601         sleep = True
  602 
  603         self.config.PLUGIN_MANAGER.load(self)
  604 
  605         if not self.config.isConfigured():
  606             logger.warning('Not configured', self)
  607             self.config.PLUGIN_MANAGER.error(1) #not configured
  608         elif not force and self.config.noSnapshotOnBattery() and tools.onBattery():
  609             self.setTakeSnapshotMessage(0, _('Deferring backup while on battery'))
  610             logger.info('Deferring backup while on battery', self)
  611             logger.warning('Backup not performed', self)
  612             ret_error = False
  613         elif not force and not self.config.backupScheduled():
  614             logger.info('Profile "%s" is not scheduled to run now.'
  615                         %self.config.profileName(), self)
  616             ret_error = False
  617         else:
  618             instance = ApplicationInstance(self.config.takeSnapshotInstanceFile(), False, flock = True)
  619             restore_instance = ApplicationInstance(self.config.restoreInstanceFile(), False)
  620             if not instance.check():
  621                 logger.warning('A backup is already running.  The pid of the \
  622 already running backup is in file %s.  Maybe delete it' % instance.pidFile , self )
  623                 self.config.PLUGIN_MANAGER.error(2) #a backup is already running
  624             elif not restore_instance.check():
  625                 logger.warning('Restore is still running. Stop backup until \
  626 restore is done. The pid of the already running restore is in %s.  Maybe delete it'\
  627                                % restore_instance.pidFile, self)
  628             else:
  629                 if self.config.noSnapshotOnBattery () and not tools.powerStatusAvailable():
  630                     logger.warning('Backups disabled on battery but power status is not available', self)
  631 
  632                 instance.startApplication()
  633                 self.flockExclusive()
  634                 logger.info('Lock', self)
  635 
  636                 now = datetime.datetime.today()
  637 
  638                 #inhibit suspend/hibernate during snapshot is running
  639                 self.config.inhibitCookie = tools.inhibitSuspend(toplevel_xid = self.config.xWindowId)
  640 
  641                 #mount
  642                 try:
  643                     hash_id = mount.Mount(cfg = self.config).mount()
  644                 except MountException as ex:
  645                     logger.error(str(ex), self)
  646                     instance.exitApplication()
  647                     logger.info('Unlock', self)
  648                     time.sleep(2)
  649                     return False, True
  650                 else:
  651                     self.config.setCurrentHashId(hash_id)
  652 
  653                 include_folders = self.config.include()
  654 
  655                 if not include_folders:
  656                     logger.info('Nothing to do', self)
  657                 elif not self.config.PLUGIN_MANAGER.processBegin():
  658                     logger.info('A plugin prevented the backup', self)
  659                 else:
  660                     #take snapshot process begin
  661                     self.setTakeSnapshotMessage(0, '...')
  662                     self.snapshotLog.new(now)
  663                     profile_id = self.config.currentProfile()
  664                     profile_name = self.config.profileName()
  665                     logger.info("Take a new snapshot. Profile: %s %s"
  666                                 %(profile_id, profile_name), self)
  667 
  668                     if not self.config.canBackup(profile_id):
  669                         if self.config.PLUGIN_MANAGER.hasGuiPlugins and self.config.notify():
  670                             self.setTakeSnapshotMessage(1,
  671                                     _('Can\'t find snapshots folder.\nIf it is on a removable drive please plug it.') +
  672                                     '\n' +
  673                                     gettext.ngettext('Waiting %s second.', 'Waiting %s seconds.', 30) % 30,
  674                                     30)
  675                         for counter in range(30, 0, -1):
  676                             time.sleep(1)
  677                             if self.config.canBackup():
  678                                 break
  679 
  680                     if not self.config.canBackup(profile_id):
  681                         logger.warning('Can\'t find snapshots folder!', self)
  682                         self.config.PLUGIN_MANAGER.error(3) #Can't find snapshots directory (is it on a removable drive ?)
  683                     else:
  684                         ret_error = False
  685                         sid = SID(now, self.config)
  686 
  687                         if sid.exists():
  688                             logger.warning("Snapshot path \"%s\" already exists" %sid.path(), self)
  689                             self.config.PLUGIN_MANAGER.error(4, sid) #This snapshots already exists
  690                         else:
  691                             try:
  692                                 ret_val, ret_error = self.takeSnapshot(sid, now, include_folders)
  693                             except:
  694                                 new = NewSnapshot(self.config)
  695                                 if new.exists():
  696                                     new.saveToContinue = False
  697                                     new.failed = True
  698                                 raise
  699 
  700                         if not ret_val:
  701                             self.remove(sid)
  702 
  703                             if ret_error:
  704                                 logger.error('Failed to take snapshot !!!', self)
  705                                 self.setTakeSnapshotMessage(1, _('Failed to take snapshot %s !!!') % sid.displayID)
  706                                 time.sleep(2)
  707                             else:
  708                                 logger.warning("No new snapshot", self)
  709                         else:
  710                             ret_error = False
  711 
  712                         if not ret_error:
  713                             self.freeSpace(now)
  714                             self.setTakeSnapshotMessage(0, _('Finalizing'))
  715 
  716                     time.sleep(2)
  717                     sleep = False
  718 
  719                     if ret_val:
  720                         self.config.PLUGIN_MANAGER.newSnapshot(sid, sid.path()) #new snapshot
  721 
  722                     self.config.PLUGIN_MANAGER.processEnd() #take snapshot process end
  723 
  724                 if sleep:
  725                     time.sleep(2)
  726                     sleep = False
  727 
  728                 if not ret_error:
  729                     self.clearTakeSnapshotMessage()
  730 
  731                 #unmount
  732                 try:
  733                     mount.Mount(cfg = self.config).umount(self.config.current_hash_id)
  734                 except MountException as ex:
  735                     logger.error(str(ex), self)
  736 
  737                 instance.exitApplication()
  738                 self.flockRelease()
  739                 logger.info('Unlock', self)
  740 
  741         if sleep:
  742             time.sleep(2) #max 1 backup / second
  743 
  744         #release inhibit suspend
  745         if self.config.inhibitCookie:
  746             self.config.inhibitCookie = tools.unInhibitSuspend(*self.config.inhibitCookie)
  747 
  748         return ret_error
  749 
  750     def filterRsyncProgress(self, line):
  751         """
  752         Filter rsync's stdout for progress informations and store them in
  753         '~/.local/share/backintime/worker<N>.progress' file.
  754 
  755         Args:
  756             line (str): stdout line from rsync
  757 
  758         Returns:
  759             str:        ``line`` if it had no progress infos. ``None`` if
  760                         ``line`` was a progress
  761         """
  762         ret = []
  763         for l in line.split('\n'):
  764             m = self.reRsyncProgress.match(l)
  765             if m:
  766                 # if m.group(5).strip():
  767                 #     return
  768                 pg = progress.ProgressFile(self.config)
  769                 pg.setIntValue('status', pg.RSYNC)
  770                 pg.setStrValue('sent', m.group(1))
  771                 pg.setIntValue('percent', int(m.group(2)))
  772                 pg.setStrValue('speed', m.group(3))
  773                 #pg.setStrValue('eta', m.group(4))
  774                 pg.save()
  775                 del(pg)
  776             else:
  777                 ret.append(l)
  778         return '\n'.join(ret)
  779 
  780     def rsyncCallback(self, line, params):
  781         """
  782         Parse rsync's stdout, send it to takeSnapshotMessage and
  783         takeSnapshotLog. Also check if there has been changes or errors in
  784         current rsync.
  785 
  786         Args:
  787             line (str):     stdout line from rsync
  788             params (list):  list of two bool '[error, changes]'. Using siteefect
  789                             on changing list items will change original
  790                             list, too. If rsync reported an error ``params[0]``
  791                             will be set to ``True``. If rsync reported a changed
  792                             file ``params[1]`` will be set to ``True``
  793         """
  794         if not line:
  795             return
  796 
  797         self.setTakeSnapshotMessage(0, _('Take snapshot') + " (rsync: %s)" % line)
  798 
  799         if line.endswith(')'):
  800             if line.startswith('rsync:'):
  801                 if not line.startswith('rsync: chgrp ') and not line.startswith('rsync: chown '):
  802                     params[0] = True
  803                     self.setTakeSnapshotMessage(1, 'Error: ' + line)
  804 
  805         if len(line) >= 13:
  806             if line.startswith('BACKINTIME: '):
  807                 if line[12] != '.' and line[12:14] != 'cd':
  808                     params[1] = True
  809                     self.snapshotLog.append('[C] ' + line[12:], 2)
  810 
  811     def makeDirs(self, path):
  812         """
  813         Wrapper for :py:func:`tools.makeDirs()`. Create directories ``path``
  814         recursive and return success. If not successful send error-message to
  815         log.
  816 
  817         Args:
  818             path (str): fullpath to directories that should be created
  819 
  820         Returns:
  821             bool:       ``True`` if successful
  822         """
  823         if not tools.makeDirs(path):
  824             logger.error("Can't create folder: %s" % path, self)
  825             self.setTakeSnapshotMessage(1, _('Can\'t create folder: %s') % path)
  826             time.sleep(2) #max 1 backup / second
  827             return False
  828         return True
  829 
  830     def backupConfig(self, sid):
  831         """
  832         Backup the config file to the snapshot and to the backup root if backup
  833         is encrypted.
  834 
  835         Args:
  836             sid (SID):  snapshot in which the config should be stored
  837         """
  838         logger.info('Save config file', self)
  839         self.setTakeSnapshotMessage(0, _('Saving config file...'))
  840         with open(self.config._LOCAL_CONFIG_PATH, 'rb') as src:
  841             with open(sid.path('config'), 'wb') as dst1:
  842                 dst1.write(src.read())
  843             if self.config.snapshotsMode() == 'local_encfs':
  844                 src.seek(0)
  845                 with open(os.path.join(self.config.localEncfsPath(), 'config'), 'wb') as dst2:
  846                     dst2.write(src.read())
  847             elif self.config.snapshotsMode() == 'ssh_encfs':
  848                 cmd = tools.rsyncPrefix(self.config, no_perms = False)
  849                 cmd.append(self.config._LOCAL_CONFIG_PATH)
  850                 cmd.append(self.rsyncRemotePath(self.config.sshSnapshotsPath()))
  851                 tools.Execute(cmd, parent = self).run()
  852 
  853     def backupInfo(self, sid):
  854         """
  855         Save infos about the snapshot into the 'info' file.
  856 
  857         Args:
  858             sid (SID):  snapshot that should get an info file
  859         """
  860         logger.info("Create info file", self)
  861         machine = self.config.host()
  862         user = self.config.user()
  863         profile_id = self.config.currentProfile()
  864         i = configfile.ConfigFile()
  865         i.setIntValue('snapshot_version', self.SNAPSHOT_VERSION)
  866         i.setStrValue('snapshot_date', sid.withoutTag)
  867         i.setStrValue('snapshot_machine', machine)
  868         i.setStrValue('snapshot_user', user)
  869         i.setIntValue('snapshot_profile_id', profile_id)
  870         i.setIntValue('snapshot_tag', sid.tag)
  871         i.setListValue('user', ('int:uid', 'str:name'), list(self.userCache.items()))
  872         i.setListValue('group', ('int:gid', 'str:name'), list(self.groupCache.items()))
  873         i.setStrValue('filesystem_mounts', json.dumps(tools.filesystemMountInfo()))
  874         sid.info = i
  875 
  876     def backupPermissions(self, sid):
  877         """
  878         Save permissions (owner, group, read-, write- and executable)
  879         for all files in Snapshot ``sid`` into snapshots fileInfoDict.
  880 
  881         Args:
  882             sid (SID):  snapshot that should be scanned
  883         """
  884         logger.info('Save permissions', self)
  885         self.setTakeSnapshotMessage(0, _('Saving permissions...'))
  886 
  887         fileInfoDict = FileInfoDict()
  888         if self.config.snapshotsMode() == 'ssh_encfs':
  889             decode = encfstools.Decode(self.config, False)
  890         else:
  891             decode = encfstools.Bounce()
  892 
  893         # backup permissions of /
  894         # bugfix for https://github.com/bit-team/backintime/issues/708
  895         self.backupPermissionsCallback(b'/', (fileInfoDict, decode))
  896 
  897         rsync = ['rsync', '--dry-run', '-r', '--out-format=%n']
  898         rsync.extend(tools.rsyncSshArgs(self.config))
  899         rsync.append(self.rsyncRemotePath(sid.pathBackup(use_mode = ['ssh', 'ssh_encfs'])) + os.sep)
  900         with TemporaryDirectory() as d:
  901             rsync.append(d + os.sep)
  902             proc = tools.Execute(rsync,
  903                                  callback = self.backupPermissionsCallback,
  904                                  user_data = (fileInfoDict, decode),
  905                                  parent = self,
  906                                  conv_str = False,
  907                                  join_stderr = False)
  908             proc.run()
  909 
  910         sid.fileInfo = fileInfoDict
  911 
  912     def backupPermissionsCallback(self, line, user_data):
  913         """
  914         Rsync callback for :py:func:`Snapshots.backupPermissions`.
  915 
  916         Args:
  917             line(bytes):        output from rsync command
  918             user_data (tuple):  two item tuple of (:py:class:`FileInfoDict`,
  919                                 :py:class:`encfstools.Decode`)
  920         """
  921         fileInfoDict, decode = user_data
  922         self.collectPermission(fileInfoDict, b'/' + decode.path(line).rstrip(b'/'))
  923 
  924     def collectPermission(self, fileinfo, path):
  925         """
  926         Collect permission infos about ``path`` and store them into
  927         ``fileinfo``.
  928 
  929         Args:
  930             fileinfo (FileInfoDict):
  931                             dict of: {path: (permission, user, group)}
  932                             Using sideefect on changing dict item will change
  933                             original dict, too.
  934             path (bytes):   full path to file or folder
  935         """
  936         assert isinstance(path, bytes), 'path is not bytes type: %s' % path
  937         if path and os.path.exists(path):
  938             info = os.stat(path)
  939             mode = info.st_mode
  940             user = self.userName(info.st_uid).encode('utf-8', 'replace')
  941             group = self.groupName(info.st_gid).encode('utf-8', 'replace')
  942             fileinfo[path] = (mode, user, group)
  943 
  944     def takeSnapshot(self, sid, now, include_folders):
  945         """
  946         This is the main backup routine. It will take a new snapshot and store
  947         permissions of included files and folders into ``fileinfo.bz2``.
  948 
  949         Args:
  950             sid (SID):                  snapshot ID which the new snapshot
  951                                         should get
  952             now (datetime.datetime):    date and time when this snapshot was
  953                                         started
  954             include_folders (list):     folders to include. list of
  955                                         tuples (item, int) where ``int`` is 0
  956                                         if ``item`` is a folder or 1 if ``item``
  957                                         is a file
  958 
  959         Returns:
  960             list:                       list of two bool
  961                                         (``ret_val``, ``ret_error``)
  962                                         where ``ret_val`` is ``True`` if a new
  963                                         snapshot has been created and
  964                                         ``ret_error`` is ``True`` if there was
  965                                         an error during taking the snapshot
  966         """
  967         self.setTakeSnapshotMessage(0, _('...'))
  968 
  969         new_snapshot = NewSnapshot(self.config)
  970         encode = self.config.ENCODE
  971         params = [False, False] # [error, changes]
  972 
  973         if new_snapshot.exists() and new_snapshot.saveToContinue:
  974             logger.info("Found leftover '%s' which can be continued." %new_snapshot.displayID, self)
  975             self.setTakeSnapshotMessage(0, _("Found leftover '%s' which can be continued.") %new_snapshot.displayID)
  976             #fix permissions
  977             for file in os.listdir(new_snapshot.path()):
  978                 file = os.path.join(new_snapshot.path(), file)
  979                 mode = os.stat(file).st_mode
  980                 try:
  981                     os.chmod(file, mode | stat.S_IWUSR)
  982                 except PermissionError:
  983                     pass
  984             # search previous log for changes and set params
  985             params[1] = new_snapshot.hasChanges
  986         elif new_snapshot.exists() and not new_snapshot.saveToContinue:
  987             logger.info("Remove leftover '%s' folder from last run" %new_snapshot.displayID)
  988             self.setTakeSnapshotMessage(0, _("Removing leftover '%s' folder from last run") %new_snapshot.displayID)
  989             self.remove(new_snapshot)
  990 
  991             if os.path.exists(new_snapshot.path()):
  992                 logger.error("Can't remove folder: %s" % new_snapshot.path(), self)
  993                 self.setTakeSnapshotMessage(1, _('Can\'t remove folder: %s') % new_snapshot.path())
  994                 time.sleep(2) #max 1 backup / second
  995                 return [False, True]
  996 
  997         if not new_snapshot.saveToContinue and not new_snapshot.makeDirs():
  998             return [False, True]
  999 
 1000         prev_sid = None
 1001         snapshots = listSnapshots(self.config)
 1002         if snapshots:
 1003             prev_sid = snapshots[0]
 1004 
 1005         #rsync prefix & suffix
 1006         rsync_prefix = tools.rsyncPrefix(self.config, no_perms = False)
 1007         if self.config.excludeBySizeEnabled():
 1008             rsync_prefix.append('--max-size=%sM' %self.config.excludeBySize())
 1009         rsync_suffix = self.rsyncSuffix(include_folders)
 1010 
 1011         # When there is no snapshots it takes the last snapshot from the other folders
 1012         # It should delete the excluded folders then
 1013         rsync_prefix.extend(('--delete', '--delete-excluded'))
 1014         rsync_prefix.append('-v')
 1015         rsync_prefix.extend(('-i', '--out-format=BACKINTIME: %i %n%L'))
 1016         if prev_sid:
 1017             link_dest = encode.path(os.path.join(prev_sid.sid, 'backup'))
 1018             link_dest = os.path.join(os.pardir, os.pardir, link_dest)
 1019             rsync_prefix.append('--link-dest=%s' %link_dest)
 1020 
 1021         #sync changed folders
 1022         logger.info("Call rsync to take the snapshot", self)
 1023         new_snapshot.saveToContinue = True
 1024         cmd = rsync_prefix + rsync_suffix
 1025         cmd.append(self.rsyncRemotePath(new_snapshot.pathBackup(use_mode = ['ssh', 'ssh_encfs'])))
 1026 
 1027         self.setTakeSnapshotMessage(0, _('Taking snapshot'))
 1028 
 1029         #run rsync
 1030         proc = tools.Execute(cmd,
 1031                              callback = self.rsyncCallback,
 1032                              user_data = params,
 1033                              filters = (self.filterRsyncProgress,),
 1034                              parent = self)
 1035         self.snapshotLog.append('[I] ' + proc.printable_cmd, 3)
 1036         proc.run()
 1037 
 1038         #cleanup
 1039         try:
 1040             os.remove(self.config.takeSnapshotProgressFile())
 1041         except Exception as e:
 1042             logger.debug('Failed to remove snapshot progress file %s: %s'
 1043                          %(self.config.takeSnapshotProgressFile(), str(e)),
 1044                          self)
 1045 
 1046         #handle errors
 1047         has_errors = False
 1048         # params[0] -> error
 1049         if params[0]:
 1050             if not self.config.continueOnErrors():
 1051                 self.remove(new_snapshot)
 1052                 return [False, True]
 1053 
 1054             has_errors = True
 1055             new_snapshot.failed = True
 1056 
 1057         # params[1] -> changes
 1058         if not params[1] and not self.config.takeSnapshotRegardlessOfChanges():
 1059             self.remove(new_snapshot)
 1060             logger.info("Nothing changed, no new snapshot necessary", self)
 1061             self.snapshotLog.append('[I] ' + _('Nothing changed, no new snapshot necessary'), 3)
 1062             if prev_sid:
 1063                 prev_sid.setLastChecked()
 1064             if not has_errors and not list(self.config.anacrontabFiles()):
 1065                 tools.writeTimeStamp(self.config.anacronSpoolFile())
 1066             return [False, False]
 1067 
 1068         self.backupConfig(new_snapshot)
 1069         self.backupPermissions(new_snapshot)
 1070 
 1071         #copy snapshot log
 1072         try:
 1073             self.snapshotLog.flush()
 1074             with open(self.snapshotLog.logFileName, 'rb') as logfile:
 1075                 new_snapshot.setLog(logfile.read())
 1076         except Exception as e:
 1077             logger.debug('Failed to write takeSnapshot log %s into compressed file %s: %s'
 1078                          %(self.config.takeSnapshotLogFile(), new_snapshot.path(SID.LOG), str(e)),
 1079                          self)
 1080 
 1081         new_snapshot.saveToContinue = False
 1082         #rename snapshot
 1083         os.rename(new_snapshot.path(), sid.path())
 1084 
 1085         if not sid.exists():
 1086             logger.error("Can't rename %s to %s" % (new_snapshot.path(), sid.path()), self)
 1087             self.setTakeSnapshotMessage(1, _('Can\'t rename %(new_path)s to %(path)s')
 1088                                                  %{'new_path': new_snapshot.path(),
 1089                                                    'path': sid.path()})
 1090             time.sleep(2) #max 1 backup / second
 1091             return [False, True]
 1092 
 1093         self.backupInfo(sid)
 1094 
 1095         if not has_errors and not list(self.config.anacrontabFiles()):
 1096             tools.writeTimeStamp(self.config.anacronSpoolFile())
 1097 
 1098         #create last_snapshot symlink
 1099         self.createLastSnapshotSymlink(sid)
 1100 
 1101         return [True, has_errors]
 1102 
 1103     def smartRemoveKeepAll(self,
 1104                            snapshots,
 1105                            min_date,
 1106                            max_date):
 1107         """
 1108         Return all snapshots between ``min_date`` and ``max_date``.
 1109 
 1110         Args:
 1111             snapshots (list):           full list of :py:class:`SID` objects
 1112             min_date (datetime.date):   minimum date for snapshots to keep
 1113             max_date (datetime.date):   maximum date for snapshots to keep
 1114 
 1115         Returns:
 1116             set:                        set of snapshots that should be keept
 1117         """
 1118         min_id = SID(min_date, self.config)
 1119         max_id = SID(max_date, self.config)
 1120 
 1121         logger.debug("Keep all >= %s and < %s" %(min_id, max_id), self)
 1122 
 1123         return set([sid for sid in snapshots if sid >= min_id and sid < max_id])
 1124 
 1125     def smartRemoveKeepFirst(self,
 1126                              snapshots,
 1127                              min_date,
 1128                              max_date,
 1129                              keep_healthy = False):
 1130         """
 1131         Return only the first snapshot between ``min_date`` and ``max_date``.
 1132 
 1133         Args:
 1134             snapshots (list):           full list of :py:class:`SID` objects
 1135             min_date (datetime.date):   minimum date for snapshots to keep
 1136             max_date (datetime.date):   maximum date for snapshots to keep
 1137             keep_healthy (bool):        return the first healthy snapshot (not
 1138                                         marked as failed) instead of the first
 1139                                         at all. If all snapshots failed this
 1140                                         will again return the very first
 1141                                         snapshot
 1142 
 1143         Returns:
 1144             set:                        set of snapshots that should be keept
 1145         """
 1146         min_id = SID(min_date, self.config)
 1147         max_id = SID(max_date, self.config)
 1148 
 1149         logger.debug("Keep first >= %s and < %s" %(min_id, max_id), self)
 1150 
 1151         for sid in snapshots:
 1152             # try to keep the first healty snapshot
 1153             if keep_healthy and sid.failed:
 1154                 logger.debug("Do not keep failed snapshot %s" %sid, self)
 1155                 continue
 1156             if sid >= min_id and sid < max_id:
 1157                 return set([sid])
 1158         # if all snapshots failed return the first snapshot
 1159         # no matter if it has errors
 1160         if keep_healthy:
 1161             return self.smartRemoveKeepFirst(snapshots,
 1162                                              min_date,
 1163                                              max_date,
 1164                                              keep_healthy = False)
 1165         return set()
 1166 
 1167     def incMonth(self, date):
 1168         """
 1169         First day of next month of ``date`` with respect on new years. So if
 1170         ``date`` is December this will return 1st of January next year.
 1171 
 1172         Args:
 1173             date (datetime.date):   old date that should be increased
 1174 
 1175         Returns:
 1176             datetime.date:          1st day of next month
 1177         """
 1178         y = date.year
 1179         m = date.month + 1
 1180         if m > 12:
 1181             m = 1
 1182             y = y + 1
 1183         return datetime.date(y, m, 1)
 1184 
 1185     def decMonth(self, date):
 1186         """
 1187         First day of previous month of ``date`` with respect on previous years.
 1188         So if ``date`` is January this will return 1st of December previous
 1189         year.
 1190 
 1191         Args:
 1192             date (datetime.date):   old date that should be decreased
 1193 
 1194         Returns:
 1195             datetime.date:          1st day of previous month
 1196         """
 1197         y = date.year
 1198         m = date.month - 1
 1199         if m < 1:
 1200             m = 12
 1201             y = y - 1
 1202         return datetime.date(y, m, 1)
 1203 
 1204     def smartRemoveList(self,
 1205                         now_full,
 1206                         keep_all,
 1207                         keep_one_per_day,
 1208                         keep_one_per_week,
 1209                         keep_one_per_month):
 1210         """
 1211         Get a list of old snapshots that should be removed based on configurable
 1212         intervals.
 1213 
 1214         Args:
 1215             now_full (datetime.datetime):   date and time when takeSnapshot was
 1216                                             started
 1217             keep_all (int):                 keep all snapshots for the
 1218                                             last ``keep_all`` days
 1219             keep_one_per_day (int):         keep one snapshot per day for the
 1220                                             last ``keep_one_per_day`` days
 1221             keep_one_per_week (int):        keep one snapshot per week for the
 1222                                             last ``keep_one_per_week`` weeks
 1223             keep_one_per_month (int):       keep one snapshot per month for the
 1224                                             last ``keep_one_per_month`` months
 1225 
 1226         Returns:
 1227             list:                           snapshots that should be removed
 1228         """
 1229         snapshots = listSnapshots(self.config)
 1230         logger.debug("Considered: %s" %snapshots, self)
 1231         if len(snapshots) <= 1:
 1232             logger.debug("There is only one snapshots, so keep it", self)
 1233             return
 1234 
 1235         if now_full is None:
 1236             now_full = datetime.datetime.today()
 1237 
 1238         now = now_full.date()
 1239 
 1240         #keep the last snapshot
 1241         keep = set([snapshots[0]])
 1242 
 1243         #keep all for the last keep_all days
 1244         if keep_all > 0:
 1245             keep |= self.smartRemoveKeepAll(snapshots,
 1246                                             now - datetime.timedelta(days=keep_all-1),
 1247                                             now + datetime.timedelta(days=1))
 1248 
 1249         #keep one per day for the last keep_one_per_day days
 1250         if keep_one_per_day > 0:
 1251             d = now
 1252             for i in range(0, keep_one_per_day):
 1253                 keep |= self.smartRemoveKeepFirst(snapshots,
 1254                                                   d,
 1255                                                   d + datetime.timedelta(days=1),
 1256                                                   keep_healthy = True)
 1257                 d -= datetime.timedelta(days=1)
 1258 
 1259         #keep one per week for the last keep_one_per_week weeks
 1260         if keep_one_per_week > 0:
 1261             d = now - datetime.timedelta(days = now.weekday() + 1)
 1262             for i in range(0, keep_one_per_week):
 1263                 keep |= self.smartRemoveKeepFirst(snapshots,
 1264                                                   d,
 1265                                                   d + datetime.timedelta(days=8),
 1266                                                   keep_healthy = True)
 1267                 d -= datetime.timedelta(days=7)
 1268 
 1269         #keep one per month for the last keep_one_per_month months
 1270         if keep_one_per_month > 0:
 1271             d1 = datetime.date(now.year, now.month, 1)
 1272             d2 = self.incMonth(d1)
 1273             for i in range(0, keep_one_per_month):
 1274                 keep |= self.smartRemoveKeepFirst(snapshots, d1, d2,
 1275                                                   keep_healthy = True)
 1276                 d2 = d1
 1277                 d1 = self.decMonth(d1)
 1278 
 1279         #keep one per year for all years
 1280         first_year = int(snapshots[-1].sid[ : 4])
 1281         for i in range(first_year, now.year+1):
 1282             keep |= self.smartRemoveKeepFirst(snapshots,
 1283                                               datetime.date(i,1,1),
 1284                                               datetime.date(i+1,1,1),
 1285                                               keep_healthy = True)
 1286 
 1287         logger.debug("Keep snapshots: %s" %keep, self)
 1288 
 1289         del_snapshots = []
 1290         for sid in snapshots:
 1291             if sid in keep:
 1292                 continue
 1293 
 1294             if self.config.dontRemoveNamedSnapshots():
 1295                 if sid.name:
 1296                     logger.debug("Keep snapshot: %s, it has a name" %sid, self)
 1297                     continue
 1298 
 1299             del_snapshots.append(sid)
 1300         return del_snapshots
 1301 
 1302     def smartRemove(self, del_snapshots, log = None):
 1303         """
 1304         Remove multiple snapshots either with
 1305         :py:func:`Snapshots.remove` or in background on the remote host
 1306         if mode is `ssh` or `ssh_encfs` and smart-remove in background is
 1307         activated.
 1308 
 1309         Args:
 1310             del_snapshots (list):   list of :py:class:`SID` that should be removed
 1311             log (method):           callable method that will handle progress log
 1312         """
 1313         if not del_snapshots:
 1314             return
 1315 
 1316         if not log:
 1317             log = lambda x: self.setTakeSnapshotMessage(0, x)
 1318 
 1319         if self.config.snapshotsMode() in ['ssh', 'ssh_encfs'] and self.config.smartRemoveRunRemoteInBackground():
 1320             logger.info('[smart remove] remove snapshots in background: %s'
 1321                         %del_snapshots, self)
 1322             lckFile = os.path.normpath(os.path.join(del_snapshots[0].path(use_mode = ['ssh', 'ssh_encfs']), os.pardir, 'smartremove.lck'))
 1323 
 1324             maxLength = self.config.sshMaxArgLength()
 1325             if not maxLength:
 1326                 import sshMaxArg
 1327                 user_host = '%s@%s' %(self.config.sshUser(), self.config.sshHost())
 1328                 maxLength = sshMaxArg.maxArgLength(self.config)
 1329                 self.config.setSshMaxArgLength(maxLength)
 1330                 self.config.save()
 1331                 sshMaxArg.reportResult(user_host, maxLength)
 1332 
 1333             additionalChars = len(self.config.sshPrefixCmd(cmd_type = str))
 1334 
 1335             head = 'screen -d -m bash -c "('
 1336             head += 'TMP=\$(mktemp -d); '                      #create temp dir used for delete files with rsync
 1337             head += 'test -z \\\"\$TMP\\\" && exit 1; '        #make sure $TMP dir was created
 1338             head += 'test -n \\\"\$(ls \$TMP)\\\" && exit 1; ' #make sure $TMP is empty
 1339             if logger.DEBUG:
 1340                 head += 'logger -t \\\"backintime smart-remove [$BASHPID]\\\" \\\"start\\\"; '
 1341             head += 'flock -x 9; '
 1342             if logger.DEBUG:
 1343                 head += 'logger -t \\\"backintime smart-remove [$BASHPID]\\\" \\\"got exclusive flock\\\"; '
 1344 
 1345             tail = 'rmdir \$TMP) 9>\\\"%s\\\""' %lckFile
 1346 
 1347             cmds = []
 1348             for sid in del_snapshots:
 1349                 remote = self.rsyncRemotePath(sid.path(use_mode = ['ssh', 'ssh_encfs']), use_mode = [], quote = '\\\"')
 1350                 rsync = ' '.join(tools.rsyncRemove(self.config, run_local = False))
 1351                 rsync += ' \\\"\$TMP/\\\" {}; '.format(remote)
 1352 
 1353                 s = 'test -e \\\"%s\\\" && (' %sid.path(use_mode = ['ssh', 'ssh_encfs'])
 1354                 if logger.DEBUG:
 1355                     s += 'logger -t \\\"backintime smart-remove [$BASHPID]\\\" '
 1356                     s += '\\\"snapshot %s still exist\\\"; ' %sid
 1357                     s += 'sleep 1; ' #add one second delay because otherwise you might not see serialized process with small snapshots
 1358                 s += rsync
 1359                 s += 'rmdir \\\"%s\\\"; ' %sid.path(use_mode = ['ssh', 'ssh_encfs'])
 1360                 if logger.DEBUG:
 1361                     s += 'logger -t \\\"backintime smart-remove [$BASHPID]\\\" '
 1362                     s += '\\\"snapshot %s remove done\\\"' %sid
 1363                 s += '); '
 1364                 cmds.append(s)
 1365 
 1366             for cmd in tools.splitCommands(cmds,
 1367                                            head = head,
 1368                                            tail = tail,
 1369                                            maxLength = maxLength - additionalChars):
 1370                 tools.Execute(self.config.sshCommand([cmd,],
 1371                                                      quote = False,
 1372                                                      nice = False,
 1373                                                      ionice = False)).run()
 1374         else:
 1375             logger.info("[smart remove] remove snapshots: %s"
 1376                         %del_snapshots, self)
 1377             for i, sid in enumerate(del_snapshots, 1):
 1378                 log(_('Smart remove') + ' %s/%s' %(i, len(del_snapshots)))
 1379                 self.remove(sid)
 1380 
 1381     def freeSpace(self, now):
 1382         """
 1383         Remove old snapshots on based on different rules (only if enabled).
 1384         First rule is to remove snapshots older than X years. Next will call
 1385         :py:func:`smartRemove` to remove snapshots based on
 1386         configurable intervals. Third rule is to remove the oldest snapshot
 1387         until there is enough free space. Last rule will remove the oldest
 1388         snapshot until there are enough free inodes.
 1389 
 1390         'last_snapshot' symlink will be fixed when done.
 1391 
 1392         Args:
 1393             now (datetime.datetime):    date and time when takeSnapshot was
 1394                                         started
 1395         """
 1396         snapshots = listSnapshots(self.config, reverse = False)
 1397         if not snapshots:
 1398             logger.debug('No snapshots. Skip freeSpace', self)
 1399             return
 1400 
 1401         last_snapshot = snapshots[-1]
 1402 
 1403         #remove old backups
 1404         if self.config.removeOldSnapshotsEnabled():
 1405             self.setTakeSnapshotMessage(0, _('Removing old snapshots'))
 1406 
 1407             oldBackupId = SID(self.config.removeOldSnapshotsDate(), self.config)
 1408             logger.debug("Remove snapshots older than: {}".format(oldBackupId.withoutTag), self)
 1409 
 1410             while True:
 1411                 if len(snapshots) <= 1:
 1412                     break
 1413                 if snapshots[0] >= oldBackupId:
 1414                     break
 1415 
 1416                 if self.config.dontRemoveNamedSnapshots():
 1417                     if snapshots[0].name:
 1418                         del snapshots[0]
 1419                         continue
 1420 
 1421                 msg = 'Remove snapshot {} because it is older than {}'
 1422                 logger.debug(msg.format(snapshots[0].withoutTag, oldBackupId.withoutTag), self)
 1423                 self.remove(snapshots[0])
 1424                 del snapshots[0]
 1425 
 1426         #smart remove
 1427         enabled, keep_all, keep_one_per_day, keep_one_per_week, keep_one_per_month = self.config.smartRemove()
 1428         if enabled:
 1429             self.setTakeSnapshotMessage(0, _('Smart remove'))
 1430             del_snapshots = self.smartRemoveList(now,
 1431                                                  keep_all,
 1432                                                  keep_one_per_day,
 1433                                                  keep_one_per_week,
 1434                                                  keep_one_per_month)
 1435             self.smartRemove(del_snapshots)
 1436 
 1437         #try to keep min free space
 1438         if self.config.minFreeSpaceEnabled():
 1439             self.setTakeSnapshotMessage(0, _('Trying to keep min free space'))
 1440 
 1441             minFreeSpace = self.config.minFreeSpaceMib()
 1442 
 1443             logger.debug("Keep min free disk space: {} MiB".format(minFreeSpace), self)
 1444 
 1445             snapshots = listSnapshots(self.config, reverse = False)
 1446 
 1447             while True:
 1448                 if len(snapshots) <= 1:
 1449                     break
 1450 
 1451                 free_space = self.statFreeSpaceLocal(self.config.snapshotsFullPath())
 1452 
 1453                 if free_space is None:
 1454                     free_space = self.statFreeSpaceSsh()
 1455 
 1456                 if free_space is None:
 1457                     logger.warning('Failed to get free space. Skipping', self)
 1458                     break
 1459 
 1460                 if free_space >= minFreeSpace:
 1461                     break
 1462 
 1463                 if self.config.dontRemoveNamedSnapshots():
 1464                     if snapshots[0].name:
 1465                         del snapshots[0]
 1466                         continue
 1467 
 1468                 msg = "free disk space: {} MiB. Remove snapshot {}"
 1469                 logger.debug(msg.format(free_space, snapshots[0].withoutTag), self)
 1470                 self.remove(snapshots[0])
 1471                 del snapshots[0]
 1472 
 1473         #try to keep free inodes
 1474         if self.config.minFreeInodesEnabled():
 1475             minFreeInodes = self.config.minFreeInodes()
 1476             self.setTakeSnapshotMessage(0, _('Trying to keep min %d%% free inodes') % minFreeInodes)
 1477             logger.debug("Keep min {}%% free inodes".format(minFreeInodes), self)
 1478 
 1479             snapshots = listSnapshots(self.config, reverse = False)
 1480 
 1481             while True:
 1482                 if len(snapshots) <= 1:
 1483                     break
 1484 
 1485                 try:
 1486                     info = os.statvfs(self.config.snapshotsPath())
 1487                     free_inodes = info.f_favail
 1488                     max_inodes  = info.f_files
 1489                 except Exception as e:
 1490                     logger.debug('Failed to get free inodes for snapshot path %s: %s'
 1491                                  %(self.config.snapshotsPath(), str(e)),
 1492                                  self)
 1493                     break
 1494 
 1495                 if free_inodes >= max_inodes * (minFreeInodes / 100.0):
 1496                     break
 1497 
 1498                 if self.config.dontRemoveNamedSnapshots():
 1499                     if snapshots[0].name:
 1500                         del snapshots[0]
 1501                         continue
 1502 
 1503                 logger.debug("free inodes: %.2f%%. Remove snapshot %s"
 1504                             %((100.0 / max_inodes * free_inodes), snapshots[0].withoutTag),
 1505                             self)
 1506                 self.remove(snapshots[0])
 1507                 del snapshots[0]
 1508 
 1509         #set correct last snapshot again
 1510         if last_snapshot is not snapshots[-1]:
 1511             self.createLastSnapshotSymlink(snapshots[-1])
 1512 
 1513     def statFreeSpaceLocal(self, path):
 1514         """
 1515         Get free space on filsystem containing ``path`` in MiB using
 1516         :py:func:`os.statvfs()`. Depending on remote SFTP server this might fail
 1517         on sshfs mounted shares.
 1518 
 1519         Args:
 1520             path (str): full path
 1521 
 1522         Returns:
 1523             int         free space in MiB
 1524         """
 1525         try:
 1526             info = os.statvfs(path)
 1527             if info.f_blocks != info.f_bavail:
 1528                 return info.f_frsize * info.f_bavail // (1024 * 1024)
 1529         except Exception as e:
 1530             logger.debug('Failed to get free space for %s: %s'
 1531                          %(path, str(e)),
 1532                          self)
 1533         logger.warning('Failed to stat snapshot path', self)
 1534 
 1535     def statFreeSpaceSsh(self):
 1536         """
 1537         Get free space on remote filsystem in MiB. This will call ``df`` on
 1538         remote host and parse its output.
 1539 
 1540         Returns:
 1541             int         free space in MiB
 1542         """
 1543         if self.config.snapshotsMode() not in ('ssh', 'ssh_encfs'):
 1544             return None
 1545 
 1546         snapshots_path_ssh = self.config.sshSnapshotsFullPath()
 1547         if not len(snapshots_path_ssh):
 1548             snapshots_path_ssh = './'
 1549         cmd = self.config.sshCommand(['df', snapshots_path_ssh],
 1550                                      nice = False,
 1551                                      ionice = False)
 1552 
 1553         df = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 1554         output = df.communicate()[0]
 1555         #Filesystem     1K-blocks      Used Available Use% Mounted on
 1556         #/tmp           127266564 115596412   5182296  96% /
 1557         #                                     ^^^^^^^
 1558         for line in output.split(b'\n'):
 1559             m = re.match(b'^.*?\s+\d+\s+\d+\s+(\d+)\s+\d+%', line, re.M)
 1560             if m:
 1561                 return int(int(m.group(1)) / 1024)
 1562         logger.warning('Failed to get free space on remote', self)
 1563 
 1564     def filter(self,
 1565                base_sid,
 1566                base_path,
 1567                snapshotsList,
 1568                list_diff_only  = False,
 1569                flag_deep_check = False,
 1570                list_equal_to = ''):
 1571         """
 1572         Filter snapshots from ``snapshotsList`` based on whether ``base_path``
 1573         file is included and optional if the snapshot is unique or equal to
 1574         ``list_equal_to``.
 1575 
 1576         Args:
 1577             base_sid (SID):         snapshot ID that contained the original
 1578                                     file ``base_path``
 1579             base_path (str):        path to file on root filesystem.
 1580             snapshotsList (list):  List of :py:class:`SID` objects that should
 1581                                     be filtered
 1582             list_diff_only (bool):  if ``True`` only return unique snapshots.
 1583                                     Which means if a file is exactly the same in
 1584                                     different snapshots only the first snapshot
 1585                                     will be listed
 1586             flag_deep_check (bool): use md5sum to check uniqueness of files.
 1587                                     More acurate but slow
 1588             list_equal_to (str):    full path to file. If not empty only return
 1589                                     snapshots which have exactly the same file
 1590                                     as this file
 1591         Returns:
 1592             list:                   filtered list of :py:class:`SID` objects
 1593         """
 1594         snapshotsFiltered = []
 1595 
 1596         base_full_path = base_sid.pathBackup(base_path)
 1597         if not os.path.lexists(base_full_path):
 1598             return []
 1599 
 1600         allSnapshotsList = [RootSnapshot(self.config)]
 1601         allSnapshotsList.extend(snapshotsList)
 1602 
 1603         #links
 1604         if os.path.islink(base_full_path):
 1605             targets = []
 1606 
 1607             for sid in allSnapshotsList:
 1608                 path = sid.pathBackup(base_path)
 1609 
 1610                 if os.path.lexists(path) and os.path.islink(path):
 1611                     if list_diff_only:
 1612                         target = os.readlink(path)
 1613                         if target in targets:
 1614                             continue
 1615                         targets.append(target)
 1616                     snapshotsFiltered.append(sid)
 1617 
 1618             return snapshotsFiltered
 1619 
 1620         #directories
 1621         if os.path.isdir(base_full_path):
 1622             for sid in allSnapshotsList:
 1623                 path = sid.pathBackup(base_path)
 1624 
 1625                 if os.path.exists(path) and not os.path.islink(path) and os.path.isdir(path):
 1626                     snapshotsFiltered.append(sid)
 1627 
 1628             return snapshotsFiltered
 1629 
 1630         #files
 1631         if not list_diff_only and not list_equal_to:
 1632             for sid in allSnapshotsList:
 1633                 path = sid.pathBackup(base_path)
 1634 
 1635                 if os.path.exists(path) and not os.path.islink(path) and os.path.isfile(path):
 1636                     snapshotsFiltered.append(sid)
 1637 
 1638             return snapshotsFiltered
 1639 
 1640         # check for duplicates
 1641         uniqueness = tools.UniquenessSet(flag_deep_check, follow_symlink = False, list_equal_to = list_equal_to)
 1642         for sid in allSnapshotsList:
 1643             path = sid.pathBackup(base_path)
 1644             if os.path.exists(path) and not os.path.islink(path) and os.path.isfile(path) and uniqueness.check(path):
 1645                 snapshotsFiltered.append(sid)
 1646 
 1647         return snapshotsFiltered
 1648 
 1649     #TODO: move this to config.Config
 1650     def rsyncRemotePath(self, path, use_mode = ['ssh', 'ssh_encfs'], quote = '"'):
 1651         """
 1652         Format the destination string for rsync depending on which profile is
 1653         used.
 1654 
 1655         Args:
 1656             path (str):         destination path
 1657             use_mode (list):    list of modes in which the result should
 1658                                 change to ``user@host:path`` instead of
 1659                                 just ``path``
 1660             quote (str):        use this to quote the path
 1661 
 1662         Returns:
 1663             str:                quoted ``path`` like '"/foo"'
 1664                                 or if the current mode is using ssh and
 1665                                 current mode is in ``use_mode`` a combination
 1666                                 of user, host and ``path``
 1667                                 like ''user@host:"/foo"''
 1668         """
 1669         mode = self.config.snapshotsMode()
 1670         if mode in ['ssh', 'ssh_encfs'] and mode in use_mode:
 1671             user = self.config.sshUser()
 1672             host = tools.escapeIPv6Address(self.config.sshHost())
 1673             return '%(u)s@%(h)s:%(q)s%(p)s%(q)s' %{'u': user,
 1674                                                    'h': host,
 1675                                                    'q': quote,
 1676                                                    'p': path}
 1677         else:
 1678             return path
 1679 
 1680     def deletePath(self, sid, path):
 1681         """
 1682         Delete ``path`` and all files and folder inside in snapshot ``sid``.
 1683 
 1684         Args:
 1685             sid (SID):  snapshot ID in which ``path`` should be deleted
 1686             path (str): path to delete
 1687         """
 1688         def errorHandler(fn, path, excinfo):
 1689             """
 1690             Error handler for :py:func:`deletePath`. This will fix permissions
 1691             and try again to remove the file.
 1692 
 1693             Args:
 1694                 fn (method):    callable which failed before
 1695                 path (str):     file to delete
 1696                 excinfo:        NotImplemented
 1697             """
 1698             dirname = os.path.dirname(path)
 1699             st = os.stat(dirname)
 1700             os.chmod(dirname, st.st_mode | stat.S_IWUSR)
 1701             st = os.stat(path)
 1702             os.chmod(path, st.st_mode | stat.S_IWUSR)
 1703             fn(path)
 1704 
 1705         full_path = sid.pathBackup(path)
 1706         dirname = os.path.dirname(full_path)
 1707         dir_st = os.stat(dirname)
 1708         os.chmod(dirname, dir_st.st_mode | stat.S_IWUSR)
 1709         if os.path.isdir(full_path) and not os.path.islink(full_path):
 1710             shutil.rmtree(full_path, onerror = errorHandler)
 1711         else:
 1712             st = os.stat(full_path)
 1713             os.chmod(full_path, st.st_mode | stat.S_IWUSR)
 1714             os.remove(full_path)
 1715         os.chmod(dirname, dir_st.st_mode)
 1716 
 1717     def createLastSnapshotSymlink(self, sid):
 1718         """
 1719         Create symlink 'last_snapshot' to snapshot ``sid``
 1720 
 1721         Args:
 1722             sid (SID):  snapshot that should be linked.
 1723 
 1724         Returns:
 1725             bool:       ``True`` if successful
 1726         """
 1727         if sid is None:
 1728             return
 1729         symlink = self.config.lastSnapshotSymlink()
 1730         try:
 1731             if os.path.islink(symlink):
 1732                 if os.path.basename(os.path.realpath(symlink)) == sid.sid:
 1733                     return True
 1734                 os.remove(symlink)
 1735             if os.path.exists(symlink):
 1736                 logger.error('Could not remove symlink %s' %symlink, self)
 1737                 return False
 1738             logger.debug('Create symlink %s => %s' %(symlink, sid), self)
 1739             os.symlink(sid.sid, symlink)
 1740             return True
 1741         except Exception as e:
 1742             logger.error('Failed to create symlink %s: %s' %(symlink, str(e)), self)
 1743             return False
 1744 
 1745     def flockExclusive(self):
 1746         """
 1747         Block :py:func:`backup` from other profiles or users
 1748         and run them serialized
 1749         """
 1750         if self.config.globalFlock():
 1751             logger.debug('Set flock %s' %self.GLOBAL_FLOCK, self)
 1752             self.flock = open(self.GLOBAL_FLOCK, 'w')
 1753             fcntl.flock(self.flock, fcntl.LOCK_EX)
 1754             #make it rw by all if that's not already done.
 1755             perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | \
 1756                     stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH
 1757             s = os.fstat(self.flock.fileno())
 1758             if not s.st_mode & perms == perms:
 1759                 logger.debug('Set flock permissions %s' %self.GLOBAL_FLOCK, self)
 1760                 os.fchmod(self.flock.fileno(), perms)
 1761 
 1762     def flockRelease(self):
 1763         """
 1764         Release lock so other snapshots can continue
 1765         """
 1766         if self.flock:
 1767             logger.debug('Release flock %s' %self.GLOBAL_FLOCK, self)
 1768             fcntl.fcntl(self.flock, fcntl.LOCK_UN)
 1769             self.flock.close()
 1770         self.flock = None
 1771 
 1772     def rsyncSuffix(self, includeFolders = None, excludeFolders = None):
 1773         """
 1774         Create suffixes for rsync.
 1775 
 1776         Args:
 1777             includeFolders (list):  folders to include. list of tuples (item, int)
 1778                                     Where ``int`` is ``0`` if ``item`` is a
 1779                                     folder or ``1`` if ``item`` is a file
 1780             excludeFolders (list):  list of folders to exclude
 1781 
 1782         Returns:
 1783             list:                   rsync include and exclude options
 1784         """
 1785         #create exclude patterns string
 1786         rsync_exclude = self.rsyncExclude(excludeFolders)
 1787 
 1788         #create include patterns list
 1789         rsync_include, rsync_include2 = self.rsyncInclude(includeFolders)
 1790 
 1791         encode = self.config.ENCODE
 1792         ret = ['--chmod=Du+wx']
 1793         ret.extend(['--exclude=' + i for i in (encode.exclude(self.config.snapshotsPath()),
 1794                                                encode.exclude(self.config._LOCAL_DATA_FOLDER),
 1795                                                encode.exclude(self.config._MOUNT_ROOT)
 1796                                                )])
 1797         # TODO: fix bug #561:
 1798         # after rsync_exclude we need to explicite include files inside excluded
 1799         # folders, recursive exclude folder-content again and finally add the
 1800         # rest from rsync_include2
 1801         ret.extend(rsync_include)
 1802         ret.extend(rsync_exclude)
 1803         ret.extend(rsync_include2)
 1804         ret.append('--exclude=*')
 1805         ret.append(encode.chroot)
 1806         return ret
 1807 
 1808     def rsyncExclude(self, excludeFolders = None):
 1809         """
 1810         Format exclude list for rsync
 1811 
 1812         Args:
 1813             excludeFolders (list):  list of folders to exclude
 1814 
 1815         Returns:
 1816             OrderedSet:             rsync exclude options
 1817         """
 1818         items = tools.OrderedSet()
 1819         encode = self.config.ENCODE
 1820         if excludeFolders is None:
 1821             excludeFolders = self.config.exclude()
 1822 
 1823         for exclude in excludeFolders:
 1824             exclude = encode.exclude(exclude)
 1825             if exclude is None:
 1826                 continue
 1827             items.add('--exclude=' + exclude)
 1828         return items
 1829 
 1830     def rsyncInclude(self, includeFolders = None):
 1831         """
 1832         Format include list for rsync. Returns a tuple of two include strings.
 1833         First string need to come before exclude, second after exclude.
 1834 
 1835         Args:
 1836             includeFolders (list):  folders to include. list of
 1837                                     tuples (item, int) where ``int`` is ``0``
 1838                                     if ``item`` is a folder or ``1`` if ``item``
 1839                                     is a file
 1840 
 1841         Returns:
 1842             tuple:                  two item tuple of
 1843                                     ``(OrderedSet('include1 opions'),
 1844                                     OrderedSet('include2 options'))``
 1845         """
 1846         items1 = tools.OrderedSet()
 1847         items2 = tools.OrderedSet()
 1848         encode = self.config.ENCODE
 1849         if includeFolders is None:
 1850             includeFolders = self.config.include()
 1851 
 1852         for include_folder in includeFolders:
 1853             folder = include_folder[0]
 1854 
 1855             if folder == "/":   # If / is selected as included folder it should be changed to ""
 1856                 #folder = ""    # because an extra / is added below. Patch thanks to Martin Hoefling
 1857                 items2.add('--include=/')
 1858                 items2.add('--include=/**')
 1859                 continue
 1860 
 1861             folder = encode.include(folder)
 1862             if include_folder[1] == 0:
 1863                 items2.add('--include={}/**'.format(folder))
 1864             else:
 1865                 items2.add('--include={}'.format(folder))
 1866                 folder = os.path.split(folder)[0]
 1867 
 1868             while True:
 1869                 if len(folder) <= 1:
 1870                     break
 1871                 items1.add('--include={}/'.format(folder))
 1872                 folder = os.path.split(folder)[0]
 1873 
 1874         return (items1, items2)
 1875 
 1876 class FileInfoDict(dict):
 1877     """
 1878     A :py:class:`dict` that maps a path (as :py:class:`bytes`) to a
 1879     tuple (:py:class:`int`, :py:class:`bytes`, :py:class:`bytes`).
 1880     """
 1881     def __init__(self):
 1882         # default permissions for /
 1883         # only used if fileinfo.bz2 does not contain a value for /
 1884         # when it was created with version <= 1.1.12
 1885         # bugfix for https://github.com/bit-team/backintime/issues/708
 1886         self[b'/'] = (16877, b'root', b'root')
 1887 
 1888     def __setitem__(self, key, value):
 1889         assert isinstance(key, bytes), "key '{}' is not bytes instance".format(key)
 1890         assert isinstance(value, tuple), "value '{}' is not tuple instance".format(value)
 1891         assert len(value) == 3, "value '{}' does not have 3 items".format(value)
 1892         assert isinstance(value[0], int), "first value '{}' is not int instance".format(value[0])
 1893         assert isinstance(value[1], bytes), "second value '{}' is not bytes instance".format(value[1])
 1894         assert isinstance(value[2], bytes), "third value '{}' is not bytes instance".format(value[2])
 1895         super(FileInfoDict, self).__setitem__(key, value)
 1896 
 1897 class SID(object):
 1898     """
 1899     Snapshot ID object used to gather all information for a snapshot
 1900 
 1901     Args:
 1902         date (:py:class:`str`, :py:class:`datetime.date` or :py:class:`datetime.datetime`):
 1903                                 used for creating this snapshot. str must be in
 1904                                 snapshot ID format (e.g 20151218-173512-123)
 1905         cfg (config.Config):    current config
 1906 
 1907     Raises:
 1908         ValueError:             if ``date`` is :py:class:`str` instance and
 1909                                 doesn't match the snapshot ID format
 1910                                 (20151218-173512-123 or 20151218-173512)
 1911         TypeError:              if ``date`` is not :py:class:`str`,
 1912                                 :py:class:`datetime.date` or
 1913                                 :py:class:`datetime.datetime` type
 1914     """
 1915     __cValidSID = re.compile(r'^\d{8}-\d{6}(?:-\d{3})?$')
 1916 
 1917     INFO     = 'info'
 1918     NAME     = 'name'
 1919     FAILED   = 'failed'
 1920     FILEINFO = 'fileinfo.bz2'
 1921     LOG      = 'takesnapshot.log.bz2'
 1922 
 1923     def __init__(self, date, cfg):
 1924         self.config = cfg
 1925         self.profileID = cfg.currentProfile()
 1926         self.isRoot = False
 1927 
 1928         if isinstance(date, datetime.datetime):
 1929             self.sid = '-'.join((date.strftime('%Y%m%d-%H%M%S'), self.config.tag(self.profileID)))
 1930             self.date = date
 1931         elif isinstance(date, datetime.date):
 1932             self.sid = '-'.join((date.strftime('%Y%m%d-000000'), self.config.tag(self.profileID)))
 1933             self.date = datetime.datetime.combine(date, datetime.datetime.min.time())
 1934         elif isinstance(date, str):
 1935             if self.__cValidSID.match(date):
 1936                 self.sid = date
 1937                 self.date = datetime.datetime(*self.split())
 1938             elif date == 'last_snapshot':
 1939                 raise LastSnapshotSymlink()
 1940             else:
 1941                 raise ValueError("'date' must be in snapshot ID format (e.g 20151218-173512-123)")
 1942         else:
 1943             raise TypeError("'date' must be an instance of str, datetime.date or datetime.datetime")
 1944 
 1945     def __repr__(self):
 1946         return self.sid
 1947 
 1948     def __eq__(self, other):
 1949         """
 1950         Compare snapshots based on self.sid
 1951 
 1952         Args:
 1953             other (:py:class:`SID`, :py:class:`str`):
 1954                         an other :py:class:`SID` or str instance
 1955 
 1956         Returns:
 1957             bool:       ``True`` if other is equal
 1958         """
 1959         if isinstance(other, SID):
 1960             return self.sid == other.sid and self.profileID == other.profileID
 1961         elif isinstance(other, str):
 1962             return self.sid == other
 1963         else:
 1964             return NotImplemented
 1965 
 1966     def __ne__(self, other):
 1967         return not self.__eq__(other)
 1968 
 1969     def __lt__(self, other):
 1970         """
 1971         Sort snapshots (alphabetical order) based on self.sid
 1972 
 1973         Args:
 1974             other (:py:class:`SID`, :py:class:`str`):
 1975                         an other :py:class:`SID` or str instance
 1976 
 1977         Returns:
 1978             bool:       ``True`` if other is lower
 1979         """
 1980         if isinstance(other, SID):
 1981             return self.sid < other.sid
 1982         elif isinstance(other, str) and self.__cValidSID.match(other):
 1983             return self.sid < other
 1984         else:
 1985             return NotImplemented
 1986 
 1987     def __le__(self, other):
 1988         if isinstance(other, SID):
 1989             return self.sid <= other.sid
 1990         elif isinstance(other, str) and self.__cValidSID.match(other):
 1991             return self.sid <= other
 1992         else:
 1993             return NotImplemented
 1994 
 1995     def __gt__(self, other):
 1996         if isinstance(other, SID):
 1997             return self.sid > other.sid
 1998         elif isinstance(other, str) and self.__cValidSID.match(other):
 1999             return self.sid > other
 2000         else:
 2001             return NotImplemented
 2002 
 2003     def __ge__(self, other):
 2004         if isinstance(other, SID):
 2005             return self.sid >= other.sid
 2006         elif isinstance(other, str) and self.__cValidSID.match(other):
 2007             return self.sid >= other
 2008         else:
 2009             return NotImplemented
 2010 
 2011     def __hash__(self):
 2012         return hash(self.sid + self.profileID)
 2013 
 2014     def split(self):
 2015         """
 2016         Split self.sid into a tuple of int's
 2017         with Year, Month, Day, Hour, Minute, Second
 2018 
 2019         Returns:
 2020             tuple:  tuple of 6 int
 2021         """
 2022         def split(s, e):
 2023             return int(self.sid[s:e])
 2024         return (split(0, 4), split(4, 6), split(6, 8), split(9, 11), split(11, 13), split(13, 15))
 2025 
 2026     @property
 2027     def displayID(self):
 2028         """
 2029         Snapshot ID in a user-readable format:
 2030         YYYY-MM-DD HH:MM:SS
 2031 
 2032         Returns:
 2033             str:    formated sID
 2034         """
 2035         return "{:04}-{:02}-{:02} {:02}:{:02}:{:02}".format(*self.split())
 2036 
 2037     @property
 2038     def displayName(self):
 2039         """
 2040         Combination of displayID, name and error indicator (if any)
 2041 
 2042         Returns:
 2043             str:    name
 2044         """
 2045         ret = self.displayID
 2046         name = self.name
 2047 
 2048         if name:
 2049             ret += ' - {}'.format(name)
 2050 
 2051         if self.failed:
 2052             ret += ' ({})'.format(_('WITH ERRORS !'))
 2053         return ret
 2054 
 2055     @property
 2056     def tag(self):
 2057         """
 2058         Snapshot ID's tag
 2059 
 2060         Returns:
 2061             str:    tag (last three digits)
 2062         """
 2063         return self.sid[16:]
 2064 
 2065     @property
 2066     def withoutTag(self):
 2067         """
 2068         Snapshot ID without tag
 2069 
 2070         Returns:
 2071             str:    YYYYMMDD-HHMMSS
 2072         """
 2073         return self.sid[0:15]
 2074 
 2075     def path(self, *path, use_mode = []):
 2076         """
 2077         Current path of this snapshot automatically altered for
 2078         remote/encrypted version of this path
 2079 
 2080         Args:
 2081             *path (str):        one or more folder/files to join at the end of
 2082                                 the path.
 2083             use_mode (list):    list of modes that should alter this path.
 2084                                 If the current mode is in this list, the path
 2085                                 will automatically altered for the
 2086                                 remote/encrypted version of this path.
 2087 
 2088         Returns:
 2089             str:                full snapshot path
 2090         """
 2091         path = [i.strip(os.sep) for i in path]
 2092         current_mode = self.config.snapshotsMode(self.profileID)
 2093         if 'ssh' in use_mode and current_mode == 'ssh':
 2094             return os.path.join(self.config.sshSnapshotsFullPath(self.profileID),
 2095                                 self.sid, *path)
 2096         if 'ssh_encfs' in use_mode and current_mode == 'ssh_encfs':
 2097             ret = os.path.join(self.config.sshSnapshotsFullPath(self.profileID),
 2098                                self.sid, *path)
 2099             return self.config.ENCODE.remote(ret)
 2100         return os.path.join(self.config.snapshotsFullPath(self.profileID),
 2101                             self.sid, *path)
 2102 
 2103     def pathBackup(self, *path, **kwargs):
 2104         """
 2105         'backup' folder inside snapshots path
 2106 
 2107         Args:
 2108             *path (str):        one or more folder/files to join at the end of
 2109                                 the path.
 2110             use_mode (list):    list of modes that should alter this path.
 2111                                 If the current mode is in this list, the path
 2112                                 will automatically altered for the
 2113                                 remote/encrypted version of this path.
 2114 
 2115         Returns:
 2116             str:                full snapshot path
 2117         """
 2118         return self.path('backup', *path, **kwargs)
 2119 
 2120     def makeDirs(self, *path):
 2121         """
 2122         Create snapshot directory
 2123 
 2124         Args:
 2125             *path (str):    one or more folder/files to join at the end
 2126                             of the path
 2127 
 2128         Returns:
 2129             bool:           ``True`` if successful
 2130         """
 2131         if not os.path.isdir(self.config.snapshotsFullPath(self.profileID)):
 2132             logger.error('Snapshots path {} doesn\'t exist. Unable to make dirs for snapshot ID {}'.format(
 2133                          self.config.snapshotsFullPath(self.profileID), self.sid),
 2134                          self)
 2135             return False
 2136 
 2137         return tools.makeDirs(self.pathBackup(*path))
 2138 
 2139     def exists(self):
 2140         """
 2141         ``True`` if the snapshot folder and the "backup" folder inside exist
 2142 
 2143         Returns:
 2144             bool:   ``True`` if exists
 2145         """
 2146         return os.path.isdir(self.path()) and os.path.isdir(self.pathBackup())
 2147 
 2148     def canOpenPath(self, path):
 2149         """
 2150         ``True`` if path is a file inside this snapshot
 2151 
 2152         Args:
 2153             path (str): path from local filesystem (no snapshot path)
 2154 
 2155         Returns:
 2156             bool:       ``True`` if file exists
 2157         """
 2158         fullPath = self.pathBackup(path)
 2159         if not os.path.exists(fullPath):
 2160             return False
 2161         if not os.path.islink(fullPath):
 2162             return True
 2163         basePath = self.pathBackup()
 2164         target = os.readlink(fullPath)
 2165         target = os.path.join(os.path.abspath(os.path.dirname(fullPath)), target)
 2166         return target.startswith(basePath)
 2167 
 2168     @property
 2169     def name(self):
 2170         """
 2171         Name of this snapshot
 2172 
 2173         Args:
 2174             name (str): new name of the snapshot
 2175 
 2176         Returns:
 2177             str:        name of this snapshot
 2178         """
 2179         nameFile = self.path(self.NAME)
 2180         if not os.path.isfile(nameFile):
 2181             return ''
 2182         try:
 2183             with open(nameFile, 'rt') as f:
 2184                 return f.read()
 2185         except Exception as e:
 2186             logger.debug('Failed to get snapshot {} name: {}'.format(
 2187                          self.sid, str(e)),
 2188                          self)
 2189 
 2190     @name.setter
 2191     def name(self, name):
 2192         nameFile = self.path(self.NAME)
 2193 
 2194         self.makeWritable()
 2195         try:
 2196             with open(nameFile, 'wt') as f:
 2197                 f.write(name)
 2198         except Exception as e:
 2199             logger.debug('Failed to set snapshot {} name: {}'.format(
 2200                          self.sid, str(e)),
 2201                          self)
 2202 
 2203     @property
 2204     def lastChecked(self):
 2205         """
 2206         Date when snapshot has finished last time.
 2207         This can be the end of creation of this snapshot or the last time when
 2208         this snapshot was checked against source without changes.
 2209 
 2210         Returns:
 2211             str:    date and time of last check (YYYY-MM-DD HH:MM:SS)
 2212         """
 2213         info = self.path(self.INFO)
 2214         if os.path.exists(info):
 2215             return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getatime(info)))
 2216         return self.displayID
 2217 
 2218     #using @property.setter would be confusing here as there is no value to give
 2219     def setLastChecked(self):
 2220         """
 2221         Set info files atime to current time to indicate this snapshot was
 2222         checked against source without changes right now.
 2223         """
 2224         info = self.path(self.INFO)
 2225         if os.path.exists(info):
 2226             os.utime(info, None)
 2227 
 2228     @property
 2229     def failed(self):
 2230         """
 2231         This snapshot has failed
 2232 
 2233         Args:
 2234             enable (bool): set or remove flag
 2235 
 2236         Returns:
 2237             bool:           ``True`` if flag is set
 2238         """
 2239         failedFile = self.path(self.FAILED)
 2240         return os.path.isfile(failedFile)
 2241 
 2242     @failed.setter
 2243     def failed(self, enable):
 2244         failedFile = self.path(self.FAILED)
 2245         if enable:
 2246             self.makeWritable()
 2247             try:
 2248                 with open(failedFile, 'wt') as f:
 2249                     f.write('')
 2250             except Exception as e:
 2251                 logger.debug('Failed to mark snapshot {} failed: {}'.format(
 2252                              self.sid, str(e)),
 2253                              self)
 2254         elif os.path.exists(failedFile):
 2255             os.remove(failedFile)
 2256 
 2257     @property
 2258     def info(self):
 2259         """
 2260         Load/save "info" file which contains additional information
 2261         about this snapshot (using configfile.ConfigFile)
 2262 
 2263         Args:
 2264             i (configfile.ConfigFile):  info that should be saved.
 2265 
 2266         Returns:
 2267             configfile.ConfigFile:  snapshots information
 2268         """
 2269         i = configfile.ConfigFile()
 2270         i.load(self.path(self.INFO))
 2271         return i
 2272 
 2273     @info.setter
 2274     def info(self, i):
 2275         assert isinstance(i, configfile.ConfigFile), 'i is not configfile.ConfigFile type: {}'.format(i)
 2276         i.save(self.path(self.INFO))
 2277 
 2278     @property
 2279     def fileInfo(self):
 2280         """
 2281         Load/save "fileinfo.bz2"
 2282 
 2283         Args:
 2284             d (FileInfoDict): dict of: {path: (permission, user, group)}
 2285 
 2286         Returns:
 2287             FileInfoDict:     dict of: {path: (permission, user, group)}
 2288         """
 2289         d = FileInfoDict()
 2290         infoFile = self.path(self.FILEINFO)
 2291         if not os.path.isfile(infoFile):
 2292             return d
 2293 
 2294         try:
 2295             with bz2.BZ2File(infoFile, 'rb') as fileinfo:
 2296                 for line in fileinfo:
 2297                     line = line.strip(b'\n')
 2298                     if not line:
 2299                         continue
 2300                     index = line.find(b'/')
 2301                     if index < 0:
 2302                         continue
 2303                     f = line[index:]
 2304                     if not f:
 2305                         continue
 2306                     info = line[:index].strip().split(b' ')
 2307                     if len(info) == 3:
 2308                         d[f] = (int(info[0]), info[1], info[2]) #perms, user, group
 2309         except (FileNotFoundError, PermissionError) as e:
 2310             logger.error('Failed to load {} from snapshot {}: {}'.format(
 2311                          self.FILEINFO, self.sid, str(e)),
 2312                          self)
 2313         return d
 2314 
 2315     @fileInfo.setter
 2316     def fileInfo(self, d):
 2317         assert isinstance(d, FileInfoDict), 'd is not FileInfoDict type: {}'.format(d)
 2318         try:
 2319             with bz2.BZ2File(self.path(self.FILEINFO), 'wb') as f:
 2320                 for path, info in d.items():
 2321                     f.write(b' '.join((str(info[0]).encode('utf-8', 'replace'),
 2322                                        info[1],
 2323                                        info[2],
 2324                                        path))
 2325                                        + b'\n')
 2326         except PermissionError as e:
 2327             logger.error('Failed to write {}: {}'.format(self.FILEINFO, str(e)))
 2328 
 2329     #TODO: use @property decorator
 2330     def log(self, mode = None, decode = None):
 2331         """
 2332         Load log from "takesnapshot.log.bz2"
 2333 
 2334         Args:
 2335             mode (int):                 Mode used for filtering. Take a look at
 2336                                         :py:class:`snapshotlog.LogFilter`
 2337             decode (encfstools.Decode): instance used for decoding lines or ``None``
 2338 
 2339         Yields:
 2340             str:                        filtered and decoded log lines
 2341         """
 2342         logFile = self.path(self.LOG)
 2343         logFilter = snapshotlog.LogFilter(mode, decode)
 2344         try:
 2345             with bz2.BZ2File(logFile, 'rb') as f:
 2346                 if logFilter.header:
 2347                     yield logFilter.header
 2348                 for line in f.readlines():
 2349                     line = logFilter.filter(line.decode('utf-8').rstrip('\n'))
 2350                     if not line is None:
 2351                         yield line
 2352         except Exception as e:
 2353             msg = ('Failed to get snapshot log from {}:'.format(logFile), str(e))
 2354             logger.debug(' '.join(msg), self)
 2355             for line in msg:
 2356                 yield line
 2357 
 2358     def setLog(self, log):
 2359         """
 2360         Write log to "takesnapshot.log.bz2"
 2361 
 2362         Args:
 2363             log: full snapshot log
 2364         """
 2365         if isinstance(log, str):
 2366             log = log.encode('utf-8', 'replace')
 2367         logFile = self.path(self.LOG)
 2368         try:
 2369             with bz2.BZ2File(logFile, 'wb') as f:
 2370                 f.write(log)
 2371         except Exception as e:
 2372             logger.error('Failed to write log into compressed file {}: {}'.format(
 2373                          logFile, str(e)),
 2374                          self)
 2375 
 2376     def makeWritable(self):
 2377         """
 2378         Make the snapshot path writable so we can change files inside
 2379 
 2380         Returns:
 2381             bool:   ``True`` if successful
 2382         """
 2383         path = self.path()
 2384         rw = os.stat(path).st_mode | stat.S_IWUSR
 2385         return os.chmod(path, rw)
 2386 
 2387 class GenericNonSnapshot(SID):
 2388     @property
 2389     def displayID(self):
 2390         return self.name
 2391 
 2392     @property
 2393     def displayName(self):
 2394         return self.name
 2395 
 2396     @property
 2397     def tag(self):
 2398         return self.name
 2399 
 2400     @property
 2401     def withoutTag(self):
 2402         return self.name
 2403 
 2404 class NewSnapshot(GenericNonSnapshot):
 2405     """
 2406     Snapshot ID object for 'new_snapshot' folder
 2407 
 2408     Args:
 2409         cfg (config.Config):    current config
 2410     """
 2411 
 2412     NEWSNAPSHOT    = 'new_snapshot'
 2413     SAVETOCONTINUE = 'save_to_continue'
 2414 
 2415     def __init__(self, cfg):
 2416         self.config = cfg
 2417         self.profileID = cfg.currentProfile()
 2418         self.isRoot = False
 2419 
 2420         self.sid = self.NEWSNAPSHOT
 2421         self.date = datetime.datetime(1, 1, 1)
 2422 
 2423         self.__le__ = self.__lt__
 2424         self.__ge__ = self.__gt__
 2425 
 2426     def __lt__(self, other):
 2427         return False
 2428 
 2429     def __gt__(self, other):
 2430         return True
 2431 
 2432     @property
 2433     def name(self):
 2434         """
 2435         Name of this snapshot
 2436 
 2437         Returns:
 2438             str:        name of this snapshot
 2439         """
 2440         return self.sid
 2441 
 2442     @property
 2443     def saveToContinue(self):
 2444         """
 2445         Check if 'save_to_continue' flag is set
 2446 
 2447         Args:
 2448             enable (bool): set or remove flag
 2449 
 2450         Returns:
 2451             bool:           ``True`` if flag is set
 2452         """
 2453         return os.path.exists(self.path(self.SAVETOCONTINUE))
 2454 
 2455     @saveToContinue.setter
 2456     def saveToContinue(self, enable):
 2457         flag = self.path(self.SAVETOCONTINUE)
 2458         if enable:
 2459             try:
 2460                 with open(flag, 'wt') as f:
 2461                     pass
 2462             except Exception as e:
 2463                 logger.error("Failed to set 'save_to_continue' flag: %s" %str(e))
 2464         elif os.path.exists(flag):
 2465             try:
 2466                 os.remove(flag)
 2467             except Exception as e:
 2468                 logger.error("Failed to remove 'save_to_continue' flag: %s" %str(e))
 2469 
 2470     @property
 2471     def hasChanges(self):
 2472         """
 2473         Check if there where changes in previous sessions.
 2474 
 2475         Returns:
 2476             bool:   ``True`` if there where changes
 2477         """
 2478         log = snapshotlog.SnapshotLog(self.config, self.profileID)
 2479         c = re.compile(r'^\[C\] ')
 2480         for line in log.get(mode = snapshotlog.LogFilter.CHANGES):
 2481             if c.match(line):
 2482                 return True
 2483         return False
 2484 
 2485 class RootSnapshot(GenericNonSnapshot):
 2486     """
 2487     Snapshot ID for the filesystem root folder ('/')
 2488 
 2489     Args:
 2490         cfg (config.Config):    current config
 2491     """
 2492     def __init__(self, cfg):
 2493         self.config = cfg
 2494         self.profileID = cfg.currentProfile()
 2495         self.isRoot = True
 2496 
 2497         self.sid = '/'
 2498         self.date = datetime.datetime(datetime.MAXYEAR, 12, 31)
 2499 
 2500         self.__le__ = self.__lt__
 2501         self.__ge__ = self.__gt__
 2502 
 2503     def __lt__(self, other):
 2504         return False
 2505 
 2506     def __gt__(self, other):
 2507         return True
 2508 
 2509     @property
 2510     def name(self):
 2511         """
 2512         Name of this snapshot
 2513 
 2514         Returns:
 2515             str:        name of this snapshot
 2516         """
 2517         return _('Now')
 2518 
 2519     def path(self, *path, use_mode = []):
 2520         """
 2521         Current path of this snapshot automatically altered for
 2522         remote/encrypted version of this path
 2523 
 2524         Args:
 2525             *path (str):        one or more folder/files to join at the end of
 2526                                 the path.
 2527             use_mode (list):    list of modes that should alter this path.
 2528                                 If the current mode is in this list, the path
 2529                                 will automatically altered for the
 2530                                 remote/encrypted version of this path.
 2531 
 2532         Returns:
 2533             str:                full snapshot path
 2534         """
 2535         current_mode = self.config.snapshotsMode(self.profileID)
 2536         if 'ssh_encfs' in use_mode and current_mode == 'ssh_encfs':
 2537             if path:
 2538                 path = self.config.ENCODE.remote(os.path.join(*path))
 2539             return os.path.join(self.config.ENCODE.chroot, path)
 2540         else:
 2541             return os.path.join(os.sep, *path)
 2542 
 2543 def iterSnapshots(cfg, includeNewSnapshot = False):
 2544     """
 2545     Iterate over snapshots in current snapshot path. Use this in a 'for' loop
 2546     for faster processing than list object
 2547 
 2548     Args:
 2549         cfg (config.Config):        current config
 2550         includeNewSnapshot (bool):  include a NewSnapshot instance if
 2551                                     'new_snapshot' folder is available.
 2552 
 2553     Yields:
 2554         SID:                        snapshot IDs
 2555     """
 2556     path = cfg.snapshotsFullPath()
 2557     if not os.path.exists(path):
 2558         return None
 2559     for item in os.listdir(path):
 2560         if item == NewSnapshot.NEWSNAPSHOT:
 2561             newSid = NewSnapshot(cfg)
 2562             if newSid.exists() and includeNewSnapshot:
 2563                 yield newSid
 2564             continue
 2565         try:
 2566             sid = SID(item, cfg)
 2567             if sid.exists():
 2568                 yield sid
 2569         except Exception as e:
 2570             if not isinstance(e, LastSnapshotSymlink):
 2571                 logger.debug("'{}' is no snapshot ID: {}".format(item, str(e)))
 2572 
 2573 def listSnapshots(cfg, includeNewSnapshot = False, reverse = True):
 2574     """
 2575     List of snapshots in current snapshot path.
 2576 
 2577     Args:
 2578         cfg (config.Config):        current config (config.Config instance)
 2579         includeNewSnapshot (bool):  include a NewSnapshot instance if
 2580                                     'new_snapshot' folder is available
 2581         reverse (bool):             sort reverse
 2582 
 2583     Returns:
 2584         list:                       list of :py:class:`SID` objects
 2585     """
 2586     ret = list(iterSnapshots(cfg, includeNewSnapshot))
 2587     ret.sort(reverse = reverse)
 2588     return ret
 2589 
 2590 def lastSnapshot(cfg):
 2591     """
 2592     Most recent snapshot.
 2593 
 2594     Args:
 2595         cfg (config.Config):    current config (config.Config instance)
 2596 
 2597     Returns:
 2598         SID:                    most recent snapshot ID
 2599     """
 2600     sids = listSnapshots(cfg)
 2601     if sids:
 2602         return sids[0]
 2603 
 2604 if __name__ == '__main__':
 2605     config = config.Config()
 2606     snapshots = Snapshots(config)
 2607     snapshots.backup()