"Fossies" - the Fresh Open Source Software Archive

Member "Tardis-1.2.1/src/Tardis/RemoteDB.py" (9 Jun 2021, 18454 Bytes) of package /linux/privat/Tardis-1.2.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "RemoteDB.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 1.1.5_vs_1.2.1.

    1 # vim: set et sw=4 sts=4 fileencoding=utf-8:
    2 #
    3 # Tardis: A Backup System
    4 # Copyright 2013-2020, Eric Koldinger, All Rights Reserved.
    5 # kolding@washington.edu
    6 #
    7 # Redistribution and use in source and binary forms, with or without
    8 # modification, are permitted provided that the following conditions are met:
    9 #
   10 #     * Redistributions of source code must retain the above copyright
   11 #       notice, this list of conditions and the following disclaimer.
   12 #     * Redistributions in binary form must reproduce the above copyright
   13 #       notice, this list of conditions and the following disclaimer in the
   14 #       documentation and/or other materials provided with the distribution.
   15 #     * Neither the name of the copyright holder nor the
   16 #       names of its contributors may be used to endorse or promote products
   17 #       derived from this software without specific prior written permission.
   18 #
   19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   20 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   21 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   22 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   23 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   24 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   25 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   26 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   27 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   28 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   29 # POSSIBILITY OF SUCH DAMAGE.
   30 
   31 import logging
   32 import tempfile
   33 import sys
   34 import urllib.request, urllib.parse, urllib.error
   35 import functools
   36 import base64
   37 
   38 from binascii import unhexlify
   39 
   40 import requests
   41 import requests_cache
   42 
   43 import Tardis
   44 import Tardis.TardisDB as TardisDB
   45 
   46 
   47 requests_cache.install_cache(backend='memory', expire_after=30.0)
   48 
   49 # Define a decorator that will wrap our functions in a retry mechanism
   50 # so that if the connection to the server fails, we can automatically
   51 # reconnect.
   52 def reconnect(func):
   53     #print "Decorating ", str(func)
   54     @functools.wraps(func)
   55     def doit(self, *args, **kwargs):
   56         try:
   57             # Try the original function
   58             return func(self, *args, **kwargs)
   59         except requests.HTTPError as e:
   60             # Got an exception.  If it's ' 401 (not authorized)
   61             # reconnecton, and try it again
   62             logger=logging.getLogger('Reconnect')
   63             logger.info("Got HTTPError: %s", e)
   64             if e.response.status_code == 401:
   65                 logger.info("Reconnecting")
   66                 self.connect()
   67                 logger.info("Retrying %s(%s %s)", str(func), str(args), str(kwargs))
   68                 return func(self, *args, **kwargs)
   69             raise e
   70     return doit
   71 
   72 def fs_encode(val):
   73     """ Turn filenames into str's (ie, series of bytes) rather than Unicode things """
   74     if not isinstance(val, bytes):
   75         #return val.encode(sys.getfilesystemencoding())
   76         return val.encode(sys.getfilesystemencoding())
   77     else:
   78         return val
   79 
   80 class RemoteDB(object):
   81     """ Proxy class to retrieve objects via HTTP queries """
   82     session = None
   83     headers = {}
   84     prevBackupSet = None
   85 
   86     def __init__(self, url, host, prevSet=None, extra=None, compress=True, verify=False):
   87         self.logger=logging.getLogger('Remote')
   88         self.logger.debug("-> %s %s", url, host)
   89         self.baseURL = url
   90         if not url.endswith('/'):
   91             self.baseURL += '/'
   92 
   93         self.verify = verify
   94         self.host = host
   95         self.headers = { "user-agent": "TardisDB-" + Tardis.__versionstring__ }
   96 
   97         self.logger.debug("Connection to %s", url)
   98 
   99         # Disable insecure requests warning, if verify is disabled.
  100         # Generates too much output
  101         if not self.verify:
  102             requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
  103 
  104         if compress:
  105             self.headers['Accept-Encoding'] = "deflate"
  106 
  107         self.prevSet = prevSet
  108         self.connect()
  109 
  110 
  111     #def __del__(self):
  112     #    try:
  113     #        self.close()
  114     #    except Exception as e:
  115     #        self.logger.warning("Caught exception closing: " + str(e))
  116     #        self.logger.exception(e)
  117 
  118     def connect(self):
  119         self.logger.debug("Creating new connection to %s for %s", self.baseURL, self.host)
  120         self.session = requests.Session()
  121         self.session.verify = self.verify
  122 
  123         postData = { 'host': self.host }
  124         self.loginData = postData
  125 
  126         response = self.session.post(self.baseURL + "login", data=postData)
  127         response.raise_for_status()
  128 
  129     def needsAuthentication(self):
  130         r = self.session.get(self.baseURL + "needsAuthentication")
  131         r.raise_for_status()
  132         return r.json()
  133 
  134     def authenticate1(self, uname, srpValueA):
  135         postData = {
  136             'srpUname':  str(base64.b64encode(bytes(uname, 'utf8')), 'utf8'),
  137             'srpValueA': str(base64.b64encode(srpValueA), 'utf8')
  138         }
  139         response = self.session.post(self.baseURL + 'authenticate1', data=postData)
  140         # Check for "not authenticated", which indicates authentication failed.
  141         if response.status_code == 401:
  142             raise TardisDB.AuthenticationFailed("Bad Password")
  143         # Catch other errors.
  144         response.raise_for_status()
  145         data = response.json()
  146         srpValueS = base64.b64decode(data['srpValueS'])
  147         srpValueB = base64.b64decode(data['srpValueB'])
  148         return srpValueS, srpValueB
  149         
  150     def authenticate2(self, srpValueM):
  151         postData = {
  152             'srpValueM': str(base64.b64encode(srpValueM), 'utf8')
  153         }
  154         response = self.session.post(self.baseURL + 'authenticate2', data=postData)
  155         # Check for "not authenticated", which indicates authentication failed.
  156         if response.status_code == 401:
  157             raise TardisDB.AuthenticationFailed("Bad Password")
  158         # Catch other errors.
  159         response.raise_for_status()
  160         data = response.json()
  161         srpValueH = base64.b64decode(data['srpValueH'])
  162         return srpValueH
  163 
  164     def close(self):
  165         self.logger.debug("Closing session")
  166         if self.session:
  167             r = self.session.get(self.baseURL + "close", headers=self.headers)
  168         r.raise_for_status()
  169         self.session = None
  170         return r.json()
  171 
  172     def _setPrevBackupSet(self):
  173         if self.prevSet:
  174             f = self.getBackupSetInfo(self.prevSet)
  175             if f:
  176                 self.prevBackupSet = f['backupset']
  177                 self.prevBackupName = f['name']
  178         else:
  179             b = self.lastBackupSet()
  180             self.prevBackupSet  = b['backupset']
  181             self.prevBackupName = b['name']
  182         self.logger.debug("Last Backup Set: %s %d", self.prevBackupName, self.prevBackupSet)
  183         return self.prevBackupSet
  184 
  185     def _bset(self, current):
  186         """ Determine the backupset we're being asked about.
  187             True == current, false = previous, otherwise a number is returned
  188         """
  189         if type(current) is bool:
  190             if current:
  191                 return str(None)
  192             else:
  193                 if self.prevBackupSet:
  194                     return str(self.prevBackupSet)
  195                 else:
  196                     return str(self._setPrevBackupSet())
  197         else:
  198             return str(current)
  199 
  200     @reconnect
  201     def listBackupSets(self):
  202         r = self.session.get(self.baseURL + "listBackupSets", headers=self.headers)
  203         r.raise_for_status()
  204         for i in r.json():
  205             self.logger.debug("Returning %s", str(i))
  206             #i['name'] = (i['name'])
  207             yield i
  208 
  209     @reconnect
  210     def lastBackupSet(self, completed=True):
  211         r = self.session.get(self.baseURL + "lastBackupSet/" + str(int(completed)), headers=self.headers)
  212         r.raise_for_status()
  213         return r.json()
  214 
  215     @reconnect
  216     def getBackupSetInfo(self, name):
  217         name = urllib.parse.quote(name, '')
  218         r = self.session.get(self.baseURL + "getBackupSetInfo/" + name, headers=self.headers)
  219         r.raise_for_status()
  220         return r.json()
  221 
  222     @reconnect
  223     def getBackupSetInfoById(self, bset):
  224         r = self.session.get(self.baseURL + "getBackupSetInfoById/" + str(bset), headers=self.headers)
  225         r.raise_for_status()
  226         return r.json()
  227 
  228     @reconnect
  229     def getBackupSetDetails(self, name):
  230         name = urllib.parse.quote(str(name), '')
  231         r = self.session.get(self.baseURL + "getBackupSetDetails/" + str(name), headers=self.headers)
  232         r.raise_for_status()
  233         return r.json()
  234 
  235     @reconnect
  236     def getBackupSetInfoForTime(self, time):
  237         r = self.session.get(self.baseURL + "getBackupSetInfoForTime/" + str(time), headers=self.headers)
  238         r.raise_for_status()
  239         return r.json()
  240 
  241     @reconnect
  242     def getFileInfoByName(self, name, parent, current=True):
  243         bset = self._bset(current)
  244         (inode, device) = parent
  245         name = urllib.parse.quote(name, '/')
  246         r = self.session.get(self.baseURL + "getFileInfoByName/" + bset + "/" + str(device) + "/" + str(inode) + "/" + name, headers=self.headers)
  247         r.raise_for_status()
  248         return r.json()
  249 
  250 
  251     @reconnect
  252     def getFileInfoByInode(self, node, current=True):
  253         bset = self._bset(current)
  254         (inode, device) = node
  255         r = self.session.get(self.baseURL + "getFileInfoByInode/" + bset + "/" + str(device) + "/" + str(inode), headers=self.headers)
  256         r.raise_for_status()
  257         return r.json()
  258 
  259     @reconnect
  260     def getFileInfoByChecksum(self, checksum, current=False):
  261         bset = self._bset(current)
  262         r = self.session.get(self.baseURL + "getFileInfoByChecksum/" + bset + "/" + str(checksum), headers=self.headers)
  263         r.raise_for_status()
  264         return r.json()
  265 
  266     @reconnect
  267     def getFileInfoByPath(self, path, current=False):
  268         bset = self._bset(current)
  269         if not path.startswith('/'):
  270             path = '/' + path
  271         path = urllib.parse.quote(path, '/')
  272         r = self.session.get(self.baseURL + "getFileInfoByPath/" + bset + path, headers=self.headers)
  273         r.raise_for_status()
  274         return r.json()
  275 
  276     @reconnect
  277     def getFileInfoByPathForRange(self, path, first, last, permchecker=None):
  278         if not path.startswith('/'):
  279             path = '/' + path
  280         path = urllib.parse.quote(path, '/')
  281         r = self.session.get(self.baseURL + "getFileInfoByPathForRange/" + str(first) + '/' + str(last) + path, headers=self.headers)
  282         r.raise_for_status()
  283         for i in r.json():
  284             yield i
  285 
  286 
  287     @reconnect
  288     def readDirectory(self, dirNode, current=False):
  289         (inode, device) = dirNode
  290         bset = self._bset(current)
  291         r = self.session.get(self.baseURL + "readDirectory/" + bset + "/" + str(device) + "/" + str(inode), headers=self.headers)
  292         r.raise_for_status()
  293         for i in r.json():
  294             i['name'] = fs_encode(i['name'])
  295             yield i
  296 
  297     @reconnect
  298     def readDirectoryForRange(self, dirNode, first, last):
  299         (inode, device) = dirNode
  300         r = self.session.get(self.baseURL + "readDirectoryForRange/" + str(device) + "/" + str(inode) + "/" + str(first) + "/" + str(last), headers=self.headers)
  301         r.raise_for_status()
  302         for i in r.json():
  303             i['name'] = fs_encode(i['name'])
  304             yield i
  305 
  306     @reconnect
  307     def checkPermissions(self, path, checker, current=False):
  308         bset = self._bset(current)
  309         r = self.session.get(self.baseURL + "getFileInfoForPath/" + bset + "/" + path, headers=self.headers)
  310         r.raise_for_status()
  311         for i in r.json():
  312             ret = checker(i['uid'], i['gid'], i['mode'])
  313             if not ret:
  314                 return False
  315         return True
  316 
  317     @reconnect
  318     def getNewFiles(self, bset, other):
  319         r = self.session.get(self.baseURL + "getNewFiles/" + str(bset) + "/" + str(other), headers=self.headers)
  320         r.raise_for_status()
  321         for i in r.json():
  322             i['name'] = str(i['name'])
  323             yield i
  324 
  325     @reconnect
  326     def getChecksumByPath(self, path, current=False, permchecker=None):
  327         bset = self._bset(current)
  328         if not path.startswith('/'):
  329             path = '/' + path
  330         path = urllib.parse.quote(path, '/')
  331         r = self.session.get(self.baseURL + "getChecksumByPath/" + bset + path, headers=self.headers)
  332         r.raise_for_status()
  333         return r.json()
  334 
  335     @reconnect
  336     def getChecksumInfo(self, checksum):
  337         r = self.session.get(self.baseURL + "getChecksumInfo/" + checksum, headers=self.headers)
  338         r.raise_for_status()
  339         return r.json()
  340 
  341     @reconnect
  342     def getChecksumInfoChain(self, checksum):
  343         r = self.session.get(self.baseURL + "getChecksumInfoChain/" + checksum, headers=self.headers)
  344         r.raise_for_status()
  345         return r.json()
  346 
  347     @reconnect
  348     def getChecksumInfoChainByPath(self, name, bset, permchecker=None):
  349         if not name.startswith('/'):
  350             name = '/' + name
  351         name = urllib.parse.quote(name, '/')
  352         r = self.session.get(self.baseURL + "getChecksumInfoChainByPath/" + str(bset) + name, headers=self.headers)
  353         r.raise_for_status()
  354         return r.json()
  355 
  356     @reconnect
  357     def getFirstBackupSet(self, name, current=False):
  358         bset = self._bset(current)
  359         r = self.session.get(self.baseURL + "getFirstBackupSet/" + str(bset) + "/" + name, headers=self.headers)
  360         r.raise_for_status()
  361         return r.json()
  362 
  363     @reconnect
  364     def getChainLength(self, checksum):
  365         r = self.session.get(self.baseURL + "getChainLength/" + checksum, headers=self.headers)
  366         r.raise_for_status()
  367         return r.json()
  368 
  369     @reconnect
  370     def getConfigValue(self, name, default=None):
  371         r = self.session.get(self.baseURL + "getConfigValue/" + name, headers=self.headers)
  372         r.raise_for_status()
  373         if r.json() == None:
  374             return default
  375         else:
  376             return r.json()
  377 
  378     @reconnect
  379     def setConfigValue(self, name, value):
  380         r = self.session.get(self.baseURL + "setConfigValue/" + name + "/" + value, headers=self.headers)
  381         r.raise_for_status()
  382         return r.json()
  383 
  384     @reconnect
  385     def setPriority(self, backupset, priority):
  386         r = self.session.get(self.baseURL + "setPriority/" + str(backupset) + "/" + str(priority), headers=self.headers)
  387         r.raise_for_status()
  388         return r.json()
  389     
  390     @reconnect
  391     def setBackupsetName(self, name, priority, current=True):
  392         backupset = self._bset(current)
  393         r = self.session.get(self.baseURL + "setBackupSetName/" + str(backupset) + "/" + name + "/" + str(priority), headers=self.headers)
  394         r.raise_for_status()
  395 
  396     @reconnect
  397     def getKeys(self):
  398         fnKey = self.getConfigValue('FilenameKey')
  399         cnKey = self.getConfigValue('ContentKey')
  400         #self.logger.info("Got keys: %s %s", fnKey, cnKey)
  401         return (fnKey, cnKey)
  402 
  403     @reconnect
  404     def setKeys(self, salt, vkey, fKey, cKey):
  405         postData = { 'Salt': base64.b64encode(salt), 'SrpVKey': base64.b64encode(vkey), 'FilenameKey': fKey, 'ContentKey': cKey }
  406         response = self.session.post(self.baseURL + "setKeys", data=postData)
  407         response.raise_for_status()
  408 
  409     @reconnect
  410     def setSrpValues(self, salt, vkey):
  411         postData = { 'salt': salt, 'vkey': vkey }
  412         response = self.session.post(self.baseURL + "setSrpValues", data=postData)
  413         response.raise_for_status()
  414         return response.json()
  415 
  416     @reconnect
  417     def getSrpValues(self):
  418         salt = unhexlify(self.getConfigValue('SRPSalt'))
  419         vKey = unhexlify(self.getConfigValue('SRPVkey'))
  420         return (salt, vKey)
  421 
  422     @reconnect
  423     def getCryptoScheme(self):
  424         return self.getConfigValue('CryptoScheme')
  425 
  426     @reconnect
  427     def listPurgeSets(self, priority, timestamp, current=False):
  428         bset = self._bset(current)
  429         r = self.session.get(self.baseURL + "listPurgeSets/" + bset + '/' + str(priority) + '/' + str(timestamp), headers=self.headers)
  430         r.raise_for_status()
  431         return r.json()
  432 
  433     @reconnect
  434     def listPurgeIncomplete(self, priority, timestamp, current=False):
  435         bset = self._bset(current)
  436         r = self.session.get(self.baseURL + "listPurgeIncomplete/" + bset + '/' + str(priority) + '/' + str(timestamp), headers=self.headers)
  437         r.raise_for_status()
  438         return r.json()
  439 
  440     @reconnect
  441     def purgeSets(self, priority, timestamp, current=False):
  442         bset = self._bset(current)
  443         r = self.session.get(self.baseURL + "purgeSets/" + bset + '/' + str(priority) + '/' + str(timestamp), verify=self.verify, headers=self.headers)
  444         r.raise_for_status()
  445         return r.json()
  446 
  447     @reconnect
  448     def purgeIncomplete(self, priority, timestamp, current=False):
  449         bset = self._bset(current)
  450         r = self.session.get(self.baseURL + "purgeIncomplete/" + bset + '/' + str(priority) + '/' + str(timestamp), headers=self.headers)
  451         r.raise_for_status()
  452         return r.json()
  453 
  454     @reconnect
  455     def deleteBackupSet(self, bset):
  456         r = self.session.get(self.baseURL + "deleteBackupSet/" + str(bset))
  457         r.raise_for_status()
  458         return r.json()
  459 
  460     @reconnect
  461     def listOrphanChecksums(self, isFile):
  462         r = self.session.get(self.baseURL + 'listOrphanChecksums/' + str(int(isFile)), headers=self.headers)
  463         r.raise_for_status()
  464         return r.json()
  465 
  466     @reconnect
  467     def open(self, checksum, mode, streaming=True):
  468         if mode[0] != 'r':
  469             raise PermissionError("Read only file system")
  470 
  471         r = self.session.get(self.baseURL + "getFileData/" + checksum, stream=True)
  472         r.raise_for_status()
  473         #self.logger.debug("%s", str(r.headers))
  474 
  475         if streaming:
  476             return r.raw
  477         else:
  478             temp = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
  479 
  480             for chunk in r.iter_content(chunk_size=1024 * 1024):
  481                 temp.write(chunk)
  482 
  483             temp.seek(0)
  484             return temp
  485 
  486     @reconnect
  487     def removeOrphans(self):
  488         r = self.session.get(self.baseURL + "removeOrphans", verify=self.verify, headers=self.headers)
  489         r.raise_for_status()
  490         return r.json()