"Fossies" - the Fresh Open Source Software Archive

Member "Tardis-1.2.1/tools/encryptDB.py" (9 Jun 2021, 14848 Bytes) of package /linux/privat/Tardis-1.2.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "encryptDB.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 1.1.5_vs_1.2.1.

    1 #! /usr/bin/env python3
    2 # vim: set et sw=4 sts=4 fileencoding=utf-8:
    3 #
    4 # Tardis: A Backup System
    5 # Copyright 2013-2020, Eric Koldinger, All Rights Reserved.
    6 # kolding@washington.edu
    7 #
    8 # Redistribution and use in source and binary forms, with or without
    9 # modification, are permitted provided that the following conditions are met:
   10 #
   11 #     * Redistributions of source code must retain the above copyright
   12 #       notice, this list of conditions and the following disclaimer.
   13 #     * Redistributions in binary form must reproduce the above copyright
   14 #       notice, this list of conditions and the following disclaimer in the
   15 #       documentation and/or other materials provided with the distribution.
   16 #     * Neither the name of the copyright holder nor the
   17 #       names of its contributors may be used to endorse or promote products
   18 #       derived from this software without specific prior written permission.
   19 #
   20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   21 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   24 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   25 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   26 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   27 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   28 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   29 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   30 # POSSIBILITY OF SUCH DAMAGE.
   31 
   32 from Tardis import Defaults, Util, TardisDB, TardisCrypto, CacheDir, librsync, Regenerator, Config
   33 import sqlite3
   34 import argparse, logging
   35 import os.path
   36 import os
   37 import sys
   38 import base64
   39 import hashlib
   40 import sys
   41 import progressbar
   42 
   43 logger = None
   44 
   45 def encryptFilenames(db, crypto):
   46     conn = db.conn
   47     c = conn.cursor()
   48     c2 = conn.cursor()
   49     names = 0
   50     r = c.execute("SELECT COUNT(*) FROM Names")
   51     z = r.fetchone()[0]
   52     logger.info("Encrypting %d filenames", z)
   53     with progressbar.ProgressBar(max_value=z) as bar:
   54         try:
   55             r = c.execute("SELECT Name, NameID FROM Names")
   56             while True:
   57                 row = r.fetchone()
   58                 if row is None:
   59                     break
   60                 (name, nameid) = row
   61                 newname = crypto.encryptFilename(name)
   62                 c2.execute('UPDATE Names SET Name = ? WHERE NameID = ?', (newname, nameid))
   63                 names = names + 1
   64                 bar.update(names)
   65             conn.commit()
   66         except Exception as e:
   67             logger.error("Caught exception encrypting filename %s: %s", name, str(e))
   68             logger.exception(e)
   69             conn.rollback()
   70     logger.info("Encrypted %d names", names)
   71 
   72 def encryptFile(checksum, cacheDir, cipher, iv, output = None):
   73     f = cacheDir.open(checksum, 'rb')
   74     if output == None:
   75         output = checksum + '.enc'
   76     o = cacheDir.open(output, 'wb')
   77     o.write(iv)
   78     nb = len(iv)
   79     cipher.update(iv)
   80     # Encrypt the chunks
   81     for chunk, eof in Util._chunks(f, 64 * 1024):
   82         ochunk = cipher.encrypt(chunk)
   83         o.write(ochunk)
   84         nb = nb + len(ochunk)
   85 
   86     # add the digest chunk
   87     ochunk = cipher.digest()
   88     o.write(ochunk)
   89     nb = nb + len(ochunk)
   90 
   91     o.close()
   92     f.close()
   93 
   94     return nb
   95 
   96 def generateFullFileInfo(checksum, regenerator, cacheDir, nameMac, signature=True, basis=None):
   97     i = regenerator.recoverChecksum(checksum, basisFile=basis)
   98     sig = None
   99     logger.debug("    Generating HMAC for %s.  Generating signature: %s", checksum, str(signature))
  100     if signature:
  101         output = cacheDir.open(checksum + ".sig", "wb+")
  102         sig = librsync.SignatureJob(output)
  103 
  104     data = i.read(64 * 1024)
  105     while data:
  106         nameMac.update(data)
  107         if sig:
  108             sig.step(data)
  109         data = i.read(64 * 1024)
  110     # Return a handle on the full file object.  Allows it to be reused in the next step
  111     i.close()
  112     return i
  113 
  114 suffixes = ['','KB','MB','GB', 'TB', 'PB']
  115 
  116 numFiles = 0
  117 
  118 def processFile(cksInfo, regenerator, cacheDir, db, crypto, pbar, basis=None):
  119     global numFiles
  120     newCks = ''
  121     try:
  122         conn = db.conn
  123         c2 = conn.cursor()
  124         checksum = cksInfo['checksum']
  125         if cksInfo['encrypted']:
  126             logger.info("    Skipping  %s", checksum)
  127             return None
  128 
  129         pbar.update(numFiles)
  130 
  131         #logger.info("  Processing %s (%s, %s)", checksum, Util.fmtSize(cksInfo['size'], formats = suffixes), Util.fmtSize(cksInfo['diskSize'], formats = suffixes))
  132         signature = not cacheDir.exists(checksum + ".sig")
  133         
  134         nameHmac = crypto.getHash()
  135         retFile = generateFullFileInfo(checksum, regenerator, cacheDir, nameHmac, signature, basis)
  136         if basis:
  137             basis.close()
  138         newCks = nameHmac.hexdigest()
  139         
  140         #logger.info("    Hashed     %s => %s (%s, %s)", checksum, newCks, Util.fmtSize(cksInfo['size'], formats = suffixes), Util.fmtSize(cksInfo['diskSize'], formats = suffixes))
  141         
  142         iv = crypto.getIV()
  143         cipher = crypto.getContentCipher(iv)
  144         hmac = crypto.getHash(func=hashlib.sha512)
  145         fSize = encryptFile(checksum, cacheDir, cipher, iv, output=newCks)
  146         #logger.info("    Encrypted  %s => %s (%s)", checksum, newCks, Util.fmtSize(fSize, formats = ['','KB','MB','GB', 'TB', 'PB']))
  147 
  148         #cacheDir.link(checksum + '.enc', newCks, soft=False)
  149         #cacheDir.link(checksum + ".sig", newCks + ".sig", soft=False)
  150         numFiles += 1
  151         cacheDir.move(checksum + ".sig", newCks + ".sig")
  152         logger.debug("    Moved sig file, updating database")
  153 
  154         c2.execute('UPDATE CheckSums SET Encrypted = 1, DiskSize = :size, Checksum = :newcks WHERE Checksum = :cks',
  155                     {"size": fSize, "newcks": newCks, "cks": checksum})
  156         c2.execute('UPDATE CheckSums SET Basis = :newcks WHERE Basis = :cks', {"newcks": newCks, "cks": checksum})
  157 
  158         logger.debug("    Ready to commit")
  159         conn.commit()
  160         logger.debug("    Commit complete, removing files")
  161         cacheDir.removeSuffixes(checksum, ['.meta', '.enc', '.sig', '.basis', ''])
  162         logger.debug("    Done with %s", checksum)
  163         return retFile
  164     except Exception as e:
  165         conn.rollback()
  166         logger.error("Unable to convert checksum: %s (%s) :: %s", checksum, newCks, e)
  167         logger.exception(e)
  168         return None
  169 
  170 def encryptFilesAtLevel(db, crypto, cacheDir, chainlength, pbar):
  171     logger.info("Encrypting files with chainlength = %d", chainlength)
  172     conn = db.conn
  173     c = conn.cursor()
  174     regenerator = Regenerator.Regenerator(cacheDir, db, crypto)
  175 
  176     r = c.execute("SELECT Checksum, Size, Basis, Compressed FROM Checksums WHERE Encrypted = 0 AND IsFile = 1 AND ChainLength = :chainlength ORDER BY CheckSum", {"chainlength": chainlength})
  177     for row in r.fetchall():
  178         try:
  179             checksum = row[0]
  180             #logger.info("Encrypting Parent %s", checksum)
  181             chain = db.getChecksumInfoChain(checksum)
  182             bFile = None
  183             while chain:
  184                 cksInfo = chain.pop()
  185                 bFile = processFile(cksInfo, regenerator, cacheDir, db, crypto, pbar, bFile)
  186         except Exception as e:
  187             logger.error("Error processing checksum: %s", checksum)
  188             logger.exception(e)
  189             #raise e
  190 
  191 def encryptFiles(db, crypto, cacheDir):
  192     conn = db.conn
  193     r = conn.execute("SELECT MAX(ChainLength) FROM CheckSums")
  194     mLevel = r.fetchone()[0]
  195     r = conn.execute("SELECT COUNT(*) FROM CheckSums WHERE Encrypted=0 AND IsFile = 1")
  196     files = r.fetchone()[0]
  197     logger.info("Encrypting %d files", files)
  198     bar = progressbar.ProgressBar(max_value=int(files))
  199 
  200     for level in range(mLevel, -1, -1):
  201         encryptFilesAtLevel(db, crypto, cacheDir, level, bar)
  202 
  203     bar.finish()
  204 
  205 
  206 def generateDirHashes(db, crypto, cacheDir):
  207     conn = db.conn
  208     r = conn.execute("SELECT COUNT(*) FROM Files WHERE Dir = 1")
  209     nDirs = r.fetchone()[0]
  210     logger.info("Hashing %d directories", nDirs)
  211     hashes = 0
  212     unique = 0
  213     with progressbar.ProgressBar(max_value=nDirs) as bar:
  214         z = conn.cursor()
  215         r = conn.execute("SELECT Inode, Device, LastSet, Names.name, Checksums.ChecksumId, Checksum "
  216                          "FROM Files "
  217                          "JOIN Names ON Names.NameId = Files.NameID "
  218                          "JOIN Checksums ON Files.ChecksumId = Checksums.ChecksumId "
  219                          "WHERE Dir = 1 "
  220                          "ORDER BY Checksum")
  221         lastHash = None
  222         batch = r.fetchmany(10000)
  223         while batch:
  224             for row in batch:
  225                 inode = row['Inode']
  226                 device = row['Device']
  227                 last = row['LastSet']
  228                 oldHash = row['Checksum']
  229                 cksId = row['ChecksumId']
  230                 files = db.readDirectory((inode, device), last)
  231                 hashes += 1
  232                 if oldHash == lastHash:
  233                     continue
  234                 lastHash = oldHash
  235                 unique += 1
  236 
  237                 #logger.debug("Rehashing directory %s (%d, %d)@%d: %s(%d)", crypto.decryptFilename(row['Name']),inode, device, last, oldHash, cksId)
  238                 #logger.debug("    Directory contents: %s", str(files))
  239                 (newHash, newSize) = Util.hashDir(crypto, files, True)
  240                 #logger.info("Rehashed %s => %s.  %d files", oldHash, newHash, newSize)
  241                 bar.update(hashes)
  242                 try:
  243                     if newHash != oldHash:
  244                         z.execute("UPDATE Checksums SET Checksum = :newHash WHERE ChecksumId = :id", {"newHash": newHash, "id": cksId})
  245                 except Exception as e:
  246                     logger.error("Caught exception: %s->%s :: %s", oldHash, newHash,str(e))
  247             batch = r.fetchmany()
  248     logger.info("Hashed %d directories (%d unique)", hashes, unique)
  249 
  250 def makeSig(checksum, regenerator, cacheDir):
  251     data = regenerator.recoverChecksum(checksum)
  252     fname = checksum + ".sig"
  253     output = cacheDir.open(fname, "wb")
  254     librsync.signature(data, output)
  255     output.close()
  256     
  257 
  258 def generateSignatures(db, crypto, cacheDir):
  259     c = db.conn.cursor()
  260 
  261     r = c.execute("SELECT COUNT(*) FROM CheckSums WHERE IsFile = 1")
  262     n = r.fetchone()[0]
  263     logger.info("Generating signature files for %d files", n)
  264 
  265     regenerator = Regenerator.Regenerator(cacheDir, db, crypto)
  266     r = c.execute("SELECT Checksum FROM Checksums WHERE IsFile = 1")
  267 
  268     sigs = 0
  269     sigsGenned = 0
  270 
  271     with progressbar.ProgressBar(max_value=int(n)) as bar:
  272         batch = r.fetchmany(4096)
  273         while batch:
  274             for row in batch:
  275                 checksum = row[0]
  276                 sigfile = checksum + '.sig'
  277                 if not cacheDir.exists(sigfile):
  278                     #logger.info("Generating signature for {}".format(checksum))
  279                     makeSig(checksum, regenerator, cacheDir)
  280                     sigsGenned += 1
  281                 sigs += 1
  282                 bar.update(sigs)
  283             batch = r.fetchmany(4096)
  284 
  285 def generateMetadata(db, cacheDir):
  286     conn = db.conn
  287     r = conn.execute("SELECT COUNT(*) FROM CheckSums WHERE IsFile = 1")
  288     n = r.fetchone()[0]
  289     c = conn.cursor()
  290     r = c.execute("SELECT Checksum, Size, Compressed, Encrypted, DiskSize, Basis FROM Checksums WHERE IsFile = 1 ORDER BY CheckSum")
  291     metas = 0
  292     logger.info("Generating metadata/recovery info for %d files", n)
  293     with progressbar.ProgressBar(max_value=int(n)) as bar:
  294         batch = r.fetchmany(4096)
  295         while batch:
  296             for row in batch:
  297                 # recordMetaData(cache, checksum, size, compressed, encrypted, disksize, basis=None, logger=None):
  298                 Util.recordMetaData(cacheDir, row[0], row[1], row[2], row[3], row[4], basis=row[5], logger=logger)
  299                 metas += 1
  300                 bar.update(metas)
  301             batch = r.fetchmany(4096)
  302 
  303 
  304 def processArgs():
  305     parser = argparse.ArgumentParser(description='Encrypt the database', add_help = False)
  306 
  307     (_, remaining) = Config.parseConfigOptions(parser)
  308     Config.addCommonOptions(parser)
  309     Config.addPasswordOptions(parser)
  310 
  311     parser.add_argument('--names',          dest='names',    action='store_true', default=False,       help='Encrypt filenames. Default=%(default)s')
  312     parser.add_argument('--dirs',           dest='dirs',     action='store_true', default=False,       help='Generate directory hashes.  Default=%(default)s')
  313     parser.add_argument('--sigs',           dest='sigs',     action='store_true', default=False,       help='Generate signature files.  Default=%(default)s')
  314     parser.add_argument('--files',          dest='files',    action='store_true', default=False,       help='Encrypt files. Default=%(default)s')
  315     parser.add_argument('--meta',           dest='meta',     action='store_true', default=False,       help='Generate metadata files.  Default=%(default)s')
  316     parser.add_argument('--all',            dest='all',      action='store_true', default=False,       help='Perform all encyrption steps. Default=%(default)s')
  317 
  318     parser.add_argument('--help', '-h',     action='help');
  319 
  320     Util.addGenCompletions(parser)
  321 
  322     args = parser.parse_args(remaining)
  323 
  324     if (not (args.names or args.files or args.dirs or args.meta or args.all or args.sigs)):
  325         parser.error("Must specify at least one --names, --files, --dirs, --meta, or --all")
  326     return args
  327 
  328 def main():
  329     global logger
  330     progressbar.streams.wrap_stderr()
  331     logging.basicConfig(level=logging.INFO)
  332     logger = logging.getLogger('')
  333     args = processArgs()
  334     password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, allowNone=False)
  335 
  336     #crypto = TardisCrypto.TardisCrypto(password, args.client)
  337 
  338     #path = os.path.join(args.database, args.client, args.dbname)
  339     #db = TardisDB.TardisDB(path, backup=False)
  340 
  341     #Util.authenticate(db, args.client, password)
  342 
  343     (db, cacheDir, crypto) = Util.setupDataConnection(args.database, args.client, password, args.keys, args.dbname, args.dbdir)
  344 
  345 
  346     #(f, c) = db.getKeys()
  347     #crypto.setKeys(f, c)
  348 
  349     #cacheDir = CacheDir.CacheDir(os.path.join(args.database, args.client))
  350 
  351     if args.names or args.all:
  352         encryptFilenames(db, crypto)
  353     if args.dirs or args.all:
  354         generateDirHashes(db, crypto, cacheDir)
  355     if args.sigs or args.all:
  356         generateSignatures(db, crypto, cacheDir)
  357     if args.files or args.all:
  358         encryptFiles(db, crypto, cacheDir)
  359     if args.meta or args.all:
  360         generateMetadata(db, cacheDir)
  361 
  362 if __name__ == "__main__":
  363     main()