"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "tools/checkDB2.py" between
Tardis-1.1.5.tar.gz and Tardis-1.2.1.tar.gz

About: Tardis is a system for making encrypted, incremental backups of filesystems.

checkDB2.py  (Tardis-1.1.5):checkDB2.py  (Tardis-1.2.1)
skipping to change at line 49 skipping to change at line 49
import Tardis import Tardis
from Tardis import Util from Tardis import Util
from Tardis import TardisCrypto from Tardis import TardisCrypto
from Tardis import Config from Tardis import Config
from Tardis import CacheDir from Tardis import CacheDir
from Tardis import CompressedBuffer from Tardis import CompressedBuffer
args = None args = None
#tardis.log: text/plain
#tardis.log.br: application/octet-stream
#tardis.log.bz2: application/x-bzip2
#tardis.log.gz: application/gzip
#tardis.log.lzma: application/x-lzma
#tardis.log.xz: application/x-xz
#tardis.log.zst: application/x-zstd
def processArgs(): def processArgs():
parser = argparse.ArgumentParser(description='Check contents of the DB again st the file system', fromfile_prefix_chars='@', formatter_class=Util.HelpFormatt er, add_help=False) parser = argparse.ArgumentParser(description='Check contents of the DB again st the file system', fromfile_prefix_chars='@', formatter_class=Util.HelpFormatt er, add_help=False)
(_, remaining) = Config.parseConfigOptions(parser) (_, remaining) = Config.parseConfigOptions(parser)
Config.addCommonOptions(parser) Config.addCommonOptions(parser)
Config.addPasswordOptions(parser) Config.addPasswordOptions(parser)
#parser.add_argument('--output', '-o', default=None, dest='output', requir ed=True, help='Output file') #parser.add_argument('--output', '-o', default=None, dest='output', requir ed=True, help='Output file')
skipping to change at line 87 skipping to change at line 95
data = rs.fetchmany(chunksize) data = rs.fetchmany(chunksize)
def decryptHeader(crypt, infile): def decryptHeader(crypt, infile):
# Get the IV, if it's not specified. # Get the IV, if it's not specified.
infile.seek(0, os.SEEK_SET) infile.seek(0, os.SEEK_SET)
iv = infile.read(crypt.ivLength) iv = infile.read(crypt.ivLength)
# Create the cipher # Create the cipher
encryptor = crypt.getContentEncryptor(iv) encryptor = crypt.getContentEncryptor(iv)
ct = infile.read(1024) ct = infile.read(64 * 1024)
pt = encryptor.decrypt(ct, False) pt = encryptor.decrypt(ct, False)
outstream = io.BytesIO(pt) outstream = io.BytesIO(pt)
return outstream return outstream
def authenticateFile(infile, size, crypt): def authenticateFile(infile, size, crypt):
# Get the IV, if it's not specified. # Get the IV, if it's not specified.
infile.seek(0, os.SEEK_SET) infile.seek(0, os.SEEK_SET)
iv = infile.read(crypt.ivLength) iv = infile.read(crypt.ivLength)
#logger.debug("Got IV: %d %s", len(iv), binascii.hexlify(iv)) #logger.debug("Got IV: %d %s", len(iv), binascii.hexlify(iv))
skipping to change at line 131 skipping to change at line 139
encryptor.verify(digest) encryptor.verify(digest)
except: except:
#logger.debug("HMAC's: File: %-128s Computed: %-128s", bina scii.hexlify(digest), binascii.hexlify(encryptor.digest())) #logger.debug("HMAC's: File: %-128s Computed: %-128s", bina scii.hexlify(digest), binascii.hexlify(encryptor.digest()))
print("HMAC's: File: %-128s Computed: %-128s", binascii.hex lify(digest), binascii.hexlify(encryptor.digest())) print("HMAC's: File: %-128s Computed: %-128s", binascii.hex lify(digest), binascii.hexlify(encryptor.digest()))
return False return False
rem -= readsize rem -= readsize
return True return True
except: except:
return False return False
#tardis.log: text/plain
#tardis.log.br: application/octet-stream
#tardis.log.bz2: application/x-bzip2
#tardis.log.gz: application/gzip
#tardis.log.lzma: application/x-lzma
#tardis.log.xz: application/x-xz
#tardis.log.zst: application/x-zstd
def checkCompression(mimetype, compresstype):
if (compresstype == 'zlib' or compresstype == True or compresstype == 1):
if mimetype != 'application/zlib':
return False
elif compresstype == 'zstd':
if mimetype != 'application/x-std':
return False
elif compresstype == 'bzip':
if mimetype != 'application/x-bzip2':
return False
elif compresstype == 'lzma':
if mimetype != 'application/x-lzma' and mimetype != 'application/x-xz':
return False
elif not (compresstype == 'none' or compresstype == False):
print(f"Unknown compression type: {compresstype}")
return True
missing = [] missing = []
zero = [] zero = []
mismatch = [] mismatch = []
notdelta = [] notdelta = []
notauth = [] notauth = []
badcomp = []
sizes = {} sizes = {}
def checkFile(cache, crypt, checksum, size, basis, compressed, encrypted, added, authCond): def checkFile(cache, crypt, checksum, size, basis, compressed, encrypted, added, authCond):
fsize = cache.size(checksum) fsize = cache.size(checksum)
if not cache.exists(checksum): if not cache.exists(checksum):
#print(f"{checksum}: does not exist") #print(f"{checksum}: does not exist")
missing.append(checksum) missing.append(checksum)
elif fsize == 0: elif fsize == 0:
print(f"{checksum} is empty") print(f"{checksum} is empty")
zero.append(checksum) zero.append(checksum)
else: else:
authenticate = (authCond == 'all') authenticate = (authCond == 'all')
if fsize != size: if fsize != size:
print(f"{checksum}: size mismatch Expected: {size}, found {fsize} ({ fsize - size})-- {added} -- {basis is not None} ") print(f"{checksum}: size mismatch Expected: {size}, found {fsize} ({ fsize - size})-- {added} -- {basis is not None} ")
mismatch.append((checksum, size, fsize)) mismatch.append((checksum, size, fsize, compressed))
sizes.setdefault((fsize - size), []).append(checksum) sizes.setdefault((fsize - size), []).append(checksum)
if authCond != 'none': if authCond != 'none':
authenticate = True authenticate = True
try:
instream = decryptHeader(crypt, cache.open(checksum, "rb"))
uc = CompressedBuffer.UncompressedBufferedReader(instream, compr
essor="none")
data = uc.read(256)
mimetype = magic.from_buffer(data, mime=True)
if not checkCompression(mimetype, compressed):
print(f"{checksum} has wrong compression type. Expected {c
ompressed}. Found {mimetype}")
badcomp.append((checksum, compressed, mimetype))
except Exception as e:
print(f"Caught exception: {compressed} {e}")
badcomp.append((checksum, compressed, "error"))
elif basis: elif basis:
#print(f"{checksum} -- {compressed} {encrypted}", flush=True) #print(f"{checksum} -- {compressed} {encrypted}", flush=True)
instream = decryptHeader(crypt, cache.open(checksum, "rb")) instream = decryptHeader(crypt, cache.open(checksum, "rb"))
uc = CompressedBuffer.UncompressedBufferedReader(instream, compresso r=compressed) uc = CompressedBuffer.UncompressedBufferedReader(instream, compresso r=compressed)
data = uc.read(256) data = uc.read(256)
kind = magic.from_buffer(data) kind = magic.from_buffer(data)
if kind != 'rdiff network-delta data': if kind != 'rdiff network-delta data':
print(f"{checksum}: Not a delta: {kind}") print(f"{checksum}: Not a delta: {kind}")
notdelta.append((checksum, kind)) notdelta.append((checksum, kind))
skipping to change at line 182 skipping to change at line 229
args = processArgs() args = processArgs()
password = Util.getPassword(args.password, args.passwordfile, args.passwordp rog, prompt="Password for %s: " % (args.client)) password = Util.getPassword(args.password, args.passwordfile, args.passwordp rog, prompt="Password for %s: " % (args.client))
(tardis, cache, crypt) = Util.setupDataConnection(args.database, args.client , password, args.keys, args.dbname, args.dbdir) (tardis, cache, crypt) = Util.setupDataConnection(args.database, args.client , password, args.keys, args.dbname, args.dbdir)
count = 0 count = 0
for (checksum, size, basis, compressed, encrypted, added) in listChecksums(t ardis): for (checksum, size, basis, compressed, encrypted, added) in listChecksums(t ardis):
count += 1 count += 1
checkFile(cache, crypt, checksum, size, basis, compressed, encrypted, ad ded, args.authenticate) checkFile(cache, crypt, checksum, size, basis, compressed, encrypted, ad ded, args.authenticate)
print(f"Files: {count} Missing Files: {len(missing)} Empty: {len(zero)} Size mismatch: {len(mismatch)} Not Delta: {len(notdelta)}") print(f"Files: {count} Missing Files: {len(missing)} Empty: {len(zero)} Size mismatch: {len(mismatch)} Not Delta: {len(notdelta)} Wrong Comp: {len(badcomp)} ")
#for i in sizes: #for i in sizes:
# print(f" Size: {i}: Count {len(sizes[i])}") # print(f" Size: {i}: Count {len(sizes[i])}")
if args.output: if args.output:
out = { out = {
"missing": missing, "missing": missing,
"empty": zero, "empty": zero,
"size": mismatch, "size": mismatch,
"notauth": notauth, "notauth": notauth,
"notdelta": notdelta "notdelta": notdelta,
"badcomp": badcomp
} }
json.dump(out, args.output, indent=2) json.dump(out, args.output, indent=2)
return 0 return 0
if __name__ == "__main__": if __name__ == "__main__":
sys.exit(main()) sys.exit(main())
 End of changes. 8 change blocks. 
4 lines changed or deleted 54 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)