"Fossies" - the Fresh Open Source Software Archive

Member "ownCloud-2.7.6.3261/src/common/checksums.cpp" (5 Feb 2021, 12426 Bytes) of package /linux/misc/ownCloud-2.7.6.3261.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "checksums.cpp" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.7.5.3180_vs_2.7.6.3261.

    1 /*
    2  * Copyright (C) by Klaas Freitag <freitag@owncloud.com>
    3  *
    4  * This library is free software; you can redistribute it and/or
    5  * modify it under the terms of the GNU Lesser General Public
    6  * License as published by the Free Software Foundation; either
    7  * version 2.1 of the License, or (at your option) any later version.
    8  *
    9  * This library is distributed in the hope that it will be useful,
   10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   12  * Lesser General Public License for more details.
   13  *
   14  * You should have received a copy of the GNU Lesser General Public
   15  * License along with this library; if not, write to the Free Software
   16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
   17  */
   18 #include "config.h"
   19 #include "filesystembase.h"
   20 #include "common/checksums.h"
   21 #include "asserts.h"
   22 
   23 #include <QLoggingCategory>
   24 #include <qtconcurrentrun.h>
   25 #include <QCryptographicHash>
   26 
   27 #ifdef ZLIB_FOUND
   28 #include <zlib.h>
   29 #endif
   30 
   31 /** \file checksums.cpp
   32  *
   33  * \brief Computing and validating file checksums
   34  *
   35  * Overview
   36  * --------
   37  *
   38  * Checksums are used in two distinct ways during synchronization:
   39  *
   40  * - to guard uploads and downloads against data corruption
   41  *   (transmission checksum)
   42  * - to quickly check whether the content of a file has changed
   43  *   to avoid redundant uploads (content checksum)
   44  *
   45  * In principle both are independent and different checksumming
   46  * algorithms can be used. To avoid redundant computations, it can
   47  * make sense to use the same checksum algorithm though.
   48  *
   49  * Transmission Checksums
   50  * ----------------------
   51  *
   52  * The usage of transmission checksums is currently optional and needs
   53  * to be explicitly enabled by adding 'transmissionChecksum=TYPE' to
   54  * the '[General]' section of the config file.
   55  *
   56  * When enabled, the checksum will be calculated on upload and sent to
   57  * the server in the OC-Checksum header with the format 'TYPE:CHECKSUM'.
   58  *
   59  * On download, the header with the same name is read and if the
   60  * received data does not have the expected checksum, the download is
   61  * rejected.
   62  *
   63  * Transmission checksums guard a specific sync action and are not stored
   64  * in the database.
   65  *
   66  * Content Checksums
   67  * -----------------
   68  *
   69  * Sometimes the metadata of a local file changes while the content stays
   70  * unchanged. Content checksums allow the sync client to avoid uploading
   71  * the same data again by comparing the file's actual checksum to the
   72  * checksum stored in the database.
   73  *
   74  * Content checksums are not sent to the server.
   75  *
   76  * Checksum Algorithms
   77  * -------------------
   78  *
   79  * - Adler32 (requires zlib)
   80  * - MD5
   81  * - SHA1
   82  * - SHA256
   83  * - SHA3-256 (requires Qt 5.9)
   84  *
   85  */
   86 
   87 namespace OCC {
   88 
   89 Q_LOGGING_CATEGORY(lcChecksums, "sync.checksums", QtInfoMsg)
   90 
   91 #define BUFSIZE qint64(500 * 1024) // 500 KiB
   92 
   93 static QByteArray calcCryptoHash(QIODevice *device, QCryptographicHash::Algorithm algo)
   94 {
   95      QByteArray arr;
   96      QCryptographicHash crypto( algo );
   97 
   98      if (crypto.addData(device)) {
   99          arr = crypto.result().toHex();
  100      }
  101      return arr;
  102  }
  103 
  104 QByteArray calcMd5(QIODevice *device)
  105 {
  106     return calcCryptoHash(device, QCryptographicHash::Md5);
  107 }
  108 
  109 QByteArray calcSha1(QIODevice *device)
  110 {
  111     return calcCryptoHash(device, QCryptographicHash::Sha1);
  112 }
  113 
  114 #ifdef ZLIB_FOUND
  115 QByteArray calcAdler32(QIODevice *device)
  116 {
  117     if (device->size() == 0)
  118     {
  119         return QByteArray();
  120     }
  121     QByteArray buf(BUFSIZE, Qt::Uninitialized);
  122 
  123     unsigned int adler = adler32(0L, Z_NULL, 0);
  124     qint64 size;
  125     while (!device->atEnd()) {
  126         size = device->read(buf.data(), BUFSIZE);
  127         if (size > 0)
  128             adler = adler32(adler, (const Bytef *)buf.data(), size);
  129     }
  130 
  131     return QByteArray::number(adler, 16);
  132 }
  133 #endif
  134 
  135 QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum)
  136 {
  137     if (checksumType.isEmpty() || checksum.isEmpty())
  138         return QByteArray();
  139     QByteArray header = checksumType;
  140     header.append(':');
  141     header.append(checksum);
  142     return header;
  143 }
  144 
  145 QByteArray findBestChecksum(const QByteArray &_checksums)
  146 {
  147     if (_checksums.isEmpty()) {
  148         return {};
  149     }
  150     const auto checksums = QString::fromUtf8(_checksums);
  151     int i = 0;
  152     // The order of the searches here defines the preference ordering.
  153     if (-1 != (i = checksums.indexOf(QLatin1String("SHA3-256:"), 0, Qt::CaseInsensitive))
  154         || -1 != (i = checksums.indexOf(QLatin1String("SHA256:"), 0, Qt::CaseInsensitive))
  155         || -1 != (i = checksums.indexOf(QLatin1String("SHA1:"), 0, Qt::CaseInsensitive))
  156         || -1 != (i = checksums.indexOf(QLatin1String("MD5:"), 0, Qt::CaseInsensitive))
  157         || -1 != (i = checksums.indexOf(QLatin1String("ADLER32:"), 0, Qt::CaseInsensitive))) {
  158         // Now i is the start of the best checksum
  159         // Grab it until the next space or end of xml or end of string.
  160         int end = _checksums.indexOf(' ', i);
  161         // workaround for https://github.com/owncloud/core/pull/38304
  162         if (end == -1) {
  163             end = _checksums.indexOf('<', i);
  164         }
  165         return _checksums.mid(i, end - i);
  166     }
  167     qCWarning(lcChecksums) << "Failed to parse" << _checksums;
  168     return {};
  169 }
  170 
  171 bool parseChecksumHeader(const QByteArray &header, QByteArray *type, QByteArray *checksum)
  172 {
  173     if (header.isEmpty()) {
  174         type->clear();
  175         checksum->clear();
  176         return true;
  177     }
  178 
  179     const auto idx = header.indexOf(':');
  180     if (idx < 0) {
  181         return false;
  182     }
  183 
  184     *type = header.left(idx);
  185     *checksum = header.mid(idx + 1);
  186     return true;
  187 }
  188 
  189 
  190 QByteArray parseChecksumHeaderType(const QByteArray &header)
  191 {
  192     const auto idx = header.indexOf(':');
  193     if (idx < 0) {
  194         return QByteArray();
  195     }
  196     return header.left(idx);
  197 }
  198 
  199 bool uploadChecksumEnabled()
  200 {
  201     static bool enabled = qEnvironmentVariableIsEmpty("OWNCLOUD_DISABLE_CHECKSUM_UPLOAD");
  202     return enabled;
  203 }
  204 
  205 static bool checksumComputationEnabled()
  206 {
  207     static bool enabled = qgetenv("OWNCLOUD_DISABLE_CHECKSUM_COMPUTATIONS").isEmpty();
  208     return enabled;
  209 }
  210 
  211 ComputeChecksum::ComputeChecksum(QObject *parent)
  212     : QObject(parent)
  213 {
  214 }
  215 
  216 ComputeChecksum::~ComputeChecksum()
  217 {
  218 }
  219 
  220 void ComputeChecksum::setChecksumType(const QByteArray &type)
  221 {
  222     _checksumType = type;
  223 }
  224 
  225 QByteArray ComputeChecksum::checksumType() const
  226 {
  227     return _checksumType;
  228 }
  229 
  230 void ComputeChecksum::start(const QString &filePath)
  231 {
  232     qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of" << filePath << "in a thread";
  233     startImpl(std::make_unique<QFile>(filePath));
  234 }
  235 
  236 void ComputeChecksum::start(std::unique_ptr<QIODevice> device)
  237 {
  238     OC_ENFORCE(device);
  239     qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of device" << device.get() << "in a thread";
  240     OC_ASSERT(!device->parent());
  241 
  242     startImpl(std::move(device));
  243 }
  244 
  245 void ComputeChecksum::startImpl(std::unique_ptr<QIODevice> device)
  246 {
  247     connect(&_watcher, &QFutureWatcherBase::finished,
  248         this, &ComputeChecksum::slotCalculationDone,
  249         Qt::UniqueConnection);
  250 
  251     // We'd prefer to move the unique_ptr into the lambda, but that's
  252     // awkward with the C++ standard we're on
  253     auto sharedDevice = QSharedPointer<QIODevice>(device.release());
  254 
  255     // Bug: The thread will keep running even if ComputeChecksum is deleted.
  256     auto type = checksumType();
  257     _watcher.setFuture(QtConcurrent::run([sharedDevice, type]() {
  258         if (!sharedDevice->open(QIODevice::ReadOnly)) {
  259             if (auto file = qobject_cast<QFile *>(sharedDevice.data())) {
  260                 qCWarning(lcChecksums) << "Could not open file" << file->fileName()
  261                         << "for reading to compute a checksum" << file->errorString();
  262             } else {
  263                 qCWarning(lcChecksums) << "Could not open device" << sharedDevice.data()
  264                         << "for reading to compute a checksum" << sharedDevice->errorString();
  265             }
  266             return QByteArray();
  267         }
  268         auto result = ComputeChecksum::computeNow(sharedDevice.data(), type);
  269         sharedDevice->close();
  270         return result;
  271     }));
  272 }
  273 
  274 QByteArray ComputeChecksum::computeNowOnFile(const QString &filePath, const QByteArray &checksumType)
  275 {
  276     QFile file(filePath);
  277     if (!file.open(QIODevice::ReadOnly)) {
  278         qCWarning(lcChecksums) << "Could not open file" << filePath << "for reading and computing checksum" << file.errorString();
  279         return QByteArray();
  280     }
  281 
  282     return computeNow(&file, checksumType);
  283 }
  284 
  285 QByteArray ComputeChecksum::computeNow(QIODevice *device, const QByteArray &checksumType)
  286 {
  287     if (!checksumComputationEnabled()) {
  288         qCWarning(lcChecksums) << "Checksum computation disabled by environment variable";
  289         return QByteArray();
  290     }
  291 
  292     if (checksumType == checkSumMD5C) {
  293         return calcMd5(device);
  294     } else if (checksumType == checkSumSHA1C) {
  295         return calcSha1(device);
  296     } else if (checksumType == checkSumSHA2C) {
  297         return calcCryptoHash(device, QCryptographicHash::Sha256);
  298     } else if (checksumType == checkSumSHA3C) {
  299         return calcCryptoHash(device, QCryptographicHash::Sha3_256);
  300     }
  301 #ifdef ZLIB_FOUND
  302     else if (checksumType == checkSumAdlerC) {
  303         return calcAdler32(device);
  304     }
  305 #endif
  306     // for an unknown checksum or no checksum, we're done right now
  307     if (!checksumType.isEmpty()) {
  308         qCWarning(lcChecksums) << "Unknown checksum type:" << checksumType;
  309     }
  310     return QByteArray();
  311 }
  312 
  313 void ComputeChecksum::slotCalculationDone()
  314 {
  315     QByteArray checksum = _watcher.future().result();
  316     if (!checksum.isNull()) {
  317         emit done(_checksumType, checksum);
  318     } else {
  319         emit done(QByteArray(), QByteArray());
  320     }
  321 }
  322 
  323 
  324 ValidateChecksumHeader::ValidateChecksumHeader(QObject *parent)
  325     : QObject(parent)
  326 {
  327 }
  328 
  329 ComputeChecksum *ValidateChecksumHeader::prepareStart(const QByteArray &checksumHeader)
  330 {
  331     // If the incoming header is empty no validation can happen. Just continue.
  332     if (checksumHeader.isEmpty()) {
  333         emit validated(QByteArray(), QByteArray());
  334         return nullptr;
  335     }
  336 
  337     if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) {
  338         qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader;
  339         emit validationFailed(tr("The checksum header is malformed."));
  340         return nullptr;
  341     }
  342 
  343     auto calculator = new ComputeChecksum(this);
  344     calculator->setChecksumType(_expectedChecksumType);
  345     connect(calculator, &ComputeChecksum::done,
  346         this, &ValidateChecksumHeader::slotChecksumCalculated);
  347     return calculator;
  348 }
  349 
  350 void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &checksumHeader)
  351 {
  352     if (auto calculator = prepareStart(checksumHeader))
  353         calculator->start(filePath);
  354 }
  355 
  356 void ValidateChecksumHeader::start(std::unique_ptr<QIODevice> device, const QByteArray &checksumHeader)
  357 {
  358     if (auto calculator = prepareStart(checksumHeader))
  359         calculator->start(std::move(device));
  360 }
  361 
  362 void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumType,
  363     const QByteArray &checksum)
  364 {
  365     if (checksumType != _expectedChecksumType) {
  366         emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(QString::fromLatin1(_expectedChecksumType)));
  367         return;
  368     }
  369     if (checksum != _expectedChecksum) {
  370         emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed. '%1' != '%2'").arg(QString::fromUtf8(_expectedChecksum), QString::fromUtf8(checksum)));
  371         return;
  372     }
  373     emit validated(checksumType, checksum);
  374 }
  375 
  376 CSyncChecksumHook::CSyncChecksumHook()
  377 {
  378 }
  379 
  380 QByteArray CSyncChecksumHook::hook(const QByteArray &path, const QByteArray &otherChecksumHeader, void * /*this_obj*/)
  381 {
  382     QByteArray type = parseChecksumHeaderType(QByteArray(otherChecksumHeader));
  383     if (type.isEmpty())
  384         return nullptr;
  385 
  386     qCInfo(lcChecksums) << "Computing" << type << "checksum of" << path << "in the csync hook";
  387     QByteArray checksum = ComputeChecksum::computeNowOnFile(QString::fromUtf8(path), type);
  388     if (checksum.isNull()) {
  389         qCWarning(lcChecksums) << "Failed to compute checksum" << type << "for" << path;
  390         return nullptr;
  391     }
  392 
  393     return makeChecksumHeader(type, checksum);
  394 }
  395 
  396 }