"Fossies" - the Fresh Open Source Software Archive

Member "ownCloud-2.7.6.3261/src/libsync/discoveryphase.cpp" (5 Feb 2021, 20225 Bytes) of package /linux/misc/ownCloud-2.7.6.3261.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "discoveryphase.cpp" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.7.5.3180_vs_2.7.6.3261.

    1 /*
    2  * Copyright (C) by Olivier Goffart <ogoffart@woboq.com>
    3  *
    4  * This program is free software; you can redistribute it and/or modify
    5  * it under the terms of the GNU General Public License as published by
    6  * the Free Software Foundation; either version 2 of the License, or
    7  * (at your option) any later version.
    8  *
    9  * This program is distributed in the hope that it will be useful, but
   10  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
   11  * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
   12  * for more details.
   13  */
   14 
   15 #include "discoveryphase.h"
   16 #include "discovery.h"
   17 
   18 #include "account.h"
   19 #include "common/asserts.h"
   20 #include "common/checksums.h"
   21 
   22 #include <csync_exclude.h>
   23 #include "vio/csync_vio_local.h"
   24 
   25 #include <QLoggingCategory>
   26 #include <QUrl>
   27 #include <QFile>
   28 #include <QFileInfo>
   29 #include <QTextCodec>
   30 #include <cstring>
   31 
   32 
   33 namespace OCC {
   34 
   35 Q_LOGGING_CATEGORY(lcDiscovery, "sync.discovery", QtInfoMsg)
   36 
   37 /* Given a sorted list of paths ending with '/', return whether or not the given path is within one of the paths of the list*/
   38 static bool findPathInList(const QStringList &list, const QString &path)
   39 {
   40     Q_ASSERT(std::is_sorted(list.begin(), list.end()));
   41 
   42     if (list.size() == 1 && list.first() == QLatin1String("/")) {
   43         // Special case for the case "/" is there, it matches everything
   44         return true;
   45     }
   46 
   47     QString pathSlash = path + QLatin1Char('/');
   48 
   49     // Since the list is sorted, we can do a binary search.
   50     // If the path is a prefix of another item or right after in the lexical order.
   51     auto it = std::lower_bound(list.begin(), list.end(), pathSlash);
   52 
   53     if (it != list.end() && *it == pathSlash) {
   54         return true;
   55     }
   56 
   57     if (it == list.begin()) {
   58         return false;
   59     }
   60     --it;
   61     Q_ASSERT(it->endsWith(QLatin1Char('/'))); // Folder::setSelectiveSyncBlackList makes sure of that
   62     return pathSlash.startsWith(*it);
   63 }
   64 
   65 bool DiscoveryPhase::isInSelectiveSyncBlackList(const QString &path) const
   66 {
   67     if (_selectiveSyncBlackList.isEmpty()) {
   68         // If there is no black list, everything is allowed
   69         return false;
   70     }
   71 
   72     // Block if it is in the black list
   73     if (findPathInList(_selectiveSyncBlackList, path)) {
   74         return true;
   75     }
   76 
   77     return false;
   78 }
   79 
   80 void DiscoveryPhase::checkSelectiveSyncNewFolder(const QString &path, RemotePermissions remotePerm,
   81     std::function<void(bool)> callback)
   82 {
   83     if (_syncOptions._confirmExternalStorage && _syncOptions._vfs->mode() == Vfs::Off
   84         && remotePerm.hasPermission(RemotePermissions::IsMounted)) {
   85         // external storage.
   86 
   87         /* Note: DiscoverySingleDirectoryJob::directoryListingIteratedSlot make sure that only the
   88          * root of a mounted storage has 'M', all sub entries have 'm' */
   89 
   90         // Only allow it if the white list contains exactly this path (not parents)
   91         // We want to ask confirmation for external storage even if the parents where selected
   92         if (_selectiveSyncWhiteList.contains(path + QLatin1Char('/'))) {
   93             return callback(false);
   94         }
   95 
   96         emit newBigFolder(path, true);
   97         return callback(true);
   98     }
   99 
  100     // If this path or the parent is in the white list, then we do not block this file
  101     if (findPathInList(_selectiveSyncWhiteList, path)) {
  102         return callback(false);
  103     }
  104 
  105     auto limit = _syncOptions._newBigFolderSizeLimit;
  106     if (limit < 0 || _syncOptions._vfs->mode() != Vfs::Off) {
  107         // no limit, everything is allowed;
  108         return callback(false);
  109     }
  110 
  111     // do a PROPFIND to know the size of this folder
  112     auto propfindJob = new PropfindJob(_account, _remoteFolder + path, this);
  113     propfindJob->setProperties(QList<QByteArray>() << "resourcetype"
  114                                                    << "http://owncloud.org/ns:size");
  115     QObject::connect(propfindJob, &PropfindJob::finishedWithError,
  116         this, [=] { return callback(false); });
  117     QObject::connect(propfindJob, &PropfindJob::result, this, [=](const QVariantMap &values) {
  118         auto result = values.value(QStringLiteral("size")).toLongLong();
  119         if (result >= limit) {
  120             // we tell the UI there is a new folder
  121             emit newBigFolder(path, false);
  122             return callback(true);
  123         } else {
  124             // it is not too big, put it in the white list (so we will not do more query for the children)
  125             // and and do not block.
  126             auto p = path;
  127             if (!p.endsWith(QLatin1Char('/')))
  128                 p += QLatin1Char('/');
  129             _selectiveSyncWhiteList.insert(
  130                 std::upper_bound(_selectiveSyncWhiteList.begin(), _selectiveSyncWhiteList.end(), p),
  131                 p);
  132             return callback(false);
  133         }
  134     });
  135     propfindJob->start();
  136 }
  137 
  138 /* Given a path on the remote, give the path as it is when the rename is done */
  139 QString DiscoveryPhase::adjustRenamedPath(const QString &original, SyncFileItem::Direction d) const
  140 {
  141     return OCC::adjustRenamedPath(d == SyncFileItem::Down ? _renamedItemsRemote : _renamedItemsLocal, original);
  142 }
  143 
  144 QString adjustRenamedPath(const QMap<QString, QString> &renamedItems, const QString &original)
  145 {
  146     int slashPos = original.size();
  147     while ((slashPos = original.lastIndexOf(QLatin1Char('/'), slashPos - 1)) > 0) {
  148         auto it = renamedItems.constFind(original.left(slashPos));
  149         if (it != renamedItems.constEnd()) {
  150             return *it + original.mid(slashPos);
  151         }
  152     }
  153     return original;
  154 }
  155 
  156 QPair<bool, QByteArray> DiscoveryPhase::findAndCancelDeletedJob(const QString &originalPath)
  157 {
  158     bool result = false;
  159     QByteArray oldEtag;
  160     auto it = _deletedItem.find(originalPath);
  161     if (it != _deletedItem.end()) {
  162         const SyncInstructions instruction = (*it)->_instruction;
  163         if (instruction == CSYNC_INSTRUCTION_IGNORE && (*it)->_type == ItemTypeVirtualFile) {
  164             // re-creation of virtual files count as a delete
  165             // a file might be in an error state and thus gets marked as CSYNC_INSTRUCTION_IGNORE
  166             // after it was initially marked as CSYNC_INSTRUCTION_REMOVE
  167             // return true, to not trigger any additional actions on that file that could elad to dataloss
  168             result = true;
  169             oldEtag = (*it)->_etag;
  170         } else {
  171             if (!(instruction == CSYNC_INSTRUCTION_REMOVE
  172                     // re-creation of virtual files count as a delete
  173                     || ((*it)->_type == ItemTypeVirtualFile && instruction == CSYNC_INSTRUCTION_NEW)
  174                     || ((*it)->_isRestoration && instruction == CSYNC_INSTRUCTION_NEW)))
  175             {
  176                 qCWarning(lcDiscovery) << "OC_ENFORCE(FAILING)" << originalPath;
  177                 qCWarning(lcDiscovery) << "instruction == CSYNC_INSTRUCTION_REMOVE" << (instruction == CSYNC_INSTRUCTION_REMOVE);
  178                 qCWarning(lcDiscovery) << "((*it)->_type == ItemTypeVirtualFile && instruction == CSYNC_INSTRUCTION_NEW)"
  179                                        << ((*it)->_type == ItemTypeVirtualFile && instruction == CSYNC_INSTRUCTION_NEW);
  180                 qCWarning(lcDiscovery) << "((*it)->_isRestoration && instruction == CSYNC_INSTRUCTION_NEW))"
  181                                        << ((*it)->_isRestoration && instruction == CSYNC_INSTRUCTION_NEW);
  182                 qCWarning(lcDiscovery) << "instruction" << instruction;
  183                 qCWarning(lcDiscovery) << "(*it)->_type" << (*it)->_type;
  184                 qCWarning(lcDiscovery) << "(*it)->_isRestoration " << (*it)->_isRestoration;
  185                 OC_ENFORCE(false);
  186             }
  187             (*it)->_instruction = CSYNC_INSTRUCTION_NONE;
  188             result = true;
  189             oldEtag = (*it)->_etag;
  190         }
  191         _deletedItem.erase(it);
  192     }
  193     if (auto *otherJob = _queuedDeletedDirectories.take(originalPath)) {
  194         oldEtag = otherJob->_dirItem->_etag;
  195         delete otherJob;
  196         result = true;
  197     }
  198     return { result, oldEtag };
  199 }
  200 
  201 void DiscoveryPhase::startJob(ProcessDirectoryJob *job)
  202 {
  203     OC_ENFORCE(!_currentRootJob);
  204     connect(job, &ProcessDirectoryJob::finished, this, [this, job] {
  205         OC_ENFORCE(_currentRootJob == sender());
  206         _currentRootJob = nullptr;
  207         if (job->_dirItem)
  208             emit itemDiscovered(job->_dirItem);
  209         job->deleteLater();
  210 
  211         // Once the main job has finished recurse here to execute the remaining
  212         // jobs for queued deleted directories.
  213         if (!_queuedDeletedDirectories.isEmpty()) {
  214             auto nextJob = _queuedDeletedDirectories.take(_queuedDeletedDirectories.firstKey());
  215             startJob(nextJob);
  216         } else {
  217             emit finished();
  218         }
  219     });
  220     _currentRootJob = job;
  221     job->start();
  222 }
  223 
  224 void DiscoveryPhase::setSelectiveSyncBlackList(const QStringList &list)
  225 {
  226     _selectiveSyncBlackList = list;
  227     std::sort(_selectiveSyncBlackList.begin(), _selectiveSyncBlackList.end());
  228 }
  229 
  230 void DiscoveryPhase::setSelectiveSyncWhiteList(const QStringList &list)
  231 {
  232     _selectiveSyncWhiteList = list;
  233     std::sort(_selectiveSyncWhiteList.begin(), _selectiveSyncWhiteList.end());
  234 }
  235 
  236 void DiscoveryPhase::scheduleMoreJobs()
  237 {
  238     auto limit = qMax(1, _syncOptions._parallelNetworkJobs);
  239     if (_currentRootJob && _currentlyActiveJobs < limit) {
  240         _currentRootJob->processSubJobs(limit - _currentlyActiveJobs);
  241     }
  242 }
  243 
  244 DiscoverySingleLocalDirectoryJob::DiscoverySingleLocalDirectoryJob(const AccountPtr &account, const QString &localPath, OCC::Vfs *vfs, QObject *parent)
  245  : QObject(parent), QRunnable(), _localPath(localPath), _account(account), _vfs(vfs)
  246 {
  247     qRegisterMetaType<QVector<LocalInfo> >("QVector<LocalInfo>");
  248 }
  249 
  250 // Use as QRunnable
  251 void DiscoverySingleLocalDirectoryJob::run() {
  252     QString localPath = _localPath;
  253     if (localPath.endsWith(QLatin1Char('/'))) // Happens if _currentFolder._local.isEmpty()
  254         localPath.chop(1);
  255 
  256     auto dh = csync_vio_local_opendir(localPath);
  257     if (!dh) {
  258         qCInfo(lcDiscovery) << "Error while opening directory" << (localPath) << errno;
  259         QString errorString = tr("Error while opening directory %1").arg(localPath);
  260         if (errno == EACCES) {
  261             errorString = tr("Directory not accessible on client, permission denied");
  262             emit finishedNonFatalError(errorString);
  263             return;
  264         } else if (errno == ENOENT) {
  265             errorString = tr("Directory not found: %1").arg(localPath);
  266         } else if (errno == ENOTDIR) {
  267             // Not a directory..
  268             // Just consider it is empty
  269             return;
  270         }
  271         emit finishedFatalError(errorString);
  272         return;
  273     }
  274 
  275     QVector<LocalInfo> results;
  276     while (true) {
  277         errno = 0;
  278         auto dirent = csync_vio_local_readdir(dh, _vfs);
  279         if (!dirent)
  280             break;
  281         if (dirent->type == ItemTypeSkip)
  282             continue;
  283         LocalInfo i;
  284         static QTextCodec *codec = QTextCodec::codecForName("UTF-8");
  285         OC_ASSERT(codec);
  286         QTextCodec::ConverterState state;
  287         i.name = codec->toUnicode(dirent->path.constData(), dirent->path.size(), &state);
  288         if (state.invalidChars > 0 || state.remainingChars > 0) {
  289             emit childIgnored(true);
  290             auto item = SyncFileItemPtr::create();
  291             //item->_file = _currentFolder._target + i.name;
  292             // FIXME ^^ do we really need to use _target or is local fine?
  293             item->_file = _localPath + i.name;
  294             item->_instruction = CSYNC_INSTRUCTION_IGNORE;
  295             item->_status = SyncFileItem::NormalError;
  296             item->_errorString = tr("Filename encoding is not valid");
  297             emit itemDiscovered(item);
  298             continue;
  299         }
  300         i.modtime = dirent->modtime;
  301         i.size = dirent->size;
  302         i.inode = dirent->inode;
  303         i.isDirectory = dirent->type == ItemTypeDirectory;
  304         i.isHidden = dirent->is_hidden;
  305         i.isSymLink = dirent->type == ItemTypeSoftLink;
  306         i.isVirtualFile = dirent->type == ItemTypeVirtualFile || dirent->type == ItemTypeVirtualFileDownload;
  307         i.type = dirent->type;
  308         results.push_back(i);
  309     }
  310     if (errno != 0) {
  311         csync_vio_local_closedir(dh);
  312 
  313         // Note: Windows vio converts any error into EACCES
  314         qCWarning(lcDiscovery) << "readdir failed for file in " << localPath << " - errno: " << errno;
  315         emit finishedFatalError(tr("Error while reading directory %1").arg(localPath));
  316         return;
  317     }
  318 
  319     errno = 0;
  320     csync_vio_local_closedir(dh);
  321     if (errno != 0) {
  322         qCWarning(lcDiscovery) << "closedir failed for file in " << localPath << " - errno: " << errno;
  323     }
  324 
  325     emit finished(results);
  326 }
  327 
  328 DiscoverySingleDirectoryJob::DiscoverySingleDirectoryJob(const AccountPtr &account, const QString &path, QObject *parent)
  329     : QObject(parent)
  330     , _subPath(path)
  331     , _account(account)
  332     , _ignoredFirst(false)
  333     , _isRootPath(false)
  334     , _isExternalStorage(false)
  335 {
  336 }
  337 
  338 void DiscoverySingleDirectoryJob::start()
  339 {
  340     // Start the actual HTTP job
  341     LsColJob *lsColJob = new LsColJob(_account, _subPath, this);
  342 
  343     QList<QByteArray> props;
  344     props << "resourcetype"
  345           << "getlastmodified"
  346           << "getcontentlength"
  347           << "getetag"
  348           << "http://owncloud.org/ns:id"
  349           << "http://owncloud.org/ns:downloadURL"
  350           << "http://owncloud.org/ns:dDC"
  351           << "http://owncloud.org/ns:permissions"
  352           << "http://owncloud.org/ns:checksums";
  353     if (_isRootPath)
  354         props << "http://owncloud.org/ns:data-fingerprint";
  355     if (_account->serverVersionInt() >= Account::makeServerVersion(10, 0, 0)) {
  356         // Server older than 10.0 have performances issue if we ask for the share-types on every PROPFIND
  357         props << "http://owncloud.org/ns:share-types";
  358     }
  359 
  360     lsColJob->setProperties(props);
  361 
  362     QObject::connect(lsColJob, &LsColJob::directoryListingIterated,
  363         this, &DiscoverySingleDirectoryJob::directoryListingIteratedSlot);
  364     QObject::connect(lsColJob, &LsColJob::finishedWithError, this, &DiscoverySingleDirectoryJob::lsJobFinishedWithErrorSlot);
  365     QObject::connect(lsColJob, &LsColJob::finishedWithoutError, this, &DiscoverySingleDirectoryJob::lsJobFinishedWithoutErrorSlot);
  366     lsColJob->start();
  367 
  368     _lsColJob = lsColJob;
  369 }
  370 
  371 void DiscoverySingleDirectoryJob::abort()
  372 {
  373     if (_lsColJob && _lsColJob->reply()) {
  374         _lsColJob->reply()->abort();
  375     }
  376 }
  377 
  378 static void propertyMapToRemoteInfo(const QMap<QString, QString> &map, RemoteInfo &result)
  379 {
  380     for (auto it = map.constBegin(); it != map.constEnd(); ++it) {
  381         QString property = it.key();
  382         QString value = it.value();
  383         if (property == QLatin1String("resourcetype")) {
  384             result.isDirectory = value.contains(QLatin1String("collection"));
  385         } else if (property == QLatin1String("getlastmodified")) {
  386             result.modtime = oc_httpdate_parse(value.toUtf8().constData());
  387         } else if (property == QLatin1String("getcontentlength")) {
  388             // See #4573, sometimes negative size values are returned
  389             bool ok = false;
  390             qlonglong ll = value.toLongLong(&ok);
  391             if (ok && ll >= 0) {
  392                 result.size = ll;
  393             } else {
  394                 result.size = 0;
  395             }
  396         } else if (property == QLatin1String("getetag")) {
  397             result.etag = Utility::normalizeEtag(value.toUtf8());
  398         } else if (property == QLatin1String("id")) {
  399             result.fileId = value.toUtf8();
  400         } else if (property == QLatin1String("downloadURL")) {
  401             result.directDownloadUrl = value;
  402         } else if (property == QLatin1String("dDC")) {
  403             result.directDownloadCookies = value;
  404         } else if (property == QLatin1String("permissions")) {
  405             result.remotePerm = RemotePermissions::fromServerString(value);
  406         } else if (property == QLatin1String("checksums")) {
  407             result.checksumHeader = findBestChecksum(value.toUtf8());
  408         } else if (property == QLatin1String("share-types") && !value.isEmpty()) {
  409             // Since QMap is sorted, "share-types" is always after "permissions".
  410             if (result.remotePerm.isNull()) {
  411                 qWarning() << "Server returned a share type, but no permissions?";
  412                 // Empty permissions will cause a sync failure
  413             } else {
  414                 // S means shared with me.
  415                 // But for our purpose, we want to know if the file is shared. It does not matter
  416                 // if we are the owner or not.
  417                 // Piggy back on the persmission field
  418                 result.remotePerm.setPermission(RemotePermissions::IsShared);
  419             }
  420         }
  421     }
  422 }
  423 
  424 void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &file, const QMap<QString, QString> &map)
  425 {
  426     if (!_ignoredFirst) {
  427         // The first entry is for the folder itself, we should process it differently.
  428         _ignoredFirst = true;
  429         if (map.contains(QStringLiteral("permissions"))) {
  430             auto perm = RemotePermissions::fromServerString(map.value(QStringLiteral("permissions")));
  431             emit firstDirectoryPermissions(perm);
  432             _isExternalStorage = perm.hasPermission(RemotePermissions::IsMounted);
  433         }
  434         if (map.contains(QStringLiteral("data-fingerprint"))) {
  435             _dataFingerprint = map.value(QStringLiteral("data-fingerprint")).toUtf8();
  436             if (_dataFingerprint.isEmpty()) {
  437                 // Placeholder that means that the server supports the feature even if it did not set one.
  438                 _dataFingerprint = "[empty]";
  439             }
  440         }
  441     } else {
  442 
  443         RemoteInfo result;
  444         int slash = file.lastIndexOf(QLatin1Char('/'));
  445         result.name = file.mid(slash + 1);
  446         result.size = -1;
  447         propertyMapToRemoteInfo(map, result);
  448         if (result.isDirectory)
  449             result.size = 0;
  450 
  451         if (_isExternalStorage && result.remotePerm.hasPermission(RemotePermissions::IsMounted)) {
  452             /* All the entries in a external storage have 'M' in their permission. However, for all
  453                purposes in the desktop client, we only need to know about the mount points.
  454                So replace the 'M' by a 'm' for every sub entries in an external storage */
  455             result.remotePerm.unsetPermission(RemotePermissions::IsMounted);
  456             result.remotePerm.setPermission(RemotePermissions::IsMountedSub);
  457         }
  458         _results.push_back(std::move(result));
  459     }
  460 
  461     //This works in concerto with the RequestEtagJob and the Folder object to check if the remote folder changed.
  462     if (map.contains(QStringLiteral("getetag"))) {
  463         if (_firstEtag.isEmpty()) {
  464             _firstEtag = QString::fromUtf8(parseEtag(map.value(QStringLiteral("getetag")).toUtf8())); // for directory itself
  465         }
  466     }
  467 }
  468 
  469 void DiscoverySingleDirectoryJob::lsJobFinishedWithoutErrorSlot()
  470 {
  471     if (!_ignoredFirst) {
  472         // This is a sanity check, if we haven't _ignoredFirst then it means we never received any directoryListingIteratedSlot
  473         // which means somehow the server XML was bogus
  474         emit finished(HttpError{ 0, tr("Server error: PROPFIND reply is not XML formatted!") });
  475         deleteLater();
  476         return;
  477     } else if (!_error.isEmpty()) {
  478         emit finished(HttpError{ 0, _error });
  479         deleteLater();
  480         return;
  481     }
  482     emit etag(_firstEtag);
  483     emit finished(_results);
  484     deleteLater();
  485 }
  486 
  487 void DiscoverySingleDirectoryJob::lsJobFinishedWithErrorSlot(QNetworkReply *r)
  488 {
  489     QString contentType = r->header(QNetworkRequest::ContentTypeHeader).toString();
  490     int httpCode = r->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
  491     QString msg = r->errorString();
  492     qCWarning(lcDiscovery) << "LSCOL job error" << r->errorString() << httpCode << r->error();
  493     if (r->error() == QNetworkReply::NoError
  494         && !contentType.contains(QLatin1String("application/xml; charset=utf-8"))) {
  495         msg = tr("Server error: PROPFIND reply is not XML formatted!");
  496 
  497     }
  498     emit finished(HttpError{ httpCode, msg });
  499     deleteLater();
  500 }
  501 }