"Fossies" - the Fresh Open Source Software Archive

Member "labplot-2.8.2/src/backend/datasources/filters/AsciiFilter.cpp" (24 Feb 2021, 87302 Bytes) of package /linux/privat/labplot-2.8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "AsciiFilter.cpp" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.8.1_vs_2.8.2.

    1 /***************************************************************************
    2 File                 : AsciiFilter.cpp
    3 Project              : LabPlot
    4 Description          : ASCII I/O-filter
    5 --------------------------------------------------------------------
    6 Copyright            : (C) 2009-2020 Stefan Gerlach (stefan.gerlach@uni.kn)
    7 Copyright            : (C) 2009-2019 Alexander Semke (alexander.semke@web.de)
    8 
    9 ***************************************************************************/
   10 
   11 /***************************************************************************
   12 *                                                                         *
   13 *  This program is free software; you can redistribute it and/or modify   *
   14 *  it under the terms of the GNU General Public License as published by   *
   15 *  the Free Software Foundation; either version 2 of the License, or      *
   16 *  (at your option) any later version.                                    *
   17 *                                                                         *
   18 *  This program is distributed in the hope that it will be useful,        *
   19 *  but WITHOUT ANY WARRANTY; without even the implied warranty of         *
   20 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          *
   21 *  GNU General Public License for more details.                           *
   22 *                                                                         *
   23 *   You should have received a copy of the GNU General Public License     *
   24 *   along with this program; if not, write to the Free Software           *
   25 *   Foundation, Inc., 51 Franklin Street, Fifth Floor,                    *
   26 *   Boston, MA  02110-1301  USA                                           *
   27 *                                                                         *
   28 ***************************************************************************/
   29 #include "backend/datasources/LiveDataSource.h"
   30 #include "backend/core/column/Column.h"
   31 #include "backend/core/Project.h"
   32 #include "backend/datasources/filters/AsciiFilter.h"
   33 #include "backend/datasources/filters/AsciiFilterPrivate.h"
   34 #include "backend/worksheet/plots/cartesian/CartesianPlot.h"
   35 #include "backend/worksheet/plots/cartesian/XYCurve.h"
   36 #include "backend/lib/macros.h"
   37 #include "backend/lib/trace.h"
   38 
   39 #ifdef HAVE_MQTT
   40 #include "backend/datasources/MQTTClient.h"
   41 #include "backend/datasources/MQTTTopic.h"
   42 #endif
   43 
   44 #include <KLocalizedString>
   45 #include <KFilterDev>
   46 #include <QDateTime>
   47 
   48 #if defined(Q_OS_LINUX) || defined(Q_OS_BSD4)
   49 #include <QProcess>
   50 #include <QStandardPaths>
   51 #endif
   52 
   53 #include <QRegularExpression>
   54 
   55 /*!
   56 \class AsciiFilter
   57 \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file.
   58 
   59 \ingroup datasources
   60 */
   61 AsciiFilter::AsciiFilter() : AbstractFileFilter(FileType::Ascii), d(new AsciiFilterPrivate(this)) {}
   62 
   63 AsciiFilter::~AsciiFilter() = default;
   64 
   65 /*!
   66   reads the content of the device \c device.
   67 */
   68 void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) {
   69     d->readDataFromDevice(device, dataSource, importMode, lines);
   70 }
   71 
   72 void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource* dataSource) {
   73     d->readFromLiveDevice(device, dataSource);
   74 }
   75 
   76 qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) {
   77     return d->readFromLiveDevice(device, dataSource, from);
   78 }
   79 
   80 #ifdef HAVE_MQTT
   81 QVector<QStringList> AsciiFilter::preview(const QString& message) {
   82     return d->preview(message);
   83 }
   84 
   85 /*!
   86   reads the content of a message received by the topic.
   87 */
   88 void AsciiFilter::readMQTTTopic(const QString& message, AbstractDataSource* dataSource) {
   89     d->readMQTTTopic(message, dataSource);
   90 }
   91 
   92 /*!
   93   After the MQTTTopic is loaded, prepares the filter for reading.
   94 */
   95 void AsciiFilter::setPreparedForMQTT(bool prepared, MQTTTopic* topic, const QString& separator) {
   96     d->setPreparedForMQTT(prepared, topic, separator);
   97 }
   98 #endif
   99 
  100 /*!
  101   returns the separator used by the filter.
  102 */
  103 QString AsciiFilter::separator() const {
  104     return d->separator();
  105 }
  106 
  107 /*!
  108   returns the separator used by the filter.
  109 */
  110 int AsciiFilter::isPrepared() {
  111     return d->isPrepared();
  112 }
  113 
  114 /*!
  115   reads the content of the file \c fileName.
  116 */
  117 void AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) {
  118     d->readDataFromFile(fileName, dataSource, importMode);
  119 }
  120 
  121 QVector<QStringList> AsciiFilter::preview(const QString& fileName, int lines) {
  122     return d->preview(fileName, lines);
  123 }
  124 
  125 QVector<QStringList> AsciiFilter::preview(QIODevice& device) {
  126     return d->preview(device);
  127 }
  128 
  129 /*!
  130 writes the content of the data source \c dataSource to the file \c fileName.
  131 */
  132 void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) {
  133     d->write(fileName, dataSource);
  134 }
  135 
  136 /*!
  137   loads the predefined filter settings for \c filterName
  138 */
  139 void AsciiFilter::loadFilterSettings(const QString& filterName) {
  140     Q_UNUSED(filterName);
  141 }
  142 
  143 /*!
  144   saves the current settings as a new filter with the name \c filterName
  145 */
  146 void AsciiFilter::saveFilterSettings(const QString& filterName) const {
  147     Q_UNUSED(filterName);
  148 }
  149 
  150 /*!
  151   returns the list with the names of all saved
  152   (system wide or user defined) filter settings.
  153 */
  154 QStringList AsciiFilter::predefinedFilters() {
  155     return QStringList();
  156 }
  157 
  158 /*!
  159   returns the list of all predefined separator characters.
  160 */
  161 QStringList AsciiFilter::separatorCharacters() {
  162     return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":"
  163             << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE" << "2xSPACE" << "3xSPACE" << "4xSPACE" << "2xTAB");
  164 }
  165 
  166 /*!
  167 returns the list of all predefined comment characters.
  168 */
  169 QStringList AsciiFilter::commentCharacters() {
  170     return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";");
  171 }
  172 
  173 /*!
  174 returns the list of all predefined data types.
  175 */
  176 QStringList AsciiFilter::dataTypes() {
  177     const QMetaObject& mo = AbstractColumn::staticMetaObject;
  178     const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode"));
  179     QStringList list;
  180     for (int i = 0; i <= 100; ++i)  // me.keyCount() does not work because we have holes in enum
  181         if (me.valueToKey(i))
  182             list << me.valueToKey(i);
  183     return list;
  184 }
  185 
  186 QString AsciiFilter::fileInfoString(const QString& fileName) {
  187     QString info(i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName)));
  188     info += QLatin1String("<br>");
  189     info += i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName));
  190     return info;
  191 }
  192 
  193 /*!
  194     returns the number of columns in the file \c fileName.
  195 */
  196 int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) {
  197     KFilterDev device(fileName);
  198     if (!device.open(QIODevice::ReadOnly)) {
  199         DEBUG("Could not open file " << STDSTRING(fileName) << " for determining number of columns");
  200         return -1;
  201     }
  202 
  203     QString line = device.readLine();
  204     line.remove(QRegularExpression(QStringLiteral("[\\n\\r]")));
  205 
  206     QStringList lineStringList;
  207     if (separator.length() > 0)
  208         lineStringList = line.split(separator);
  209     else
  210         lineStringList = line.split(QRegularExpression(QStringLiteral("\\s+")));
  211     DEBUG("number of columns : " << lineStringList.size());
  212 
  213     return lineStringList.size();
  214 }
  215 
  216 size_t AsciiFilter::lineNumber(const QString& fileName) {
  217     KFilterDev device(fileName);
  218 
  219     if (!device.open(QIODevice::ReadOnly)) {
  220         DEBUG("Could not open file " << STDSTRING(fileName) << " to determine number of lines");
  221 
  222         return 0;
  223     }
  224 //  if (!device.canReadLine())
  225 //      return -1;
  226 
  227     size_t lineCount = 0;
  228 #if defined(Q_OS_LINUX) || defined(Q_OS_BSD4)
  229     //on linux and BSD use wc, if available, which is much faster than counting lines in the file
  230     if (device.compressionType() == KCompressionDevice::None && !QStandardPaths::findExecutable(QLatin1String("wc")).isEmpty()) {
  231         QProcess wc;
  232         wc.start(QLatin1String("wc"), QStringList() << QLatin1String("-l") << fileName);
  233         size_t lineCount = 0;
  234         while (wc.waitForReadyRead()) {
  235             QString line(wc.readLine());
  236             // wc on macOS has leading spaces: use SkipEmptyParts
  237             lineCount = line.split(' ', QString::SkipEmptyParts)[0].toInt();
  238         }
  239         return lineCount;
  240     }
  241 #endif
  242 
  243     while (!device.atEnd()) {
  244         device.readLine();
  245         lineCount++;
  246     }
  247 
  248     return lineCount;
  249 }
  250 
  251 /*!
  252   returns the number of lines in the device \c device and 0 if sequential.
  253   resets the position to 0!
  254 */
  255 size_t AsciiFilter::lineNumber(QIODevice& device) const {
  256     if (device.isSequential())
  257         return 0;
  258 //  if (!device.canReadLine())
  259 //      DEBUG("WARNING in AsciiFilter::lineNumber(): device cannot 'readLine()' but using it anyway.");
  260 
  261     size_t lineCount = 0;
  262     device.seek(0);
  263     if (d->readingFile)
  264         lineCount = lineNumber(d->readingFileName);
  265     else {
  266         while (!device.atEnd()) {
  267             device.readLine();
  268             lineCount++;
  269         }
  270     }
  271     device.seek(0);
  272 
  273     return lineCount;
  274 }
  275 
  276 void AsciiFilter::setCommentCharacter(const QString& s) {
  277     d->commentCharacter = s;
  278 }
  279 QString AsciiFilter::commentCharacter() const {
  280     return d->commentCharacter;
  281 }
  282 
  283 void AsciiFilter::setSeparatingCharacter(const QString& s) {
  284     d->separatingCharacter = s;
  285 }
  286 QString AsciiFilter::separatingCharacter() const {
  287     return d->separatingCharacter;
  288 }
  289 
  290 void AsciiFilter::setDateTimeFormat(const QString &f) {
  291     d->dateTimeFormat = f;
  292 }
  293 QString AsciiFilter::dateTimeFormat() const {
  294     return d->dateTimeFormat;
  295 }
  296 
  297 void AsciiFilter::setNumberFormat(QLocale::Language lang) {
  298     d->numberFormat = lang;
  299     d->locale = QLocale(lang);
  300 }
  301 QLocale::Language AsciiFilter::numberFormat() const {
  302     return d->numberFormat;
  303 }
  304 
  305 void AsciiFilter::setAutoModeEnabled(const bool b) {
  306     d->autoModeEnabled = b;
  307 }
  308 bool AsciiFilter::isAutoModeEnabled() const {
  309     return d->autoModeEnabled;
  310 }
  311 
  312 void AsciiFilter::setHeaderEnabled(const bool b) {
  313     d->headerEnabled = b;
  314 }
  315 bool AsciiFilter::isHeaderEnabled() const {
  316     return d->headerEnabled;
  317 }
  318 
  319 void AsciiFilter::setSkipEmptyParts(const bool b) {
  320     d->skipEmptyParts = b;
  321 }
  322 bool AsciiFilter::skipEmptyParts() const {
  323     return d->skipEmptyParts;
  324 }
  325 
  326 void AsciiFilter::setCreateIndexEnabled(bool b) {
  327     d->createIndexEnabled = b;
  328 }
  329 
  330 bool AsciiFilter::createIndexEnabled() const {
  331     return d->createIndexEnabled;
  332 }
  333 
  334 void AsciiFilter::setCreateTimestampEnabled(bool b) {
  335     d->createTimestampEnabled = b;
  336 }
  337 
  338 bool AsciiFilter::createTimestampEnabled() const {
  339     return d->createTimestampEnabled;
  340 }
  341 
  342 void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) {
  343     d->simplifyWhitespacesEnabled = b;
  344 }
  345 bool AsciiFilter::simplifyWhitespacesEnabled() const {
  346     return d->simplifyWhitespacesEnabled;
  347 }
  348 
  349 void AsciiFilter::setNaNValueToZero(bool b) {
  350     if (b)
  351         d->nanValue = 0;
  352     else
  353         d->nanValue = std::numeric_limits<double>::quiet_NaN();
  354 }
  355 bool AsciiFilter::NaNValueToZeroEnabled() const {
  356     return (d->nanValue == 0);
  357 }
  358 
  359 void AsciiFilter::setRemoveQuotesEnabled(bool b) {
  360     d->removeQuotesEnabled = b;
  361 }
  362 bool AsciiFilter::removeQuotesEnabled() const {
  363     return d->removeQuotesEnabled;
  364 }
  365 
  366 void AsciiFilter::setVectorNames(const QString& s) {
  367     d->vectorNames.clear();
  368     if (!s.simplified().isEmpty())
  369         d->vectorNames = s.simplified().split(' ');
  370 }
  371 void AsciiFilter::setVectorNames(const QStringList& list) {
  372     d->vectorNames = list;
  373 }
  374 QStringList AsciiFilter::vectorNames() const {
  375     return d->vectorNames;
  376 }
  377 
  378 QVector<AbstractColumn::ColumnMode> AsciiFilter::columnModes() {
  379     return d->columnModes;
  380 }
  381 
  382 void AsciiFilter::setStartRow(const int r) {
  383     d->startRow = r;
  384 }
  385 int AsciiFilter::startRow() const {
  386     return d->startRow;
  387 }
  388 
  389 void AsciiFilter::setEndRow(const int r) {
  390     d->endRow = r;
  391 }
  392 int AsciiFilter::endRow() const {
  393     return d->endRow;
  394 }
  395 
  396 void AsciiFilter::setStartColumn(const int c) {
  397     d->startColumn = c;
  398 }
  399 int AsciiFilter::startColumn() const {
  400     return d->startColumn;
  401 }
  402 
  403 void AsciiFilter::setEndColumn(const int c) {
  404     d->endColumn = c;
  405 }
  406 int AsciiFilter::endColumn() const {
  407     return d->endColumn;
  408 }
  409 
  410 //#####################################################################
  411 //################### Private implementation ##########################
  412 //#####################################################################
  413 AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner) {
  414 }
  415 
  416 int AsciiFilterPrivate::isPrepared() {
  417     return m_prepared;
  418 }
  419 
  420 /*!
  421  * \brief Returns the separator used by the filter
  422  * \return
  423  */
  424 QString AsciiFilterPrivate::separator() const {
  425     return m_separator;
  426 }
  427 
  428 
  429 //#####################################################################
  430 //############################# Read ##################################
  431 //#####################################################################
  432 /*!
  433  * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise.
  434  */
  435 int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) {
  436     DEBUG("AsciiFilterPrivate::prepareDeviceToRead(): is sequential = " << device.isSequential() << ", can readLine = " << device.canReadLine());
  437 
  438     if (!device.open(QIODevice::ReadOnly))
  439         return -1;
  440 
  441     if (device.atEnd() && !device.isSequential()) // empty file
  442         return 1;
  443 
  444 /////////////////////////////////////////////////////////////////
  445     // Parse the first line:
  446     // Determine the number of columns, create the columns and use (if selected) the first row to name them
  447     QString firstLine;
  448 
  449     // skip the comment lines and read the first line
  450     if (!commentCharacter.isEmpty()) {
  451         do {
  452             if (!device.canReadLine())
  453                 DEBUG(Q_FUNC_INFO << ", WARNING: device cannot 'readLine()' but using it anyway.");
  454 
  455             if (device.atEnd()) {
  456                 DEBUG("device at end! Giving up.");
  457                 if (device.isSequential())
  458                     break;
  459                 else
  460                     return 1;
  461             }
  462 
  463             firstLine = device.readLine();
  464         } while (firstLine.startsWith(commentCharacter) || firstLine.simplified().isEmpty());
  465     } else
  466         firstLine = device.readLine();
  467 
  468     // navigate to the line where we asked to start reading from
  469     DEBUG(Q_FUNC_INFO << ", Skipping " << startRow - 1 << " lines");
  470     for (int i = 0; i < startRow - 1; ++i) {
  471         if (!device.canReadLine())
  472             DEBUG(Q_FUNC_INFO << ", WARNING: device cannot 'readLine()' but using it anyway.");
  473 
  474         if (device.atEnd()) {
  475             DEBUG("device at end! Giving up.");
  476             if (device.isSequential())
  477                 break;
  478             else
  479                 return 1;
  480         }
  481 
  482         firstLine = device.readLine();
  483         DEBUG(" line = " << STDSTRING(firstLine));
  484     }
  485 
  486     DEBUG(" device position after first line and comments = " << device.pos());
  487     firstLine.remove(QRegularExpression(QStringLiteral("[\\n\\r]")));   // remove any newline
  488     if (removeQuotesEnabled)
  489         firstLine = firstLine.remove(QLatin1Char('"'));
  490 
  491     //TODO: this doesn't work, the split below introduces whitespaces again
  492 //  if (simplifyWhitespacesEnabled)
  493 //      firstLine = firstLine.simplified();
  494     DEBUG("First line: \'" << STDSTRING(firstLine) << '\'');
  495 
  496     // determine separator and split first line
  497     QStringList firstLineStringList;
  498     if (separatingCharacter == "auto") {
  499         DEBUG("automatic separator");
  500         if (firstLine.indexOf(QLatin1Char('\t')) != -1) {
  501             //in case we have a mix of tabs and spaces in the header, give the tab character the preference
  502             m_separator = QLatin1Char('\t');
  503             firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
  504         } else {
  505             const QRegularExpression regExp(QStringLiteral("[,;:]?\\s+"));
  506             firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts);
  507 
  508             if (!firstLineStringList.isEmpty()) {
  509                 int length1 = firstLineStringList.at(0).length();
  510                 if (firstLineStringList.size() > 1)
  511                     m_separator = firstLine.mid(length1, 1);
  512                 else
  513                     m_separator = ' ';
  514             }
  515         }
  516     } else {    // use given separator
  517         // replace symbolic "TAB" with '\t'
  518         m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive);
  519         m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive);
  520         // replace symbolic "SPACE" with ' '
  521         m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String("  "), Qt::CaseInsensitive);
  522         m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String("   "), Qt::CaseInsensitive);
  523         m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String("    "), Qt::CaseInsensitive);
  524         m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive);
  525         firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
  526     }
  527     DEBUG(Q_FUNC_INFO << ", separator: \'" << STDSTRING(m_separator) << '\'');
  528     DEBUG(Q_FUNC_INFO << ", number of columns: " << firstLineStringList.size());
  529     QDEBUG(Q_FUNC_INFO << ", first line: " << firstLineStringList);
  530     DEBUG(Q_FUNC_INFO << ", headerEnabled: " << headerEnabled);
  531 
  532     //optionally, remove potential spaces in the first line
  533     //TODO: this part should be obsolete actually if we do firstLine = firstLine.simplified(); above...
  534     if (simplifyWhitespacesEnabled) {
  535         for (int i = 0; i < firstLineStringList.size(); ++i)
  536             firstLineStringList[i] = firstLineStringList[i].simplified();
  537     }
  538 
  539     //in GUI in AsciiOptionsWidget we start counting from 1, subtract 1 here to start from zero
  540     m_actualStartRow = startRow - 1;
  541 
  542     if (headerEnabled) {    // use first line to name vectors (starting from startColumn)
  543         vectorNames = firstLineStringList.mid(startColumn - 1);
  544         ++m_actualStartRow;
  545     }
  546 
  547     // set range to read
  548     if (endColumn == -1) {
  549         if (headerEnabled || vectorNames.size() == 0)
  550             endColumn = firstLineStringList.size(); // last column
  551         else
  552             //number of vector names provided in the import dialog (not more than the maximal number of columns in the file)
  553             endColumn = qMin(vectorNames.size(), firstLineStringList.size());
  554     }
  555 
  556     if (endColumn < startColumn)
  557         m_actualCols = 0;
  558     else
  559         m_actualCols = endColumn - startColumn + 1;
  560 
  561     //add index column
  562     if (createTimestampEnabled) {
  563         vectorNames.prepend(i18n("Timestamp"));
  564         m_actualCols++;
  565     }
  566 
  567     //add index column
  568     if (createIndexEnabled) {
  569         vectorNames.prepend(i18n("Index"));
  570         m_actualCols++;
  571     }
  572 
  573     QDEBUG(Q_FUNC_INFO << ", vector names =" << vectorNames);
  574 
  575 //TEST: readline-seek-readline fails
  576     /*  qint64 testpos = device.pos();
  577         DEBUG("read data line @ pos " << testpos << " : " << STDSTRING(device.readLine()));
  578         device.seek(testpos);
  579         testpos = device.pos();
  580         DEBUG("read data line again @ pos " << testpos << "  : " << STDSTRING(device.readLine()));
  581     */
  582 /////////////////////////////////////////////////////////////////
  583 
  584     // parse first data line to determine data type for each column
  585     // if the first line was already parsed as the header, read the next line
  586     if (headerEnabled && !device.isSequential())
  587         firstLineStringList = getLineString(device);
  588 
  589     columnModes.resize(m_actualCols);
  590     int col = 0;
  591     if (createIndexEnabled) {
  592         columnModes[0] = AbstractColumn::ColumnMode::Integer;
  593         ++col;
  594     }
  595 
  596     if (createTimestampEnabled) {
  597         columnModes[col] = AbstractColumn::ColumnMode::DateTime;
  598         ++col;
  599     }
  600 
  601     for (auto& valueString : firstLineStringList) { // parse columns available in first data line
  602         const int index{ col - startColumn + 1 };
  603         if (index < (int)createIndexEnabled + (int)createTimestampEnabled) {
  604             col++;
  605             continue;
  606         }
  607         if (index == m_actualCols)
  608             break;
  609 
  610         if (simplifyWhitespacesEnabled)
  611             valueString = valueString.simplified();
  612         if (removeQuotesEnabled)
  613             valueString.remove(QLatin1Char('"'));
  614         columnModes[index] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat);
  615         col++;
  616     }
  617     for (const auto mode : columnModes)
  618         DEBUG(Q_FUNC_INFO << ", column mode = " << static_cast<int>(mode));
  619 
  620     // parsing more lines to better determine data types
  621     for (unsigned int i = 0; i < m_dataTypeLines; ++i) {
  622         if (device.atEnd()) // EOF reached
  623             break;
  624         firstLineStringList = getLineString(device);
  625 
  626         col = (int)createIndexEnabled + (int)createTimestampEnabled;
  627 
  628         for (auto& valueString : firstLineStringList) {
  629             const int index{col - startColumn + 1};
  630             if (index < (int)createIndexEnabled + (int)createTimestampEnabled) {
  631                 col++;
  632                 continue;
  633             }
  634             if (index == m_actualCols)
  635                 break;
  636 
  637             if (simplifyWhitespacesEnabled)
  638                 valueString = valueString.simplified();
  639             if (removeQuotesEnabled)
  640                 valueString.remove(QLatin1Char('"'));
  641             auto mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat);
  642 
  643             // numeric: integer -> numeric
  644             if (mode == AbstractColumn::ColumnMode::Numeric && columnModes[index] == AbstractColumn::ColumnMode::Integer)
  645                 columnModes[index] = mode;
  646             // text: non text -> text
  647             if (mode == AbstractColumn::ColumnMode::Text && columnModes[index] != AbstractColumn::ColumnMode::Text)
  648                 columnModes[index] = mode;
  649             // numeric: text -> numeric/integer
  650             if (mode != AbstractColumn::ColumnMode::Text && columnModes[index] == AbstractColumn::ColumnMode::Text)
  651                 columnModes[index] = mode;
  652             col++;
  653         }
  654     }
  655     for (const auto mode : columnModes)
  656         DEBUG(Q_FUNC_INFO << ", column mode = " << static_cast<int>(mode));
  657 
  658     // ATTENTION: This resets the position in the device to 0
  659     m_actualRows = (int)q->lineNumber(device);
  660 
  661     const int actualEndRow = (endRow == -1 || endRow > m_actualRows) ? m_actualRows : endRow;
  662     if (actualEndRow > m_actualStartRow)
  663         m_actualRows = actualEndRow - m_actualStartRow;
  664     else
  665         m_actualRows = 0;
  666 
  667     DEBUG("start/end column: " << startColumn << ' ' << endColumn);
  668     DEBUG("start/end row: " << m_actualStartRow << ' ' << actualEndRow);
  669     DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows);
  670 
  671     if (m_actualRows == 0 && !device.isSequential())
  672         return 1;
  673 
  674     return 0;
  675 }
  676 
  677 /*!
  678     reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source.
  679 */
  680 void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) {
  681     DEBUG(Q_FUNC_INFO << ", fileName = \'" << STDSTRING(fileName) << "\', dataSource = "
  682           << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode));
  683 
  684     //dirty hack: set readingFile and readingFileName in order to know in lineNumber(QIODevice)
  685     //that we're reading from a file and to benefit from much faster wc on linux
  686     //TODO: redesign the APIs and remove this later
  687     readingFile = true;
  688     readingFileName = fileName;
  689     KFilterDev device(fileName);
  690     readDataFromDevice(device, dataSource, importMode);
  691     readingFile = false;
  692 }
  693 
  694 qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) {
  695     DEBUG(Q_FUNC_INFO << ", bytes available = " << device.bytesAvailable() << ", from = " << from);
  696     if (device.bytesAvailable() <= 0) {
  697         DEBUG(" No new data available");
  698         return 0;
  699     }
  700 
  701     //TODO: may be also a matrix?
  702     auto* spreadsheet = dynamic_cast<LiveDataSource*>(dataSource);
  703     if (!spreadsheet)
  704         return 0;
  705 
  706     if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe)
  707         if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16))
  708             return 0;
  709 
  710     if (!m_prepared) {
  711         DEBUG(" Preparing ..");
  712 
  713         switch (spreadsheet->sourceType()) {
  714         case LiveDataSource::SourceType::FileOrPipe: {
  715             const int deviceError = prepareDeviceToRead(device);
  716             if (deviceError != 0) {
  717                 DEBUG(" Device error = " << deviceError);
  718                 return 0;
  719             }
  720             break;
  721         }
  722         case LiveDataSource::SourceType::NetworkTcpSocket:
  723         case LiveDataSource::SourceType::NetworkUdpSocket:
  724         case LiveDataSource::SourceType::LocalSocket:
  725         case LiveDataSource::SourceType::SerialPort:
  726             m_actualRows = 1;
  727             m_actualCols = 1;
  728             columnModes.clear();
  729             if (createIndexEnabled) {
  730                 columnModes << AbstractColumn::ColumnMode::Integer;
  731                 vectorNames << i18n("Index");
  732                 m_actualCols++;
  733             }
  734 
  735             if (createTimestampEnabled) {
  736                 columnModes << AbstractColumn::ColumnMode::DateTime;
  737                 vectorNames << i18n("Timestamp");
  738                 m_actualCols++;
  739             }
  740 
  741             //add column for the actual value
  742             columnModes << AbstractColumn::ColumnMode::Numeric;
  743             vectorNames << i18n("Value");
  744 
  745             QDEBUG("    vector names = " << vectorNames);
  746             break;
  747         case LiveDataSource::SourceType::MQTT:
  748             break;
  749         }
  750 
  751         // prepare import for spreadsheet
  752         spreadsheet->setUndoAware(false);
  753         spreadsheet->resize(AbstractFileFilter::ImportMode::Replace, vectorNames, m_actualCols);
  754 
  755         //columns in a file data source don't have any manual changes.
  756         //make the available columns undo unaware and suppress the "data changed" signal.
  757         //data changes will be propagated via an explicit Column::setChanged() call once new data was read.
  758         for (int i = 0; i < spreadsheet->childCount<Column>(); i++) {
  759             spreadsheet->child<Column>(i)->setUndoAware(false);
  760             spreadsheet->child<Column>(i)->setSuppressDataChangedSignal(true);
  761         }
  762 
  763         int keepNValues = spreadsheet->keepNValues();
  764         if (keepNValues == 0)
  765             spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1);
  766         else {
  767             spreadsheet->setRowCount(keepNValues);
  768             m_actualRows = keepNValues;
  769         }
  770 
  771         m_dataContainer.resize(m_actualCols);
  772         initDataContainers(spreadsheet);
  773 
  774         DEBUG(" data source resized to col: " << m_actualCols);
  775         DEBUG(" data source rowCount: " << spreadsheet->rowCount());
  776         DEBUG(" Prepared!");
  777     }
  778 
  779     qint64 bytesread = 0;
  780 
  781 #ifdef PERFTRACE_LIVE_IMPORT
  782     PERFTRACE("AsciiLiveDataImportTotal: ");
  783 #endif
  784     LiveDataSource::ReadingType readingType;
  785     if (!m_prepared) {
  786         readingType = LiveDataSource::ReadingType::TillEnd;
  787     } else {
  788         //we have to read all the data when reading from end
  789         //so we set readingType to TillEnd
  790         if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd)
  791             readingType = LiveDataSource::ReadingType::TillEnd;
  792         //if we read the whole file we just start from the beginning of it
  793         //and read till end
  794         else if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
  795             readingType = LiveDataSource::ReadingType::TillEnd;
  796         else
  797             readingType = spreadsheet->readingType();
  798     }
  799     DEBUG(" Reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType));
  800 
  801     //move to the last read position, from == total bytes read
  802     //since the other source types are sequential we cannot seek on them
  803     if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe)
  804         device.seek(from);
  805 
  806     //count the new lines, increase actualrows on each
  807     //now we read all the new lines, if we want to use sample rate
  808     //then here we can do it, if we have actually sample rate number of lines :-?
  809     int newLinesForSampleSizeNotTillEnd = 0;
  810     int newLinesTillEnd = 0;
  811     QVector<QString> newData;
  812     if (readingType != LiveDataSource::ReadingType::TillEnd)
  813         newData.resize(spreadsheet->sampleSize());
  814 
  815     int newDataIdx = 0;
  816     {
  817 #ifdef PERFTRACE_LIVE_IMPORT
  818         PERFTRACE("AsciiLiveDataImportReadingFromFile: ");
  819 #endif
  820         DEBUG(" source type = " << ENUM_TO_STRING(LiveDataSource, SourceType, spreadsheet->sourceType()));
  821         while (!device.atEnd()) {
  822             if (readingType != LiveDataSource::ReadingType::TillEnd) {
  823                 switch (spreadsheet->sourceType()) {    // different sources need different read methods
  824                 case LiveDataSource::SourceType::LocalSocket:
  825                     newData[newDataIdx++] = device.readAll();
  826                     break;
  827                 case LiveDataSource::SourceType::NetworkUdpSocket:
  828                     newData[newDataIdx++] = device.read(device.bytesAvailable());
  829                     break;
  830                 case LiveDataSource::SourceType::FileOrPipe:
  831                     newData.push_back(device.readLine());
  832                     break;
  833                 case LiveDataSource::SourceType::NetworkTcpSocket:
  834                 //TODO: check serial port
  835                 case LiveDataSource::SourceType::SerialPort:
  836                     newData[newDataIdx++] = device.read(device.bytesAvailable());
  837                     break;
  838                 case LiveDataSource::SourceType::MQTT:
  839                     break;
  840                 }
  841             } else {    // ReadingType::TillEnd
  842                 switch (spreadsheet->sourceType()) {    // different sources need different read methods
  843                 case LiveDataSource::SourceType::LocalSocket:
  844                     newData.push_back(device.readAll());
  845                     break;
  846                 case LiveDataSource::SourceType::NetworkUdpSocket:
  847                     newData.push_back(device.read(device.bytesAvailable()));
  848                     break;
  849                 case LiveDataSource::SourceType::FileOrPipe:
  850                     newData.push_back(device.readLine());
  851                     break;
  852                 case LiveDataSource::SourceType::NetworkTcpSocket:
  853                 //TODO: check serial port
  854                 case LiveDataSource::SourceType::SerialPort:
  855                     newData.push_back(device.read(device.bytesAvailable()));
  856                     break;
  857                 case LiveDataSource::SourceType::MQTT:
  858                     break;
  859                 }
  860             }
  861             newLinesTillEnd++;
  862 
  863             if (readingType != LiveDataSource::ReadingType::TillEnd) {
  864                 newLinesForSampleSizeNotTillEnd++;
  865                 //for Continuous reading and FromEnd we read sample rate number of lines if possible
  866                 //here TillEnd and Whole file behave the same
  867                 if (newLinesForSampleSizeNotTillEnd == spreadsheet->sampleSize())
  868                     break;
  869             }
  870         }
  871         QDEBUG("    data read: " << newData);
  872     }
  873 
  874     //now we reset the readingType
  875     if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd)
  876         readingType = spreadsheet->readingType();
  877 
  878     //we had less new lines than the sample size specified
  879     if (readingType != LiveDataSource::ReadingType::TillEnd)
  880         QDEBUG("    Removed empty lines: " << newData.removeAll(QString()));
  881 
  882     //back to the last read position before counting when reading from files
  883     if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe)
  884         device.seek(from);
  885 
  886     const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount();
  887 
  888     int currentRow = 0; // indexes the position in the vector(column)
  889     int linesToRead = 0;
  890     int keepNValues = spreadsheet->keepNValues();
  891 
  892     DEBUG(" Increase row count. keepNValues = " << keepNValues);
  893     if (m_prepared) {
  894         //increase row count if we don't have a fixed size
  895         //but only after the preparation step
  896         if (keepNValues == 0) {
  897             DEBUG(" keep All values");
  898             if (readingType != LiveDataSource::ReadingType::TillEnd)
  899                 m_actualRows += qMin(newData.size(), spreadsheet->sampleSize());
  900             else {
  901                 //we don't increase it if we reread the whole file, we reset it
  902                 if (!(spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile))
  903                     m_actualRows += newData.size();
  904                 else
  905                     m_actualRows = newData.size();
  906             }
  907 
  908             //appending
  909             if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
  910                 linesToRead = m_actualRows;
  911             else
  912                 linesToRead = m_actualRows - spreadsheetRowCountBeforeResize;
  913         } else {    // fixed size
  914             DEBUG(" keep " << keepNValues << " values");
  915             if (readingType == LiveDataSource::ReadingType::TillEnd) {
  916                 //we had more lines than the fixed size, so we read m_actualRows number of lines
  917                 if (newLinesTillEnd > m_actualRows) {
  918                     linesToRead = m_actualRows;
  919                     //TODO after reading we should skip the next data lines
  920                     //because it's TillEnd actually
  921                 } else
  922                     linesToRead = newLinesTillEnd;
  923             } else {
  924                 //we read max sample size number of lines when the reading mode
  925                 //is ContinuouslyFixed or FromEnd, WholeFile is disabled
  926                 linesToRead = qMin(spreadsheet->sampleSize(), newLinesTillEnd);
  927             }
  928         }
  929 
  930         if (linesToRead == 0)
  931             return 0;
  932     } else  // not prepared
  933         linesToRead = newLinesTillEnd;
  934 
  935     DEBUG(" lines to read = " << linesToRead);
  936     DEBUG(" actual rows (w/o header) = " << m_actualRows);
  937 
  938     //TODO
  939 //  if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe || spreadsheet->sourceType() == LiveDataSource::SourceType::NetworkUdpSocket) {
  940 //      if (m_actualRows < linesToRead) {
  941 //          DEBUG(" SET lines to read to " << m_actualRows);
  942 //          linesToRead = m_actualRows;
  943 //      }
  944 //  }
  945 
  946     //new rows/resize columns if we don't have a fixed size
  947     //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize
  948     if (keepNValues == 0) {
  949 #ifdef PERFTRACE_LIVE_IMPORT
  950         PERFTRACE("AsciiLiveDataImportResizing: ");
  951 #endif
  952         if (spreadsheet->rowCount() < m_actualRows)
  953             spreadsheet->setRowCount(m_actualRows);
  954 
  955         if (!m_prepared)
  956             currentRow = 0;
  957         else {
  958             // indexes the position in the vector(column)
  959             if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
  960                 currentRow = 0;
  961             else
  962                 currentRow = spreadsheetRowCountBeforeResize;
  963         }
  964 
  965         // if we have fixed size, we do this only once in preparation, here we can use
  966         // m_prepared and we need something to decide whether it has a fixed size or increasing
  967         initDataContainers(spreadsheet);
  968     } else {    // fixed size
  969         //when we have a fixed size we have to pop sampleSize number of lines if specified
  970         //here popping, setting currentRow
  971         if (!m_prepared) {
  972             if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
  973                 currentRow = 0;
  974             else
  975                 currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows);
  976         } else {
  977             if (readingType == LiveDataSource::ReadingType::TillEnd) {
  978                 if (newLinesTillEnd > m_actualRows) {
  979                     currentRow = 0;
  980                 } else {
  981                     if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
  982                         currentRow = 0;
  983                     else
  984                         currentRow = m_actualRows - newLinesTillEnd;
  985                 }
  986             } else {
  987                 //we read max sample size number of lines when the reading mode
  988                 //is ContinuouslyFixed or FromEnd
  989                 currentRow = m_actualRows - qMin(spreadsheet->sampleSize(), newLinesTillEnd);
  990             }
  991         }
  992 
  993         if (m_prepared) {
  994 #ifdef PERFTRACE_LIVE_IMPORT
  995             PERFTRACE("AsciiLiveDataImportPopping: ");
  996 #endif
  997             // enable data change signal
  998             for (int col = 0; col < m_actualCols; ++col)
  999                 spreadsheet->child<Column>(col)->setSuppressDataChangedSignal(false);
 1000 
 1001             for (int row = 0; row < linesToRead; ++row) {
 1002                 for (int col = 0; col < m_actualCols; ++col) {
 1003                     switch (columnModes[col]) {
 1004                     case AbstractColumn::ColumnMode::Numeric: {
 1005                         auto* vector = static_cast<QVector<double>* >(spreadsheet->child<Column>(col)->data());
 1006                         vector->pop_front();
 1007                         vector->resize(m_actualRows);
 1008                         m_dataContainer[col] = static_cast<void *>(vector);
 1009                         break;
 1010                     }
 1011                     case AbstractColumn::ColumnMode::Integer: {
 1012                         auto* vector = static_cast<QVector<int>* >(spreadsheet->child<Column>(col)->data());
 1013                         vector->pop_front();
 1014                         vector->resize(m_actualRows);
 1015                         m_dataContainer[col] = static_cast<void *>(vector);
 1016                         break;
 1017                     }
 1018                     case AbstractColumn::ColumnMode::BigInt: {
 1019                         auto* vector = static_cast<QVector<qint64>* >(spreadsheet->child<Column>(col)->data());
 1020                         vector->pop_front();
 1021                         vector->resize(m_actualRows);
 1022                         m_dataContainer[col] = static_cast<void *>(vector);
 1023                         break;
 1024                     }
 1025                     case AbstractColumn::ColumnMode::Text: {
 1026                         auto* vector = static_cast<QVector<QString>*>(spreadsheet->child<Column>(col)->data());
 1027                         vector->pop_front();
 1028                         vector->resize(m_actualRows);
 1029                         m_dataContainer[col] = static_cast<void *>(vector);
 1030                         break;
 1031                     }
 1032                     case AbstractColumn::ColumnMode::DateTime: {
 1033                         auto* vector = static_cast<QVector<QDateTime>* >(spreadsheet->child<Column>(col)->data());
 1034                         vector->pop_front();
 1035                         vector->resize(m_actualRows);
 1036                         m_dataContainer[col] = static_cast<void *>(vector);
 1037                         break;
 1038                     }
 1039                     //TODO
 1040                     case AbstractColumn::ColumnMode::Month:
 1041                     case AbstractColumn::ColumnMode::Day:
 1042                         break;
 1043                     }
 1044                 }
 1045             }
 1046         }
 1047     }
 1048 
 1049     // from the last row we read the new data in the spreadsheet
 1050     DEBUG(" Reading from line "  << currentRow << " till end line " << newLinesTillEnd);
 1051     DEBUG(" Lines to read:" << linesToRead <<", actual rows:" << m_actualRows << ", actual cols:" << m_actualCols);
 1052     newDataIdx = 0;
 1053     if (readingType == LiveDataSource::ReadingType::FromEnd) {
 1054         if (m_prepared) {
 1055             if (newData.size() > spreadsheet->sampleSize())
 1056                 newDataIdx = newData.size() - spreadsheet->sampleSize();
 1057             //since we skip a couple of lines, we need to count those bytes too
 1058             for (int i = 0; i < newDataIdx; ++i)
 1059                 bytesread += newData.at(i).size();
 1060         }
 1061     }
 1062     DEBUG(" newDataIdx: " << newDataIdx);
 1063 
 1064     static int indexColumnIdx = 1;
 1065     {
 1066 #ifdef PERFTRACE_LIVE_IMPORT
 1067         PERFTRACE("AsciiLiveDataImportFillingContainers: ");
 1068 #endif
 1069         int row = 0;
 1070 
 1071         if (readingType == LiveDataSource::ReadingType::TillEnd || (readingType == LiveDataSource::ReadingType::ContinuousFixed)) {
 1072             if (headerEnabled) {
 1073                 if (!m_prepared) {
 1074                     row = 1;
 1075                     bytesread += newData.at(0).size();
 1076                 }
 1077             }
 1078         }
 1079         if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) {
 1080             if (readingType == LiveDataSource::ReadingType::WholeFile) {
 1081                 if (headerEnabled) {
 1082                     row = 1;
 1083                     bytesread += newData.at(0).size();
 1084                 }
 1085             }
 1086         }
 1087 
 1088         for (; row < linesToRead; ++row) {
 1089             DEBUG("\n   Reading row " << row + 1 << " of " << linesToRead);
 1090             QString line;
 1091             if (readingType == LiveDataSource::ReadingType::FromEnd)
 1092                 line = newData.at(newDataIdx++);
 1093             else
 1094                 line = newData.at(row);
 1095             //when we read the whole file we don't care about the previous position
 1096             //so we don't have to count those bytes
 1097             if (readingType != LiveDataSource::ReadingType::WholeFile) {
 1098                 if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) {
 1099                     bytesread += line.size();
 1100                 }
 1101             }
 1102 
 1103             if (removeQuotesEnabled)
 1104                 line.remove(QLatin1Char('"'));
 1105 
 1106             if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
 1107                 continue;
 1108 
 1109             QStringList lineStringList;
 1110             // only FileOrPipe support multiple columns
 1111             if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe)
 1112                 lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
 1113             else
 1114                 lineStringList << line;
 1115             QDEBUG("    line = " << lineStringList << ", separator = \'" << m_separator << "\'");
 1116 
 1117             DEBUG(" Line bytes: " << line.size() << " line: " << STDSTRING(line));
 1118             if (simplifyWhitespacesEnabled) {
 1119                 for (int i = 0; i < lineStringList.size(); ++i)
 1120                     lineStringList[i] = lineStringList[i].simplified();
 1121             }
 1122 
 1123             //add index if required
 1124             int offset = 0;
 1125             if (createIndexEnabled) {
 1126                 int index = (spreadsheet->keepNValues() == 0) ? currentRow + 1 : indexColumnIdx++;
 1127                 static_cast<QVector<int>*>(m_dataContainer[0])->operator[](currentRow) = index;
 1128                 ++offset;
 1129             }
 1130 
 1131             //add current timestamp if required
 1132             if (createTimestampEnabled) {
 1133                 static_cast<QVector<QDateTime>*>(m_dataContainer[offset])->operator[](currentRow) = QDateTime::currentDateTime();
 1134                 ++offset;
 1135             }
 1136 
 1137             //parse columns
 1138             for (int n = offset; n < m_actualCols; ++n) {
 1139                 QString valueString;
 1140                 if (n - offset < lineStringList.size())
 1141                     valueString = lineStringList.at(n - offset);
 1142 
 1143                 setValue(n, currentRow, valueString);
 1144             }
 1145             currentRow++;
 1146         }
 1147     }
 1148 
 1149     if (m_prepared) {
 1150         //notify all affected columns and plots about the changes
 1151         PERFTRACE("AsciiLiveDataImport, notify affected columns and plots");
 1152 
 1153         //determine the dependent plots
 1154         QVector<CartesianPlot*> plots;
 1155         for (int n = 0; n < m_actualCols; ++n)
 1156             spreadsheet->column(n)->addUsedInPlots(plots);
 1157 
 1158         //suppress retransform in the dependent plots
 1159         for (auto* plot : plots)
 1160             plot->setSuppressDataChangedSignal(true);
 1161 
 1162         for (int n = 0; n < m_actualCols; ++n)
 1163             spreadsheet->column(n)->setChanged();
 1164 
 1165         //retransform the dependent plots
 1166         for (auto* plot : plots) {
 1167             plot->setSuppressDataChangedSignal(false);
 1168             plot->dataChanged();
 1169         }
 1170     } else
 1171         m_prepared = true;
 1172 
 1173     DEBUG("AsciiFilterPrivate::readFromLiveDevice() DONE");
 1174     return bytesread;
 1175 }
 1176 
 1177 /*!
 1178     reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source.
 1179 */
 1180 void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) {
 1181     DEBUG(Q_FUNC_INFO << ", dataSource = " << dataSource
 1182           << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines);
 1183 
 1184     if (!m_prepared) {
 1185         const int deviceError = prepareDeviceToRead(device);
 1186         if (deviceError != 0) {
 1187             DEBUG(Q_FUNC_INFO << ", DEVICE ERROR = " << deviceError);
 1188             return;
 1189         }
 1190 
 1191         // matrix data has only one column mode
 1192         if (dynamic_cast<Matrix*>(dataSource)) {
 1193             auto mode = columnModes[0];
 1194             //TODO: remove this when Matrix supports text type
 1195             if (mode == AbstractColumn::ColumnMode::Text)
 1196                 mode = AbstractColumn::ColumnMode::Numeric;
 1197             for (auto& c : columnModes)
 1198                 if (c != mode)
 1199                     c = mode;
 1200         }
 1201 
 1202         Q_ASSERT(dataSource);
 1203         m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes);
 1204         m_prepared = true;
 1205     }
 1206 
 1207     DEBUG("locale = " << STDSTRING(QLocale::languageToString(numberFormat)));
 1208 
 1209     // Read the data
 1210     int currentRow = 0; // indexes the position in the vector(column)
 1211     if (lines == -1)
 1212         lines = m_actualRows;
 1213 
 1214     //skip data lines, if required
 1215     DEBUG(" Skipping " << m_actualStartRow << " lines");
 1216     for (int i = 0; i < m_actualStartRow; ++i)
 1217         device.readLine();
 1218 
 1219     DEBUG(" Reading " << qMin(lines, m_actualRows)  << " lines, " << m_actualCols << " columns");
 1220 
 1221     if (qMin(lines, m_actualRows) == 0 || m_actualCols == 0)
 1222         return;
 1223 
 1224     QString line;
 1225     QString valueString;
 1226     //Don't put the definition QStringList lineStringList outside of the for-loop,
 1227     //the compiler doesn't seem to optimize the destructor of QList well enough in this case.
 1228 
 1229     lines = qMin(lines, m_actualRows);
 1230     int progressIndex = 0;
 1231     const qreal progressInterval = 0.01*lines; //update on every 1% only
 1232 
 1233     for (int i = 0; i < lines; ++i) {
 1234         line = device.readLine();
 1235 
 1236         // remove any newline
 1237         line.remove(QLatin1Char('\n'));
 1238         line.remove(QLatin1Char('\r'));
 1239 
 1240         if (removeQuotesEnabled)
 1241             line.remove(QLatin1Char('"'));
 1242 
 1243         if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
 1244             continue;
 1245 
 1246         QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
 1247 //      DEBUG(" Line bytes: " << line.size() << " line: " << STDSTRING(line));
 1248 
 1249         if (simplifyWhitespacesEnabled) {
 1250             for (int i = 0; i < lineStringList.size(); ++i)
 1251                 lineStringList[i] = lineStringList[i].simplified();
 1252         }
 1253 
 1254         // remove left white spaces
 1255         if (skipEmptyParts) {
 1256             for (int n = 0; n < lineStringList.size(); ++n) {
 1257                 valueString = lineStringList.at(n);
 1258                 if (!QString::compare(valueString, " ")) {
 1259                     lineStringList.removeAt(n);
 1260                     n--;
 1261                 }
 1262             }
 1263         }
 1264 
 1265         //parse columns
 1266         DEBUG(Q_FUNC_INFO << ", actual cols = " << m_actualCols)
 1267         for (int n = 0; n < m_actualCols; ++n) {
 1268             // index column if required
 1269             if (n == 0 && createIndexEnabled) {
 1270                 static_cast<QVector<int>*>(m_dataContainer[0])->operator[](currentRow) = i + 1;
 1271                 continue;
 1272             }
 1273 
 1274             //column counting starts with 1, subtract 1 as well as another 1 for the index column if required
 1275             int col = createIndexEnabled ? n + startColumn - 2: n + startColumn - 1;
 1276             QString valueString;
 1277             if (col < lineStringList.size())
 1278                 valueString = lineStringList.at(col);
 1279 
 1280             setValue(n, currentRow, valueString);
 1281         }
 1282         currentRow++;
 1283 
 1284         //ask to update the progress bar only if we have more than 1000 lines
 1285         //only in 1% steps
 1286         progressIndex++;
 1287         if (lines > 1000 && progressIndex > progressInterval) {
 1288             emit q->completed(100 * currentRow/lines);
 1289             progressIndex = 0;
 1290             QApplication::processEvents(QEventLoop::AllEvents, 0);
 1291         }
 1292     }
 1293 
 1294     DEBUG(Q_FUNC_INFO <<", Read " << currentRow << " lines");
 1295 
 1296     //we might have skipped empty lines above. shrink the spreadsheet if the number of read lines (=currentRow)
 1297     //is smaller than the initial size of the spreadsheet (=m_actualRows).
 1298     //TODO: should also be relevant for Matrix
 1299     auto* s = dynamic_cast<Spreadsheet*>(dataSource);
 1300     if (s && currentRow != m_actualRows && importMode == AbstractFileFilter::ImportMode::Replace)
 1301         s->setRowCount(currentRow);
 1302 
 1303     Q_ASSERT(dataSource);
 1304     dataSource->finalizeImport(m_columnOffset, startColumn, startColumn + m_actualCols - 1, dateTimeFormat, importMode);
 1305 }
 1306 
 1307 //#####################################################################
 1308 //############################ Preview ################################
 1309 //#####################################################################
 1310 
 1311 /*!
 1312  * preview for special devices (local/UDP/TCP socket or serial port)
 1313  */
 1314 QVector<QStringList> AsciiFilterPrivate::preview(QIODevice &device) {
 1315     DEBUG(Q_FUNC_INFO << ", bytesAvailable = " << device.bytesAvailable() << ", isSequential = " << device.isSequential());
 1316     QVector<QStringList> dataStrings;
 1317 
 1318     if (!(device.bytesAvailable() > 0)) {
 1319         DEBUG("No new data available");
 1320         return dataStrings;
 1321     }
 1322 
 1323     if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16))
 1324         return dataStrings;
 1325 
 1326     int linesToRead = 0;
 1327     QVector<QString> newData;
 1328 
 1329     //TODO: serial port "read(nBytes)"?
 1330     while (!device.atEnd()) {
 1331         if (device.canReadLine())
 1332             newData.push_back(device.readLine());
 1333         else    // UDP fails otherwise
 1334             newData.push_back(device.readAll());
 1335         linesToRead++;
 1336     }
 1337     QDEBUG("    data = " << newData);
 1338 
 1339     if (linesToRead == 0)
 1340         return dataStrings;
 1341 
 1342     vectorNames.clear();
 1343     columnModes.clear();
 1344 
 1345     if (createIndexEnabled) {
 1346         columnModes << AbstractColumn::ColumnMode::Integer;
 1347         vectorNames << i18n("Index");
 1348     }
 1349 
 1350     if (createTimestampEnabled) {
 1351         columnModes << AbstractColumn::ColumnMode::DateTime;
 1352         vectorNames << i18n("Timestamp");
 1353     }
 1354 
 1355     //parse the first data line to determine data type for each column
 1356 #if (QT_VERSION >= QT_VERSION_CHECK(5, 14, 0))
 1357     QStringList firstLineStringList = newData.at(0).split(' ', Qt::SkipEmptyParts);
 1358 #else
 1359     QStringList firstLineStringList = newData.at(0).split(' ', QString::SkipEmptyParts);
 1360 #endif
 1361     int i = 1;
 1362     for (auto& valueString : firstLineStringList) {
 1363         if (simplifyWhitespacesEnabled)
 1364             valueString = valueString.simplified();
 1365         if (removeQuotesEnabled)
 1366             valueString.remove(QLatin1Char('"'));
 1367         if (skipEmptyParts && !QString::compare(valueString, " "))  // handle left white spaces
 1368             continue;
 1369 
 1370         vectorNames << i18n("Value %1", i);
 1371         columnModes << AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat);
 1372         ++i;
 1373     }
 1374 
 1375     int offset = int(createIndexEnabled) + int(createTimestampEnabled);
 1376     QString line;
 1377 
 1378     //loop over all lines in the new data in the device and parse the available columns
 1379     for (int i = 0; i < linesToRead; ++i) {
 1380         line = newData.at(i);
 1381 
 1382         // remove any newline
 1383         line = line.remove('\n');
 1384         line = line.remove('\r');
 1385 
 1386         if (simplifyWhitespacesEnabled)
 1387             line = line.simplified();
 1388 
 1389         if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
 1390             continue;
 1391 
 1392         QStringList lineString;
 1393 
 1394         // index column if required
 1395         if (createIndexEnabled)
 1396             lineString += QString::number(i + 1);
 1397 
 1398         // timestamp column if required
 1399         if (createTimestampEnabled)
 1400             lineString += QDateTime::currentDateTime().toString();
 1401 
 1402 #if (QT_VERSION >= QT_VERSION_CHECK(5, 14, 0))
 1403         QStringList lineStringList = line.split(' ', Qt::SkipEmptyParts);
 1404 #else
 1405         QStringList lineStringList = line.split(' ', QString::SkipEmptyParts);
 1406 #endif
 1407         QDEBUG(" line = " << lineStringList);
 1408 
 1409         //parse columns
 1410         DEBUG(Q_FUNC_INFO << ", number of columns = " << lineStringList.size())
 1411         for (int n = 0; n < lineStringList.size(); ++n) {
 1412             if (n < lineStringList.size()) {
 1413                 QString valueString = lineStringList.at(n);
 1414                 if (removeQuotesEnabled)
 1415                     valueString.remove(QLatin1Char('"'));
 1416 
 1417                 if (skipEmptyParts && !QString::compare(valueString, " "))  // handle left white spaces
 1418                     continue;
 1419 
 1420                 lineString += previewValue(valueString, columnModes[n+offset]);
 1421             } else  // missing columns in this line
 1422                 lineString += QString();
 1423         }
 1424 
 1425         dataStrings << lineString;
 1426     }
 1427 
 1428     return dataStrings;
 1429 }
 1430 
 1431 /*!
 1432  * generates the preview for the file \c fileName reading the provided number of \c lines.
 1433  */
 1434 QVector<QStringList> AsciiFilterPrivate::preview(const QString& fileName, int lines) {
 1435     QVector<QStringList> dataStrings;
 1436 
 1437     //dirty hack: set readingFile and readingFileName in order to know in lineNumber(QIODevice)
 1438     //that we're reading from a file and to benefit from much faster wc on linux
 1439     //TODO: redesign the APIs and remove this later
 1440     readingFile = true;
 1441     readingFileName = fileName;
 1442     KFilterDev device(fileName);
 1443     const int deviceError = prepareDeviceToRead(device);
 1444     readingFile = false;
 1445 
 1446     if (deviceError != 0) {
 1447         DEBUG("Device error = " << deviceError);
 1448         return dataStrings;
 1449     }
 1450 
 1451     //number formatting
 1452     DEBUG("locale = " << STDSTRING(QLocale::languageToString(numberFormat)));
 1453 
 1454     // Read the data
 1455     if (lines == -1)
 1456         lines = m_actualRows;
 1457 
 1458     // set column names for preview
 1459     if (!headerEnabled) {
 1460         int start = 0;
 1461         if (createIndexEnabled)
 1462             start = 1;
 1463         for (int i = start; i < m_actualCols; i++)
 1464             vectorNames << "Column " + QString::number(i + 1);
 1465     }
 1466     QDEBUG("    column names = " << vectorNames);
 1467 
 1468     //skip data lines, if required
 1469     DEBUG(" Skipping " << m_actualStartRow << " lines");
 1470     for (int i = 0; i < m_actualStartRow; ++i)
 1471         device.readLine();
 1472 
 1473     DEBUG(" Generating preview for " << qMin(lines, m_actualRows)  << " lines");
 1474     QString line;
 1475 
 1476     //loop over the preview lines in the file and parse the available columns
 1477     for (int i = 0; i < qMin(lines, m_actualRows); ++i) {
 1478         line = device.readLine();
 1479 
 1480         // remove any newline
 1481         line = line.remove('\n');
 1482         line = line.remove('\r');
 1483 
 1484         if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
 1485             continue;
 1486 
 1487         QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
 1488         QDEBUG(" line = " << lineStringList);
 1489         DEBUG(" Line bytes: " << line.size() << " line: " << STDSTRING(line));
 1490 
 1491         if (simplifyWhitespacesEnabled) {
 1492             for (int i = 0; i < lineStringList.size(); ++i)
 1493                 lineStringList[i] = lineStringList[i].simplified();
 1494         }
 1495 
 1496         QStringList lineString;
 1497 
 1498         //parse columns
 1499         for (int n = 0; n < m_actualCols; ++n) {
 1500             // index column if required
 1501             if (n == 0 && createIndexEnabled) {
 1502                 lineString += QString::number(i + 1);
 1503                 continue;
 1504             }
 1505 
 1506             //column counting starts with 1, subtract 1 as well as another 1 for the index column if required
 1507             int col = createIndexEnabled ? n + startColumn - 2 : n + startColumn - 1;
 1508 
 1509             if (col < lineStringList.size()) {
 1510                 QString valueString = lineStringList.at(col);
 1511                 if (removeQuotesEnabled)
 1512                     valueString.remove(QLatin1Char('"'));
 1513 
 1514                 if (skipEmptyParts && !QString::compare(valueString, " "))  // handle left white spaces
 1515                     continue;
 1516 
 1517                 lineString += previewValue(valueString, columnModes[n]);
 1518             } else  // missing columns in this line
 1519                 lineString += QString();
 1520         }
 1521 
 1522         dataStrings << lineString;
 1523     }
 1524 
 1525     return dataStrings;
 1526 }
 1527 
 1528 //#####################################################################
 1529 //####################### Helper functions ############################
 1530 //#####################################################################
 1531 /*!
 1532  * converts \c valueString to the date type according to \c mode and \c locale
 1533  * and returns its string representation.
 1534  */
 1535 QString AsciiFilterPrivate::previewValue(const QString& valueString, AbstractColumn::ColumnMode mode) {
 1536     DEBUG(Q_FUNC_INFO << ", valueString = " << valueString.toStdString() << ", mode = " << (int)mode)
 1537 
 1538     QString result;
 1539     switch (mode) {
 1540     case AbstractColumn::ColumnMode::Numeric: {
 1541         bool isNumber;
 1542         const double value = locale.toDouble(valueString, &isNumber);
 1543         result = QString::number(isNumber ? value : nanValue, 'g', 15);
 1544         break;
 1545     }
 1546     case AbstractColumn::ColumnMode::Integer: {
 1547         bool isNumber;
 1548         const int value = locale.toInt(valueString, &isNumber);
 1549         result = QString::number(isNumber ? value : 0);
 1550         break;
 1551     }
 1552     case AbstractColumn::ColumnMode::BigInt: {
 1553         bool isNumber;
 1554         const qint64 value = locale.toLongLong(valueString, &isNumber);
 1555         result = QString::number(isNumber ? value : 0);
 1556         break;
 1557     }
 1558     case AbstractColumn::ColumnMode::DateTime: {
 1559         QDateTime valueDateTime = parseDateTime(valueString, dateTimeFormat);
 1560         result = valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" ");
 1561         break;
 1562     }
 1563     case AbstractColumn::ColumnMode::Text:
 1564         result = valueString;
 1565         break;
 1566     case AbstractColumn::ColumnMode::Month: // never happens
 1567     case AbstractColumn::ColumnMode::Day:
 1568         break;
 1569     }
 1570     return result;
 1571 }
 1572 
 1573 //set value depending on data type
 1574 void AsciiFilterPrivate::setValue(int col, int row, const QString& valueString) {
 1575     if (!valueString.isEmpty()) {
 1576         switch (columnModes.at(col)) {
 1577         case AbstractColumn::ColumnMode::Numeric: {
 1578             bool isNumber;
 1579             const double value = locale.toDouble(valueString, &isNumber);
 1580             static_cast<QVector<double>*>(m_dataContainer[col])->operator[](row) = (isNumber ? value : nanValue);
 1581             break;
 1582         }
 1583         case AbstractColumn::ColumnMode::Integer: {
 1584             bool isNumber;
 1585             const int value = locale.toInt(valueString, &isNumber);
 1586             static_cast<QVector<int>*>(m_dataContainer[col])->operator[](row) = (isNumber ? value : 0);
 1587             break;
 1588         }
 1589         case AbstractColumn::ColumnMode::BigInt: {
 1590             bool isNumber;
 1591             const qint64 value = locale.toLongLong(valueString, &isNumber);
 1592             static_cast<QVector<qint64>*>(m_dataContainer[col])->operator[](row) = (isNumber ? value : 0);
 1593             break;
 1594         }
 1595         case AbstractColumn::ColumnMode::DateTime: {
 1596             QDateTime valueDateTime = parseDateTime(valueString, dateTimeFormat);
 1597             static_cast<QVector<QDateTime>*>(m_dataContainer[col])->operator[](row) = valueDateTime.isValid() ? valueDateTime : QDateTime();
 1598             break;
 1599         }
 1600         case AbstractColumn::ColumnMode::Text: {
 1601             auto* colData = static_cast<QVector<QString>*>(m_dataContainer[col]);
 1602             colData->operator[](row) = valueString;
 1603             break;
 1604         }
 1605         case AbstractColumn::ColumnMode::Month: // never happens
 1606         case AbstractColumn::ColumnMode::Day:
 1607             break;
 1608         }
 1609     } else {    // missing columns in this line
 1610         switch (columnModes.at(col)) {
 1611         case AbstractColumn::ColumnMode::Numeric:
 1612             static_cast<QVector<double>*>(m_dataContainer[col])->operator[](row) = nanValue;
 1613             break;
 1614         case AbstractColumn::ColumnMode::Integer:
 1615             static_cast<QVector<int>*>(m_dataContainer[col])->operator[](row) = 0;
 1616             break;
 1617         case AbstractColumn::ColumnMode::BigInt:
 1618             static_cast<QVector<qint64>*>(m_dataContainer[col])->operator[](row) = 0;
 1619             break;
 1620         case AbstractColumn::ColumnMode::DateTime:
 1621             static_cast<QVector<QDateTime>*>(m_dataContainer[col])->operator[](row) = QDateTime();
 1622             break;
 1623         case AbstractColumn::ColumnMode::Text:
 1624             static_cast<QVector<QString>*>(m_dataContainer[col])->operator[](row).clear();
 1625             break;
 1626         case AbstractColumn::ColumnMode::Month: // never happens
 1627         case AbstractColumn::ColumnMode::Day:
 1628             break;
 1629         }
 1630     }
 1631 }
 1632 
 1633 void AsciiFilterPrivate::initDataContainers(Spreadsheet* spreadsheet) {
 1634     DEBUG(" Initializing the data containers ..");
 1635     for (int n = 0; n < m_actualCols; ++n) {
 1636         // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp)
 1637         spreadsheet->child<Column>(n)->setColumnMode(columnModes[n]);
 1638         switch (columnModes[n]) {
 1639         case AbstractColumn::ColumnMode::Numeric: {
 1640             auto* vector = static_cast<QVector<double>* >(spreadsheet->child<Column>(n)->data());
 1641             vector->reserve(m_actualRows);
 1642             vector->resize(m_actualRows);
 1643             m_dataContainer[n] = static_cast<void *>(vector);
 1644             break;
 1645         }
 1646         case AbstractColumn::ColumnMode::Integer: {
 1647             auto* vector = static_cast<QVector<int>* >(spreadsheet->child<Column>(n)->data());
 1648             vector->resize(m_actualRows);
 1649             m_dataContainer[n] = static_cast<void *>(vector);
 1650             break;
 1651         }
 1652         case AbstractColumn::ColumnMode::BigInt: {
 1653             auto* vector = static_cast<QVector<qint64>* >(spreadsheet->child<Column>(n)->data());
 1654             vector->resize(m_actualRows);
 1655             m_dataContainer[n] = static_cast<void *>(vector);
 1656             break;
 1657         }
 1658         case AbstractColumn::ColumnMode::Text: {
 1659             auto* vector = static_cast<QVector<QString>*>(spreadsheet->child<Column>(n)->data());
 1660             vector->resize(m_actualRows);
 1661             m_dataContainer[n] = static_cast<void *>(vector);
 1662             break;
 1663         }
 1664         case AbstractColumn::ColumnMode::DateTime: {
 1665             auto* vector = static_cast<QVector<QDateTime>* >(spreadsheet->child<Column>(n)->data());
 1666             vector->resize(m_actualRows);
 1667             m_dataContainer[n] = static_cast<void *>(vector);
 1668             break;
 1669         }
 1670         //TODO
 1671         case AbstractColumn::ColumnMode::Month:
 1672         case AbstractColumn::ColumnMode::Day:
 1673             break;
 1674         }
 1675     }
 1676 }
 1677 
 1678 /*!
 1679  * get a single line from device
 1680  */
 1681 QStringList AsciiFilterPrivate::getLineString(QIODevice& device) {
 1682     QString line;
 1683     do {    // skip comment lines in data lines
 1684         if (!device.canReadLine())
 1685             DEBUG("WARNING in AsciiFilterPrivate::getLineString(): device cannot 'readLine()' but using it anyway.");
 1686 //          line = device.readAll();
 1687         line = device.readLine();
 1688     } while (!commentCharacter.isEmpty() && line.startsWith(commentCharacter));
 1689 
 1690     line.remove(QRegularExpression(QStringLiteral("[\\n\\r]")));    // remove any newline
 1691     DEBUG("data line : \'" << STDSTRING(line) << '\'');
 1692     QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
 1693     //TODO: remove quotes here?
 1694     if (simplifyWhitespacesEnabled) {
 1695         for (int i = 0; i < lineStringList.size(); ++i)
 1696             lineStringList[i] = lineStringList[i].simplified();
 1697     }
 1698     QDEBUG("data line, parsed: " << lineStringList);
 1699 
 1700     return lineStringList;
 1701 }
 1702 
 1703 /*!
 1704     writes the content of \c dataSource to the file \c fileName.
 1705 */
 1706 void AsciiFilterPrivate::write(const QString& fileName, AbstractDataSource* dataSource) {
 1707     Q_UNUSED(fileName);
 1708     Q_UNUSED(dataSource);
 1709 
 1710     //TODO: save data to ascii file
 1711 }
 1712 
 1713 /*!
 1714  * create datetime from \c string using \c format considering corner cases
 1715  */
 1716 QDateTime AsciiFilterPrivate::parseDateTime(const QString& string, const QString& format) {
 1717     //DEBUG("string = " << STDSTRING(string) << ", format = " << STDSTRING(format))
 1718     QString fixedString(string);
 1719     QString fixedFormat(format);
 1720     if (!format.contains("yy")) {   // no year given: set temporary to 2000 (must be a leap year to parse "Feb 29")
 1721         fixedString.append(" 2000");
 1722         fixedFormat.append(" yyyy");
 1723     }
 1724 
 1725     QDateTime dateTime = QDateTime::fromString(fixedString, fixedFormat);
 1726     //QDEBUG("fromString() =" << dateTime)
 1727     // interpret 2-digit year smaller than 50 as 20XX
 1728     if (dateTime.date().year() < 1950 && !format.contains("yyyy"))
 1729         dateTime = dateTime.addYears(100);
 1730     //QDEBUG("dateTime fixed =" << dateTime)
 1731     //DEBUG("dateTime.toString =" << STDSTRING(dateTime.toString(format)))
 1732 
 1733     return dateTime;
 1734 }
 1735 
 1736 
 1737 //##############################################################################
 1738 //##################  Serialization/Deserialization  ###########################
 1739 //##############################################################################
 1740 /*!
 1741   Saves as XML.
 1742  */
 1743 void AsciiFilter::save(QXmlStreamWriter* writer) const {
 1744     writer->writeStartElement( "asciiFilter");
 1745     writer->writeAttribute( "commentCharacter", d->commentCharacter);
 1746     writer->writeAttribute( "separatingCharacter", d->separatingCharacter);
 1747     writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled));
 1748     writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled));
 1749     writer->writeAttribute( "createTimestamp", QString::number(d->createTimestampEnabled));
 1750     writer->writeAttribute( "header", QString::number(d->headerEnabled));
 1751     writer->writeAttribute( "vectorNames", d->vectorNames.join(' '));
 1752     writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts));
 1753     writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled));
 1754     writer->writeAttribute( "nanValue", QString::number(d->nanValue));
 1755     writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled));
 1756     writer->writeAttribute( "startRow", QString::number(d->startRow));
 1757     writer->writeAttribute( "endRow", QString::number(d->endRow));
 1758     writer->writeAttribute( "startColumn", QString::number(d->startColumn));
 1759     writer->writeAttribute( "endColumn", QString::number(d->endColumn));
 1760     writer->writeEndElement();
 1761 }
 1762 
 1763 /*!
 1764   Loads from XML.
 1765 */
 1766 bool AsciiFilter::load(XmlStreamReader* reader) {
 1767     KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used");
 1768     QXmlStreamAttributes attribs = reader->attributes();
 1769     QString str;
 1770 
 1771     READ_STRING_VALUE("commentCharacter", commentCharacter);
 1772     READ_STRING_VALUE("separatingCharacter", separatingCharacter);
 1773 
 1774     READ_INT_VALUE("createIndex", createIndexEnabled, bool);
 1775     READ_INT_VALUE("createTimestamp", createTimestampEnabled, bool);
 1776     READ_INT_VALUE("autoMode", autoModeEnabled, bool);
 1777     READ_INT_VALUE("header", headerEnabled, bool);
 1778 
 1779     str = attribs.value("vectorNames").toString();
 1780     d->vectorNames = str.split(' '); //may be empty
 1781 
 1782     READ_INT_VALUE("simplifyWhitespaces", simplifyWhitespacesEnabled, bool);
 1783     READ_DOUBLE_VALUE("nanValue", nanValue);
 1784     READ_INT_VALUE("removeQuotes", removeQuotesEnabled, bool);
 1785     READ_INT_VALUE("skipEmptyParts", skipEmptyParts, bool);
 1786     READ_INT_VALUE("startRow", startRow, int);
 1787     READ_INT_VALUE("endRow", endRow, int);
 1788     READ_INT_VALUE("startColumn", startColumn, int);
 1789     READ_INT_VALUE("endColumn", endColumn, int);
 1790     return true;
 1791 }
 1792 
 1793 //##############################################################################
 1794 //########################## MQTT releated code  ###############################
 1795 //##############################################################################
 1796 #ifdef HAVE_MQTT
 1797 int AsciiFilterPrivate::prepareToRead(const QString& message) {
 1798     QStringList lines = message.split('\n');
 1799     if (lines.isEmpty())
 1800         return 1;
 1801 
 1802     // Parse the first line:
 1803     // Determine the number of columns, create the columns and use (if selected) the first row to name them
 1804     QString firstLine = lines.at(0);
 1805     if (simplifyWhitespacesEnabled)
 1806         firstLine = firstLine.simplified();
 1807     DEBUG("First line: \'" << STDSTRING(firstLine) << '\'');
 1808 
 1809     // determine separator and split first line
 1810     QStringList firstLineStringList;
 1811     if (separatingCharacter == "auto") {
 1812         DEBUG("automatic separator");
 1813         const QRegularExpression regExp(QStringLiteral("[,;:]?\\s+"));
 1814         firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts);
 1815     } else {    // use given separator
 1816         // replace symbolic "TAB" with '\t'
 1817         m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive);
 1818         m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive);
 1819         // replace symbolic "SPACE" with ' '
 1820         m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String("  "), Qt::CaseInsensitive);
 1821         m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String("   "), Qt::CaseInsensitive);
 1822         m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String("    "), Qt::CaseInsensitive);
 1823         m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive);
 1824         firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
 1825     }
 1826     DEBUG("separator: \'" << STDSTRING(m_separator) << '\'');
 1827     DEBUG("number of columns: " << firstLineStringList.size());
 1828     QDEBUG("first line: " << firstLineStringList);
 1829 
 1830     //all columns are read plus the optional column for the index and for the timestamp
 1831     m_actualCols = firstLineStringList.size() + int(createIndexEnabled) + int(createTimestampEnabled);
 1832 
 1833     //column names:
 1834     //when reading the message strings for different topics, it's not possible to specify vector names
 1835     //since the different topics can have different content and different number of columns/vectors
 1836     //->we always set the vector names here to fixed values
 1837     vectorNames.clear();
 1838     columnModes.clear();
 1839 
 1840     //add index column
 1841     if (createIndexEnabled) {
 1842         vectorNames << i18n("Index");
 1843         columnModes << AbstractColumn::ColumnMode::Integer;
 1844     }
 1845 
 1846     //add timestamp column
 1847     if (createTimestampEnabled) {
 1848         vectorNames << i18n("Timestamp");
 1849         columnModes << AbstractColumn::ColumnMode::DateTime;
 1850     }
 1851 
 1852     //parse the first data line to determine data type for each column
 1853     int i = 1;
 1854     for (auto& valueString : firstLineStringList) {
 1855         if (simplifyWhitespacesEnabled)
 1856             valueString = valueString.simplified();
 1857         if (removeQuotesEnabled)
 1858             valueString.remove(QLatin1Char('"'));
 1859 
 1860         vectorNames << i18n("Value %1", i);
 1861         columnModes << AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat);
 1862         ++i;
 1863     }
 1864 
 1865     m_actualStartRow = startRow;
 1866     m_actualRows = lines.size();
 1867 
 1868 //  QDEBUG("column modes = " << columnModes);
 1869     DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows);
 1870 
 1871     return 0;
 1872 }
 1873 
 1874 /*!
 1875  * generates the preview for the string \s message.
 1876  */
 1877 QVector<QStringList> AsciiFilterPrivate::preview(const QString& message) {
 1878     QVector<QStringList> dataStrings;
 1879     prepareToRead(message);
 1880 
 1881     //number formatting
 1882     DEBUG("locale = " << STDSTRING(QLocale::languageToString(numberFormat)));
 1883 
 1884     // Read the data
 1885     QStringList lines = message.split('\n');
 1886 
 1887     //loop over all lines in the message and parse the available columns
 1888     int i = 0;
 1889     for (auto line : lines) {
 1890         if (simplifyWhitespacesEnabled)
 1891             line = line.simplified();
 1892 
 1893         if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
 1894             continue;
 1895 
 1896         const QStringList& lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
 1897         QDEBUG(" line = " << lineStringList);
 1898 
 1899         QStringList lineString;
 1900 
 1901         // index column if required
 1902         if (createIndexEnabled)
 1903             lineString += QString::number(i + 1);
 1904 
 1905         // timestamp column if required
 1906         if (createTimestampEnabled)
 1907             lineString += QDateTime::currentDateTime().toString();
 1908 
 1909         int offset = int(createIndexEnabled) + int(createTimestampEnabled);
 1910 
 1911         //parse columns
 1912         for (int n = 0; n < m_actualCols - offset; ++n) {
 1913             if (n < lineStringList.size()) {
 1914                 QString valueString = lineStringList.at(n);
 1915                 if (removeQuotesEnabled)
 1916                     valueString.remove(QLatin1Char('"'));
 1917                 if (skipEmptyParts && !QString::compare(valueString, " "))  // handle left white spaces
 1918                     continue;
 1919 
 1920                 lineString += previewValue(valueString, columnModes[n+offset]);
 1921             } else  // missing columns in this line
 1922                 lineString += QString();
 1923         }
 1924 
 1925         ++i;
 1926         dataStrings << lineString;
 1927     }
 1928 
 1929     return dataStrings;
 1930 }
 1931 
 1932 /*!
 1933  * \brief reads the content of a message received by the topic.
 1934  * Uses the settings defined in the MQTTTopic's MQTTClient
 1935  * \param message
 1936  * \param topic
 1937  * \param dataSource
 1938  */
 1939 void AsciiFilterPrivate::readMQTTTopic(const QString& message, AbstractDataSource* dataSource) {
 1940     //If the message is empty, there is nothing to do
 1941     if (message.isEmpty()) {
 1942         DEBUG("No new data available");
 1943         return;
 1944     }
 1945 
 1946     MQTTTopic* spreadsheet = dynamic_cast<MQTTTopic*>(dataSource);
 1947     if (!spreadsheet)
 1948         return;
 1949 
 1950     const int keepNValues = spreadsheet->mqttClient()->keepNValues();
 1951 
 1952     if (!m_prepared) {
 1953         DEBUG("Start preparing filter for: " << STDSTRING(spreadsheet->topicName()));
 1954 
 1955         //Prepare the filter
 1956         const int mqttPrepareError = prepareToRead(message);
 1957         if (mqttPrepareError != 0) {
 1958             DEBUG("Mqtt Prepare Error = " << mqttPrepareError);
 1959             return;
 1960         }
 1961 
 1962         // prepare import for spreadsheet
 1963         spreadsheet->setUndoAware(false);
 1964         spreadsheet->resize(AbstractFileFilter::ImportMode::Replace, vectorNames, m_actualCols);
 1965 
 1966         //columns in a MQTTTopic don't have any manual changes.
 1967         //make the available columns undo unaware and suppress the "data changed" signal.
 1968         //data changes will be propagated via an explicit Column::setChanged() call once new data was read.
 1969         for (int i = 0; i < spreadsheet->childCount<Column>(); i++) {
 1970             spreadsheet->child<Column>(i)->setUndoAware(false);
 1971             spreadsheet->child<Column>(i)->setSuppressDataChangedSignal(true);
 1972         }
 1973 
 1974         if (keepNValues == 0)
 1975             spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1);
 1976         else {
 1977             spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues());
 1978             m_actualRows = spreadsheet->mqttClient()->keepNValues();
 1979         }
 1980 
 1981         m_dataContainer.resize(m_actualCols);
 1982         initDataContainers(spreadsheet);
 1983     }
 1984 
 1985     MQTTClient::ReadingType readingType;
 1986     if (!m_prepared) {
 1987         //if filter is not prepared we read till the end
 1988         readingType = MQTTClient::ReadingType::TillEnd;
 1989     } else {
 1990         //we have to read all the data when reading from end
 1991         //so we set readingType to TillEnd
 1992         if (spreadsheet->mqttClient()->readingType() == MQTTClient::ReadingType::FromEnd)
 1993             readingType = MQTTClient::ReadingType::TillEnd;
 1994         else
 1995             readingType = spreadsheet->mqttClient()->readingType();
 1996     }
 1997 
 1998     //count the new lines, increase actualrows on each
 1999     //now we read all the new lines, if we want to use sample rate
 2000     //then here we can do it, if we have actually sample rate number of lines :-?
 2001     int newLinesForSampleSizeNotTillEnd = 0;
 2002     int newLinesTillEnd = 0;
 2003     QVector<QString> newData;
 2004     if (readingType != MQTTClient::ReadingType::TillEnd) {
 2005         newData.reserve(spreadsheet->mqttClient()->sampleSize());
 2006         newData.resize(spreadsheet->mqttClient()->sampleSize());
 2007     }
 2008 
 2009     //TODO: bool sampleSizeReached = false;
 2010     const QStringList newDataList = message.split(QRegularExpression(QStringLiteral("\n|\r\n|\r")),
 2011                                                     QString::SkipEmptyParts);
 2012     for (auto& line : newDataList) {
 2013         newData.push_back(line);
 2014         newLinesTillEnd++;
 2015 
 2016         if (readingType != MQTTClient::ReadingType::TillEnd) {
 2017             newLinesForSampleSizeNotTillEnd++;
 2018             //for Continuous reading and FromEnd we read sample rate number of lines if possible
 2019             if (newLinesForSampleSizeNotTillEnd == spreadsheet->mqttClient()->sampleSize()) {
 2020                 //TODO: sampleSizeReached = true;
 2021                 break;
 2022             }
 2023         }
 2024     }
 2025 
 2026     qDebug()<<"Processing message done";
 2027     //now we reset the readingType
 2028     if (spreadsheet->mqttClient()->readingType() == MQTTClient::ReadingType::FromEnd)
 2029         readingType = static_cast<MQTTClient::ReadingType>(spreadsheet->mqttClient()->readingType());
 2030 
 2031     //we had less new lines than the sample rate specified
 2032     if (readingType != MQTTClient::ReadingType::TillEnd)
 2033         qDebug() << "Removed empty lines: " << newData.removeAll(QString());
 2034 
 2035     const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount();
 2036 
 2037     if (m_prepared ) {
 2038         if (keepNValues == 0)
 2039             m_actualRows = spreadsheetRowCountBeforeResize;
 2040         else {
 2041             //if the keepNValues changed since the last read we have to manage the columns accordingly
 2042             if (m_actualRows != spreadsheet->mqttClient()->keepNValues()) {
 2043                 if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
 2044                     spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues());
 2045                     qDebug()<<"rowcount set to: " << spreadsheet->mqttClient()->keepNValues();
 2046                 }
 2047 
 2048                 //Calculate the difference between the old and new keepNValues
 2049                 int rowDiff = 0;
 2050                 if (m_actualRows > spreadsheet->mqttClient()->keepNValues())
 2051                     rowDiff = m_actualRows -  spreadsheet->mqttClient()->keepNValues();
 2052 
 2053                 if (m_actualRows < spreadsheet->mqttClient()->keepNValues())
 2054                     rowDiff = spreadsheet->mqttClient()->keepNValues() - m_actualRows;
 2055 
 2056                 for (int n = 0; n < columnModes.size(); ++n) {
 2057                     // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp)
 2058                     switch (columnModes[n]) {
 2059                     case AbstractColumn::ColumnMode::Numeric: {
 2060                         QVector<double>*  vector = static_cast<QVector<double>* >(spreadsheet->child<Column>(n)->data());
 2061                         m_dataContainer[n] = static_cast<void *>(vector);
 2062 
 2063                         //if the keepNValues got smaller then we move the last keepNValues count of data
 2064                         //in the first keepNValues places
 2065                         if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
 2066                             for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
 2067                                 static_cast<QVector<double>*>(m_dataContainer[n])->operator[] (i) =
 2068                                     static_cast<QVector<double>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
 2069                             }
 2070                         }
 2071 
 2072                         //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
 2073                         //then fill the remaining lines with NaN
 2074                         if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
 2075                             vector->reserve( spreadsheet->mqttClient()->keepNValues());
 2076                             vector->resize( spreadsheet->mqttClient()->keepNValues());
 2077 
 2078                             for (int i = 1; i <= m_actualRows; i++) {
 2079                                 static_cast<QVector<double>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
 2080                                     static_cast<QVector<double>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
 2081                             }
 2082                             for (int i = 0; i < rowDiff; i++)
 2083                                 static_cast<QVector<double>*>(m_dataContainer[n])->operator[](i) = nanValue;
 2084                         }
 2085                         break;
 2086                     }
 2087                     case AbstractColumn::ColumnMode::Integer: {
 2088                         QVector<int>* vector = static_cast<QVector<int>* >(spreadsheet->child<Column>(n)->data());
 2089                         m_dataContainer[n] = static_cast<void *>(vector);
 2090 
 2091                         //if the keepNValues got smaller then we move the last keepNValues count of data
 2092                         //in the first keepNValues places
 2093                         if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
 2094                             for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
 2095                                 static_cast<QVector<int>*>(m_dataContainer[n])->operator[] (i) =
 2096                                     static_cast<QVector<int>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
 2097                             }
 2098                         }
 2099 
 2100                         //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
 2101                         //then fill the remaining lines with 0
 2102                         if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
 2103                             vector->reserve( spreadsheet->mqttClient()->keepNValues());
 2104                             vector->resize( spreadsheet->mqttClient()->keepNValues());
 2105                             for (int i = 1; i <= m_actualRows; i++) {
 2106                                 static_cast<QVector<int>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
 2107                                     static_cast<QVector<int>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
 2108                             }
 2109                             for (int i = 0; i < rowDiff; i++)
 2110                                 static_cast<QVector<int>*>(m_dataContainer[n])->operator[](i) = 0;
 2111                         }
 2112                         break;
 2113                     }
 2114                     case AbstractColumn::ColumnMode::BigInt: {
 2115                         QVector<qint64>* vector = static_cast<QVector<qint64>* >(spreadsheet->child<Column>(n)->data());
 2116                         m_dataContainer[n] = static_cast<void *>(vector);
 2117 
 2118                         //if the keepNValues got smaller then we move the last keepNValues count of data
 2119                         //in the first keepNValues places
 2120                         if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
 2121                             for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
 2122                                 static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[] (i) =
 2123                                     static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
 2124                             }
 2125                         }
 2126 
 2127                         //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
 2128                         //then fill the remaining lines with 0
 2129                         if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
 2130                             vector->reserve( spreadsheet->mqttClient()->keepNValues());
 2131                             vector->resize( spreadsheet->mqttClient()->keepNValues());
 2132                             for (int i = 1; i <= m_actualRows; i++) {
 2133                                 static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
 2134                                     static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
 2135                             }
 2136                             for (int i = 0; i < rowDiff; i++)
 2137                                 static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[](i) = 0;
 2138                         }
 2139                         break;
 2140                     }
 2141                     case AbstractColumn::ColumnMode::Text: {
 2142                         QVector<QString>* vector = static_cast<QVector<QString>*>(spreadsheet->child<Column>(n)->data());
 2143                         m_dataContainer[n] = static_cast<void *>(vector);
 2144 
 2145                         //if the keepNValues got smaller then we move the last keepNValues count of data
 2146                         //in the first keepNValues places
 2147                         if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
 2148                             for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
 2149                                 static_cast<QVector<QString>*>(m_dataContainer[n])->operator[] (i) =
 2150                                     static_cast<QVector<QString>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
 2151                             }
 2152                         }
 2153 
 2154                         //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
 2155                         //then fill the remaining lines with empty lines
 2156                         if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
 2157                             vector->reserve( spreadsheet->mqttClient()->keepNValues());
 2158                             vector->resize( spreadsheet->mqttClient()->keepNValues());
 2159                             for (int i = 1; i <= m_actualRows; i++) {
 2160                                 static_cast<QVector<QString>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
 2161                                     static_cast<QVector<QString>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
 2162                             }
 2163                             for (int i = 0; i < rowDiff; i++)
 2164                                 static_cast<QVector<QString>*>(m_dataContainer[n])->operator[](i).clear();
 2165                         }
 2166                         break;
 2167                     }
 2168                     case AbstractColumn::ColumnMode::DateTime: {
 2169                         QVector<QDateTime>* vector = static_cast<QVector<QDateTime>* >(spreadsheet->child<Column>(n)->data());
 2170                         m_dataContainer[n] = static_cast<void *>(vector);
 2171 
 2172                         //if the keepNValues got smaller then we move the last keepNValues count of data
 2173                         //in the first keepNValues places
 2174                         if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
 2175                             for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
 2176                                 static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[] (i) =
 2177                                     static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
 2178                             }
 2179                         }
 2180 
 2181                         //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
 2182                         //then fill the remaining lines with null datetime
 2183                         if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
 2184                             vector->reserve( spreadsheet->mqttClient()->keepNValues());
 2185                             vector->resize( spreadsheet->mqttClient()->keepNValues());
 2186                             for (int i = 1; i <= m_actualRows; i++) {
 2187                                 static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
 2188                                     static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
 2189                             }
 2190                             for (int i = 0; i < rowDiff; i++)
 2191                                 static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[](i) = QDateTime();
 2192                         }
 2193                         break;
 2194                     }
 2195                     //TODO
 2196                     case AbstractColumn::ColumnMode::Month:
 2197                     case AbstractColumn::ColumnMode::Day:
 2198                         break;
 2199                     }
 2200                 }
 2201                 //if the keepNValues got smaller resize the spreadsheet
 2202                 if (m_actualRows > spreadsheet->mqttClient()->keepNValues())
 2203                     spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues());
 2204 
 2205                 //set the new row count
 2206                 m_actualRows = spreadsheet->mqttClient()->keepNValues();
 2207                 qDebug()<<"actual rows: "<<m_actualRows;
 2208             }
 2209         }
 2210     }
 2211 
 2212     qDebug()<<"starting m_actual rows calculated: " << m_actualRows <<", new data size: "<<newData.size();
 2213 
 2214     int currentRow = 0; // indexes the position in the vector(column)
 2215     int linesToRead = 0;
 2216 
 2217     if (m_prepared) {
 2218         //increase row count if we don't have a fixed size
 2219         //but only after the preparation step
 2220         if (keepNValues == 0) {
 2221             if (readingType != MQTTClient::ReadingType::TillEnd)
 2222                 m_actualRows += qMin(newData.size(), spreadsheet->mqttClient()->sampleSize());
 2223             else {
 2224                 m_actualRows += newData.size();
 2225             }
 2226         }
 2227 
 2228         //fixed size
 2229         if (keepNValues != 0) {
 2230             if (readingType == MQTTClient::ReadingType::TillEnd) {
 2231                 //we had more lines than the fixed size, so we read m_actualRows number of lines
 2232                 if (newLinesTillEnd > m_actualRows) {
 2233                     linesToRead = m_actualRows;
 2234                 } else
 2235                     linesToRead = newLinesTillEnd;
 2236             } else {
 2237                 //we read max sample size number of lines when the reading mode
 2238                 //is ContinuouslyFixed or FromEnd
 2239                 if (spreadsheet->mqttClient()->sampleSize() <= spreadsheet->mqttClient()->keepNValues())
 2240                     linesToRead = qMin(spreadsheet->mqttClient()->sampleSize(), newLinesTillEnd);
 2241                 else
 2242                     linesToRead = qMin(spreadsheet->mqttClient()->keepNValues(), newLinesTillEnd);
 2243             }
 2244         } else
 2245             linesToRead = m_actualRows - spreadsheetRowCountBeforeResize;
 2246 
 2247         if (linesToRead == 0)
 2248             return;
 2249     } else {
 2250         if (keepNValues != 0)
 2251             linesToRead = newLinesTillEnd > m_actualRows ? m_actualRows : newLinesTillEnd;
 2252         else
 2253             linesToRead = newLinesTillEnd;
 2254     }
 2255     qDebug()<<"linestoread = " << linesToRead;
 2256 
 2257     //new rows/resize columns if we don't have a fixed size
 2258     if (keepNValues == 0) {
 2259 
 2260 #ifdef PERFTRACE_LIVE_IMPORT
 2261         PERFTRACE("AsciiLiveDataImportResizing: ");
 2262 #endif
 2263         if (spreadsheet->rowCount() < m_actualRows)
 2264             spreadsheet->setRowCount(m_actualRows);
 2265 
 2266         if (!m_prepared)
 2267             currentRow = 0;
 2268         else {
 2269             // indexes the position in the vector(column)
 2270             currentRow = spreadsheetRowCountBeforeResize;
 2271         }
 2272 
 2273         // if we have fixed size, we do this only once in preparation, here we can use
 2274         // m_prepared and we need something to decide whether it has a fixed size or increasing
 2275 
 2276         initDataContainers(spreadsheet);
 2277     } else {
 2278         //when we have a fixed size we have to pop sampleSize number of lines if specified
 2279         //here popping, setting currentRow
 2280         if (!m_prepared)
 2281             currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows);
 2282         else {
 2283             if (readingType == MQTTClient::ReadingType::TillEnd) {
 2284                 if (newLinesTillEnd > m_actualRows)
 2285                     currentRow = 0;
 2286                 else
 2287                     currentRow = m_actualRows - newLinesTillEnd;
 2288             } else {
 2289                 //we read max sample rate number of lines when the reading mode
 2290                 //is ContinuouslyFixed or FromEnd
 2291                 currentRow = m_actualRows - linesToRead;
 2292             }
 2293         }
 2294 
 2295         if (m_prepared) {
 2296 #ifdef PERFTRACE_LIVE_IMPORT
 2297             PERFTRACE("AsciiLiveDataImportPopping: ");
 2298 #endif
 2299             for (int row = 0; row < linesToRead; ++row) {
 2300                 for (int col = 0;  col < m_actualCols; ++col) {
 2301                     switch (columnModes[col]) {
 2302                     case AbstractColumn::ColumnMode::Numeric: {
 2303                         QVector<double>* vector = static_cast<QVector<double>* >(spreadsheet->child<Column>(col)->data());
 2304                         vector->pop_front();
 2305                         vector->reserve(m_actualRows);
 2306                         vector->resize(m_actualRows);
 2307                         m_dataContainer[col] = static_cast<void *>(vector);
 2308                         break;
 2309                     }
 2310                     case AbstractColumn::ColumnMode::Integer: {
 2311                         QVector<int>* vector = static_cast<QVector<int>* >(spreadsheet->child<Column>(col)->data());
 2312                         vector->pop_front();
 2313                         vector->reserve(m_actualRows);
 2314                         vector->resize(m_actualRows);
 2315                         m_dataContainer[col] = static_cast<void *>(vector);
 2316                         break;
 2317                     }
 2318                     case AbstractColumn::ColumnMode::BigInt: {
 2319                         QVector<qint64>* vector = static_cast<QVector<qint64>* >(spreadsheet->child<Column>(col)->data());
 2320                         vector->pop_front();
 2321                         vector->reserve(m_actualRows);
 2322                         vector->resize(m_actualRows);
 2323                         m_dataContainer[col] = static_cast<void *>(vector);
 2324                         break;
 2325                     }
 2326                     case AbstractColumn::ColumnMode::Text: {
 2327                         QVector<QString>* vector = static_cast<QVector<QString>*>(spreadsheet->child<Column>(col)->data());
 2328                         vector->pop_front();
 2329                         vector->reserve(m_actualRows);
 2330                         vector->resize(m_actualRows);
 2331                         m_dataContainer[col] = static_cast<void *>(vector);
 2332                         break;
 2333                     }
 2334                     case AbstractColumn::ColumnMode::DateTime: {
 2335                         QVector<QDateTime>* vector = static_cast<QVector<QDateTime>* >(spreadsheet->child<Column>(col)->data());
 2336                         vector->pop_front();
 2337                         vector->reserve(m_actualRows);
 2338                         vector->resize(m_actualRows);
 2339                         m_dataContainer[col] = static_cast<void *>(vector);
 2340                         break;
 2341                     }
 2342                     //TODO
 2343                     case AbstractColumn::ColumnMode::Month:
 2344                     case AbstractColumn::ColumnMode::Day:
 2345                         break;
 2346                     }
 2347                 }
 2348             }
 2349         }
 2350     }
 2351 
 2352     // from the last row we read the new data in the spreadsheet
 2353     qDebug() << "reading from line: "  << currentRow << " lines till end: " << newLinesTillEnd;
 2354     qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows;
 2355     int newDataIdx = 0;
 2356     //From end means that we read the last sample size amount of data
 2357     if (readingType == MQTTClient::ReadingType::FromEnd) {
 2358         if (m_prepared) {
 2359             if (newData.size() > spreadsheet->mqttClient()->sampleSize())
 2360                 newDataIdx = newData.size() - spreadsheet->mqttClient()->sampleSize();
 2361         }
 2362     }
 2363 
 2364     qDebug() << "newDataIdx: " << newDataIdx;
 2365 
 2366     //read the data
 2367     static int indexColumnIdx = 0;
 2368     {
 2369 #ifdef PERFTRACE_LIVE_IMPORT
 2370         PERFTRACE("AsciiLiveDataImportFillingContainers: ");
 2371 #endif
 2372         int row = 0;
 2373         for (; row < linesToRead; ++row) {
 2374             QString line;
 2375             if (readingType == MQTTClient::ReadingType::FromEnd)
 2376                 line = newData.at(newDataIdx++);
 2377             else
 2378                 line = newData.at(row);
 2379 
 2380             if (removeQuotesEnabled)
 2381                 line.remove(QLatin1Char('"'));
 2382 
 2383             if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter)))
 2384                 continue;
 2385 
 2386             QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
 2387 
 2388             if (simplifyWhitespacesEnabled) {
 2389                 for (int i = 0; i < lineStringList.size(); ++i)
 2390                     lineStringList[i] = lineStringList[i].simplified();
 2391             }
 2392 
 2393             //add index if required
 2394             int offset = 0;
 2395             if (createIndexEnabled) {
 2396                 int index = (keepNValues == 0) ? currentRow + 1 : indexColumnIdx++;
 2397                 static_cast<QVector<int>*>(m_dataContainer[0])->operator[](currentRow) = index;
 2398                 ++offset;
 2399             }
 2400 
 2401             //add current timestamp if required
 2402             if (createTimestampEnabled) {
 2403                 static_cast<QVector<QDateTime>*>(m_dataContainer[offset])->operator[](currentRow) = QDateTime::currentDateTime();
 2404                 ++offset;
 2405             }
 2406 
 2407             //parse the columns
 2408             for (int n = 0; n < m_actualCols - offset; ++n) {
 2409                 int col = n + offset;
 2410                 QString valueString;
 2411                 if (n < lineStringList.size())
 2412                     valueString = lineStringList.at(n);
 2413 
 2414                 setValue(col, currentRow, valueString);
 2415             }
 2416             currentRow++;
 2417         }
 2418     }
 2419 
 2420     if (m_prepared) {
 2421         //notify all affected columns and plots about the changes
 2422         PERFTRACE("AsciiLiveDataImport, notify affected columns and plots");
 2423 
 2424         const Project* project = spreadsheet->project();
 2425         QVector<const XYCurve*> curves = project->children<const XYCurve>(AbstractAspect::ChildIndexFlag::Recursive);
 2426         QVector<CartesianPlot*> plots;
 2427 
 2428         for (int n = 0; n < m_actualCols; ++n) {
 2429             Column* column = spreadsheet->column(n);
 2430 
 2431             //determine the plots where the column is consumed
 2432             for (const auto* curve : curves) {
 2433                 if (curve->xColumn() == column || curve->yColumn() == column) {
 2434                     CartesianPlot* plot = static_cast<CartesianPlot*>(curve->parentAspect());
 2435                     if (plots.indexOf(plot) == -1) {
 2436                         plots << plot;
 2437                         plot->setSuppressDataChangedSignal(true);
 2438                     }
 2439                 }
 2440             }
 2441 
 2442             column->setChanged();
 2443         }
 2444 
 2445         //loop over all affected plots and retransform them
 2446         for (auto* const plot : plots) {
 2447             //TODO setting this back to true triggers again a lot of retransforms in the plot (one for each curve).
 2448             //              plot->setSuppressDataChangedSignal(false);
 2449             plot->dataChanged();
 2450         }
 2451     } else
 2452         m_prepared = true;
 2453 
 2454     DEBUG("AsciiFilterPrivate::readFromMQTTTopic() DONE");
 2455 }
 2456 
 2457 /*!
 2458  * \brief After the MQTTTopic was loaded, the filter is prepared for reading
 2459  * \param prepared
 2460  * \param topic
 2461  * \param separator
 2462  */
 2463 void AsciiFilterPrivate::setPreparedForMQTT(bool prepared, MQTTTopic* topic, const QString& separator) {
 2464     m_prepared = prepared;
 2465     //If originally it was prepared we have to restore the settings
 2466     if (prepared) {
 2467         m_separator = separator;
 2468         m_actualCols = endColumn - startColumn + 1;
 2469         m_actualRows = topic->rowCount();
 2470         //set the column modes
 2471         columnModes.resize(topic->columnCount());
 2472         for (int i = 0; i < topic->columnCount(); ++i)
 2473             columnModes[i] = topic->column(i)->columnMode();
 2474 
 2475         //set the data containers
 2476         m_dataContainer.resize(m_actualCols);
 2477         initDataContainers(topic);
 2478     }
 2479 }
 2480 #endif