labplot  2.8.2
About: LabPlot is an application for plotting and analysis of 2D and 3D functions and data. It is a complete rewrite of LabPlot1 and lacks in the first release a lot of features available in the predecessor. On the other hand, the GUI and the usability is more superior.
  Fossies Dox: labplot-2.8.2.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

AsciiFilter.cpp
Go to the documentation of this file.
1 /***************************************************************************
2 File : AsciiFilter.cpp
3 Project : LabPlot
4 Description : ASCII I/O-filter
5 --------------------------------------------------------------------
6 Copyright : (C) 2009-2020 Stefan Gerlach (stefan.gerlach@uni.kn)
7 Copyright : (C) 2009-2019 Alexander Semke (alexander.semke@web.de)
8 
9 ***************************************************************************/
10 
11 /***************************************************************************
12 * *
13 * This program is free software; you can redistribute it and/or modify *
14 * it under the terms of the GNU General Public License as published by *
15 * the Free Software Foundation; either version 2 of the License, or *
16 * (at your option) any later version. *
17 * *
18 * This program is distributed in the hope that it will be useful, *
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
21 * GNU General Public License for more details. *
22 * *
23 * You should have received a copy of the GNU General Public License *
24 * along with this program; if not, write to the Free Software *
25 * Foundation, Inc., 51 Franklin Street, Fifth Floor, *
26 * Boston, MA 02110-1301 USA *
27 * *
28 ***************************************************************************/
31 #include "backend/core/Project.h"
36 #include "backend/lib/macros.h"
37 #include "backend/lib/trace.h"
38 
39 #ifdef HAVE_MQTT
42 #endif
43 
44 #include <KLocalizedString>
45 #include <KFilterDev>
46 #include <QDateTime>
47 
48 #if defined(Q_OS_LINUX) || defined(Q_OS_BSD4)
49 #include <QProcess>
50 #include <QStandardPaths>
51 #endif
52 
53 #include <QRegularExpression>
54 
55 /*!
56 \class AsciiFilter
57 \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file.
58 
59 \ingroup datasources
60 */
62 
63 AsciiFilter::~AsciiFilter() = default;
64 
65 /*!
66  reads the content of the device \c device.
67 */
68 void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) {
69  d->readDataFromDevice(device, dataSource, importMode, lines);
70 }
71 
72 void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource* dataSource) {
73  d->readFromLiveDevice(device, dataSource);
74 }
75 
76 qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) {
77  return d->readFromLiveDevice(device, dataSource, from);
78 }
79 
80 #ifdef HAVE_MQTT
81 QVector<QStringList> AsciiFilter::preview(const QString& message) {
82  return d->preview(message);
83 }
84 
85 /*!
86  reads the content of a message received by the topic.
87 */
88 void AsciiFilter::readMQTTTopic(const QString& message, AbstractDataSource* dataSource) {
89  d->readMQTTTopic(message, dataSource);
90 }
91 
92 /*!
93  After the MQTTTopic is loaded, prepares the filter for reading.
94 */
95 void AsciiFilter::setPreparedForMQTT(bool prepared, MQTTTopic* topic, const QString& separator) {
96  d->setPreparedForMQTT(prepared, topic, separator);
97 }
98 #endif
99 
100 /*!
101  returns the separator used by the filter.
102 */
103 QString AsciiFilter::separator() const {
104  return d->separator();
105 }
106 
107 /*!
108  returns the separator used by the filter.
109 */
111  return d->isPrepared();
112 }
113 
114 /*!
115  reads the content of the file \c fileName.
116 */
117 void AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) {
118  d->readDataFromFile(fileName, dataSource, importMode);
119 }
120 
121 QVector<QStringList> AsciiFilter::preview(const QString& fileName, int lines) {
122  return d->preview(fileName, lines);
123 }
124 
126  return d->preview(device);
127 }
128 
129 /*!
130 writes the content of the data source \c dataSource to the file \c fileName.
131 */
132 void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) {
133  d->write(fileName, dataSource);
134 }
135 
136 /*!
137  loads the predefined filter settings for \c filterName
138 */
139 void AsciiFilter::loadFilterSettings(const QString& filterName) {
140  Q_UNUSED(filterName);
141 }
142 
143 /*!
144  saves the current settings as a new filter with the name \c filterName
145 */
146 void AsciiFilter::saveFilterSettings(const QString& filterName) const {
147  Q_UNUSED(filterName);
148 }
149 
150 /*!
151  returns the list with the names of all saved
152  (system wide or user defined) filter settings.
153 */
155  return QStringList();
156 }
157 
158 /*!
159  returns the list of all predefined separator characters.
160 */
162  return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":"
163  << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE" << "2xSPACE" << "3xSPACE" << "4xSPACE" << "2xTAB");
164 }
165 
166 /*!
167 returns the list of all predefined comment characters.
168 */
170  return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";");
171 }
172 
173 /*!
174 returns the list of all predefined data types.
175 */
176 QStringList AsciiFilter::dataTypes() {
177  const QMetaObject& mo = AbstractColumn::staticMetaObject;
178  const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode"));
179  QStringList list;
180  for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum
181  if (me.valueToKey(i))
182  list << me.valueToKey(i);
183  return list;
184 }
185 
186 QString AsciiFilter::fileInfoString(const QString& fileName) {
187  QString info(i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName)));
188  info += QLatin1String("<br>");
189  info += i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName));
190  return info;
191 }
192 
193 /*!
194  returns the number of columns in the file \c fileName.
195 */
196 int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) {
197  KFilterDev device(fileName);
198  if (!device.open(QIODevice::ReadOnly)) {
199  DEBUG("Could not open file " << STDSTRING(fileName) << " for determining number of columns");
200  return -1;
201  }
202 
203  QString line = device.readLine();
204  line.remove(QRegularExpression(QStringLiteral("[\\n\\r]")));
205 
206  QStringList lineStringList;
207  if (separator.length() > 0)
208  lineStringList = line.split(separator);
209  else
210  lineStringList = line.split(QRegularExpression(QStringLiteral("\\s+")));
211  DEBUG("number of columns : " << lineStringList.size());
212 
213  return lineStringList.size();
214 }
215 
216 size_t AsciiFilter::lineNumber(const QString& fileName) {
217  KFilterDev device(fileName);
218 
219  if (!device.open(QIODevice::ReadOnly)) {
220  DEBUG("Could not open file " << STDSTRING(fileName) << " to determine number of lines");
221 
222  return 0;
223  }
224 // if (!device.canReadLine())
225 // return -1;
226 
227  size_t lineCount = 0;
228 #if defined(Q_OS_LINUX) || defined(Q_OS_BSD4)
229  //on linux and BSD use wc, if available, which is much faster than counting lines in the file
230  if (device.compressionType() == KCompressionDevice::None && !QStandardPaths::findExecutable(QLatin1String("wc")).isEmpty()) {
231  QProcess wc;
232  wc.start(QLatin1String("wc"), QStringList() << QLatin1String("-l") << fileName);
233  size_t lineCount = 0;
234  while (wc.waitForReadyRead()) {
235  QString line(wc.readLine());
236  // wc on macOS has leading spaces: use SkipEmptyParts
237  lineCount = line.split(' ', QString::SkipEmptyParts)[0].toInt();
238  }
239  return lineCount;
240  }
241 #endif
242 
243  while (!device.atEnd()) {
244  device.readLine();
245  lineCount++;
246  }
247 
248  return lineCount;
249 }
250 
251 /*!
252  returns the number of lines in the device \c device and 0 if sequential.
253  resets the position to 0!
254 */
255 size_t AsciiFilter::lineNumber(QIODevice& device) const {
256  if (device.isSequential())
257  return 0;
258 // if (!device.canReadLine())
259 // DEBUG("WARNING in AsciiFilter::lineNumber(): device cannot 'readLine()' but using it anyway.");
260 
261  size_t lineCount = 0;
262  device.seek(0);
263  if (d->readingFile)
264  lineCount = lineNumber(d->readingFileName);
265  else {
266  while (!device.atEnd()) {
267  device.readLine();
268  lineCount++;
269  }
270  }
271  device.seek(0);
272 
273  return lineCount;
274 }
275 
276 void AsciiFilter::setCommentCharacter(const QString& s) {
277  d->commentCharacter = s;
278 }
280  return d->commentCharacter;
281 }
282 
283 void AsciiFilter::setSeparatingCharacter(const QString& s) {
284  d->separatingCharacter = s;
285 }
287  return d->separatingCharacter;
288 }
289 
290 void AsciiFilter::setDateTimeFormat(const QString &f) {
291  d->dateTimeFormat = f;
292 }
294  return d->dateTimeFormat;
295 }
296 
297 void AsciiFilter::setNumberFormat(QLocale::Language lang) {
298  d->numberFormat = lang;
299  d->locale = QLocale(lang);
300 }
301 QLocale::Language AsciiFilter::numberFormat() const {
302  return d->numberFormat;
303 }
304 
306  d->autoModeEnabled = b;
307 }
309  return d->autoModeEnabled;
310 }
311 
312 void AsciiFilter::setHeaderEnabled(const bool b) {
313  d->headerEnabled = b;
314 }
316  return d->headerEnabled;
317 }
318 
319 void AsciiFilter::setSkipEmptyParts(const bool b) {
320  d->skipEmptyParts = b;
321 }
323  return d->skipEmptyParts;
324 }
325 
327  d->createIndexEnabled = b;
328 }
329 
331  return d->createIndexEnabled;
332 }
333 
335  d->createTimestampEnabled = b;
336 }
337 
339  return d->createTimestampEnabled;
340 }
341 
343  d->simplifyWhitespacesEnabled = b;
344 }
346  return d->simplifyWhitespacesEnabled;
347 }
348 
350  if (b)
351  d->nanValue = 0;
352  else
353  d->nanValue = std::numeric_limits<double>::quiet_NaN();
354 }
356  return (d->nanValue == 0);
357 }
358 
360  d->removeQuotesEnabled = b;
361 }
363  return d->removeQuotesEnabled;
364 }
365 
366 void AsciiFilter::setVectorNames(const QString& s) {
367  d->vectorNames.clear();
368  if (!s.simplified().isEmpty())
369  d->vectorNames = s.simplified().split(' ');
370 }
371 void AsciiFilter::setVectorNames(const QStringList& list) {
372  d->vectorNames = list;
373 }
374 QStringList AsciiFilter::vectorNames() const {
375  return d->vectorNames;
376 }
377 
379  return d->columnModes;
380 }
381 
382 void AsciiFilter::setStartRow(const int r) {
383  d->startRow = r;
384 }
386  return d->startRow;
387 }
388 
389 void AsciiFilter::setEndRow(const int r) {
390  d->endRow = r;
391 }
392 int AsciiFilter::endRow() const {
393  return d->endRow;
394 }
395 
396 void AsciiFilter::setStartColumn(const int c) {
397  d->startColumn = c;
398 }
400  return d->startColumn;
401 }
402 
403 void AsciiFilter::setEndColumn(const int c) {
404  d->endColumn = c;
405 }
407  return d->endColumn;
408 }
409 
410 //#####################################################################
411 //################### Private implementation ##########################
412 //#####################################################################
414 }
415 
417  return m_prepared;
418 }
419 
420 /*!
421  * \brief Returns the separator used by the filter
422  * \return
423  */
425  return m_separator;
426 }
427 
428 
429 //#####################################################################
430 //############################# Read ##################################
431 //#####################################################################
432 /*!
433  * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise.
434  */
436  DEBUG("AsciiFilterPrivate::prepareDeviceToRead(): is sequential = " << device.isSequential() << ", can readLine = " << device.canReadLine());
437 
438  if (!device.open(QIODevice::ReadOnly))
439  return -1;
440 
441  if (device.atEnd() && !device.isSequential()) // empty file
442  return 1;
443 
444 /////////////////////////////////////////////////////////////////
445  // Parse the first line:
446  // Determine the number of columns, create the columns and use (if selected) the first row to name them
447  QString firstLine;
448 
449  // skip the comment lines and read the first line
450  if (!commentCharacter.isEmpty()) {
451  do {
452  if (!device.canReadLine())
453  DEBUG(Q_FUNC_INFO << ", WARNING: device cannot 'readLine()' but using it anyway.");
454 
455  if (device.atEnd()) {
456  DEBUG("device at end! Giving up.");
457  if (device.isSequential())
458  break;
459  else
460  return 1;
461  }
462 
463  firstLine = device.readLine();
464  } while (firstLine.startsWith(commentCharacter) || firstLine.simplified().isEmpty());
465  } else
466  firstLine = device.readLine();
467 
468  // navigate to the line where we asked to start reading from
469  DEBUG(Q_FUNC_INFO << ", Skipping " << startRow - 1 << " lines");
470  for (int i = 0; i < startRow - 1; ++i) {
471  if (!device.canReadLine())
472  DEBUG(Q_FUNC_INFO << ", WARNING: device cannot 'readLine()' but using it anyway.");
473 
474  if (device.atEnd()) {
475  DEBUG("device at end! Giving up.");
476  if (device.isSequential())
477  break;
478  else
479  return 1;
480  }
481 
482  firstLine = device.readLine();
483  DEBUG(" line = " << STDSTRING(firstLine));
484  }
485 
486  DEBUG(" device position after first line and comments = " << device.pos());
487  firstLine.remove(QRegularExpression(QStringLiteral("[\\n\\r]"))); // remove any newline
489  firstLine = firstLine.remove(QLatin1Char('"'));
490 
491  //TODO: this doesn't work, the split below introduces whitespaces again
492 // if (simplifyWhitespacesEnabled)
493 // firstLine = firstLine.simplified();
494  DEBUG("First line: \'" << STDSTRING(firstLine) << '\'');
495 
496  // determine separator and split first line
497  QStringList firstLineStringList;
498  if (separatingCharacter == "auto") {
499  DEBUG("automatic separator");
500  if (firstLine.indexOf(QLatin1Char('\t')) != -1) {
501  //in case we have a mix of tabs and spaces in the header, give the tab character the preference
502  m_separator = QLatin1Char('\t');
503  firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
504  } else {
505  const QRegularExpression regExp(QStringLiteral("[,;:]?\\s+"));
506  firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts);
507 
508  if (!firstLineStringList.isEmpty()) {
509  int length1 = firstLineStringList.at(0).length();
510  if (firstLineStringList.size() > 1)
511  m_separator = firstLine.mid(length1, 1);
512  else
513  m_separator = ' ';
514  }
515  }
516  } else { // use given separator
517  // replace symbolic "TAB" with '\t'
518  m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive);
519  m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive);
520  // replace symbolic "SPACE" with ' '
521  m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive);
522  m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive);
523  m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive);
524  m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive);
525  firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
526  }
527  DEBUG(Q_FUNC_INFO << ", separator: \'" << STDSTRING(m_separator) << '\'');
528  DEBUG(Q_FUNC_INFO << ", number of columns: " << firstLineStringList.size());
529  QDEBUG(Q_FUNC_INFO << ", first line: " << firstLineStringList);
530  DEBUG(Q_FUNC_INFO << ", headerEnabled: " << headerEnabled);
531 
532  //optionally, remove potential spaces in the first line
533  //TODO: this part should be obsolete actually if we do firstLine = firstLine.simplified(); above...
535  for (int i = 0; i < firstLineStringList.size(); ++i)
536  firstLineStringList[i] = firstLineStringList[i].simplified();
537  }
538 
539  //in GUI in AsciiOptionsWidget we start counting from 1, subtract 1 here to start from zero
541 
542  if (headerEnabled) { // use first line to name vectors (starting from startColumn)
543  vectorNames = firstLineStringList.mid(startColumn - 1);
545  }
546 
547  // set range to read
548  if (endColumn == -1) {
549  if (headerEnabled || vectorNames.size() == 0)
550  endColumn = firstLineStringList.size(); // last column
551  else
552  //number of vector names provided in the import dialog (not more than the maximal number of columns in the file)
553  endColumn = qMin(vectorNames.size(), firstLineStringList.size());
554  }
555 
556  if (endColumn < startColumn)
557  m_actualCols = 0;
558  else
560 
561  //add index column
563  vectorNames.prepend(i18n("Timestamp"));
564  m_actualCols++;
565  }
566 
567  //add index column
568  if (createIndexEnabled) {
569  vectorNames.prepend(i18n("Index"));
570  m_actualCols++;
571  }
572 
573  QDEBUG(Q_FUNC_INFO << ", vector names =" << vectorNames);
574 
575 //TEST: readline-seek-readline fails
576  /* qint64 testpos = device.pos();
577  DEBUG("read data line @ pos " << testpos << " : " << STDSTRING(device.readLine()));
578  device.seek(testpos);
579  testpos = device.pos();
580  DEBUG("read data line again @ pos " << testpos << " : " << STDSTRING(device.readLine()));
581  */
582 /////////////////////////////////////////////////////////////////
583 
584  // parse first data line to determine data type for each column
585  // if the first line was already parsed as the header, read the next line
586  if (headerEnabled && !device.isSequential())
587  firstLineStringList = getLineString(device);
588 
589  columnModes.resize(m_actualCols);
590  int col = 0;
591  if (createIndexEnabled) {
593  ++col;
594  }
595 
598  ++col;
599  }
600 
601  for (auto& valueString : firstLineStringList) { // parse columns available in first data line
602  const int index{ col - startColumn + 1 };
603  if (index < (int)createIndexEnabled + (int)createTimestampEnabled) {
604  col++;
605  continue;
606  }
607  if (index == m_actualCols)
608  break;
609 
611  valueString = valueString.simplified();
613  valueString.remove(QLatin1Char('"'));
615  col++;
616  }
617  for (const auto mode : columnModes)
618  DEBUG(Q_FUNC_INFO << ", column mode = " << static_cast<int>(mode));
619 
620  // parsing more lines to better determine data types
621  for (unsigned int i = 0; i < m_dataTypeLines; ++i) {
622  if (device.atEnd()) // EOF reached
623  break;
624  firstLineStringList = getLineString(device);
625 
626  col = (int)createIndexEnabled + (int)createTimestampEnabled;
627 
628  for (auto& valueString : firstLineStringList) {
629  const int index{col - startColumn + 1};
630  if (index < (int)createIndexEnabled + (int)createTimestampEnabled) {
631  col++;
632  continue;
633  }
634  if (index == m_actualCols)
635  break;
636 
638  valueString = valueString.simplified();
640  valueString.remove(QLatin1Char('"'));
642 
643  // numeric: integer -> numeric
645  columnModes[index] = mode;
646  // text: non text -> text
648  columnModes[index] = mode;
649  // numeric: text -> numeric/integer
651  columnModes[index] = mode;
652  col++;
653  }
654  }
655  for (const auto mode : columnModes)
656  DEBUG(Q_FUNC_INFO << ", column mode = " << static_cast<int>(mode));
657 
658  // ATTENTION: This resets the position in the device to 0
659  m_actualRows = (int)q->lineNumber(device);
660 
661  const int actualEndRow = (endRow == -1 || endRow > m_actualRows) ? m_actualRows : endRow;
662  if (actualEndRow > m_actualStartRow)
663  m_actualRows = actualEndRow - m_actualStartRow;
664  else
665  m_actualRows = 0;
666 
667  DEBUG("start/end column: " << startColumn << ' ' << endColumn);
668  DEBUG("start/end row: " << m_actualStartRow << ' ' << actualEndRow);
669  DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows);
670 
671  if (m_actualRows == 0 && !device.isSequential())
672  return 1;
673 
674  return 0;
675 }
676 
677 /*!
678  reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source.
679 */
680 void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) {
681  DEBUG(Q_FUNC_INFO << ", fileName = \'" << STDSTRING(fileName) << "\', dataSource = "
682  << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode));
683 
684  //dirty hack: set readingFile and readingFileName in order to know in lineNumber(QIODevice)
685  //that we're reading from a file and to benefit from much faster wc on linux
686  //TODO: redesign the APIs and remove this later
687  readingFile = true;
688  readingFileName = fileName;
689  KFilterDev device(fileName);
690  readDataFromDevice(device, dataSource, importMode);
691  readingFile = false;
692 }
693 
694 qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) {
695  DEBUG(Q_FUNC_INFO << ", bytes available = " << device.bytesAvailable() << ", from = " << from);
696  if (device.bytesAvailable() <= 0) {
697  DEBUG(" No new data available");
698  return 0;
699  }
700 
701  //TODO: may be also a matrix?
702  auto* spreadsheet = dynamic_cast<LiveDataSource*>(dataSource);
703  if (!spreadsheet)
704  return 0;
705 
706  if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe)
707  if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16))
708  return 0;
709 
710  if (!m_prepared) {
711  DEBUG(" Preparing ..");
712 
713  switch (spreadsheet->sourceType()) {
715  const int deviceError = prepareDeviceToRead(device);
716  if (deviceError != 0) {
717  DEBUG(" Device error = " << deviceError);
718  return 0;
719  }
720  break;
721  }
726  m_actualRows = 1;
727  m_actualCols = 1;
728  columnModes.clear();
729  if (createIndexEnabled) {
731  vectorNames << i18n("Index");
732  m_actualCols++;
733  }
734 
737  vectorNames << i18n("Timestamp");
738  m_actualCols++;
739  }
740 
741  //add column for the actual value
743  vectorNames << i18n("Value");
744 
745  QDEBUG(" vector names = " << vectorNames);
746  break;
748  break;
749  }
750 
751  // prepare import for spreadsheet
752  spreadsheet->setUndoAware(false);
754 
755  //columns in a file data source don't have any manual changes.
756  //make the available columns undo unaware and suppress the "data changed" signal.
757  //data changes will be propagated via an explicit Column::setChanged() call once new data was read.
758  for (int i = 0; i < spreadsheet->childCount<Column>(); i++) {
759  spreadsheet->child<Column>(i)->setUndoAware(false);
760  spreadsheet->child<Column>(i)->setSuppressDataChangedSignal(true);
761  }
762 
763  int keepNValues = spreadsheet->keepNValues();
764  if (keepNValues == 0)
765  spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1);
766  else {
767  spreadsheet->setRowCount(keepNValues);
768  m_actualRows = keepNValues;
769  }
770 
772  initDataContainers(spreadsheet);
773 
774  DEBUG(" data source resized to col: " << m_actualCols);
775  DEBUG(" data source rowCount: " << spreadsheet->rowCount());
776  DEBUG(" Prepared!");
777  }
778 
779  qint64 bytesread = 0;
780 
781 #ifdef PERFTRACE_LIVE_IMPORT
782  PERFTRACE("AsciiLiveDataImportTotal: ");
783 #endif
784  LiveDataSource::ReadingType readingType;
785  if (!m_prepared) {
787  } else {
788  //we have to read all the data when reading from end
789  //so we set readingType to TillEnd
790  if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd)
792  //if we read the whole file we just start from the beginning of it
793  //and read till end
794  else if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
796  else
797  readingType = spreadsheet->readingType();
798  }
799  DEBUG(" Reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType));
800 
801  //move to the last read position, from == total bytes read
802  //since the other source types are sequential we cannot seek on them
803  if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe)
804  device.seek(from);
805 
806  //count the new lines, increase actualrows on each
807  //now we read all the new lines, if we want to use sample rate
808  //then here we can do it, if we have actually sample rate number of lines :-?
809  int newLinesForSampleSizeNotTillEnd = 0;
810  int newLinesTillEnd = 0;
811  QVector<QString> newData;
812  if (readingType != LiveDataSource::ReadingType::TillEnd)
813  newData.resize(spreadsheet->sampleSize());
814 
815  int newDataIdx = 0;
816  {
817 #ifdef PERFTRACE_LIVE_IMPORT
818  PERFTRACE("AsciiLiveDataImportReadingFromFile: ");
819 #endif
820  DEBUG(" source type = " << ENUM_TO_STRING(LiveDataSource, SourceType, spreadsheet->sourceType()));
821  while (!device.atEnd()) {
822  if (readingType != LiveDataSource::ReadingType::TillEnd) {
823  switch (spreadsheet->sourceType()) { // different sources need different read methods
825  newData[newDataIdx++] = device.readAll();
826  break;
828  newData[newDataIdx++] = device.read(device.bytesAvailable());
829  break;
831  newData.push_back(device.readLine());
832  break;
834  //TODO: check serial port
836  newData[newDataIdx++] = device.read(device.bytesAvailable());
837  break;
839  break;
840  }
841  } else { // ReadingType::TillEnd
842  switch (spreadsheet->sourceType()) { // different sources need different read methods
844  newData.push_back(device.readAll());
845  break;
847  newData.push_back(device.read(device.bytesAvailable()));
848  break;
850  newData.push_back(device.readLine());
851  break;
853  //TODO: check serial port
855  newData.push_back(device.read(device.bytesAvailable()));
856  break;
858  break;
859  }
860  }
861  newLinesTillEnd++;
862 
863  if (readingType != LiveDataSource::ReadingType::TillEnd) {
864  newLinesForSampleSizeNotTillEnd++;
865  //for Continuous reading and FromEnd we read sample rate number of lines if possible
866  //here TillEnd and Whole file behave the same
867  if (newLinesForSampleSizeNotTillEnd == spreadsheet->sampleSize())
868  break;
869  }
870  }
871  QDEBUG(" data read: " << newData);
872  }
873 
874  //now we reset the readingType
875  if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd)
876  readingType = spreadsheet->readingType();
877 
878  //we had less new lines than the sample size specified
879  if (readingType != LiveDataSource::ReadingType::TillEnd)
880  QDEBUG(" Removed empty lines: " << newData.removeAll(QString()));
881 
882  //back to the last read position before counting when reading from files
883  if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe)
884  device.seek(from);
885 
886  const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount();
887 
888  int currentRow = 0; // indexes the position in the vector(column)
889  int linesToRead = 0;
890  int keepNValues = spreadsheet->keepNValues();
891 
892  DEBUG(" Increase row count. keepNValues = " << keepNValues);
893  if (m_prepared) {
894  //increase row count if we don't have a fixed size
895  //but only after the preparation step
896  if (keepNValues == 0) {
897  DEBUG(" keep All values");
898  if (readingType != LiveDataSource::ReadingType::TillEnd)
899  m_actualRows += qMin(newData.size(), spreadsheet->sampleSize());
900  else {
901  //we don't increase it if we reread the whole file, we reset it
902  if (!(spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile))
903  m_actualRows += newData.size();
904  else
905  m_actualRows = newData.size();
906  }
907 
908  //appending
909  if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
910  linesToRead = m_actualRows;
911  else
912  linesToRead = m_actualRows - spreadsheetRowCountBeforeResize;
913  } else { // fixed size
914  DEBUG(" keep " << keepNValues << " values");
915  if (readingType == LiveDataSource::ReadingType::TillEnd) {
916  //we had more lines than the fixed size, so we read m_actualRows number of lines
917  if (newLinesTillEnd > m_actualRows) {
918  linesToRead = m_actualRows;
919  //TODO after reading we should skip the next data lines
920  //because it's TillEnd actually
921  } else
922  linesToRead = newLinesTillEnd;
923  } else {
924  //we read max sample size number of lines when the reading mode
925  //is ContinuouslyFixed or FromEnd, WholeFile is disabled
926  linesToRead = qMin(spreadsheet->sampleSize(), newLinesTillEnd);
927  }
928  }
929 
930  if (linesToRead == 0)
931  return 0;
932  } else // not prepared
933  linesToRead = newLinesTillEnd;
934 
935  DEBUG(" lines to read = " << linesToRead);
936  DEBUG(" actual rows (w/o header) = " << m_actualRows);
937 
938  //TODO
939 // if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe || spreadsheet->sourceType() == LiveDataSource::SourceType::NetworkUdpSocket) {
940 // if (m_actualRows < linesToRead) {
941 // DEBUG(" SET lines to read to " << m_actualRows);
942 // linesToRead = m_actualRows;
943 // }
944 // }
945 
946  //new rows/resize columns if we don't have a fixed size
947  //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize
948  if (keepNValues == 0) {
949 #ifdef PERFTRACE_LIVE_IMPORT
950  PERFTRACE("AsciiLiveDataImportResizing: ");
951 #endif
952  if (spreadsheet->rowCount() < m_actualRows)
953  spreadsheet->setRowCount(m_actualRows);
954 
955  if (!m_prepared)
956  currentRow = 0;
957  else {
958  // indexes the position in the vector(column)
959  if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
960  currentRow = 0;
961  else
962  currentRow = spreadsheetRowCountBeforeResize;
963  }
964 
965  // if we have fixed size, we do this only once in preparation, here we can use
966  // m_prepared and we need something to decide whether it has a fixed size or increasing
967  initDataContainers(spreadsheet);
968  } else { // fixed size
969  //when we have a fixed size we have to pop sampleSize number of lines if specified
970  //here popping, setting currentRow
971  if (!m_prepared) {
972  if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
973  currentRow = 0;
974  else
975  currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows);
976  } else {
977  if (readingType == LiveDataSource::ReadingType::TillEnd) {
978  if (newLinesTillEnd > m_actualRows) {
979  currentRow = 0;
980  } else {
981  if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)
982  currentRow = 0;
983  else
984  currentRow = m_actualRows - newLinesTillEnd;
985  }
986  } else {
987  //we read max sample size number of lines when the reading mode
988  //is ContinuouslyFixed or FromEnd
989  currentRow = m_actualRows - qMin(spreadsheet->sampleSize(), newLinesTillEnd);
990  }
991  }
992 
993  if (m_prepared) {
994 #ifdef PERFTRACE_LIVE_IMPORT
995  PERFTRACE("AsciiLiveDataImportPopping: ");
996 #endif
997  // enable data change signal
998  for (int col = 0; col < m_actualCols; ++col)
999  spreadsheet->child<Column>(col)->setSuppressDataChangedSignal(false);
1000 
1001  for (int row = 0; row < linesToRead; ++row) {
1002  for (int col = 0; col < m_actualCols; ++col) {
1003  switch (columnModes[col]) {
1005  auto* vector = static_cast<QVector<double>* >(spreadsheet->child<Column>(col)->data());
1006  vector->pop_front();
1007  vector->resize(m_actualRows);
1008  m_dataContainer[col] = static_cast<void *>(vector);
1009  break;
1010  }
1012  auto* vector = static_cast<QVector<int>* >(spreadsheet->child<Column>(col)->data());
1013  vector->pop_front();
1014  vector->resize(m_actualRows);
1015  m_dataContainer[col] = static_cast<void *>(vector);
1016  break;
1017  }
1019  auto* vector = static_cast<QVector<qint64>* >(spreadsheet->child<Column>(col)->data());
1020  vector->pop_front();
1021  vector->resize(m_actualRows);
1022  m_dataContainer[col] = static_cast<void *>(vector);
1023  break;
1024  }
1026  auto* vector = static_cast<QVector<QString>*>(spreadsheet->child<Column>(col)->data());
1027  vector->pop_front();
1028  vector->resize(m_actualRows);
1029  m_dataContainer[col] = static_cast<void *>(vector);
1030  break;
1031  }
1033  auto* vector = static_cast<QVector<QDateTime>* >(spreadsheet->child<Column>(col)->data());
1034  vector->pop_front();
1035  vector->resize(m_actualRows);
1036  m_dataContainer[col] = static_cast<void *>(vector);
1037  break;
1038  }
1039  //TODO
1042  break;
1043  }
1044  }
1045  }
1046  }
1047  }
1048 
1049  // from the last row we read the new data in the spreadsheet
1050  DEBUG(" Reading from line " << currentRow << " till end line " << newLinesTillEnd);
1051  DEBUG(" Lines to read:" << linesToRead <<", actual rows:" << m_actualRows << ", actual cols:" << m_actualCols);
1052  newDataIdx = 0;
1053  if (readingType == LiveDataSource::ReadingType::FromEnd) {
1054  if (m_prepared) {
1055  if (newData.size() > spreadsheet->sampleSize())
1056  newDataIdx = newData.size() - spreadsheet->sampleSize();
1057  //since we skip a couple of lines, we need to count those bytes too
1058  for (int i = 0; i < newDataIdx; ++i)
1059  bytesread += newData.at(i).size();
1060  }
1061  }
1062  DEBUG(" newDataIdx: " << newDataIdx);
1063 
1064  static int indexColumnIdx = 1;
1065  {
1066 #ifdef PERFTRACE_LIVE_IMPORT
1067  PERFTRACE("AsciiLiveDataImportFillingContainers: ");
1068 #endif
1069  int row = 0;
1070 
1072  if (headerEnabled) {
1073  if (!m_prepared) {
1074  row = 1;
1075  bytesread += newData.at(0).size();
1076  }
1077  }
1078  }
1079  if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) {
1080  if (readingType == LiveDataSource::ReadingType::WholeFile) {
1081  if (headerEnabled) {
1082  row = 1;
1083  bytesread += newData.at(0).size();
1084  }
1085  }
1086  }
1087 
1088  for (; row < linesToRead; ++row) {
1089  DEBUG("\n Reading row " << row + 1 << " of " << linesToRead);
1090  QString line;
1091  if (readingType == LiveDataSource::ReadingType::FromEnd)
1092  line = newData.at(newDataIdx++);
1093  else
1094  line = newData.at(row);
1095  //when we read the whole file we don't care about the previous position
1096  //so we don't have to count those bytes
1097  if (readingType != LiveDataSource::ReadingType::WholeFile) {
1098  if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) {
1099  bytesread += line.size();
1100  }
1101  }
1102 
1103  if (removeQuotesEnabled)
1104  line.remove(QLatin1Char('"'));
1105 
1106  if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
1107  continue;
1108 
1109  QStringList lineStringList;
1110  // only FileOrPipe support multiple columns
1111  if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe)
1112  lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
1113  else
1114  lineStringList << line;
1115  QDEBUG(" line = " << lineStringList << ", separator = \'" << m_separator << "\'");
1116 
1117  DEBUG(" Line bytes: " << line.size() << " line: " << STDSTRING(line));
1119  for (int i = 0; i < lineStringList.size(); ++i)
1120  lineStringList[i] = lineStringList[i].simplified();
1121  }
1122 
1123  //add index if required
1124  int offset = 0;
1125  if (createIndexEnabled) {
1126  int index = (spreadsheet->keepNValues() == 0) ? currentRow + 1 : indexColumnIdx++;
1127  static_cast<QVector<int>*>(m_dataContainer[0])->operator[](currentRow) = index;
1128  ++offset;
1129  }
1130 
1131  //add current timestamp if required
1132  if (createTimestampEnabled) {
1133  static_cast<QVector<QDateTime>*>(m_dataContainer[offset])->operator[](currentRow) = QDateTime::currentDateTime();
1134  ++offset;
1135  }
1136 
1137  //parse columns
1138  for (int n = offset; n < m_actualCols; ++n) {
1139  QString valueString;
1140  if (n - offset < lineStringList.size())
1141  valueString = lineStringList.at(n - offset);
1142 
1143  setValue(n, currentRow, valueString);
1144  }
1145  currentRow++;
1146  }
1147  }
1148 
1149  if (m_prepared) {
1150  //notify all affected columns and plots about the changes
1151  PERFTRACE("AsciiLiveDataImport, notify affected columns and plots");
1152 
1153  //determine the dependent plots
1155  for (int n = 0; n < m_actualCols; ++n)
1156  spreadsheet->column(n)->addUsedInPlots(plots);
1157 
1158  //suppress retransform in the dependent plots
1159  for (auto* plot : plots)
1160  plot->setSuppressDataChangedSignal(true);
1161 
1162  for (int n = 0; n < m_actualCols; ++n)
1163  spreadsheet->column(n)->setChanged();
1164 
1165  //retransform the dependent plots
1166  for (auto* plot : plots) {
1167  plot->setSuppressDataChangedSignal(false);
1168  plot->dataChanged();
1169  }
1170  } else
1171  m_prepared = true;
1172 
1173  DEBUG("AsciiFilterPrivate::readFromLiveDevice() DONE");
1174  return bytesread;
1175 }
1176 
1177 /*!
1178  reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source.
1179 */
1180 void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) {
1181  DEBUG(Q_FUNC_INFO << ", dataSource = " << dataSource
1182  << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines);
1183 
1184  if (!m_prepared) {
1185  const int deviceError = prepareDeviceToRead(device);
1186  if (deviceError != 0) {
1187  DEBUG(Q_FUNC_INFO << ", DEVICE ERROR = " << deviceError);
1188  return;
1189  }
1190 
1191  // matrix data has only one column mode
1192  if (dynamic_cast<Matrix*>(dataSource)) {
1193  auto mode = columnModes[0];
1194  //TODO: remove this when Matrix supports text type
1197  for (auto& c : columnModes)
1198  if (c != mode)
1199  c = mode;
1200  }
1201 
1202  Q_ASSERT(dataSource);
1204  m_prepared = true;
1205  }
1206 
1207  DEBUG("locale = " << STDSTRING(QLocale::languageToString(numberFormat)));
1208 
1209  // Read the data
1210  int currentRow = 0; // indexes the position in the vector(column)
1211  if (lines == -1)
1212  lines = m_actualRows;
1213 
1214  //skip data lines, if required
1215  DEBUG(" Skipping " << m_actualStartRow << " lines");
1216  for (int i = 0; i < m_actualStartRow; ++i)
1217  device.readLine();
1218 
1219  DEBUG(" Reading " << qMin(lines, m_actualRows) << " lines, " << m_actualCols << " columns");
1220 
1221  if (qMin(lines, m_actualRows) == 0 || m_actualCols == 0)
1222  return;
1223 
1224  QString line;
1225  QString valueString;
1226  //Don't put the definition QStringList lineStringList outside of the for-loop,
1227  //the compiler doesn't seem to optimize the destructor of QList well enough in this case.
1228 
1229  lines = qMin(lines, m_actualRows);
1230  int progressIndex = 0;
1231  const qreal progressInterval = 0.01*lines; //update on every 1% only
1232 
1233  for (int i = 0; i < lines; ++i) {
1234  line = device.readLine();
1235 
1236  // remove any newline
1237  line.remove(QLatin1Char('\n'));
1238  line.remove(QLatin1Char('\r'));
1239 
1240  if (removeQuotesEnabled)
1241  line.remove(QLatin1Char('"'));
1242 
1243  if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
1244  continue;
1245 
1246  QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
1247 // DEBUG(" Line bytes: " << line.size() << " line: " << STDSTRING(line));
1248 
1250  for (int i = 0; i < lineStringList.size(); ++i)
1251  lineStringList[i] = lineStringList[i].simplified();
1252  }
1253 
1254  // remove left white spaces
1255  if (skipEmptyParts) {
1256  for (int n = 0; n < lineStringList.size(); ++n) {
1257  valueString = lineStringList.at(n);
1258  if (!QString::compare(valueString, " ")) {
1259  lineStringList.removeAt(n);
1260  n--;
1261  }
1262  }
1263  }
1264 
1265  //parse columns
1266  DEBUG(Q_FUNC_INFO << ", actual cols = " << m_actualCols)
1267  for (int n = 0; n < m_actualCols; ++n) {
1268  // index column if required
1269  if (n == 0 && createIndexEnabled) {
1270  static_cast<QVector<int>*>(m_dataContainer[0])->operator[](currentRow) = i + 1;
1271  continue;
1272  }
1273 
1274  //column counting starts with 1, subtract 1 as well as another 1 for the index column if required
1275  int col = createIndexEnabled ? n + startColumn - 2: n + startColumn - 1;
1276  QString valueString;
1277  if (col < lineStringList.size())
1278  valueString = lineStringList.at(col);
1279 
1280  setValue(n, currentRow, valueString);
1281  }
1282  currentRow++;
1283 
1284  //ask to update the progress bar only if we have more than 1000 lines
1285  //only in 1% steps
1286  progressIndex++;
1287  if (lines > 1000 && progressIndex > progressInterval) {
1288  emit q->completed(100 * currentRow/lines);
1289  progressIndex = 0;
1290  QApplication::processEvents(QEventLoop::AllEvents, 0);
1291  }
1292  }
1293 
1294  DEBUG(Q_FUNC_INFO <<", Read " << currentRow << " lines");
1295 
1296  //we might have skipped empty lines above. shrink the spreadsheet if the number of read lines (=currentRow)
1297  //is smaller than the initial size of the spreadsheet (=m_actualRows).
1298  //TODO: should also be relevant for Matrix
1299  auto* s = dynamic_cast<Spreadsheet*>(dataSource);
1300  if (s && currentRow != m_actualRows && importMode == AbstractFileFilter::ImportMode::Replace)
1301  s->setRowCount(currentRow);
1302 
1303  Q_ASSERT(dataSource);
1305 }
1306 
1307 //#####################################################################
1308 //############################ Preview ################################
1309 //#####################################################################
1310 
1311 /*!
1312  * preview for special devices (local/UDP/TCP socket or serial port)
1313  */
1315  DEBUG(Q_FUNC_INFO << ", bytesAvailable = " << device.bytesAvailable() << ", isSequential = " << device.isSequential());
1316  QVector<QStringList> dataStrings;
1317 
1318  if (!(device.bytesAvailable() > 0)) {
1319  DEBUG("No new data available");
1320  return dataStrings;
1321  }
1322 
1323  if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16))
1324  return dataStrings;
1325 
1326  int linesToRead = 0;
1327  QVector<QString> newData;
1328 
1329  //TODO: serial port "read(nBytes)"?
1330  while (!device.atEnd()) {
1331  if (device.canReadLine())
1332  newData.push_back(device.readLine());
1333  else // UDP fails otherwise
1334  newData.push_back(device.readAll());
1335  linesToRead++;
1336  }
1337  QDEBUG(" data = " << newData);
1338 
1339  if (linesToRead == 0)
1340  return dataStrings;
1341 
1342  vectorNames.clear();
1343  columnModes.clear();
1344 
1345  if (createIndexEnabled) {
1347  vectorNames << i18n("Index");
1348  }
1349 
1350  if (createTimestampEnabled) {
1352  vectorNames << i18n("Timestamp");
1353  }
1354 
1355  //parse the first data line to determine data type for each column
1356 #if (QT_VERSION >= QT_VERSION_CHECK(5, 14, 0))
1357  QStringList firstLineStringList = newData.at(0).split(' ', Qt::SkipEmptyParts);
1358 #else
1359  QStringList firstLineStringList = newData.at(0).split(' ', QString::SkipEmptyParts);
1360 #endif
1361  int i = 1;
1362  for (auto& valueString : firstLineStringList) {
1364  valueString = valueString.simplified();
1365  if (removeQuotesEnabled)
1366  valueString.remove(QLatin1Char('"'));
1367  if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces
1368  continue;
1369 
1370  vectorNames << i18n("Value %1", i);
1372  ++i;
1373  }
1374 
1375  int offset = int(createIndexEnabled) + int(createTimestampEnabled);
1376  QString line;
1377 
1378  //loop over all lines in the new data in the device and parse the available columns
1379  for (int i = 0; i < linesToRead; ++i) {
1380  line = newData.at(i);
1381 
1382  // remove any newline
1383  line = line.remove('\n');
1384  line = line.remove('\r');
1385 
1387  line = line.simplified();
1388 
1389  if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
1390  continue;
1391 
1392  QStringList lineString;
1393 
1394  // index column if required
1395  if (createIndexEnabled)
1396  lineString += QString::number(i + 1);
1397 
1398  // timestamp column if required
1400  lineString += QDateTime::currentDateTime().toString();
1401 
1402 #if (QT_VERSION >= QT_VERSION_CHECK(5, 14, 0))
1403  QStringList lineStringList = line.split(' ', Qt::SkipEmptyParts);
1404 #else
1405  QStringList lineStringList = line.split(' ', QString::SkipEmptyParts);
1406 #endif
1407  QDEBUG(" line = " << lineStringList);
1408 
1409  //parse columns
1410  DEBUG(Q_FUNC_INFO << ", number of columns = " << lineStringList.size())
1411  for (int n = 0; n < lineStringList.size(); ++n) {
1412  if (n < lineStringList.size()) {
1413  QString valueString = lineStringList.at(n);
1414  if (removeQuotesEnabled)
1415  valueString.remove(QLatin1Char('"'));
1416 
1417  if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces
1418  continue;
1419 
1420  lineString += previewValue(valueString, columnModes[n+offset]);
1421  } else // missing columns in this line
1422  lineString += QString();
1423  }
1424 
1425  dataStrings << lineString;
1426  }
1427 
1428  return dataStrings;
1429 }
1430 
1431 /*!
1432  * generates the preview for the file \c fileName reading the provided number of \c lines.
1433  */
1434 QVector<QStringList> AsciiFilterPrivate::preview(const QString& fileName, int lines) {
1435  QVector<QStringList> dataStrings;
1436 
1437  //dirty hack: set readingFile and readingFileName in order to know in lineNumber(QIODevice)
1438  //that we're reading from a file and to benefit from much faster wc on linux
1439  //TODO: redesign the APIs and remove this later
1440  readingFile = true;
1441  readingFileName = fileName;
1442  KFilterDev device(fileName);
1443  const int deviceError = prepareDeviceToRead(device);
1444  readingFile = false;
1445 
1446  if (deviceError != 0) {
1447  DEBUG("Device error = " << deviceError);
1448  return dataStrings;
1449  }
1450 
1451  //number formatting
1452  DEBUG("locale = " << STDSTRING(QLocale::languageToString(numberFormat)));
1453 
1454  // Read the data
1455  if (lines == -1)
1456  lines = m_actualRows;
1457 
1458  // set column names for preview
1459  if (!headerEnabled) {
1460  int start = 0;
1461  if (createIndexEnabled)
1462  start = 1;
1463  for (int i = start; i < m_actualCols; i++)
1464  vectorNames << "Column " + QString::number(i + 1);
1465  }
1466  QDEBUG(" column names = " << vectorNames);
1467 
1468  //skip data lines, if required
1469  DEBUG(" Skipping " << m_actualStartRow << " lines");
1470  for (int i = 0; i < m_actualStartRow; ++i)
1471  device.readLine();
1472 
1473  DEBUG(" Generating preview for " << qMin(lines, m_actualRows) << " lines");
1474  QString line;
1475 
1476  //loop over the preview lines in the file and parse the available columns
1477  for (int i = 0; i < qMin(lines, m_actualRows); ++i) {
1478  line = device.readLine();
1479 
1480  // remove any newline
1481  line = line.remove('\n');
1482  line = line.remove('\r');
1483 
1484  if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
1485  continue;
1486 
1487  QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
1488  QDEBUG(" line = " << lineStringList);
1489  DEBUG(" Line bytes: " << line.size() << " line: " << STDSTRING(line));
1490 
1492  for (int i = 0; i < lineStringList.size(); ++i)
1493  lineStringList[i] = lineStringList[i].simplified();
1494  }
1495 
1496  QStringList lineString;
1497 
1498  //parse columns
1499  for (int n = 0; n < m_actualCols; ++n) {
1500  // index column if required
1501  if (n == 0 && createIndexEnabled) {
1502  lineString += QString::number(i + 1);
1503  continue;
1504  }
1505 
1506  //column counting starts with 1, subtract 1 as well as another 1 for the index column if required
1507  int col = createIndexEnabled ? n + startColumn - 2 : n + startColumn - 1;
1508 
1509  if (col < lineStringList.size()) {
1510  QString valueString = lineStringList.at(col);
1511  if (removeQuotesEnabled)
1512  valueString.remove(QLatin1Char('"'));
1513 
1514  if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces
1515  continue;
1516 
1517  lineString += previewValue(valueString, columnModes[n]);
1518  } else // missing columns in this line
1519  lineString += QString();
1520  }
1521 
1522  dataStrings << lineString;
1523  }
1524 
1525  return dataStrings;
1526 }
1527 
1528 //#####################################################################
1529 //####################### Helper functions ############################
1530 //#####################################################################
1531 /*!
1532  * converts \c valueString to the date type according to \c mode and \c locale
1533  * and returns its string representation.
1534  */
1536  DEBUG(Q_FUNC_INFO << ", valueString = " << valueString.toStdString() << ", mode = " << (int)mode)
1537 
1538  QString result;
1539  switch (mode) {
1541  bool isNumber;
1542  const double value = locale.toDouble(valueString, &isNumber);
1543  result = QString::number(isNumber ? value : nanValue, 'g', 15);
1544  break;
1545  }
1547  bool isNumber;
1548  const int value = locale.toInt(valueString, &isNumber);
1549  result = QString::number(isNumber ? value : 0);
1550  break;
1551  }
1553  bool isNumber;
1554  const qint64 value = locale.toLongLong(valueString, &isNumber);
1555  result = QString::number(isNumber ? value : 0);
1556  break;
1557  }
1559  QDateTime valueDateTime = parseDateTime(valueString, dateTimeFormat);
1560  result = valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" ");
1561  break;
1562  }
1564  result = valueString;
1565  break;
1566  case AbstractColumn::ColumnMode::Month: // never happens
1568  break;
1569  }
1570  return result;
1571 }
1572 
1573 //set value depending on data type
1574 void AsciiFilterPrivate::setValue(int col, int row, const QString& valueString) {
1575  if (!valueString.isEmpty()) {
1576  switch (columnModes.at(col)) {
1578  bool isNumber;
1579  const double value = locale.toDouble(valueString, &isNumber);
1580  static_cast<QVector<double>*>(m_dataContainer[col])->operator[](row) = (isNumber ? value : nanValue);
1581  break;
1582  }
1584  bool isNumber;
1585  const int value = locale.toInt(valueString, &isNumber);
1586  static_cast<QVector<int>*>(m_dataContainer[col])->operator[](row) = (isNumber ? value : 0);
1587  break;
1588  }
1590  bool isNumber;
1591  const qint64 value = locale.toLongLong(valueString, &isNumber);
1592  static_cast<QVector<qint64>*>(m_dataContainer[col])->operator[](row) = (isNumber ? value : 0);
1593  break;
1594  }
1596  QDateTime valueDateTime = parseDateTime(valueString, dateTimeFormat);
1597  static_cast<QVector<QDateTime>*>(m_dataContainer[col])->operator[](row) = valueDateTime.isValid() ? valueDateTime : QDateTime();
1598  break;
1599  }
1601  auto* colData = static_cast<QVector<QString>*>(m_dataContainer[col]);
1602  colData->operator[](row) = valueString;
1603  break;
1604  }
1605  case AbstractColumn::ColumnMode::Month: // never happens
1607  break;
1608  }
1609  } else { // missing columns in this line
1610  switch (columnModes.at(col)) {
1612  static_cast<QVector<double>*>(m_dataContainer[col])->operator[](row) = nanValue;
1613  break;
1615  static_cast<QVector<int>*>(m_dataContainer[col])->operator[](row) = 0;
1616  break;
1618  static_cast<QVector<qint64>*>(m_dataContainer[col])->operator[](row) = 0;
1619  break;
1621  static_cast<QVector<QDateTime>*>(m_dataContainer[col])->operator[](row) = QDateTime();
1622  break;
1624  static_cast<QVector<QString>*>(m_dataContainer[col])->operator[](row).clear();
1625  break;
1626  case AbstractColumn::ColumnMode::Month: // never happens
1628  break;
1629  }
1630  }
1631 }
1632 
1634  DEBUG(" Initializing the data containers ..");
1635  for (int n = 0; n < m_actualCols; ++n) {
1636  // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp)
1637  spreadsheet->child<Column>(n)->setColumnMode(columnModes[n]);
1638  switch (columnModes[n]) {
1640  auto* vector = static_cast<QVector<double>* >(spreadsheet->child<Column>(n)->data());
1641  vector->reserve(m_actualRows);
1642  vector->resize(m_actualRows);
1643  m_dataContainer[n] = static_cast<void *>(vector);
1644  break;
1645  }
1647  auto* vector = static_cast<QVector<int>* >(spreadsheet->child<Column>(n)->data());
1648  vector->resize(m_actualRows);
1649  m_dataContainer[n] = static_cast<void *>(vector);
1650  break;
1651  }
1653  auto* vector = static_cast<QVector<qint64>* >(spreadsheet->child<Column>(n)->data());
1654  vector->resize(m_actualRows);
1655  m_dataContainer[n] = static_cast<void *>(vector);
1656  break;
1657  }
1659  auto* vector = static_cast<QVector<QString>*>(spreadsheet->child<Column>(n)->data());
1660  vector->resize(m_actualRows);
1661  m_dataContainer[n] = static_cast<void *>(vector);
1662  break;
1663  }
1665  auto* vector = static_cast<QVector<QDateTime>* >(spreadsheet->child<Column>(n)->data());
1666  vector->resize(m_actualRows);
1667  m_dataContainer[n] = static_cast<void *>(vector);
1668  break;
1669  }
1670  //TODO
1673  break;
1674  }
1675  }
1676 }
1677 
1678 /*!
1679  * get a single line from device
1680  */
1681 QStringList AsciiFilterPrivate::getLineString(QIODevice& device) {
1682  QString line;
1683  do { // skip comment lines in data lines
1684  if (!device.canReadLine())
1685  DEBUG("WARNING in AsciiFilterPrivate::getLineString(): device cannot 'readLine()' but using it anyway.");
1686 // line = device.readAll();
1687  line = device.readLine();
1688  } while (!commentCharacter.isEmpty() && line.startsWith(commentCharacter));
1689 
1690  line.remove(QRegularExpression(QStringLiteral("[\\n\\r]"))); // remove any newline
1691  DEBUG("data line : \'" << STDSTRING(line) << '\'');
1692  QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
1693  //TODO: remove quotes here?
1695  for (int i = 0; i < lineStringList.size(); ++i)
1696  lineStringList[i] = lineStringList[i].simplified();
1697  }
1698  QDEBUG("data line, parsed: " << lineStringList);
1699 
1700  return lineStringList;
1701 }
1702 
1703 /*!
1704  writes the content of \c dataSource to the file \c fileName.
1705 */
1706 void AsciiFilterPrivate::write(const QString& fileName, AbstractDataSource* dataSource) {
1707  Q_UNUSED(fileName);
1708  Q_UNUSED(dataSource);
1709 
1710  //TODO: save data to ascii file
1711 }
1712 
1713 /*!
1714  * create datetime from \c string using \c format considering corner cases
1715  */
1716 QDateTime AsciiFilterPrivate::parseDateTime(const QString& string, const QString& format) {
1717  //DEBUG("string = " << STDSTRING(string) << ", format = " << STDSTRING(format))
1718  QString fixedString(string);
1719  QString fixedFormat(format);
1720  if (!format.contains("yy")) { // no year given: set temporary to 2000 (must be a leap year to parse "Feb 29")
1721  fixedString.append(" 2000");
1722  fixedFormat.append(" yyyy");
1723  }
1724 
1725  QDateTime dateTime = QDateTime::fromString(fixedString, fixedFormat);
1726  //QDEBUG("fromString() =" << dateTime)
1727  // interpret 2-digit year smaller than 50 as 20XX
1728  if (dateTime.date().year() < 1950 && !format.contains("yyyy"))
1729  dateTime = dateTime.addYears(100);
1730  //QDEBUG("dateTime fixed =" << dateTime)
1731  //DEBUG("dateTime.toString =" << STDSTRING(dateTime.toString(format)))
1732 
1733  return dateTime;
1734 }
1735 
1736 
1737 //##############################################################################
1738 //################## Serialization/Deserialization ###########################
1739 //##############################################################################
1740 /*!
1741  Saves as XML.
1742  */
1743 void AsciiFilter::save(QXmlStreamWriter* writer) const {
1744  writer->writeStartElement( "asciiFilter");
1745  writer->writeAttribute( "commentCharacter", d->commentCharacter);
1746  writer->writeAttribute( "separatingCharacter", d->separatingCharacter);
1747  writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled));
1748  writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled));
1749  writer->writeAttribute( "createTimestamp", QString::number(d->createTimestampEnabled));
1750  writer->writeAttribute( "header", QString::number(d->headerEnabled));
1751  writer->writeAttribute( "vectorNames", d->vectorNames.join(' '));
1752  writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts));
1753  writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled));
1754  writer->writeAttribute( "nanValue", QString::number(d->nanValue));
1755  writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled));
1756  writer->writeAttribute( "startRow", QString::number(d->startRow));
1757  writer->writeAttribute( "endRow", QString::number(d->endRow));
1758  writer->writeAttribute( "startColumn", QString::number(d->startColumn));
1759  writer->writeAttribute( "endColumn", QString::number(d->endColumn));
1760  writer->writeEndElement();
1761 }
1762 
1763 /*!
1764  Loads from XML.
1765 */
1767  KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used");
1768  QXmlStreamAttributes attribs = reader->attributes();
1769  QString str;
1770 
1771  READ_STRING_VALUE("commentCharacter", commentCharacter);
1772  READ_STRING_VALUE("separatingCharacter", separatingCharacter);
1773 
1774  READ_INT_VALUE("createIndex", createIndexEnabled, bool);
1775  READ_INT_VALUE("createTimestamp", createTimestampEnabled, bool);
1776  READ_INT_VALUE("autoMode", autoModeEnabled, bool);
1777  READ_INT_VALUE("header", headerEnabled, bool);
1778 
1779  str = attribs.value("vectorNames").toString();
1780  d->vectorNames = str.split(' '); //may be empty
1781 
1782  READ_INT_VALUE("simplifyWhitespaces", simplifyWhitespacesEnabled, bool);
1783  READ_DOUBLE_VALUE("nanValue", nanValue);
1784  READ_INT_VALUE("removeQuotes", removeQuotesEnabled, bool);
1785  READ_INT_VALUE("skipEmptyParts", skipEmptyParts, bool);
1786  READ_INT_VALUE("startRow", startRow, int);
1787  READ_INT_VALUE("endRow", endRow, int);
1788  READ_INT_VALUE("startColumn", startColumn, int);
1789  READ_INT_VALUE("endColumn", endColumn, int);
1790  return true;
1791 }
1792 
1793 //##############################################################################
1794 //########################## MQTT releated code ###############################
1795 //##############################################################################
1796 #ifdef HAVE_MQTT
1797 int AsciiFilterPrivate::prepareToRead(const QString& message) {
1798  QStringList lines = message.split('\n');
1799  if (lines.isEmpty())
1800  return 1;
1801 
1802  // Parse the first line:
1803  // Determine the number of columns, create the columns and use (if selected) the first row to name them
1804  QString firstLine = lines.at(0);
1806  firstLine = firstLine.simplified();
1807  DEBUG("First line: \'" << STDSTRING(firstLine) << '\'');
1808 
1809  // determine separator and split first line
1810  QStringList firstLineStringList;
1811  if (separatingCharacter == "auto") {
1812  DEBUG("automatic separator");
1813  const QRegularExpression regExp(QStringLiteral("[,;:]?\\s+"));
1814  firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts);
1815  } else { // use given separator
1816  // replace symbolic "TAB" with '\t'
1817  m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive);
1818  m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive);
1819  // replace symbolic "SPACE" with ' '
1820  m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive);
1821  m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive);
1822  m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive);
1823  m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive);
1824  firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
1825  }
1826  DEBUG("separator: \'" << STDSTRING(m_separator) << '\'');
1827  DEBUG("number of columns: " << firstLineStringList.size());
1828  QDEBUG("first line: " << firstLineStringList);
1829 
1830  //all columns are read plus the optional column for the index and for the timestamp
1831  m_actualCols = firstLineStringList.size() + int(createIndexEnabled) + int(createTimestampEnabled);
1832 
1833  //column names:
1834  //when reading the message strings for different topics, it's not possible to specify vector names
1835  //since the different topics can have different content and different number of columns/vectors
1836  //->we always set the vector names here to fixed values
1837  vectorNames.clear();
1838  columnModes.clear();
1839 
1840  //add index column
1841  if (createIndexEnabled) {
1842  vectorNames << i18n("Index");
1844  }
1845 
1846  //add timestamp column
1847  if (createTimestampEnabled) {
1848  vectorNames << i18n("Timestamp");
1850  }
1851 
1852  //parse the first data line to determine data type for each column
1853  int i = 1;
1854  for (auto& valueString : firstLineStringList) {
1856  valueString = valueString.simplified();
1857  if (removeQuotesEnabled)
1858  valueString.remove(QLatin1Char('"'));
1859 
1860  vectorNames << i18n("Value %1", i);
1862  ++i;
1863  }
1864 
1866  m_actualRows = lines.size();
1867 
1868 // QDEBUG("column modes = " << columnModes);
1869  DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows);
1870 
1871  return 0;
1872 }
1873 
1874 /*!
1875  * generates the preview for the string \s message.
1876  */
1877 QVector<QStringList> AsciiFilterPrivate::preview(const QString& message) {
1878  QVector<QStringList> dataStrings;
1879  prepareToRead(message);
1880 
1881  //number formatting
1882  DEBUG("locale = " << STDSTRING(QLocale::languageToString(numberFormat)));
1883 
1884  // Read the data
1885  QStringList lines = message.split('\n');
1886 
1887  //loop over all lines in the message and parse the available columns
1888  int i = 0;
1889  for (auto line : lines) {
1891  line = line.simplified();
1892 
1893  if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines
1894  continue;
1895 
1896  const QStringList& lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
1897  QDEBUG(" line = " << lineStringList);
1898 
1899  QStringList lineString;
1900 
1901  // index column if required
1902  if (createIndexEnabled)
1903  lineString += QString::number(i + 1);
1904 
1905  // timestamp column if required
1907  lineString += QDateTime::currentDateTime().toString();
1908 
1909  int offset = int(createIndexEnabled) + int(createTimestampEnabled);
1910 
1911  //parse columns
1912  for (int n = 0; n < m_actualCols - offset; ++n) {
1913  if (n < lineStringList.size()) {
1914  QString valueString = lineStringList.at(n);
1915  if (removeQuotesEnabled)
1916  valueString.remove(QLatin1Char('"'));
1917  if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces
1918  continue;
1919 
1920  lineString += previewValue(valueString, columnModes[n+offset]);
1921  } else // missing columns in this line
1922  lineString += QString();
1923  }
1924 
1925  ++i;
1926  dataStrings << lineString;
1927  }
1928 
1929  return dataStrings;
1930 }
1931 
1932 /*!
1933  * \brief reads the content of a message received by the topic.
1934  * Uses the settings defined in the MQTTTopic's MQTTClient
1935  * \param message
1936  * \param topic
1937  * \param dataSource
1938  */
1939 void AsciiFilterPrivate::readMQTTTopic(const QString& message, AbstractDataSource* dataSource) {
1940  //If the message is empty, there is nothing to do
1941  if (message.isEmpty()) {
1942  DEBUG("No new data available");
1943  return;
1944  }
1945 
1946  MQTTTopic* spreadsheet = dynamic_cast<MQTTTopic*>(dataSource);
1947  if (!spreadsheet)
1948  return;
1949 
1950  const int keepNValues = spreadsheet->mqttClient()->keepNValues();
1951 
1952  if (!m_prepared) {
1953  DEBUG("Start preparing filter for: " << STDSTRING(spreadsheet->topicName()));
1954 
1955  //Prepare the filter
1956  const int mqttPrepareError = prepareToRead(message);
1957  if (mqttPrepareError != 0) {
1958  DEBUG("Mqtt Prepare Error = " << mqttPrepareError);
1959  return;
1960  }
1961 
1962  // prepare import for spreadsheet
1963  spreadsheet->setUndoAware(false);
1965 
1966  //columns in a MQTTTopic don't have any manual changes.
1967  //make the available columns undo unaware and suppress the "data changed" signal.
1968  //data changes will be propagated via an explicit Column::setChanged() call once new data was read.
1969  for (int i = 0; i < spreadsheet->childCount<Column>(); i++) {
1970  spreadsheet->child<Column>(i)->setUndoAware(false);
1971  spreadsheet->child<Column>(i)->setSuppressDataChangedSignal(true);
1972  }
1973 
1974  if (keepNValues == 0)
1975  spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1);
1976  else {
1977  spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues());
1978  m_actualRows = spreadsheet->mqttClient()->keepNValues();
1979  }
1980 
1981  m_dataContainer.resize(m_actualCols);
1982  initDataContainers(spreadsheet);
1983  }
1984 
1985  MQTTClient::ReadingType readingType;
1986  if (!m_prepared) {
1987  //if filter is not prepared we read till the end
1988  readingType = MQTTClient::ReadingType::TillEnd;
1989  } else {
1990  //we have to read all the data when reading from end
1991  //so we set readingType to TillEnd
1992  if (spreadsheet->mqttClient()->readingType() == MQTTClient::ReadingType::FromEnd)
1993  readingType = MQTTClient::ReadingType::TillEnd;
1994  else
1995  readingType = spreadsheet->mqttClient()->readingType();
1996  }
1997 
1998  //count the new lines, increase actualrows on each
1999  //now we read all the new lines, if we want to use sample rate
2000  //then here we can do it, if we have actually sample rate number of lines :-?
2001  int newLinesForSampleSizeNotTillEnd = 0;
2002  int newLinesTillEnd = 0;
2003  QVector<QString> newData;
2004  if (readingType != MQTTClient::ReadingType::TillEnd) {
2005  newData.reserve(spreadsheet->mqttClient()->sampleSize());
2006  newData.resize(spreadsheet->mqttClient()->sampleSize());
2007  }
2008 
2009  //TODO: bool sampleSizeReached = false;
2010  const QStringList newDataList = message.split(QRegularExpression(QStringLiteral("\n|\r\n|\r")),
2011  QString::SkipEmptyParts);
2012  for (auto& line : newDataList) {
2013  newData.push_back(line);
2014  newLinesTillEnd++;
2015 
2016  if (readingType != MQTTClient::ReadingType::TillEnd) {
2017  newLinesForSampleSizeNotTillEnd++;
2018  //for Continuous reading and FromEnd we read sample rate number of lines if possible
2019  if (newLinesForSampleSizeNotTillEnd == spreadsheet->mqttClient()->sampleSize()) {
2020  //TODO: sampleSizeReached = true;
2021  break;
2022  }
2023  }
2024  }
2025 
2026  qDebug()<<"Processing message done";
2027  //now we reset the readingType
2028  if (spreadsheet->mqttClient()->readingType() == MQTTClient::ReadingType::FromEnd)
2029  readingType = static_cast<MQTTClient::ReadingType>(spreadsheet->mqttClient()->readingType());
2030 
2031  //we had less new lines than the sample rate specified
2032  if (readingType != MQTTClient::ReadingType::TillEnd)
2033  qDebug() << "Removed empty lines: " << newData.removeAll(QString());
2034 
2035  const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount();
2036 
2037  if (m_prepared ) {
2038  if (keepNValues == 0)
2039  m_actualRows = spreadsheetRowCountBeforeResize;
2040  else {
2041  //if the keepNValues changed since the last read we have to manage the columns accordingly
2042  if (m_actualRows != spreadsheet->mqttClient()->keepNValues()) {
2043  if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
2044  spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues());
2045  qDebug()<<"rowcount set to: " << spreadsheet->mqttClient()->keepNValues();
2046  }
2047 
2048  //Calculate the difference between the old and new keepNValues
2049  int rowDiff = 0;
2050  if (m_actualRows > spreadsheet->mqttClient()->keepNValues())
2051  rowDiff = m_actualRows - spreadsheet->mqttClient()->keepNValues();
2052 
2053  if (m_actualRows < spreadsheet->mqttClient()->keepNValues())
2054  rowDiff = spreadsheet->mqttClient()->keepNValues() - m_actualRows;
2055 
2056  for (int n = 0; n < columnModes.size(); ++n) {
2057  // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp)
2058  switch (columnModes[n]) {
2060  QVector<double>* vector = static_cast<QVector<double>* >(spreadsheet->child<Column>(n)->data());
2061  m_dataContainer[n] = static_cast<void *>(vector);
2062 
2063  //if the keepNValues got smaller then we move the last keepNValues count of data
2064  //in the first keepNValues places
2065  if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
2066  for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
2067  static_cast<QVector<double>*>(m_dataContainer[n])->operator[] (i) =
2068  static_cast<QVector<double>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
2069  }
2070  }
2071 
2072  //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
2073  //then fill the remaining lines with NaN
2074  if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
2075  vector->reserve( spreadsheet->mqttClient()->keepNValues());
2076  vector->resize( spreadsheet->mqttClient()->keepNValues());
2077 
2078  for (int i = 1; i <= m_actualRows; i++) {
2079  static_cast<QVector<double>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
2080  static_cast<QVector<double>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
2081  }
2082  for (int i = 0; i < rowDiff; i++)
2083  static_cast<QVector<double>*>(m_dataContainer[n])->operator[](i) = nanValue;
2084  }
2085  break;
2086  }
2088  QVector<int>* vector = static_cast<QVector<int>* >(spreadsheet->child<Column>(n)->data());
2089  m_dataContainer[n] = static_cast<void *>(vector);
2090 
2091  //if the keepNValues got smaller then we move the last keepNValues count of data
2092  //in the first keepNValues places
2093  if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
2094  for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
2095  static_cast<QVector<int>*>(m_dataContainer[n])->operator[] (i) =
2096  static_cast<QVector<int>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
2097  }
2098  }
2099 
2100  //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
2101  //then fill the remaining lines with 0
2102  if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
2103  vector->reserve( spreadsheet->mqttClient()->keepNValues());
2104  vector->resize( spreadsheet->mqttClient()->keepNValues());
2105  for (int i = 1; i <= m_actualRows; i++) {
2106  static_cast<QVector<int>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
2107  static_cast<QVector<int>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
2108  }
2109  for (int i = 0; i < rowDiff; i++)
2110  static_cast<QVector<int>*>(m_dataContainer[n])->operator[](i) = 0;
2111  }
2112  break;
2113  }
2115  QVector<qint64>* vector = static_cast<QVector<qint64>* >(spreadsheet->child<Column>(n)->data());
2116  m_dataContainer[n] = static_cast<void *>(vector);
2117 
2118  //if the keepNValues got smaller then we move the last keepNValues count of data
2119  //in the first keepNValues places
2120  if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
2121  for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
2122  static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[] (i) =
2123  static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
2124  }
2125  }
2126 
2127  //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
2128  //then fill the remaining lines with 0
2129  if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
2130  vector->reserve( spreadsheet->mqttClient()->keepNValues());
2131  vector->resize( spreadsheet->mqttClient()->keepNValues());
2132  for (int i = 1; i <= m_actualRows; i++) {
2133  static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
2134  static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
2135  }
2136  for (int i = 0; i < rowDiff; i++)
2137  static_cast<QVector<qint64>*>(m_dataContainer[n])->operator[](i) = 0;
2138  }
2139  break;
2140  }
2142  QVector<QString>* vector = static_cast<QVector<QString>*>(spreadsheet->child<Column>(n)->data());
2143  m_dataContainer[n] = static_cast<void *>(vector);
2144 
2145  //if the keepNValues got smaller then we move the last keepNValues count of data
2146  //in the first keepNValues places
2147  if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
2148  for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
2149  static_cast<QVector<QString>*>(m_dataContainer[n])->operator[] (i) =
2150  static_cast<QVector<QString>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
2151  }
2152  }
2153 
2154  //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
2155  //then fill the remaining lines with empty lines
2156  if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
2157  vector->reserve( spreadsheet->mqttClient()->keepNValues());
2158  vector->resize( spreadsheet->mqttClient()->keepNValues());
2159  for (int i = 1; i <= m_actualRows; i++) {
2160  static_cast<QVector<QString>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
2161  static_cast<QVector<QString>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
2162  }
2163  for (int i = 0; i < rowDiff; i++)
2164  static_cast<QVector<QString>*>(m_dataContainer[n])->operator[](i).clear();
2165  }
2166  break;
2167  }
2169  QVector<QDateTime>* vector = static_cast<QVector<QDateTime>* >(spreadsheet->child<Column>(n)->data());
2170  m_dataContainer[n] = static_cast<void *>(vector);
2171 
2172  //if the keepNValues got smaller then we move the last keepNValues count of data
2173  //in the first keepNValues places
2174  if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) {
2175  for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) {
2176  static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[] (i) =
2177  static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i);
2178  }
2179  }
2180 
2181  //if the keepNValues got bigger we move the existing values to the last m_actualRows positions
2182  //then fill the remaining lines with null datetime
2183  if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) {
2184  vector->reserve( spreadsheet->mqttClient()->keepNValues());
2185  vector->resize( spreadsheet->mqttClient()->keepNValues());
2186  for (int i = 1; i <= m_actualRows; i++) {
2187  static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) =
2188  static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff);
2189  }
2190  for (int i = 0; i < rowDiff; i++)
2191  static_cast<QVector<QDateTime>*>(m_dataContainer[n])->operator[](i) = QDateTime();
2192  }
2193  break;
2194  }
2195  //TODO
2198  break;
2199  }
2200  }
2201  //if the keepNValues got smaller resize the spreadsheet
2202  if (m_actualRows > spreadsheet->mqttClient()->keepNValues())
2203  spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues());
2204 
2205  //set the new row count
2206  m_actualRows = spreadsheet->mqttClient()->keepNValues();
2207  qDebug()<<"actual rows: "<<m_actualRows;
2208  }
2209  }
2210  }
2211 
2212  qDebug()<<"starting m_actual rows calculated: " << m_actualRows <<", new data size: "<<newData.size();
2213 
2214  int currentRow = 0; // indexes the position in the vector(column)
2215  int linesToRead = 0;
2216 
2217  if (m_prepared) {
2218  //increase row count if we don't have a fixed size
2219  //but only after the preparation step
2220  if (keepNValues == 0) {
2221  if (readingType != MQTTClient::ReadingType::TillEnd)
2222  m_actualRows += qMin(newData.size(), spreadsheet->mqttClient()->sampleSize());
2223  else {
2224  m_actualRows += newData.size();
2225  }
2226  }
2227 
2228  //fixed size
2229  if (keepNValues != 0) {
2230  if (readingType == MQTTClient::ReadingType::TillEnd) {
2231  //we had more lines than the fixed size, so we read m_actualRows number of lines
2232  if (newLinesTillEnd > m_actualRows) {
2233  linesToRead = m_actualRows;
2234  } else
2235  linesToRead = newLinesTillEnd;
2236  } else {
2237  //we read max sample size number of lines when the reading mode
2238  //is ContinuouslyFixed or FromEnd
2239  if (spreadsheet->mqttClient()->sampleSize() <= spreadsheet->mqttClient()->keepNValues())
2240  linesToRead = qMin(spreadsheet->mqttClient()->sampleSize(), newLinesTillEnd);
2241  else
2242  linesToRead = qMin(spreadsheet->mqttClient()->keepNValues(), newLinesTillEnd);
2243  }
2244  } else
2245  linesToRead = m_actualRows - spreadsheetRowCountBeforeResize;
2246 
2247  if (linesToRead == 0)
2248  return;
2249  } else {
2250  if (keepNValues != 0)
2251  linesToRead = newLinesTillEnd > m_actualRows ? m_actualRows : newLinesTillEnd;
2252  else
2253  linesToRead = newLinesTillEnd;
2254  }
2255  qDebug()<<"linestoread = " << linesToRead;
2256 
2257  //new rows/resize columns if we don't have a fixed size
2258  if (keepNValues == 0) {
2259 
2260 #ifdef PERFTRACE_LIVE_IMPORT
2261  PERFTRACE("AsciiLiveDataImportResizing: ");
2262 #endif
2263  if (spreadsheet->rowCount() < m_actualRows)
2264  spreadsheet->setRowCount(m_actualRows);
2265 
2266  if (!m_prepared)
2267  currentRow = 0;
2268  else {
2269  // indexes the position in the vector(column)
2270  currentRow = spreadsheetRowCountBeforeResize;
2271  }
2272 
2273  // if we have fixed size, we do this only once in preparation, here we can use
2274  // m_prepared and we need something to decide whether it has a fixed size or increasing
2275 
2276  initDataContainers(spreadsheet);
2277  } else {
2278  //when we have a fixed size we have to pop sampleSize number of lines if specified
2279  //here popping, setting currentRow
2280  if (!m_prepared)
2281  currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows);
2282  else {
2283  if (readingType == MQTTClient::ReadingType::TillEnd) {
2284  if (newLinesTillEnd > m_actualRows)
2285  currentRow = 0;
2286  else
2287  currentRow = m_actualRows - newLinesTillEnd;
2288  } else {
2289  //we read max sample rate number of lines when the reading mode
2290  //is ContinuouslyFixed or FromEnd
2291  currentRow = m_actualRows - linesToRead;
2292  }
2293  }
2294 
2295  if (m_prepared) {
2296 #ifdef PERFTRACE_LIVE_IMPORT
2297  PERFTRACE("AsciiLiveDataImportPopping: ");
2298 #endif
2299  for (int row = 0; row < linesToRead; ++row) {
2300  for (int col = 0; col < m_actualCols; ++col) {
2301  switch (columnModes[col]) {
2303  QVector<double>* vector = static_cast<QVector<double>* >(spreadsheet->child<Column>(col)->data());
2304  vector->pop_front();
2305  vector->reserve(m_actualRows);
2306  vector->resize(m_actualRows);
2307  m_dataContainer[col] = static_cast<void *>(vector);
2308  break;
2309  }
2311  QVector<int>* vector = static_cast<QVector<int>* >(spreadsheet->child<Column>(col)->data());
2312  vector->pop_front();
2313  vector->reserve(m_actualRows);
2314  vector->resize(m_actualRows);
2315  m_dataContainer[col] = static_cast<void *>(vector);
2316  break;
2317  }
2319  QVector<qint64>* vector = static_cast<QVector<qint64>* >(spreadsheet->child<Column>(col)->data());
2320  vector->pop_front();
2321  vector->reserve(m_actualRows);
2322  vector->resize(m_actualRows);
2323  m_dataContainer[col] = static_cast<void *>(vector);
2324  break;
2325  }
2327  QVector<QString>* vector = static_cast<QVector<QString>*>(spreadsheet->child<Column>(col)->data());
2328  vector->pop_front();
2329  vector->reserve(m_actualRows);
2330  vector->resize(m_actualRows);
2331  m_dataContainer[col] = static_cast<void *>(vector);
2332  break;
2333  }
2335  QVector<QDateTime>* vector = static_cast<QVector<QDateTime>* >(spreadsheet->child<Column>(col)->data());
2336  vector->pop_front();
2337  vector->reserve(m_actualRows);
2338  vector->resize(m_actualRows);
2339  m_dataContainer[col] = static_cast<void *>(vector);
2340  break;
2341  }
2342  //TODO
2345  break;
2346  }
2347  }
2348  }
2349  }
2350  }
2351 
2352  // from the last row we read the new data in the spreadsheet
2353  qDebug() << "reading from line: " << currentRow << " lines till end: " << newLinesTillEnd;
2354  qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows;
2355  int newDataIdx = 0;
2356  //From end means that we read the last sample size amount of data
2357  if (readingType == MQTTClient::ReadingType::FromEnd) {
2358  if (m_prepared) {
2359  if (newData.size() > spreadsheet->mqttClient()->sampleSize())
2360  newDataIdx = newData.size() - spreadsheet->mqttClient()->sampleSize();
2361  }
2362  }
2363 
2364  qDebug() << "newDataIdx: " << newDataIdx;
2365 
2366  //read the data
2367  static int indexColumnIdx = 0;
2368  {
2369 #ifdef PERFTRACE_LIVE_IMPORT
2370  PERFTRACE("AsciiLiveDataImportFillingContainers: ");
2371 #endif
2372  int row = 0;
2373  for (; row < linesToRead; ++row) {
2374  QString line;
2375  if (readingType == MQTTClient::ReadingType::FromEnd)
2376  line = newData.at(newDataIdx++);
2377  else
2378  line = newData.at(row);
2379 
2380  if (removeQuotesEnabled)
2381  line.remove(QLatin1Char('"'));
2382 
2383  if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter)))
2384  continue;
2385 
2386  QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts);
2387 
2389  for (int i = 0; i < lineStringList.size(); ++i)
2390  lineStringList[i] = lineStringList[i].simplified();
2391  }
2392 
2393  //add index if required
2394  int offset = 0;
2395  if (createIndexEnabled) {
2396  int index = (keepNValues == 0) ? currentRow + 1 : indexColumnIdx++;
2397  static_cast<QVector<int>*>(m_dataContainer[0])->operator[](currentRow) = index;
2398  ++offset;
2399  }
2400 
2401  //add current timestamp if required
2402  if (createTimestampEnabled) {
2403  static_cast<QVector<QDateTime>*>(m_dataContainer[offset])->operator[](currentRow) = QDateTime::currentDateTime();
2404  ++offset;
2405  }
2406 
2407  //parse the columns
2408  for (int n = 0; n < m_actualCols - offset; ++n) {
2409  int col = n + offset;
2410  QString valueString;
2411  if (n < lineStringList.size())
2412  valueString = lineStringList.at(n);
2413 
2414  setValue(col, currentRow, valueString);
2415  }
2416  currentRow++;
2417  }
2418  }
2419 
2420  if (m_prepared) {
2421  //notify all affected columns and plots about the changes
2422  PERFTRACE("AsciiLiveDataImport, notify affected columns and plots");
2423 
2424  const Project* project = spreadsheet->project();
2427 
2428  for (int n = 0; n < m_actualCols; ++n) {
2429  Column* column = spreadsheet->column(n);
2430 
2431  //determine the plots where the column is consumed
2432  for (const auto* curve : curves) {
2433  if (curve->xColumn() == column || curve->yColumn() == column) {
2434  CartesianPlot* plot = static_cast<CartesianPlot*>(curve->parentAspect());
2435  if (plots.indexOf(plot) == -1) {
2436  plots << plot;
2437  plot->setSuppressDataChangedSignal(true);
2438  }
2439  }
2440  }
2441 
2442  column->setChanged();
2443  }
2444 
2445  //loop over all affected plots and retransform them
2446  for (auto* const plot : plots) {
2447  //TODO setting this back to true triggers again a lot of retransforms in the plot (one for each curve).
2448  // plot->setSuppressDataChangedSignal(false);
2449  plot->dataChanged();
2450  }
2451  } else
2452  m_prepared = true;
2453 
2454  DEBUG("AsciiFilterPrivate::readFromMQTTTopic() DONE");
2455 }
2456 
2457 /*!
2458  * \brief After the MQTTTopic was loaded, the filter is prepared for reading
2459  * \param prepared
2460  * \param topic
2461  * \param separator
2462  */
2463 void AsciiFilterPrivate::setPreparedForMQTT(bool prepared, MQTTTopic* topic, const QString& separator) {
2464  m_prepared = prepared;
2465  //If originally it was prepared we have to restore the settings
2466  if (prepared) {
2469  m_actualRows = topic->rowCount();
2470  //set the column modes
2471  columnModes.resize(topic->columnCount());
2472  for (int i = 0; i < topic->columnCount(); ++i)
2473  columnModes[i] = topic->column(i)->columnMode();
2474 
2475  //set the data containers
2476  m_dataContainer.resize(m_actualCols);
2477  initDataContainers(topic);
2478  }
2479 }
2480 #endif
void setUndoAware(bool)
@ Recursive
Recursively handle all descendents, not just immediate children.
int childCount(ChildIndexFlags flags={}) const
Return the number of child Aspects inheriting from given class.
T * child(int index, ChildIndexFlags flags={}) const
QVector< AbstractAspect * > children(AspectType type, ChildIndexFlags flags={}) const
virtual Project * project()
Return the Project this Aspect belongs to, or 0 if it is currently not part of one.
Interface for the data sources.
virtual int prepareImport(std::vector< void * > &dataContainer, AbstractFileFilter::ImportMode, int actualRows, int actualCols, QStringList colNameList=QStringList(), QVector< AbstractColumn::ColumnMode >=QVector< AbstractColumn::ColumnMode >())=0
virtual void finalizeImport(int columnOffset=0, int startColumn=0, int endColumn=0, const QString &dateTimeFormat=QString(), AbstractFileFilter::ImportMode importMode=AbstractFileFilter::ImportMode::Replace)=0
Interface for the input/output file filters.
static AbstractColumn::ColumnMode columnMode(const QString &valueString, const QString &dateTimeFormat, QLocale::Language)
void completed(int) const
int ranging from 0 to 100 notifies about the status of a read/write process
qint64 readFromLiveDevice(QIODevice &, AbstractDataSource *, qint64 from=-1)
QString previewValue(const QString &, AbstractColumn::ColumnMode)
QStringList getLineString(QIODevice &)
QLocale::Language numberFormat
void write(const QString &fileName, AbstractDataSource *)
void readDataFromDevice(QIODevice &, AbstractDataSource *=nullptr, AbstractFileFilter::ImportMode=AbstractFileFilter::ImportMode::Replace, int lines=-1)
void setValue(int col, int row, const QString &value)
QDateTime parseDateTime(const QString &string, const QString &format)
std::vector< void * > m_dataContainer
int prepareDeviceToRead(QIODevice &)
QVector< QStringList > preview(const QString &fileName, int lines)
AsciiFilterPrivate(AsciiFilter *)
static const unsigned int m_dataTypeLines
void initDataContainers(Spreadsheet *)
const AsciiFilter * q
void readDataFromFile(const QString &fileName, AbstractDataSource *=nullptr, AbstractFileFilter::ImportMode=AbstractFileFilter::ImportMode::Replace)
QVector< AbstractColumn::ColumnMode > columnModes
QString separator() const
Returns the separator used by the filter.
Manages the import/export of data organized as columns (vectors) from/to an ASCII-file.
Definition: AsciiFilter.h:42
void setCommentCharacter(const QString &)
bool isHeaderEnabled() const
static QString fileInfoString(const QString &)
QVector< QStringList > preview(const QString &fileName, int lines)
void setEndColumn(const int)
void setHeaderEnabled(const bool)
QStringList vectorNames() const
void readDataFromFile(const QString &fileName, AbstractDataSource *=nullptr, AbstractFileFilter::ImportMode=AbstractFileFilter::ImportMode::Replace) override
bool skipEmptyParts() const
void setVectorNames(const QString &)
void setRemoveQuotesEnabled(const bool)
bool createIndexEnabled() const
int endColumn() const
void save(QXmlStreamWriter *) const override
QLocale::Language numberFormat() const
void setStartRow(const int)
static QStringList dataTypes()
QString separator() const
qint64 readFromLiveDevice(QIODevice &device, AbstractDataSource *, qint64 from=-1)
Definition: AsciiFilter.cpp:76
int isPrepared()
void setSkipEmptyParts(const bool)
void setStartColumn(const int)
void write(const QString &fileName, AbstractDataSource *) override
bool simplifyWhitespacesEnabled() const
void setSeparatingCharacter(const QString &)
~AsciiFilter() override
static size_t lineNumber(const QString &fileName)
static int columnNumber(const QString &fileName, const QString &separator=QString())
bool removeQuotesEnabled() const
void saveFilterSettings(const QString &) const override
static QStringList commentCharacters()
bool load(XmlStreamReader *) override
static QStringList separatorCharacters()
QString commentCharacter() const
static QStringList predefinedFilters()
bool isAutoModeEnabled() const
void setEndRow(const int)
void setNumberFormat(QLocale::Language)
void readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource *dataSource)
Definition: AsciiFilter.cpp:72
int startRow() const
int startColumn() const
void setAutoModeEnabled(const bool)
void loadFilterSettings(const QString &) override
QString separatingCharacter() const
QVector< AbstractColumn::ColumnMode > columnModes()
void setCreateIndexEnabled(const bool)
bool createTimestampEnabled() const
void setSimplifyWhitespacesEnabled(const bool)
void setCreateTimestampEnabled(const bool)
void setNaNValueToZero(const bool)
QString dateTimeFormat() const
void setDateTimeFormat(const QString &)
std::unique_ptr< AsciiFilterPrivate > const d
Definition: AsciiFilter.h:128
bool NaNValueToZeroEnabled() const
void readDataFromDevice(QIODevice &device, AbstractDataSource *, AbstractFileFilter::ImportMode=AbstractFileFilter::ImportMode::Replace, int lines=-1)
Definition: AsciiFilter.cpp:68
int endRow() const
A xy-plot.
Definition: CartesianPlot.h:58
void setSuppressDataChangedSignal(bool)
Aspect that manages a column.
Definition: Column.h:42
void setChanged()
Definition: Column.cpp:998
void setSuppressDataChangedSignal(const bool)
Definition: Column.cpp:246
AbstractColumn::ColumnMode columnMode() const override
Return the column mode.
Definition: Column.cpp:1388
Represents data stored in a file. Reading and writing is done with the help of appropriate I/O-filter...
SourceType sourceType() const
int sampleSize() const
Returns the size rate.
Definition: MQTTClient.cpp:175
ReadingType readingType() const
Returns the MQTTClient's reading type.
Definition: MQTTClient.cpp:190
int keepNValues() const
Returns how many values we should store.
Definition: MQTTClient.cpp:150
Represents a topic of a subscription made in MQTTClient.
Definition: MQTTTopic.h:39
QString topicName() const
Returns the name of the MQTTTopic.
Definition: MQTTTopic.cpp:152
MQTTClient * mqttClient() const
Returns the MQTTClient the topic belongs to.
Definition: MQTTTopic.cpp:167
Definition: Matrix.h:41
Represents a project.
Definition: Project.h:42
Aspect providing a spreadsheet table with column logic.
Definition: Spreadsheet.h:40
int columnCount() const
int resize(AbstractFileFilter::ImportMode, QStringList colNameList, int cols)
int rowCount() const
Column * column(int index) const
void setRowCount(int)
A 2D-curve, provides an interface for editing many properties of the curve.
Definition: XYCurve.h:46
XML stream parser that supports errors as well as warnings. This class also adds line and column numb...
#define READ_DOUBLE_VALUE(name, var)
Definition: macros.h:475
#define STDSTRING(qstr)
Definition: macros.h:67
#define ENUM_TO_STRING(class, enum, value)
Definition: macros.h:69
#define DEBUG(x)
Definition: macros.h:50
#define READ_INT_VALUE(name, var, type)
Definition: macros.h:468
#define QDEBUG(x)
Definition: macros.h:47
#define READ_STRING_VALUE(name, var)
Definition: macros.h:482
@ None
Definition: OriginObj.h:71
#define i18n(m)
Definition: nsl_common.h:38
#define PERFTRACE(msg)
Definition: trace.h:57