labplot  2.8.2
About: LabPlot is an application for plotting and analysis of 2D and 3D functions and data. It is a complete rewrite of LabPlot1 and lacks in the first release a lot of features available in the predecessor. On the other hand, the GUI and the usability is more superior.
  Fossies Dox: labplot-2.8.2.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

DatasetHandler.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  File : DatasetHandler.cpp
3  Project : LabPlot
4  Description : Processes a dataset's metadata file
5  --------------------------------------------------------------------
6  Copyright : (C) 2019 Kovacs Ferencz (kferike98@gmail.com)
7  Copyright : (C) 2019 by Alexander Semke (alexander.semke@web.de)
8 
9 ***************************************************************************/
10 
11 /***************************************************************************
12 * *
13 * This program is free software; you can redistribute it and/or modify *
14 * it under the terms of the GNU General Public License as published by *
15 * the Free Software Foundation; either version 2 of the License, or *
16 * (at your option) any later version. *
17 * *
18 * This program is distributed in the hope that it will be useful, *
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
21 * GNU General Public License for more details. *
22 * *
23 * You should have received a copy of the GNU General Public License *
24 * along with this program; if not, write to the Free Software *
25 * Foundation, Inc., 51 Franklin Street, Fifth Floor, *
26 * Boston, MA 02110-1301 USA *
27 * *
28 ***************************************************************************/
29 
32 
33 #include <QDir>
34 #include <QFile>
35 #include <QJsonArray>
36 #include <QJsonObject>
37 #include <QMessageBox>
38 #include <QStandardPaths>
39 #include <QtNetwork/QNetworkAccessManager>
40 #include <QtNetwork/QNetworkReply>
41 
42 #include <KLocalizedString>
43 
44 /*!
45  \class DatasetHandler
46  \brief Provides functionality to process a metadata file of a dataset, configure a spreadsheet and filter based on it, download the dataset
47  and load it into the spreadsheet.
48 
49  \ingroup datasources
50 */
51 DatasetHandler::DatasetHandler(Spreadsheet* spreadsheet) : m_spreadsheet(spreadsheet),
52  m_filter(new AsciiFilter),
53  m_downloadManager(new QNetworkAccessManager) {
54  connect(m_downloadManager, &QNetworkAccessManager::finished, this, &DatasetHandler::downloadFinished);
56 }
57 
59  delete m_downloadManager;
60  delete m_filter;
61 }
62 
63 /**
64  * @brief Initiates processing the metadata file,, located at the given path, belonging to a dataset.
65  * @param path the path to the metadata file
66  */
67 void DatasetHandler::processMetadata(const QJsonObject& object) {
68  m_object = new QJsonObject(object);
69  DEBUG("Start processing dataset...");
70 
71  if (!m_object->isEmpty()) {
75  }
76 }
77 
78 /**
79  * @brief Marks the metadata file being invalid by setting the value of a flag, also pops up a messagebox.
80  */
82  m_invalidMetadataFile = true;
83  QMessageBox::critical(nullptr, i18n("Invalid metadata file"), i18n("The metadata file for the selected dataset is invalid."));
84 }
85 
86 /**
87  * @brief Configures the filter, that will be used later, based on the metadata file.
88  */
90  //set some default values common to many datasets
93  m_filter->setHeaderEnabled(false);
94 
95  //read properties specified in the dataset description
96  if (!m_object->isEmpty()) {
97  if (m_object->contains("separator"))
98  m_filter->setSeparatingCharacter(m_object->value("separator").toString());
99 
100  if (m_object->contains("comment_character"))
101  m_filter->setCommentCharacter(m_object->value("comment_character").toString());
102 
103  if (m_object->contains("create_index_column"))
104  m_filter->setCreateIndexEnabled(m_object->value("create_index_column").toBool());
105 
106  if (m_object->contains("skip_empty_parts"))
107  m_filter->setSkipEmptyParts(m_object->value("skip_empty_parts").toBool());
108 
109  if (m_object->contains("simplify_whitespaces"))
110  m_filter->setSimplifyWhitespacesEnabled(m_object->value("simplify_whitespaces").toBool());
111 
112  if (m_object->contains("remove_quotes"))
113  m_filter->setRemoveQuotesEnabled(m_object->value("remove_quotes").toBool());
114 
115  if (m_object->contains("use_first_row_for_vectorname"))
116  m_filter->setHeaderEnabled(m_object->value("use_first_row_for_vectorname").toBool());
117 
118  if (m_object->contains("number_format"))
119  m_filter->setNumberFormat(QLocale::Language(m_object->value("number_format").toInt()));
120 
121  if (m_object->contains("DateTime_format"))
122  m_filter->setDateTimeFormat(m_object->value("DateTime_format").toString());
123 
124  if (m_object->contains("columns")) {
125  const QJsonArray& columnsArray = m_object->value("columns").toArray();
126  QStringList columnNames;
127  for (const auto& col : columnsArray)
128  columnNames << col.toString();
129 
130  m_filter->setVectorNames(columnNames);
131  }
132  } else {
133  DEBUG("Empty object");
135  }
136 }
137 
138 /**
139  * @brief Configures the spreadsheet based on the metadata file.
140  */
142  DEBUG("Start preparing spreadsheet");
143  if (!m_object->isEmpty()) {
144  if (m_object->contains("name"))
145  m_spreadsheet->setName( m_object->value("name").toString());
146  else
148 
149  if (m_object->contains("description_url")) {
150  auto* manager = new QNetworkAccessManager(this);
151  connect(manager, &QNetworkAccessManager::finished, [this] (QNetworkReply* reply) {
152  if (reply->error() == QNetworkReply::NoError) {
153  QByteArray ba = reply->readAll();
154  QString info(ba);
155  m_spreadsheet->setComment(info);
156  } else {
157  DEBUG("Failed to fetch the description.");
158  if (m_object->contains("description"))
159  m_spreadsheet->setComment(m_object->value("description").toString());
160  }
161  reply->deleteLater();
162  }
163  );
164  manager->get(QNetworkRequest(QUrl(m_object->value("description_url").toString())));
165  } else if (m_object->contains("description"))
166  m_spreadsheet->setComment(m_object->value("description").toString());
167  } else {
169  }
170 }
171 
172 /**
173  * @brief Extracts the download URL of the dataset and initiates the process of download.
174  */
176  DEBUG("Start downloading dataset");
177  if (!m_object->isEmpty()) {
178  if (m_object->contains("url")) {
179  const QString& url = m_object->value("url").toString();
180  doDownload(url);
181  }
182  else {
183  QMessageBox::critical(nullptr, i18n("Invalid metadata file"), i18n("There is no download URL present in the metadata file!"));
184  }
185 
186  } else {
188  }
189 }
190 
191 /**
192  * @brief Starts the download of the dataset.
193  * @param url the download URL of the dataset
194  */
195 void DatasetHandler::doDownload(const QUrl& url) {
196  DEBUG("Download request");
197  QNetworkRequest request(url);
198  m_currentDownload = m_downloadManager->get(request);
199  connect(m_currentDownload, &QNetworkReply::downloadProgress, [this] (qint64 bytesReceived, qint64 bytesTotal) {
200  double progress;
201  if (bytesTotal == -1)
202  progress = 0;
203  else
204  progress = 100 * (static_cast<double>(bytesReceived) / static_cast<double>(bytesTotal));
205  qDebug() << "Progress: " << progress;
206  emit downloadProgress(progress);
207  });
208 }
209 
210 /**
211  * @brief Called when the download of the dataset is finished.
212  */
213 void DatasetHandler::downloadFinished(QNetworkReply* reply) {
214  DEBUG("Download finished");
215  const QUrl& url = reply->url();
216  if (reply->error()) {
217  qDebug("Download of %s failed: %s\n",
218  url.toEncoded().constData(),
219  qPrintable(reply->errorString()));
220  } else {
221  if (isHttpRedirect(reply)) {
222  qDebug("Request was redirected.\n");
223  } else {
224  QString filename = saveFileName(url);
225  if (saveToDisk(filename, reply)) {
226  qDebug("Download of %s succeeded (saved to %s)\n",
227  url.toEncoded().constData(), qPrintable(filename));
228  m_fileName = filename;
229  emit downloadCompleted();
230  }
231  }
232  }
233 
234  m_currentDownload = nullptr;
235  reply->deleteLater();
236 }
237 
238 /**
239  * @brief Checks whether the GET request was redirected or not.
240  */
241 bool DatasetHandler::isHttpRedirect(QNetworkReply* reply) {
242  const int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
243  // TODO enum/defines for status codes ?
244  return statusCode == 301 || statusCode == 302 || statusCode == 303
245  || statusCode == 305 || statusCode == 307 || statusCode == 308;
246 }
247 
248 /**
249  * @brief Returns the name and path of the file that will contain the content of the reply (based on the URL).
250  * @param url
251  */
252 QString DatasetHandler::saveFileName(const QUrl& url) {
253  const QString path = url.path();
254 
255  //get the extension of the downloaded file
256  const QString downloadFileName = QFileInfo(path).fileName();
257  int lastIndex = downloadFileName.lastIndexOf(".");
258  const QString fileExtension = lastIndex >= 0 ? downloadFileName.right(downloadFileName.length() - lastIndex) : "";
259 
260  QString basename = m_object->value("filename").toString() + fileExtension;
261 
262  if (basename.isEmpty())
263  basename = "url";
264 
265  QDir downloadDir(QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) + QLatin1String("/datasets_local/"));
266  if (!downloadDir.exists()) {
267  if (!downloadDir.mkpath(downloadDir.path())) {
268  qDebug()<<"Failed to create the directory " << downloadDir.path();
269  return QString();
270  }
271  }
272 
273  QString fileName = downloadDir.path() + QLatin1Char('/') + basename;
274  QFileInfo fileInfo (fileName);
275  if (QFile::exists(fileName)) {
276  if (fileInfo.lastModified().addDays(1) < QDateTime::currentDateTime()){
277  QFile removeFile (fileName);
278  removeFile.remove();
279  } else {
280  qDebug() << "Dataset file already exists, no need to download it again";
281  }
282  }
283  return fileName;
284 }
285 
286 /**
287  * @brief Saves the content of the network reply to the given path under the given name.
288  */
289 bool DatasetHandler::saveToDisk(const QString& filename, QIODevice* data) {
290  QFile file(filename);
291  if (!file.open(QIODevice::WriteOnly)) {
292  qDebug("Could not open %s for writing: %s\n",
293  qPrintable(filename),
294  qPrintable(file.errorString()));
295  return false;
296  }
297 
298  file.write(data->readAll());
299  file.close();
300 
301  return true;
302 }
303 
304 /**
305  * @brief Processes the downloaded dataset with the help of the already configured filter.
306  */
309 
310  //set column comments/descriptions, if available
311  //TODO:
312 // if (!m_object->isEmpty()) {
313 // int index = 0;
314 // const int columnsCount = m_spreadsheet->columnCount();
315 // while(m_object->contains(i18n("column_description_%1", index)) && (index < columnsCount)) {
316 // m_spreadsheet->column(index)->setComment(m_object->value(i18n("column_description_%1", index)).toString());
317 // ++index;
318 // }
319 // }
320 }
#define C(a, b)
Definition: Faddeeva.cc:255
bool setName(const QString &, bool autoUnique=true)
AbstractAspect::setName sets the name of the abstract aspect.
void setComment(const QString &)
Manages the import/export of data organized as columns (vectors) from/to an ASCII-file.
Definition: AsciiFilter.h:42
void setCommentCharacter(const QString &)
void setHeaderEnabled(const bool)
void readDataFromFile(const QString &fileName, AbstractDataSource *=nullptr, AbstractFileFilter::ImportMode=AbstractFileFilter::ImportMode::Replace) override
void setVectorNames(const QString &)
void setRemoveQuotesEnabled(const bool)
void setSkipEmptyParts(const bool)
void setSeparatingCharacter(const QString &)
void setNumberFormat(QLocale::Language)
void setCreateIndexEnabled(const bool)
void setSimplifyWhitespacesEnabled(const bool)
void setDateTimeFormat(const QString &)
void markMetadataAsInvalid()
Marks the metadata file being invalid by setting the value of a flag, also pops up a messagebox.
QString saveFileName(const QUrl &)
Returns the name and path of the file that will contain the content of the reply (based on the URL).
void downloadProgress(int progress)
QNetworkAccessManager * m_downloadManager
QNetworkReply * m_currentDownload
bool m_invalidMetadataFile
bool saveToDisk(const QString &filename, QIODevice *)
Saves the content of the network reply to the given path under the given name.
void downloadCompleted()
AsciiFilter * m_filter
void processDataset()
Processes the downloaded dataset with the help of the already configured filter.
void configureFilter()
Configures the filter, that will be used later, based on the metadata file.
bool isHttpRedirect(QNetworkReply *)
Checks whether the GET request was redirected or not.
void configureSpreadsheet()
Configures the spreadsheet based on the metadata file.
void prepareForDataset()
Extracts the download URL of the dataset and initiates the process of download.
QString m_fileName
DatasetHandler(Spreadsheet *)
QJsonObject * m_object
void processMetadata(const QJsonObject &)
Initiates processing the metadata file,, located at the given path, belonging to a dataset.
Spreadsheet * m_spreadsheet
void doDownload(const QUrl &)
Starts the download of the dataset.
void downloadFinished(QNetworkReply *)
Called when the download of the dataset is finished.
Aspect providing a spreadsheet table with column logic.
Definition: Spreadsheet.h:40
#define DEBUG(x)
Definition: macros.h:50
@ NoError
Definition: qxtnamespace.h:64
#define i18n(m)
Definition: nsl_common.h:38