summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVolker Krause <[email protected]>2017-12-12 22:18:33 +0100
committerVolker Krause <[email protected]>2017-12-12 22:21:38 +0100
commit94437d7a2efd0ac58d780c4712e60e46c8aef239 (patch)
tree26fb11a40b119b4f03a8f03a35154d26d2a6b7dd
parentcf416ee40f8587de3c8239ca5aabfaded77b62c6 (diff)
Add filtering of incomplete/invalid data to post-processing
That way we will never show incomplete data if any of the extractors fail to find mandatory information.
-rw-r--r--plugins/messageviewer/bodypartformatter/autotests/CMakeLists.txt5
-rw-r--r--plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-augment.post.json36
-rw-r--r--plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-augment.pre.json24
-rw-r--r--plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-filter.post.json2
-rw-r--r--plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-filter.pre.json73
-rw-r--r--plugins/messageviewer/bodypartformatter/autotests/postprocessortest.cpp83
-rw-r--r--plugins/messageviewer/bodypartformatter/semantic/datatypes.h14
-rw-r--r--plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.cpp37
-rw-r--r--plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.h4
-rw-r--r--plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.cpp55
-rw-r--r--plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.h1
11 files changed, 324 insertions, 10 deletions
diff --git a/plugins/messageviewer/bodypartformatter/autotests/CMakeLists.txt b/plugins/messageviewer/bodypartformatter/autotests/CMakeLists.txt
index 7f16969..a6aa98c 100644
--- a/plugins/messageviewer/bodypartformatter/autotests/CMakeLists.txt
+++ b/plugins/messageviewer/bodypartformatter/autotests/CMakeLists.txt
@@ -63,3 +63,8 @@ ecm_add_test(
NAME_PREFIX "messageviewerplugins-"
LINK_LIBRARIES Qt5::Test semantic_extractor
)
+ecm_add_test(
+ postprocessortest.cpp
+ NAME_PREFIX "messageviewerplugins-"
+ LINK_LIBRARIES Qt5::Test semantic_extractor
+)
diff --git a/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-augment.post.json b/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-augment.post.json
new file mode 100644
index 0000000..c0aa47f
--- /dev/null
+++ b/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-augment.post.json
@@ -0,0 +1,36 @@
+[{
+ "@context": "http://schema.org",
+ "@type": "FlightReservation",
+ "reservationFor": {
+ "@type": "Flight",
+ "airline": {
+ "@type": "Airline",
+ "iataCode": "UA",
+ "name": "United"
+ },
+ "arrivalAirport": {
+ "@type": "Airport",
+ "geo": {
+ "@type": "GeoCoordinates",
+ "latitude": -77.45580291748047,
+ "longitude": 38.944400787353516
+ },
+ "iataCode": "IAD",
+ "name": "Washington Dulles International Airport"
+ },
+ "arrivalTime": "2027-03-05T06:30:00-05:00",
+ "departureAirport": {
+ "@type": "Airport",
+ "geo": {
+ "@type": "GeoCoordinates",
+ "latitude": -122.30899810791016,
+ "longitude": 47.44889831542969
+ },
+ "iataCode": "SEA",
+ "name": "Seattle-Tacoma International Airport"
+ },
+ "departureTime": "2027-03-04T20:15:00-08:00",
+ "flightNumber": "110"
+ },
+ "reservationNumber": "RXJ34P"
+}]
diff --git a/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-augment.pre.json b/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-augment.pre.json
new file mode 100644
index 0000000..f424d6a
--- /dev/null
+++ b/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-augment.pre.json
@@ -0,0 +1,24 @@
+[{
+ "@context": "http://schema.org",
+ "@type": "FlightReservation",
+ "reservationNumber": "RXJ34P",
+ "reservationFor": {
+ "@type": "Flight",
+ "flightNumber": "110",
+ "airline": {
+ "@type": "Airline",
+ "name": "United",
+ "iataCode": "UA"
+ },
+ "departureAirport": {
+ "@type": "Airport",
+ "name": "Seattle-Tacoma International Airport"
+ },
+ "departureTime": "2027-03-04T20:15:00",
+ "arrivalAirport": {
+ "@type": "Airport",
+ "name": "Washington Dulles International Airport"
+ },
+ "arrivalTime": "2027-03-05T06:30:00"
+ }
+}]
diff --git a/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-filter.post.json b/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-filter.post.json
new file mode 100644
index 0000000..0d4f101
--- /dev/null
+++ b/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-filter.post.json
@@ -0,0 +1,2 @@
+[
+]
diff --git a/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-filter.pre.json b/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-filter.pre.json
new file mode 100644
index 0000000..3a60cde
--- /dev/null
+++ b/plugins/messageviewer/bodypartformatter/autotests/postprocessordata/flight-filter.pre.json
@@ -0,0 +1,73 @@
+[{
+ "@context": "http://schema.org",
+ "@type": "FlightReservation",
+ "reservationNumber": "RXJ34P",
+ "reservationFor": {
+ "@type": "Flight",
+ "flightNumber": "110",
+ "airline": {
+ "@type": "Airline",
+ "name": "United",
+ "iataCode": "UA"
+ },
+ "departureAirport": {
+ "@type": "Airport",
+ "name": "San Francisco Airport",
+ "iataCode": "SFO"
+ },
+ "departureTime": "2027-03-04T20:15:00-08:00",
+ "arrivalTime": "2027-03-05T06:30:00-05:00"
+ }
+}, {
+ "@context": "http://schema.org",
+ "@type": "FlightReservation",
+ "reservationNumber": "XXX123"
+}, {
+ "@context": "http://schema.org",
+ "@type": "FlightReservation",
+ "reservationNumber": "RXJ34P",
+ "reservationFor": {
+ "@type": "Flight",
+ "flightNumber": "110",
+ "airline": {
+ "@type": "Airline",
+ "name": "United",
+ "iataCode": "UA"
+ },
+ "departureAirport": {
+ "@type": "Airport",
+ "name": "San Francisco Airport",
+ "iataCode": "SFO"
+ },
+ "arrivalAirport": {
+ "@type": "Airport",
+ "name": "John F. Kennedy International Airport"
+ },
+ "arrivalTime": "2027-03-05T06:30:00-05:00"
+ }
+}, {
+ "@context": "http://schema.org",
+ "@type": "FlightReservation",
+ "reservationNumber": "RXJ34P",
+ "reservationFor": {
+ "@type": "Flight",
+ "flightNumber": "110",
+ "airline": {
+ "@type": "Airline",
+ "name": "United",
+ "iataCode": "UA"
+ },
+ "departureAirport": {
+ "@type": "Airport",
+ "name": "San Francisco Airport",
+ "iataCode": "SFO"
+ },
+ "departureTime": "2027-03-04T20:15:00-08:00",
+ "arrivalAirport": {
+ "@type": "Airport",
+ "name": "John F. Kennedy International Airport"
+ }
+ }
+}
+]
+
diff --git a/plugins/messageviewer/bodypartformatter/autotests/postprocessortest.cpp b/plugins/messageviewer/bodypartformatter/autotests/postprocessortest.cpp
new file mode 100644
index 0000000..c1d90ff
--- /dev/null
+++ b/plugins/messageviewer/bodypartformatter/autotests/postprocessortest.cpp
@@ -0,0 +1,83 @@
+/*
+ Copyright (c) 2017 Volker Krause <[email protected]>
+
+ This library is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Library General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or (at your
+ option) any later version.
+
+ This library is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
+ License for more details.
+
+ You should have received a copy of the GNU Library General Public License
+ along with this library; see the file COPYING.LIB. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ 02110-1301, USA.
+*/
+
+#include "extractorpostprocessor.h"
+#include "jsonlddocument.h"
+
+#include <QDebug>
+#include <QDir>
+#include <QFile>
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QObject>
+#include <QTest>
+
+class PostprocessorTest : public QObject
+{
+ Q_OBJECT
+private Q_SLOTS:
+ void testPostProc_data()
+ {
+ QTest::addColumn<QString>("preFile");
+ QTest::addColumn<QString>("postFile");
+
+ QDir dir(QStringLiteral(SOURCE_DIR "/postprocessordata"));
+ const auto lst = dir.entryList(QStringList(QStringLiteral("*.pre.json")), QDir::Files | QDir::Readable | QDir::NoSymLinks);
+ for (const auto &file : lst) {
+ const auto refFile = dir.path() + QLatin1Char('/') + file.left(file.size() - 8) + QStringLiteral("post.json");
+ if (!QFile::exists(refFile)) {
+ qDebug() << "reference file" << refFile << "does not exist, skipping test file" << file;
+ continue;
+ }
+ QTest::newRow(file.toLatin1()) << QString(dir.path() + QLatin1Char('/') + file) << refFile;
+ }
+ }
+
+ void testPostProc()
+ {
+ QFETCH(QString, preFile);
+ QFETCH(QString, postFile);
+
+ QFile f(preFile);
+ QVERIFY(f.open(QFile::ReadOnly));
+ const auto inArray = QJsonDocument::fromJson(f.readAll()).array();
+ QVERIFY(!inArray.isEmpty());
+ const auto preData = JsonLdDocument::fromJson(inArray);
+ QCOMPARE(inArray.size(), preData.size());
+
+ ExtractorPostprocessor postproc;
+ postproc.process(preData);
+ const auto outArray = JsonLdDocument::toJson(postproc.result());
+ QCOMPARE(outArray.size(), postproc.result().size());
+
+ QFile ref(postFile);
+ QVERIFY(ref.open(QFile::ReadOnly));
+ const auto refArray = QJsonDocument::fromJson(ref.readAll()).array();
+
+ if (outArray != refArray) {
+ qDebug().noquote() << QJsonDocument(outArray).toJson();
+ }
+ QCOMPARE(refArray.size(), postproc.result().size());
+ QCOMPARE(outArray, refArray);
+ }
+};
+
+QTEST_APPLESS_MAIN(PostprocessorTest)
+
+#include "postprocessortest.moc"
diff --git a/plugins/messageviewer/bodypartformatter/semantic/datatypes.h b/plugins/messageviewer/bodypartformatter/semantic/datatypes.h
index dbad4e1..12872a4 100644
--- a/plugins/messageviewer/bodypartformatter/semantic/datatypes.h
+++ b/plugins/messageviewer/bodypartformatter/semantic/datatypes.h
@@ -26,7 +26,7 @@
#define SEMANTIC_GADGET \
Q_GADGET \
- Q_PROPERTY(QString className READ className CONSTANT) \
+ Q_PROPERTY(QString className READ className STORED false CONSTANT) \
inline QString className() const { return QString::fromUtf8(staticMetaObject.className()); }
#define SEMANTIC_PROPERTY(Type, Name) \
@@ -100,8 +100,8 @@ class Flight
SEMANTIC_PROPERTY(Airport, arrivalAirport)
SEMANTIC_PROPERTY(QDateTime, arrivalTime)
- Q_PROPERTY(QString departureTimeLocalized READ departureTimeLocalized CONSTANT)
- Q_PROPERTY(QString arrivalTimeLocalized READ arrivalTimeLocalized CONSTANT)
+ Q_PROPERTY(QString departureTimeLocalized READ departureTimeLocalized STORED false CONSTANT)
+ Q_PROPERTY(QString arrivalTimeLocalized READ arrivalTimeLocalized STORED false CONSTANT)
private:
QString departureTimeLocalized() const;
QString arrivalTimeLocalized() const;
@@ -124,8 +124,8 @@ class TrainTrip
SEMANTIC_PROPERTY(QDateTime, departureTime)
SEMANTIC_PROPERTY(QString, trainNumber)
- Q_PROPERTY(QString departureTimeLocalized READ departureTimeLocalized CONSTANT)
- Q_PROPERTY(QString arrivalTimeLocalized READ arrivalTimeLocalized CONSTANT)
+ Q_PROPERTY(QString departureTimeLocalized READ departureTimeLocalized STORED false CONSTANT)
+ Q_PROPERTY(QString arrivalTimeLocalized READ arrivalTimeLocalized STORED false CONSTANT)
private:
QString departureTimeLocalized() const;
@@ -145,8 +145,8 @@ class LodgingReservation : protected Reservation
SEMANTIC_PROPERTY(QDateTime, checkinDate)
SEMANTIC_PROPERTY(QDateTime, checkoutDate)
- Q_PROPERTY(QString checkinDateLocalized READ checkinDateLocalized CONSTANT)
- Q_PROPERTY(QString checkoutDateLocalized READ checkoutDateLocalized CONSTANT)
+ Q_PROPERTY(QString checkinDateLocalized READ checkinDateLocalized STORED false CONSTANT)
+ Q_PROPERTY(QString checkoutDateLocalized READ checkoutDateLocalized STORED false CONSTANT)
private:
QString checkinDateLocalized() const;
QString checkoutDateLocalized() const;
diff --git a/plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.cpp b/plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.cpp
index 8f56ec6..bead93e 100644
--- a/plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.cpp
+++ b/plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.cpp
@@ -28,10 +28,12 @@
void ExtractorPostprocessor::process(const QVector<QVariant> &data)
{
m_data.reserve(data.size());
- for (const auto &d : data) {
+ for (auto d : data) {
if (d.userType() == qMetaTypeId<FlightReservation>()) {
- m_data.push_back(processFlightReservation(d));
- } else {
+ d = processFlightReservation(d);
+ }
+
+ if (filterReservation(d)) {
m_data.push_back(d);
}
}
@@ -123,3 +125,32 @@ void ExtractorPostprocessor::processFlightTime(QVariant &flight, const char *tim
JsonLdDocument::writeProperty(flight, timePropName, QDateTime());
JsonLdDocument::writeProperty(flight, timePropName, dt);
}
+
+bool ExtractorPostprocessor::filterReservation(const QVariant &res) const
+{
+ const auto resFor = JsonLdDocument::readProperty(res, "reservationFor");
+ if (resFor.isNull()) {
+ return false;
+ }
+
+ if (resFor.userType() == qMetaTypeId<Flight>()) {
+ return filterFlight(resFor);
+ }
+ return true;
+}
+
+bool ExtractorPostprocessor::filterFlight(const QVariant &flight) const
+{
+ const auto depDt = JsonLdDocument::readProperty(flight, "departureTime").toDateTime();
+ const auto arrDt = JsonLdDocument::readProperty(flight, "arrivalTime").toDateTime();
+ return filterAirport(JsonLdDocument::readProperty(flight, "departureAirport"))
+ && filterAirport(JsonLdDocument::readProperty(flight, "arrivalAirport"))
+ && depDt.isValid() && arrDt.isValid();
+}
+
+bool ExtractorPostprocessor::filterAirport(const QVariant &airport) const
+{
+ const auto iataCode = JsonLdDocument::readProperty(airport, "iataCode").toString();
+ const auto name = JsonLdDocument::readProperty(airport, "name").toString();
+ return !iataCode.isEmpty() || !name.isEmpty();
+}
diff --git a/plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.h b/plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.h
index 72e37e0..c8a026a 100644
--- a/plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.h
+++ b/plugins/messageviewer/bodypartformatter/semantic/extractorpostprocessor.h
@@ -38,6 +38,10 @@ private:
QVariant processAirport(QVariant airport) const;
void processFlightTime(QVariant &flight, const char *timePropName, const char *airportPropName) const;
+ bool filterReservation(const QVariant &res) const;
+ bool filterFlight(const QVariant &flight) const;
+ bool filterAirport(const QVariant &airport) const;
+
QVector<QVariant> m_data;
};
diff --git a/plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.cpp b/plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.cpp
index 94766d4..8bff1f1 100644
--- a/plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.cpp
+++ b/plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.cpp
@@ -114,6 +114,61 @@ QVector<QVariant> JsonLdDocument::fromJson(const QJsonArray &array)
return l;
}
+static QJsonValue toJson(const QVariant &v)
+{
+ const auto mo = QMetaType(v.userType()).metaObject();
+ if (!mo) {
+ // basic types
+ switch (v.type()) {
+ case QVariant::String:
+ return v.toString();
+ case QVariant::Double:
+ return v.toDouble();
+ case QVariant::Int:
+ return v.toInt();
+ case QVariant::DateTime:
+ return v.toDateTime().toString(Qt::ISODate);
+ default:
+ break;
+ }
+ if (v.userType() == qMetaTypeId<float>()) {
+ return v.toFloat();
+ }
+ qCDebug(SEMANTIC_LOG) << "unhandled value:" << v;
+ return {};
+ }
+
+ // composite types
+ QJsonObject obj;
+ obj.insert(QStringLiteral("@type"), QString::fromUtf8(mo->className()));
+ for (int i = 0; i < mo->propertyCount(); ++i) {
+ const auto prop = mo->property(i);
+ if (!prop.isStored()) {
+ continue;
+ }
+ const auto value = prop.readOnGadget(v.constData());
+ if (!value.isNull()) {
+ obj.insert(QString::fromUtf8(prop.name()), toJson(value));
+ }
+ }
+ return obj;
+}
+
+QJsonArray JsonLdDocument::toJson(const QVector<QVariant> &data)
+{
+ QJsonArray a;
+ for (const auto &d : data) {
+ const auto value = toJson(d);
+ if (!value.isObject()) {
+ continue;
+ }
+ auto obj = value.toObject();
+ obj.insert(QStringLiteral("@context"), QStringLiteral("http://schema.org"));
+ a.push_back(obj);
+ }
+ return a;
+}
+
QVariant JsonLdDocument::readProperty(const QVariant &obj, const char *name)
{
const auto mo = QMetaType(obj.userType()).metaObject();
diff --git a/plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.h b/plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.h
index e635b7d..6eb15b1 100644
--- a/plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.h
+++ b/plugins/messageviewer/bodypartformatter/semantic/jsonlddocument.h
@@ -28,6 +28,7 @@ class QJsonArray;
/** Serialization/deserialization code for JSON-LD data. */
namespace JsonLdDocument {
QVector<QVariant> fromJson(const QJsonArray &array);
+QJsonArray toJson(const QVector<QVariant> &data);
/** Read property @p name on object @p obj. */
QVariant readProperty(const QVariant &obj, const char *name);