Skip to content

Commit

Permalink
try to enhance new approach to datetime parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@martinrotter
martinrotter committed May 17, 2024
1 parent 526c4d6 commit f8e09fa
Show file tree
Hide file tree
Showing 20 changed files with 115 additions and 73 deletions.
8 changes: 4 additions & 4 deletions src/librssguard-gmail/src/gmailnetworkfactory.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ void GmailNetworkFactory::onAuthFailed() {
}});
}

bool GmailNetworkFactory::fillFullMessage(Message& msg, const QJsonObject& json, const QString& feed_id) const {
bool GmailNetworkFactory::fillFullMessage(Message& msg, const QJsonObject& json, const QString& feed_id) {
// Assign correct main labels/states.
auto labelids = json[QSL("labelIds")].toArray().toVariantList();

Expand Down Expand Up @@ -569,10 +569,10 @@ bool GmailNetworkFactory::fillFullMessage(Message& msg, const QJsonObject& json,
msg.m_url = QSL("https://mail.google.com/mail/u/0/#all/%1").arg(msg.m_customId);

msg.m_createdFromFeed = true;
msg.m_created = TextFactory::parseDateTime(headers[QSL("Date")]);
msg.m_created = TextFactory::parseDateTime(headers[QSL("Date")], &m_dateTimeFormat);

if (!msg.m_created.isValid()) {
msg.m_created = TextFactory::parseDateTime(headers[QSL("date")]);
msg.m_created = TextFactory::parseDateTime(headers[QSL("date")], &m_dateTimeFormat);
}

if (msg.m_title.isEmpty()) {
Expand Down Expand Up @@ -695,7 +695,7 @@ QMap<QString, QString> GmailNetworkFactory::getMessageMetadata(const QString& ms

QList<Message> GmailNetworkFactory::obtainAndDecodeFullMessages(const QStringList& message_ids,
const QString& feed_id,
const QNetworkProxy& custom_proxy) const {
const QNetworkProxy& custom_proxy) {
QHash<QString, Message> msgs;
int next_message = 0;
QString bearer = m_oauth2->bearer();
Expand Down
5 changes: 3 additions & 2 deletions src/librssguard-gmail/src/gmailnetworkfactory.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,10 @@ class GmailNetworkFactory : public QObject {
void onAuthFailed();

private:
bool fillFullMessage(Message& msg, const QJsonObject& json, const QString& feed_id) const;
bool fillFullMessage(Message& msg, const QJsonObject& json, const QString& feed_id);
QList<Message> obtainAndDecodeFullMessages(const QStringList& message_ids,
const QString& feed_id,
const QNetworkProxy& custom_proxy) const;
const QNetworkProxy& custom_proxy);
QStringList decodeLiteMessages(const QString& messages_json_data, QString& next_page_token) const;
QString sanitizeEmailAuthor(const QString& author) const;

Expand All @@ -86,6 +86,7 @@ class GmailNetworkFactory : public QObject {
private:
GmailServiceRoot* m_service;
QString m_username;
QString m_dateTimeFormat;
int m_batchSize;
bool m_downloadOnlyUnreadMessages;
OAuth2Service* m_oauth2;
Expand Down
4 changes: 2 additions & 2 deletions src/librssguard-standard/src/parsers/atomparser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,14 +334,14 @@ QString AtomParser::xmlMessageDescription(const QDomElement& msg_element) const
return summary;
}

QDateTime AtomParser::xmlMessageDateCreated(const QDomElement& msg_element) const {
QDateTime AtomParser::xmlMessageDateCreated(const QDomElement& msg_element) {
QString updated = xmlTextsFromPath(msg_element, m_atomNamespace, QSL("updated"), true).join(QSL(", "));

if (updated.simplified().isEmpty()) {
updated = xmlTextsFromPath(msg_element, m_atomNamespace, QSL("modified"), true).join(QSL(", "));
}

return TextFactory::parseDateTime(updated);
return TextFactory::parseDateTime(updated, &m_dateTimeFormat);
}

QString AtomParser::xmlMessageId(const QDomElement& msg_element) const {
Expand Down
2 changes: 1 addition & 1 deletion src/librssguard-standard/src/parsers/atomparser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class AtomParser : public FeedParser {

virtual QString xmlMessageTitle(const QDomElement& msg_element) const;
virtual QString xmlMessageDescription(const QDomElement& msg_element) const;
virtual QDateTime xmlMessageDateCreated(const QDomElement& msg_element) const;
virtual QDateTime xmlMessageDateCreated(const QDomElement& msg_element);
virtual QString xmlMessageId(const QDomElement& msg_element) const;
virtual QString xmlMessageUrl(const QDomElement& msg_element) const;
virtual QList<Enclosure> xmlMessageEnclosures(const QDomElement& msg_element) const;
Expand Down
18 changes: 14 additions & 4 deletions src/librssguard-standard/src/parsers/feedparser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
#include <QFile>
#include <QRegularExpression>

FeedParser::FeedParser() {}

FeedParser::FeedParser(QString data, DataType is_xml)
: m_dataType(is_xml), m_data(std::move(data)), m_mrssNamespace(QSL("http://search.yahoo.com/mrss/")) {
if (m_data.isEmpty()) {
Expand Down Expand Up @@ -104,7 +106,7 @@ QString FeedParser::jsonMessageAuthor(const QJsonObject& msg_element) const {
return {};
}

QDateTime FeedParser::jsonMessageDateCreated(const QJsonObject& msg_element) const {
QDateTime FeedParser::jsonMessageDateCreated(const QJsonObject& msg_element) {
return {};
}

Expand Down Expand Up @@ -136,15 +138,15 @@ QString FeedParser::objMessageUrl(const QVariant& msg_element) const {
return {};
}

QString FeedParser::objMessageDescription(const QVariant& msg_element) const {
QString FeedParser::objMessageDescription(const QVariant& msg_element) {
return {};
}

QString FeedParser::objMessageAuthor(const QVariant& msg_element) const {
return {};
}

QDateTime FeedParser::objMessageDateCreated(const QVariant& msg_element) const {
QDateTime FeedParser::objMessageDateCreated(const QVariant& msg_element) {
return {};
}

Expand Down Expand Up @@ -398,6 +400,14 @@ QStringList FeedParser::xmlTextsFromPath(const QDomElement& element,
return result;
}

QString FeedParser::dateTimeFormat() const {
return m_dateTimeFormat;
}

void FeedParser::setDateTimeFormat(const QString& dt_format) {
m_dateTimeFormat = dt_format;
}

QString FeedParser::feedAuthor() const {
return QL1S("");
}
Expand All @@ -422,7 +432,7 @@ QString FeedParser::xmlMessageAuthor(const QDomElement& msg_element) const {
return {};
}

QDateTime FeedParser::xmlMessageDateCreated(const QDomElement& msg_element) const {
QDateTime FeedParser::xmlMessageDateCreated(const QDomElement& msg_element) {
return {};
}

Expand Down
13 changes: 9 additions & 4 deletions src/librssguard-standard/src/parsers/feedparser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class FeedParser {
Other
};

FeedParser();
explicit FeedParser(QString data, DataType is_xml = DataType::Xml);
virtual ~FeedParser();

Expand All @@ -36,6 +37,9 @@ class FeedParser {
// Returns list of all messages from the feed.
virtual QList<Message> messages();

QString dateTimeFormat() const;
void setDateTimeFormat(const QString& dt_format);

protected:
virtual QString feedAuthor() const;

Expand All @@ -45,7 +49,7 @@ class FeedParser {
virtual QString xmlMessageUrl(const QDomElement& msg_element) const;
virtual QString xmlMessageDescription(const QDomElement& msg_element) const;
virtual QString xmlMessageAuthor(const QDomElement& msg_element) const;
virtual QDateTime xmlMessageDateCreated(const QDomElement& msg_element) const;
virtual QDateTime xmlMessageDateCreated(const QDomElement& msg_element);
virtual QString xmlMessageId(const QDomElement& msg_element) const;
virtual QList<Enclosure> xmlMessageEnclosures(const QDomElement& msg_element) const;
virtual QList<MessageCategory> xmlMessageCategories(const QDomElement& msg_element) const;
Expand All @@ -57,7 +61,7 @@ class FeedParser {
virtual QString jsonMessageUrl(const QJsonObject& msg_element) const;
virtual QString jsonMessageDescription(const QJsonObject& msg_element) const;
virtual QString jsonMessageAuthor(const QJsonObject& msg_element) const;
virtual QDateTime jsonMessageDateCreated(const QJsonObject& msg_element) const;
virtual QDateTime jsonMessageDateCreated(const QJsonObject& msg_element) ;
virtual QString jsonMessageId(const QJsonObject& msg_element) const;
virtual QList<Enclosure> jsonMessageEnclosures(const QJsonObject& msg_element) const;
virtual QList<MessageCategory> jsonMessageCategories(const QJsonObject& msg_element) const;
Expand All @@ -67,9 +71,9 @@ class FeedParser {
virtual QVariantList objMessageElements();
virtual QString objMessageTitle(const QVariant& msg_element) const;
virtual QString objMessageUrl(const QVariant& msg_element) const;
virtual QString objMessageDescription(const QVariant& msg_element) const;
virtual QString objMessageDescription(const QVariant& msg_element) ;
virtual QString objMessageAuthor(const QVariant& msg_element) const;
virtual QDateTime objMessageDateCreated(const QVariant& msg_element) const;
virtual QDateTime objMessageDateCreated(const QVariant& msg_element) ;
virtual QString objMessageId(const QVariant& msg_element) const;
virtual QList<Enclosure> objMessageEnclosures(const QVariant& msg_element) const;
virtual QList<MessageCategory> objMessageCategories(const QVariant& msg_element) const;
Expand All @@ -87,6 +91,7 @@ class FeedParser {
protected:
DataType m_dataType;
QString m_data;
QString m_dateTimeFormat;
QDomDocument m_xml;
QJsonDocument m_json;
QString m_mrssNamespace;
Expand Down
38 changes: 11 additions & 27 deletions src/librssguard-standard/src/parsers/icalparser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,17 +106,17 @@ QString IcalParser::objMessageUrl(const QVariant& msg_element) const {
return comp.url();
}

QString IcalParser::objMessageDescription(const QVariant& msg_element) const {
QString IcalParser::objMessageDescription(const QVariant& msg_element) {
const IcalendarComponent& comp_base = msg_element.value<IcalendarComponent>();
const EventComponent& comp = static_cast<const EventComponent&>(comp_base);

bool has_dt;
auto son = comp.startsOn(m_iCalendar.m_tzs, &has_dt).toLocalTime();
auto son = comp.startsOn(m_iCalendar.m_tzs, &has_dt, &m_dateTimeFormat).toLocalTime();

QString formaton = has_dt ? QLocale().dateTimeFormat(QLocale::FormatType::LongFormat)
: QLocale().dateFormat(QLocale::FormatType::LongFormat);

auto soff = comp.endsOn(m_iCalendar.m_tzs, &has_dt).toLocalTime();
auto soff = comp.endsOn(m_iCalendar.m_tzs, &has_dt, &m_dateTimeFormat).toLocalTime();
QString formatoff = has_dt ? QLocale().dateTimeFormat(QLocale::FormatType::LongFormat)
: QLocale().dateFormat(QLocale::FormatType::LongFormat);

Expand All @@ -142,11 +142,11 @@ QString IcalParser::objMessageAuthor(const QVariant& msg_element) const {
return comp.organizer();
}

QDateTime IcalParser::objMessageDateCreated(const QVariant& msg_element) const {
QDateTime IcalParser::objMessageDateCreated(const QVariant& msg_element) {
const IcalendarComponent& comp_base = msg_element.value<IcalendarComponent>();
const EventComponent& comp = static_cast<const EventComponent&>(comp_base);

QDateTime dat = comp.startsOn(m_iCalendar.m_tzs);
QDateTime dat = comp.startsOn(m_iCalendar.m_tzs, nullptr, &m_dateTimeFormat);

return dat;
}
Expand Down Expand Up @@ -347,13 +347,12 @@ QDateTime IcalendarComponent::fixupDate(QDateTime dat,
}
}

QDateTime EventComponent::startsOn(const QMap<QString, QTimeZone>& time_zones, bool* had_dt) const {
QDateTime EventComponent::startsOn(const QMap<QString, QTimeZone>& time_zones, bool* had_dt, QString* dt_format) const {
QString modifiers;
QString dt_format;
bool has_dt;
QDateTime dat = TextFactory::parseDateTime(getPropertyValue(QSL("DTSTART"), modifiers).toString(), &dt_format);
QDateTime dat = TextFactory::parseDateTime(getPropertyValue(QSL("DTSTART"), modifiers).toString(), dt_format);

dat = fixupDate(dat, dt_format, time_zones, modifiers, &has_dt);
dat = fixupDate(dat, *dt_format, time_zones, modifiers, &has_dt);

if (had_dt != nullptr) {
*had_dt = has_dt;
Expand All @@ -362,13 +361,12 @@ QDateTime EventComponent::startsOn(const QMap<QString, QTimeZone>& time_zones, b
return dat;
}

QDateTime EventComponent::endsOn(const QMap<QString, QTimeZone>& time_zones, bool* had_dt) const {
QDateTime EventComponent::endsOn(const QMap<QString, QTimeZone>& time_zones, bool* had_dt, QString* dt_format) const {
QString modifiers;
QString dt_format;
bool has_dt;
QDateTime dat = TextFactory::parseDateTime(getPropertyValue(QSL("DTEND"), modifiers).toString(), &dt_format);
QDateTime dat = TextFactory::parseDateTime(getPropertyValue(QSL("DTEND"), modifiers).toString(), dt_format);

dat = fixupDate(dat, dt_format, time_zones, modifiers, &has_dt);
dat = fixupDate(dat, *dt_format, time_zones, modifiers, &has_dt);

if (had_dt != nullptr) {
*had_dt = has_dt;
Expand Down Expand Up @@ -396,17 +394,3 @@ QString EventComponent::location() const {
QString EventComponent::description() const {
return getPropertyValue(QSL("DESCRIPTION")).toString();
}

QDateTime EventComponent::created(const QMap<QString, QTimeZone>& time_zones) const {
QString modifiers;
QDateTime dat = TextFactory::parseDateTime(getPropertyValue(QSL("CREATED"), modifiers).toString());

return fixupDate(dat, {}, time_zones, modifiers);
}

QDateTime EventComponent::lastModified(const QMap<QString, QTimeZone>& time_zones) const {
QString modifiers;
QDateTime dat = TextFactory::parseDateTime(getPropertyValue(QSL("LAST-MODIFIED"), modifiers).toString());

return fixupDate(dat, {}, time_zones, modifiers);
}
14 changes: 8 additions & 6 deletions src/librssguard-standard/src/parsers/icalparser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,17 @@ Q_DECLARE_METATYPE(IcalendarComponent)

class EventComponent : public IcalendarComponent {
public:
QDateTime startsOn(const QMap<QString, QTimeZone>& time_zones = {}, bool* had_dt = nullptr) const;
QDateTime endsOn(const QMap<QString, QTimeZone>& time_zones = {}, bool* had_dt = nullptr) const;
QDateTime startsOn(const QMap<QString, QTimeZone>& time_zones = {},
bool* had_dt = nullptr,
QString* dt_format = nullptr) const;
QDateTime endsOn(const QMap<QString, QTimeZone>& time_zones = {},
bool* had_dt = nullptr,
QString* dt_format = nullptr) const;
QString title() const;
QString url() const;
QString organizer() const;
QString location() const;
QString description() const;
QDateTime created(const QMap<QString, QTimeZone>& time_zones = {}) const;
QDateTime lastModified(const QMap<QString, QTimeZone>& time_zones = {}) const;
};

Q_DECLARE_METATYPE(EventComponent)
Expand Down Expand Up @@ -81,9 +83,9 @@ class IcalParser : public FeedParser {
virtual QVariantList objMessageElements();
virtual QString objMessageTitle(const QVariant& msg_element) const;
virtual QString objMessageUrl(const QVariant& msg_element) const;
virtual QString objMessageDescription(const QVariant& msg_element) const;
virtual QString objMessageDescription(const QVariant& msg_element);
virtual QString objMessageAuthor(const QVariant& msg_element) const;
virtual QDateTime objMessageDateCreated(const QVariant& msg_element) const;
virtual QDateTime objMessageDateCreated(const QVariant& msg_element);
virtual QString objMessageId(const QVariant& msg_element) const;
virtual QList<Enclosure> objMessageEnclosures(const QVariant& msg_element) const;
virtual QList<MessageCategory> objMessageCategories(const QVariant& msg_element) const;
Expand Down
5 changes: 3 additions & 2 deletions src/librssguard-standard/src/parsers/jsonparser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,10 +199,11 @@ QString JsonParser::jsonMessageAuthor(const QJsonObject& msg_element) const {
}
}

QDateTime JsonParser::jsonMessageDateCreated(const QJsonObject& msg_element) const {
QDateTime JsonParser::jsonMessageDateCreated(const QJsonObject& msg_element) {
return TextFactory::parseDateTime(msg_element.contains(QSL("date_modified"))
? msg_element[QSL("date_modified")].toString()
: msg_element[QSL("date_published")].toString());
: msg_element[QSL("date_published")].toString(),
&m_dateTimeFormat);
}

QString JsonParser::jsonMessageId(const QJsonObject& msg_element) const {
Expand Down
2 changes: 1 addition & 1 deletion src/librssguard-standard/src/parsers/jsonparser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class JsonParser : public FeedParser {
virtual QString jsonMessageUrl(const QJsonObject& msg_element) const;
virtual QString jsonMessageDescription(const QJsonObject& msg_element) const;
virtual QString jsonMessageAuthor(const QJsonObject& msg_element) const;
virtual QDateTime jsonMessageDateCreated(const QJsonObject& msg_element) const;
virtual QDateTime jsonMessageDateCreated(const QJsonObject& msg_element) ;
virtual QString jsonMessageId(const QJsonObject& msg_element) const;
virtual QList<Enclosure> jsonMessageEnclosures(const QJsonObject& msg_element) const;
virtual QString jsonMessageRawContents(const QJsonObject& msg_element) const;
Expand Down
5 changes: 3 additions & 2 deletions src/librssguard-standard/src/parsers/rdfparser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,11 +257,12 @@ QString RdfParser::xmlMessageAuthor(const QDomElement& msg_element) const {
return msg_element.elementsByTagNameNS(m_dcElNamespace, QSL("creator")).at(0).toElement().text();
}

QDateTime RdfParser::xmlMessageDateCreated(const QDomElement& msg_element) const {
QDateTime RdfParser::xmlMessageDateCreated(const QDomElement& msg_element) {
return TextFactory::parseDateTime(msg_element.elementsByTagNameNS(m_dcElNamespace, QSL("date"))
.at(0)
.toElement()
.text());
.text(),
&m_dateTimeFormat);
}

QString RdfParser::xmlMessageId(const QDomElement& msg_element) const {
Expand Down
2 changes: 1 addition & 1 deletion src/librssguard-standard/src/parsers/rdfparser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class RdfParser : public FeedParser {
virtual QString xmlMessageTitle(const QDomElement& msg_element) const;
virtual QString xmlMessageDescription(const QDomElement& msg_element) const;
virtual QString xmlMessageAuthor(const QDomElement& msg_element) const;
virtual QDateTime xmlMessageDateCreated(const QDomElement& msg_element) const;
virtual QDateTime xmlMessageDateCreated(const QDomElement& msg_element);
virtual QString xmlMessageId(const QDomElement& msg_element) const;
virtual QString xmlMessageUrl(const QDomElement& msg_element) const;
virtual QList<Enclosure> xmlMessageEnclosures(const QDomElement& msg_element) const;
Expand Down
10 changes: 6 additions & 4 deletions src/librssguard-standard/src/parsers/rssparser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,15 +280,17 @@ QString RssParser::xmlMessageAuthor(const QDomElement& msg_element) const {
return author;
}

QDateTime RssParser::xmlMessageDateCreated(const QDomElement& msg_element) const {
QDateTime date_created = TextFactory::parseDateTime(msg_element.namedItem(QSL("pubDate")).toElement().text());
QDateTime RssParser::xmlMessageDateCreated(const QDomElement& msg_element) {
QDateTime date_created =
TextFactory::parseDateTime(msg_element.namedItem(QSL("pubDate")).toElement().text(), &m_dateTimeFormat);

if (date_created.isNull()) {
date_created = TextFactory::parseDateTime(msg_element.namedItem(QSL("date")).toElement().text());
date_created = TextFactory::parseDateTime(msg_element.namedItem(QSL("date")).toElement().text(), &m_dateTimeFormat);
}

if (date_created.isNull()) {
date_created = TextFactory::parseDateTime(msg_element.namedItem(QSL("dc:modified")).toElement().text());
date_created =
TextFactory::parseDateTime(msg_element.namedItem(QSL("dc:modified")).toElement().text(), &m_dateTimeFormat);
}

return date_created;
Expand Down
2 changes: 1 addition & 1 deletion src/librssguard-standard/src/parsers/rssparser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ class RssParser : public FeedParser {
virtual QString xmlMessageTitle(const QDomElement& msg_element) const;
virtual QString xmlMessageDescription(const QDomElement& msg_element) const;
virtual QString xmlMessageAuthor(const QDomElement& msg_element) const;
virtual QDateTime xmlMessageDateCreated(const QDomElement& msg_element) const;
virtual QDateTime xmlMessageDateCreated(const QDomElement& msg_element);
virtual QString xmlMessageId(const QDomElement& msg_element) const;
virtual QString xmlMessageUrl(const QDomElement& msg_element) const;
virtual QList<Enclosure> xmlMessageEnclosures(const QDomElement& msg_element) const;
Expand Down

0 comments on commit f8e09fa

Please sign in to comment.