void FunctionsTest::testDateTimeFromString()
{
// Timestamps
QCOMPARE(qDateTimeFromString("1492192180").toUTC(), QDateTime(QDate(2017, 4, 14), QTime(17, 49, 40), Qt::UTC));
// Standart dates
QCOMPARE(qDateTimeFromString("2017/04/14 17:49:40").toUTC(), QDateTime(QDate(2017, 4, 14), QTime(17, 49, 40), Qt::UTC));
QCOMPARE(qDateTimeFromString("2017-04-14 17:49:40").toUTC(), QDateTime(QDate(2017, 4, 14), QTime(17, 49, 40), Qt::UTC));
QCOMPARE(qDateTimeFromString("2017/04/14 17:49").toUTC(), QDateTime(QDate(2017, 4, 14), QTime(17, 49), Qt::UTC));
QCOMPARE(qDateTimeFromString("2017-04-14 17:49").toUTC(), QDateTime(QDate(2017, 4, 14), QTime(17, 49), Qt::UTC));
// Danbooru dates
QCOMPARE(qDateTimeFromString("2017-04-14T17:49:40.498-04:00").toUTC(), QDateTime(QDate(2017, 4, 14), QTime(17 + 4, 49, 40), Qt::UTC));
// Gelbooru dates
QCOMPARE(qDateTimeFromString("Tue Apr 4 17:49:40 2017").toUTC(), QDateTime(QDate(2017, 4, 4), QTime(17, 49, 40), Qt::UTC));
QCOMPARE(qDateTimeFromString("Fri Apr 14 17:49:40 2017").toUTC(), QDateTime(QDate(2017, 4, 14), QTime(17, 49, 40), Qt::UTC));
QCOMPARE(qDateTimeFromString("Fri Apr 14 17:49:40 -0500 2017").toUTC(), QDateTime(QDate(2017, 4, 14), QTime(17 + 5, 49, 40), Qt::UTC));
QCOMPARE(qDateTimeFromString("Fri Apr 14 23:49:40 -0500 2017").toUTC(), QDateTime(QDate(2017, 4, 15), QTime(4, 49, 40), Qt::UTC));
}
ParsedDetails HtmlApi::parseDetails(const QString &source, Site *site) const
{
Q_UNUSED(site);
ParsedDetails ret;
// Pools
if (contains("Regex/Pools"))
{
QRegularExpression rx(value("Regex/Pools"));
auto matches = rx.globalMatch(source);
while (matches.hasNext())
{
auto match = matches.next();
QString previous = match.captured(1), id = match.captured(2), name = match.captured(3), next = match.captured(4);
ret.pools.append(Pool(id.toInt(), name, 0, next.toInt(), previous.toInt()));
}
}
// Tags
QString rxtags;
if (contains("Regex/ImageTags"))
{ rxtags = value("Regex/ImageTags"); }
else if (contains("Regex/Tags"))
{ rxtags = value("Regex/Tags"); }
if (!rxtags.isEmpty())
{ ret.tags = Tag::FromRegexp(rxtags, source); }
// Image url
if (contains("Regex/ImageUrl"))
{
QRegularExpression rx(value("Regex/ImageUrl"));
auto matches = rx.globalMatch(source);
while (matches.hasNext())
{
auto match = matches.next();
ret.imageUrl = match.captured(1).replace("&", "&");
}
}
// Image date
if (contains("Regex/ImageDate"))
{
QRegularExpression rx(value("Regex/ImageDate"));
auto matches = rx.globalMatch(source);
while (matches.hasNext())
{
auto match = matches.next();
ret.createdAt = qDateTimeFromString(match.captured(1));
}
}
return ret;
}
QVariant embeddedResult::data(int field)
{
if (!isSelect() || field >= d->fields.count()) {
printf("embeddedResult::data: column %d out of range", field);
return QVariant();
}
if (!d->driver)
return QVariant();
int fieldLength = 0;
const embeddedResultPrivate::QMyField &f = d->fields.at(field);
QString val;
if (d->preparedQuery) {
if (f.nullIndicator)
return QVariant(f.type);
if (f.type != QVariant::ByteArray)
val = toUnicode(d->driver->d->tc, f.outField, f.bufLength);
} else {
if (d->row[field] == NULL) {
// NULL value
return QVariant(f.type);
}
fieldLength = mysql_fetch_lengths(d->result)[field];
if (f.type != QVariant::ByteArray)
val = toUnicode(d->driver->d->tc, d->row[field], fieldLength);
}
switch(f.type) {
case QVariant::LongLong:
return QVariant(val.toLongLong());
case QVariant::ULongLong:
return QVariant(val.toULongLong());
case QVariant::Int:
return QVariant(val.toInt());
case QVariant::UInt:
return QVariant(val.toUInt());
case QVariant::Double: {
QVariant v;
bool ok=false;
double dbl = val.toDouble(&ok);
switch(numericalPrecisionPolicy()) {
case QSql::LowPrecisionInt32:
v=QVariant(dbl).toInt();
break;
case QSql::LowPrecisionInt64:
v = QVariant(dbl).toLongLong();
break;
case QSql::LowPrecisionDouble:
v = QVariant(dbl);
break;
case QSql::HighPrecision:
default:
v = val;
ok = true;
break;
}
if(ok)
return v;
else
return QVariant();
}
return QVariant(val.toDouble());
case QVariant::Date:
return qDateFromString(val);
case QVariant::Time:
return qTimeFromString(val);
case QVariant::DateTime:
return qDateTimeFromString(val);
case QVariant::ByteArray: {
QByteArray ba;
if (d->preparedQuery) {
ba = QByteArray(f.outField, f.bufLength);
} else {
ba = QByteArray(d->row[field], fieldLength);
}
return QVariant(ba);
}
default:
case QVariant::String:
return QVariant(val);
}
printf("embeddedResult::data: unknown data type");
return QVariant();
}
Image::Image(Site *site, QMap<QString, QString> details, Profile *profile, Page* parent)
: m_profile(profile), m_id(0), m_parentSite(site), m_extensionRotator(nullptr)
{
m_settings = m_profile->getSettings();
// Parents
if (m_parentSite == nullptr)
{
log(QStringLiteral("Image has nullptr parent, aborting creation."));
return;
}
// Other details
m_isGallery = details.contains("type") && details["type"] == "gallery";
m_md5 = details.contains("md5") ? details["md5"] : "";
m_author = details.contains("author") ? details["author"] : "";
m_name = details.contains("name") ? details["name"] : "";
m_status = details.contains("status") ? details["status"] : "";
m_search = parent != nullptr ? parent->search() : (details.contains("search") ? details["search"].split(' ') : QStringList());
m_id = details.contains("id") ? details["id"].toULongLong() : 0;
m_score = details.contains("score") ? details["score"].toInt() : 0;
m_hasScore = details.contains("score");
m_parentId = details.contains("parent_id") ? details["parent_id"].toInt() : 0;
m_fileSize = details.contains("file_size") ? details["file_size"].toInt() : 0;
m_authorId = details.contains("creator_id") ? details["creator_id"].toInt() : 0;
m_hasChildren = details.contains("has_children") && details["has_children"] == "true";
m_hasNote = details.contains("has_note") && details["has_note"] == "true";
m_hasComments = details.contains("has_comments") && details["has_comments"] == "true";
m_fileUrl = details.contains("file_url") ? m_parentSite->fixUrl(details["file_url"]) : QUrl();
m_sampleUrl = details.contains("sample_url") ? m_parentSite->fixUrl(details["sample_url"]) : QUrl();
m_previewUrl = details.contains("preview_url") ? m_parentSite->fixUrl(details["preview_url"]) : QUrl();
m_size = QSize(details.contains("width") ? details["width"].toInt() : 0, details.contains("height") ? details["height"].toInt() : 0);
m_source = details.contains("source") ? details["source"] : "";
// Page url
if (details.contains("page_url"))
{ m_pageUrl = details["page_url"]; }
else
{
Api *api = m_parentSite->detailsApi();
if (api != Q_NULLPTR)
{ m_pageUrl = api->detailsUrl(m_id, m_md5, m_parentSite).url; }
}
m_pageUrl = site->fixUrl(m_pageUrl).toString();
// Rating
setRating(details.contains("rating") ? details["rating"] : "");
// Tags
QStringList types = QStringList() << "general" << "artist" << "character" << "copyright" << "model" << "species" << "meta";
for (const QString &typ : types)
{
QString key = "tags_" + typ;
if (!details.contains(key))
continue;
TagType ttype(typ);
QStringList t = details[key].split(' ', QString::SkipEmptyParts);
for (QString tg : t)
{
tg.replace("&", "&");
m_tags.append(Tag(tg, ttype));
}
}
if (m_tags.isEmpty() && details.contains("tags"))
{
QString tgs = QString(details["tags"]).replace(QRegularExpression("[\r\n\t]+"), " ");
// Automatically find tag separator and split the list
int commas = tgs.count(", ");
int spaces = tgs.count(" ");
const QStringList &t = commas >= 10 || (commas > 0 && (spaces - commas) / commas < 2)
? tgs.split(", ", QString::SkipEmptyParts)
: tgs.split(" ", QString::SkipEmptyParts);
for (QString tg : t)
{
tg.replace("&", "&");
int colon = tg.indexOf(':');
if (colon != -1)
{
QString tp = tg.left(colon).toLower();
if (tp == "user")
{ m_author = tg.mid(colon + 1); }
else if (tp == "score")
{ m_score = tg.midRef(colon + 1).toInt(); }
else if (tp == "size")
{
QStringList size = tg.mid(colon + 1).split('x');
if (size.size() == 2)
m_size = QSize(size[0].toInt(), size[1].toInt());
}
else if (tp == "rating")
{ setRating(tg.mid(colon + 1)); }
else
{ m_tags.append(Tag(tg)); }
}
else
{ m_tags.append(Tag(tg)); }
}
}
// Complete missing tag type information
m_parentSite->tagDatabase()->load();
QStringList unknownTags;
for (Tag const &tag : qAsConst(m_tags))
if (tag.type().name() == "unknown")
unknownTags.append(tag.text());
QMap<QString, TagType> dbTypes = m_parentSite->tagDatabase()->getTagTypes(unknownTags);
for (Tag &tag : m_tags)
if (dbTypes.contains(tag.text()))
tag.setType(dbTypes[tag.text()]);
// Get file url and try to improve it to save bandwidth
m_url = m_fileUrl.toString();
QString ext = getExtension(m_url);
if (details.contains("ext") && !details["ext"].isEmpty())
{
QString realExt = details["ext"];
if (ext != realExt)
{ setFileExtension(realExt); }
}
else if (ext == QLatin1String("jpg") && !m_previewUrl.isEmpty())
{
bool fixed = false;
QString previewExt = getExtension(details["preview_url"]);
if (!m_sampleUrl.isEmpty())
{
// Guess extension from sample url
QString sampleExt = getExtension(details["sample_url"]);
if (sampleExt != QLatin1String("jpg") && sampleExt != QLatin1String("png") && sampleExt != ext && previewExt == ext)
{
m_url = setExtension(m_url, sampleExt);
fixed = true;
}
}
// Guess the extension from the tags
if (!fixed)
{
if ((hasTag(QStringLiteral("swf")) || hasTag(QStringLiteral("flash"))) && ext != QLatin1String("swf"))
{ setFileExtension(QStringLiteral("swf")); }
else if ((hasTag(QStringLiteral("gif")) || hasTag(QStringLiteral("animated_gif"))) && ext != QLatin1String("webm") && ext != QLatin1String("mp4"))
{ setFileExtension(QStringLiteral("gif")); }
else if (hasTag(QStringLiteral("mp4")) && ext != QLatin1String("gif") && ext != QLatin1String("webm"))
{ setFileExtension(QStringLiteral("mp4")); }
else if (hasTag(QStringLiteral("animated_png")) && ext != QLatin1String("webm") && ext != QLatin1String("mp4"))
{ setFileExtension(QStringLiteral("png")); }
else if ((hasTag(QStringLiteral("webm")) || hasTag(QStringLiteral("animated"))) && ext != QLatin1String("gif") && ext != QLatin1String("mp4"))
{ setFileExtension(QStringLiteral("webm")); }
}
}
else if (details.contains("image") && details["image"].contains("MB // gif\" height=\"") && !m_url.endsWith(".gif", Qt::CaseInsensitive))
{ m_url = setExtension(m_url, QStringLiteral("gif")); }
// Remove ? in urls
m_url = removeCacheUrl(m_url);
m_fileUrl = removeCacheUrl(m_fileUrl.toString());
m_sampleUrl = removeCacheUrl(m_sampleUrl.toString());
m_previewUrl = removeCacheUrl(m_previewUrl.toString());
// We use the sample URL as the URL for zip files (ugoira) or if the setting is set
bool downloadOriginals = m_settings->value("Save/downloadoriginals", true).toBool();
if (!m_sampleUrl.isEmpty() && (getExtension(m_url) == "zip" || !downloadOriginals))
m_url = m_sampleUrl.toString();
// Creation date
m_createdAt = QDateTime();
if (details.contains("created_at"))
{ m_createdAt = qDateTimeFromString(details["created_at"]); }
else if (details.contains("date"))
{ m_createdAt = QDateTime::fromString(details["date"], Qt::ISODate); }
// Setup extension rotator
bool animated = hasTag("gif") || hasTag("animated_gif") || hasTag("mp4") || hasTag("animated_png") || hasTag("webm") || hasTag("animated");
QStringList extensions = animated
? QStringList() << "webm" << "mp4" << "gif" << "jpg" << "png" << "jpeg" << "swf"
: QStringList() << "jpg" << "png" << "gif" << "jpeg" << "webm" << "swf" << "mp4";
m_extensionRotator = new ExtensionRotator(getExtension(m_url), extensions, this);
// Tech details
m_parent = parent;
m_loadDetails = nullptr;
m_loadImage = nullptr;
m_loadingPreview = false;
m_loadingDetails = false;
m_loadedDetails = false;
m_loadedImage = false;
m_loadingImage = false;
m_tryingSample = false;
m_pools = QList<Pool>();
}
