void tst_QNetworkCacheMetaData::isValid_data()
{
QTest::addColumn<QNetworkCacheMetaData>("data");
QTest::addColumn<bool>("isValid");
QNetworkCacheMetaData metaData;
QTest::newRow("null") << metaData << false;
QNetworkCacheMetaData data1;
data1.setUrl(QUrl(EXAMPLE_URL));
QTest::newRow("valid-1") << data1 << true;
QNetworkCacheMetaData data2;
QNetworkCacheMetaData::RawHeaderList headers;
headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
data2.setRawHeaders(headers);
QTest::newRow("valid-2") << data2 << true;
QNetworkCacheMetaData data3;
data3.setLastModified(QDateTime::currentDateTime());
QTest::newRow("valid-3") << data3 << true;
QNetworkCacheMetaData data4;
data4.setExpirationDate(QDateTime::currentDateTime());
QTest::newRow("valid-4") << data4 << true;
QNetworkCacheMetaData data5;
data5.setSaveToDisk(false);
QTest::newRow("valid-5") << data5 << true;
}
// This function simulates a partially or fully occupied disk cache
// like a normal user of a cache might encounter is real-life browsing.
// The point of this is to trigger degradation in file-system and media performance
// that occur due to the quantity and layout of data.
void tst_qnetworkdiskcache::injectFakeData()
{
QNetworkCacheMetaData::RawHeaderList headers;
headers.append(qMakePair(QByteArray("X-TestHeader"),QByteArray("HeaderValue")));
//Prep cache dir with fake data using QNetworkDiskCache APIs
for (quint32 i = 0; i < NumFakeCacheObjects; i++) {
//prepare metata for url
QNetworkCacheMetaData meta;
QString fakeURL;
QTextStream stream(&fakeURL);
stream << fakeURLbase << i;
QUrl url(fakeURL);
meta.setUrl(url);
meta.setRawHeaders(headers);
meta.setSaveToDisk(true);
//commit payload and metadata to disk
QIODevice *device = cache->prepare(meta);
device->write(payload);
cache->insert(device);
}
}
QIODevice *TabCache::prepare(const QNetworkCacheMetaData &metaData) {
QLOG_DEBUG() << "TabCache::prepare";
QNetworkCacheMetaData local{metaData};
//Default policy based on received HTTP headers is to not save to disk.
//Override here, and set proper expiration so items are put in cache
//(setSaveToDisk) and valid when retrieved using metaData call -
//(setExpirationDate)
local.setSaveToDisk(true);
// Need to set some reasonable length of time in which our cache entries
// will expire. It's possible we'll want to allow users to customize this.
local.setExpirationDate(QDateTime().currentDateTime().addDays(kCacheExpireInDays));
QNetworkCacheMetaData::RawHeaderList headers;
for (auto &header: local.rawHeaders()) {
// Modify Cache-Control headers - basically need to drop 'no-store'
// as we want to store to cache and 'must-revalidate' as we don't have
// ETag or last modified headers available to attempt re-validation. To
// be on the safe side though just drop Cache-Control and Pragma headers.
if (header.first == "Cache-Control") continue;
if (header.first == "Pragma") continue;
headers.push_back(header);
}
local.setRawHeaders(headers);
return QNetworkDiskCache::prepare(local);
}
/** \fn MythDownloadManager::GetLastModified(const QString &url)
* \brief Gets the Last Modified timestamp for a URI
* \param url URI to test.
* \return Timestamp the URI was last modified or now if an error occurred
*/
QDateTime MythDownloadManager::GetLastModified(const QString &url)
{
static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
VERBOSE(VB_FILE+VB_EXTRA, LOC + QString("GetLastModified('%1')").arg(url));
QDateTime result;
QDateTime now = QDateTime::currentDateTime();
QNetworkCacheMetaData urlData = m_manager->cache()->metaData(QUrl(url));
if (urlData.isValid())
{
if (urlData.lastModified().secsTo(now) <= 60)
{
result = urlData.lastModified();
}
else
{
// If the last modification date is older than 60 seconds, and
// we loaded the page over 60 seconds ago, then redownload the
// page to re-verify it's last modified date.
QNetworkCacheMetaData::RawHeaderList headers =
urlData.rawHeaders();
bool found = false;
QNetworkCacheMetaData::RawHeaderList::iterator it
= headers.begin();
for (; !found && it != headers.end(); ++it)
{
if ((*it).first == "Date")
{
found = true;
QDateTime loadDate =
QDateTime::fromString((*it).second, dateFormat);
loadDate.setTimeSpec(Qt::UTC);
if (loadDate.secsTo(now) <= 60)
result = urlData.lastModified();
}
}
}
}
if (!result.isValid())
{
MythDownloadInfo *dlInfo = new MythDownloadInfo;
dlInfo->m_url = url;
dlInfo->m_syncMode = true;
if (downloadNow(dlInfo, false) && dlInfo->m_reply)
{
QVariant lastMod =
dlInfo->m_reply->header(QNetworkRequest::LastModifiedHeader);
if (lastMod.isValid())
result = lastMod.toDateTime();
}
delete dlInfo;
}
return result;
}
void tst_QNetworkCacheMetaData::rawHeaders_data()
{
QTest::addColumn<QNetworkCacheMetaData::RawHeaderList>("rawHeaders");
QTest::newRow("null") << QNetworkCacheMetaData::RawHeaderList();
QNetworkCacheMetaData::RawHeaderList headers;
headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
QTest::newRow("valie") << headers;
}
bool QNetworkAccessCacheBackend::sendCacheContents()
{
setCachingEnabled(false);
QAbstractNetworkCache *nc = networkCache();
if (!nc)
return false;
QNetworkCacheMetaData item = nc->metaData(url());
if (!item.isValid())
return false;
QNetworkCacheMetaData::AttributesMap attributes = item.attributes();
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, attributes.value(QNetworkRequest::HttpStatusCodeAttribute));
setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
// set the raw headers
QNetworkCacheMetaData::RawHeaderList rawHeaders = item.rawHeaders();
QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
end = rawHeaders.constEnd();
for ( ; it != end; ++it) {
if (it->first.toLower() == "cache-control" &&
it->second.toLower().contains("must-revalidate")) {
return false;
}
setRawHeader(it->first, it->second);
}
// handle a possible redirect
QVariant redirectionTarget = attributes.value(QNetworkRequest::RedirectionTargetAttribute);
if (redirectionTarget.isValid()) {
setAttribute(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
redirectionRequested(redirectionTarget.toUrl());
}
// signal we're open
metaDataChanged();
if (operation() == QNetworkAccessManager::GetOperation) {
QIODevice *contents = nc->data(url());
if (!contents)
return false;
contents->setParent(this);
writeDownstreamData(contents);
}
#if defined(QNETWORKACCESSCACHEBACKEND_DEBUG)
qDebug() << "Successfully sent cache:" << url();
#endif
return true;
}
void tst_QNetworkCacheMetaData::operatorEqual_data()
{
QTest::addColumn<QNetworkCacheMetaData>("other");
QTest::newRow("null") << QNetworkCacheMetaData();
QNetworkCacheMetaData data;
data.setUrl(QUrl(EXAMPLE_URL));
QNetworkCacheMetaData::RawHeaderList headers;
headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
data.setRawHeaders(headers);
data.setLastModified(QDateTime::currentDateTime());
data.setExpirationDate(QDateTime::currentDateTime());
data.setSaveToDisk(false);
QTest::newRow("valid") << data;
}
void tst_QNetworkCacheMetaData::operatorEqualEqual_data()
{
QTest::addColumn<QNetworkCacheMetaData>("a");
QTest::addColumn<QNetworkCacheMetaData>("b");
QTest::addColumn<bool>("operatorEqualEqual");
QTest::newRow("null") << QNetworkCacheMetaData() << QNetworkCacheMetaData() << true;
QNetworkCacheMetaData data1;
data1.setUrl(QUrl(EXAMPLE_URL));
QTest::newRow("valid-1-1") << data1 << QNetworkCacheMetaData() << false;
QTest::newRow("valid-1-2") << data1 << data1 << true;
QNetworkCacheMetaData data2;
QNetworkCacheMetaData::RawHeaderList headers;
headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
data2.setRawHeaders(headers);
QTest::newRow("valid-2-1") << data2 << QNetworkCacheMetaData() << false;
QTest::newRow("valid-2-2") << data2 << data2 << true;
QTest::newRow("valid-2-3") << data2 << data1 << false;
QNetworkCacheMetaData data3;
data3.setLastModified(QDateTime::currentDateTime());
QTest::newRow("valid-3-1") << data3 << QNetworkCacheMetaData() << false;
QTest::newRow("valid-3-2") << data3 << data3 << true;
QTest::newRow("valid-3-3") << data3 << data1 << false;
QTest::newRow("valid-3-4") << data3 << data2 << false;
QNetworkCacheMetaData data4;
data4.setExpirationDate(QDateTime::currentDateTime());
QTest::newRow("valid-4-1") << data4 << QNetworkCacheMetaData() << false;
QTest::newRow("valid-4-2") << data4 << data4 << true;
QTest::newRow("valid-4-3") << data4 << data1 << false;
QTest::newRow("valid-4-4") << data4 << data2 << false;
QTest::newRow("valid-4-5") << data4 << data3 << false;
QNetworkCacheMetaData data5;
data5.setSaveToDisk(false);
QTest::newRow("valid-5-1") << data5 << QNetworkCacheMetaData() << false;
QTest::newRow("valid-5-2") << data5 << data5 << true;
QTest::newRow("valid-5-3") << data5 << data1 << false;
QTest::newRow("valid-5-4") << data5 << data2 << false;
QTest::newRow("valid-5-5") << data5 << data3 << false;
QTest::newRow("valid-5-6") << data5 << data4 << false;
}
/*
A simple web page that can be used to test us: http://www.procata.com/cachetest/
*/
bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData)
{
setCachingEnabled(false);
if (!metaData.isValid())
return false;
QAbstractNetworkCache *nc = networkCache();
Q_ASSERT(nc);
QIODevice *contents = nc->data(url());
if (!contents) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Can not send cache, the contents are 0" << url();
#endif
return false;
}
contents->setParent(this);
QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (status < 100)
status = 200; // fake it
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status);
setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true);
QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders();
QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
end = rawHeaders.constEnd();
for ( ; it != end; ++it)
setRawHeader(it->first, it->second);
checkForRedirect(status);
writeDownstreamData(contents);
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes";
#endif
if (httpReply)
disconnect(httpReply, SIGNAL(finished()), this, SLOT(replyFinished()));
return true;
}
void tst_QNetworkCacheMetaData::stream()
{
QNetworkCacheMetaData data;
data.setUrl(QUrl(EXAMPLE_URL));
QNetworkCacheMetaData::RawHeaderList headers;
headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
data.setRawHeaders(headers);
data.setLastModified(QDateTime::currentDateTime());
data.setExpirationDate(QDateTime::currentDateTime());
data.setSaveToDisk(false);
QBuffer buffer;
buffer.open(QIODevice::ReadWrite);
QDataStream stream(&buffer);
stream << data;
buffer.seek(0);
QNetworkCacheMetaData data2;
stream >> data2;
QCOMPARE(data2, data);
}
void QgsWFSRequest::replyFinished()
{
if ( !mIsAborted && mReply )
{
if ( mReply->error() == QNetworkReply::NoError )
{
QgsDebugMsg( "reply ok" );
QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
if ( !redirect.isNull() )
{
QgsDebugMsg( "Request redirected." );
const QUrl& toUrl = redirect.toUrl();
mReply->request();
if ( toUrl == mReply->url() )
{
mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
mResponse.clear();
}
else
{
QNetworkRequest request( toUrl );
if ( !mUri.auth().setAuthorization( request ) )
{
mResponse.clear();
mErrorMessage = errorMessageFailedAuth();
mErrorCode = QgsWFSRequest::NetworkError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
emit downloadFinished();
return;
}
request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
mReply->deleteLater();
mReply = nullptr;
QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) );
mReply = QgsNetworkAccessManager::instance()->get( request );
connect( mReply, SIGNAL( finished() ), this, SLOT( replyFinished() ) );
connect( mReply, SIGNAL( downloadProgress( qint64, qint64 ) ), this, SLOT( replyProgress( qint64, qint64 ) ) );
return;
}
}
else
{
const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();
if ( nam->cache() )
{
QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );
QNetworkCacheMetaData::RawHeaderList hl;
Q_FOREACH ( const QNetworkCacheMetaData::RawHeader &h, cmd.rawHeaders() )
{
if ( h.first != "Cache-Control" )
hl.append( h );
}
cmd.setRawHeaders( hl );
QgsDebugMsg( QString( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ) );
if ( cmd.expirationDate().isNull() )
{
cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) );
}
nam->cache()->updateMetaData( cmd );
}
else
{
QgsDebugMsg( "No cache!" );
}
#ifdef QGISDEBUG
bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
QgsDebugMsg( QString( "Reply was cached: %1" ).arg( fromCache ) );
#endif
mResponse = mReply->readAll();
if ( mResponse.isEmpty() && !mGotNonEmptyResponse )
{
mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
mErrorCode = QgsWFSRequest::ServerExceptionError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
}
}
/** \fn MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo)
* \brief Downloads a QNetworkRequest via the QNetworkAccessManager
* \param dlInfo MythDownloadInfo information for download
*/
void MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo)
{
static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
QUrl qurl(dlInfo->m_url);
QNetworkRequest request;
if (dlInfo->m_request)
{
request = *dlInfo->m_request;
delete dlInfo->m_request;
dlInfo->m_request = NULL;
}
else
request.setUrl(qurl);
if (!dlInfo->m_reload)
{
// Prefer the in-cache item if one exists and it is less than 60
// seconds old and has not expired in the last 10 seconds.
QDateTime now = QDateTime::currentDateTime();
QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
if ((urlData.isValid()) &&
((!urlData.expirationDate().isValid()) ||
(urlData.expirationDate().secsTo(now) < 10)))
{
QNetworkCacheMetaData::RawHeaderList headers =
urlData.rawHeaders();
bool found = false;
QNetworkCacheMetaData::RawHeaderList::iterator it
= headers.begin();
for (; !found && it != headers.end(); ++it)
{
if ((*it).first == "Date")
{
found = true;
QDateTime loadDate =
QDateTime::fromString((*it).second, dateFormat);
loadDate.setTimeSpec(Qt::UTC);
if (loadDate.secsTo(now) < 60)
dlInfo->m_preferCache = true;
}
}
}
}
if (dlInfo->m_preferCache)
request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
QNetworkRequest::PreferCache);
request.setRawHeader("User-Agent",
"MythDownloadManager v" MYTH_BINARY_VERSION);
if (dlInfo->m_post)
dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
else
dlInfo->m_reply = m_manager->get(request);
m_downloadReplies[dlInfo->m_reply] = dlInfo;
connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
SLOT(downloadError(QNetworkReply::NetworkError)));
connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
this, SLOT(downloadProgress(qint64, qint64)));
}
void QgsWfsRequest::replyFinished()
{
if ( !mIsAborted && mReply )
{
if ( mReply->error() == QNetworkReply::NoError )
{
QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 4 );
QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
if ( !redirect.isNull() )
{
QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 4 );
const QUrl &toUrl = redirect.toUrl();
mReply->request();
if ( toUrl == mReply->url() )
{
mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
mResponse.clear();
}
else
{
QNetworkRequest request( toUrl );
QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsWfsRequest" ) );
if ( !mUri.auth().setAuthorization( request ) )
{
mResponse.clear();
mErrorMessage = errorMessageFailedAuth();
mErrorCode = QgsWfsRequest::NetworkError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
emit downloadFinished();
return;
}
request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
mReply->deleteLater();
mReply = nullptr;
QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 4 );
mReply = QgsNetworkAccessManager::instance()->get( request );
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mUri.auth().setAuthorizationReply( mReply ) )
{
mResponse.clear();
mErrorMessage = errorMessageFailedAuth();
mErrorCode = QgsWfsRequest::NetworkError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
emit downloadFinished();
return;
}
connect( mReply, &QNetworkReply::finished, this, &QgsWfsRequest::replyFinished, Qt::DirectConnection );
connect( mReply, &QNetworkReply::downloadProgress, this, &QgsWfsRequest::replyProgress, Qt::DirectConnection );
return;
}
}
else
{
const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();
if ( nam->cache() )
{
QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );
QNetworkCacheMetaData::RawHeaderList hl;
const auto constRawHeaders = cmd.rawHeaders();
for ( const QNetworkCacheMetaData::RawHeader &h : constRawHeaders )
{
if ( h.first != "Cache-Control" )
hl.append( h );
}
cmd.setRawHeaders( hl );
QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 4 );
if ( cmd.expirationDate().isNull() )
{
cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) );
}
nam->cache()->updateMetaData( cmd );
}
else
{
QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 4 );
}
#ifdef QGISDEBUG
bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 4 );
#endif
mResponse = mReply->readAll();
if ( mResponse.isEmpty() && !mGotNonEmptyResponse )
{
mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
mErrorCode = QgsWfsRequest::ServerExceptionError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
}
}
}
else
{
mErrorMessage = errorMessageWithReason( mReply->errorString() );
mErrorCode = QgsWfsRequest::ServerExceptionError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
mResponse.clear();
}
}
if ( mTimedout )
mErrorCode = QgsWfsRequest::TimeoutError;
if ( mReply )
{
mReply->deleteLater();
mReply = nullptr;
}
emit downloadFinished();
}
