void tst_QNetworkCacheMetaData::isValid_data() { QTest::addColumn<QNetworkCacheMetaData>("data"); QTest::addColumn<bool>("isValid"); QNetworkCacheMetaData metaData; QTest::newRow("null") << metaData << false; QNetworkCacheMetaData data1; data1.setUrl(QUrl(EXAMPLE_URL)); QTest::newRow("valid-1") << data1 << true; QNetworkCacheMetaData data2; QNetworkCacheMetaData::RawHeaderList headers; headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar")); data2.setRawHeaders(headers); QTest::newRow("valid-2") << data2 << true; QNetworkCacheMetaData data3; data3.setLastModified(QDateTime::currentDateTime()); QTest::newRow("valid-3") << data3 << true; QNetworkCacheMetaData data4; data4.setExpirationDate(QDateTime::currentDateTime()); QTest::newRow("valid-4") << data4 << true; QNetworkCacheMetaData data5; data5.setSaveToDisk(false); QTest::newRow("valid-5") << data5 << true; }
// This function simulates a partially or fully occupied disk cache // like a normal user of a cache might encounter is real-life browsing. // The point of this is to trigger degradation in file-system and media performance // that occur due to the quantity and layout of data. void tst_qnetworkdiskcache::injectFakeData() { QNetworkCacheMetaData::RawHeaderList headers; headers.append(qMakePair(QByteArray("X-TestHeader"),QByteArray("HeaderValue"))); //Prep cache dir with fake data using QNetworkDiskCache APIs for (quint32 i = 0; i < NumFakeCacheObjects; i++) { //prepare metata for url QNetworkCacheMetaData meta; QString fakeURL; QTextStream stream(&fakeURL); stream << fakeURLbase << i; QUrl url(fakeURL); meta.setUrl(url); meta.setRawHeaders(headers); meta.setSaveToDisk(true); //commit payload and metadata to disk QIODevice *device = cache->prepare(meta); device->write(payload); cache->insert(device); } }
QIODevice *TabCache::prepare(const QNetworkCacheMetaData &metaData) { QLOG_DEBUG() << "TabCache::prepare"; QNetworkCacheMetaData local{metaData}; //Default policy based on received HTTP headers is to not save to disk. //Override here, and set proper expiration so items are put in cache //(setSaveToDisk) and valid when retrieved using metaData call - //(setExpirationDate) local.setSaveToDisk(true); // Need to set some reasonable length of time in which our cache entries // will expire. It's possible we'll want to allow users to customize this. local.setExpirationDate(QDateTime().currentDateTime().addDays(kCacheExpireInDays)); QNetworkCacheMetaData::RawHeaderList headers; for (auto &header: local.rawHeaders()) { // Modify Cache-Control headers - basically need to drop 'no-store' // as we want to store to cache and 'must-revalidate' as we don't have // ETag or last modified headers available to attempt re-validation. To // be on the safe side though just drop Cache-Control and Pragma headers. if (header.first == "Cache-Control") continue; if (header.first == "Pragma") continue; headers.push_back(header); } local.setRawHeaders(headers); return QNetworkDiskCache::prepare(local); }
/** \fn MythDownloadManager::GetLastModified(const QString &url) * \brief Gets the Last Modified timestamp for a URI * \param url URI to test. * \return Timestamp the URI was last modified or now if an error occurred */ QDateTime MythDownloadManager::GetLastModified(const QString &url) { static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'"; VERBOSE(VB_FILE+VB_EXTRA, LOC + QString("GetLastModified('%1')").arg(url)); QDateTime result; QDateTime now = QDateTime::currentDateTime(); QNetworkCacheMetaData urlData = m_manager->cache()->metaData(QUrl(url)); if (urlData.isValid()) { if (urlData.lastModified().secsTo(now) <= 60) { result = urlData.lastModified(); } else { // If the last modification date is older than 60 seconds, and // we loaded the page over 60 seconds ago, then redownload the // page to re-verify it's last modified date. QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders(); bool found = false; QNetworkCacheMetaData::RawHeaderList::iterator it = headers.begin(); for (; !found && it != headers.end(); ++it) { if ((*it).first == "Date") { found = true; QDateTime loadDate = QDateTime::fromString((*it).second, dateFormat); loadDate.setTimeSpec(Qt::UTC); if (loadDate.secsTo(now) <= 60) result = urlData.lastModified(); } } } } if (!result.isValid()) { MythDownloadInfo *dlInfo = new MythDownloadInfo; dlInfo->m_url = url; dlInfo->m_syncMode = true; if (downloadNow(dlInfo, false) && dlInfo->m_reply) { QVariant lastMod = dlInfo->m_reply->header(QNetworkRequest::LastModifiedHeader); if (lastMod.isValid()) result = lastMod.toDateTime(); } delete dlInfo; } return result; }
void tst_QNetworkCacheMetaData::rawHeaders_data() { QTest::addColumn<QNetworkCacheMetaData::RawHeaderList>("rawHeaders"); QTest::newRow("null") << QNetworkCacheMetaData::RawHeaderList(); QNetworkCacheMetaData::RawHeaderList headers; headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar")); QTest::newRow("valie") << headers; }
bool QNetworkAccessCacheBackend::sendCacheContents() { setCachingEnabled(false); QAbstractNetworkCache *nc = networkCache(); if (!nc) return false; QNetworkCacheMetaData item = nc->metaData(url()); if (!item.isValid()) return false; QNetworkCacheMetaData::AttributesMap attributes = item.attributes(); setAttribute(QNetworkRequest::HttpStatusCodeAttribute, attributes.value(QNetworkRequest::HttpStatusCodeAttribute)); setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); // set the raw headers QNetworkCacheMetaData::RawHeaderList rawHeaders = item.rawHeaders(); QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), end = rawHeaders.constEnd(); for ( ; it != end; ++it) { if (it->first.toLower() == "cache-control" && it->second.toLower().contains("must-revalidate")) { return false; } setRawHeader(it->first, it->second); } // handle a possible redirect QVariant redirectionTarget = attributes.value(QNetworkRequest::RedirectionTargetAttribute); if (redirectionTarget.isValid()) { setAttribute(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget); redirectionRequested(redirectionTarget.toUrl()); } // signal we're open metaDataChanged(); if (operation() == QNetworkAccessManager::GetOperation) { QIODevice *contents = nc->data(url()); if (!contents) return false; contents->setParent(this); writeDownstreamData(contents); } #if defined(QNETWORKACCESSCACHEBACKEND_DEBUG) qDebug() << "Successfully sent cache:" << url(); #endif return true; }
void tst_QNetworkCacheMetaData::operatorEqual_data() { QTest::addColumn<QNetworkCacheMetaData>("other"); QTest::newRow("null") << QNetworkCacheMetaData(); QNetworkCacheMetaData data; data.setUrl(QUrl(EXAMPLE_URL)); QNetworkCacheMetaData::RawHeaderList headers; headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar")); data.setRawHeaders(headers); data.setLastModified(QDateTime::currentDateTime()); data.setExpirationDate(QDateTime::currentDateTime()); data.setSaveToDisk(false); QTest::newRow("valid") << data; }
void tst_QNetworkCacheMetaData::operatorEqualEqual_data() { QTest::addColumn<QNetworkCacheMetaData>("a"); QTest::addColumn<QNetworkCacheMetaData>("b"); QTest::addColumn<bool>("operatorEqualEqual"); QTest::newRow("null") << QNetworkCacheMetaData() << QNetworkCacheMetaData() << true; QNetworkCacheMetaData data1; data1.setUrl(QUrl(EXAMPLE_URL)); QTest::newRow("valid-1-1") << data1 << QNetworkCacheMetaData() << false; QTest::newRow("valid-1-2") << data1 << data1 << true; QNetworkCacheMetaData data2; QNetworkCacheMetaData::RawHeaderList headers; headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar")); data2.setRawHeaders(headers); QTest::newRow("valid-2-1") << data2 << QNetworkCacheMetaData() << false; QTest::newRow("valid-2-2") << data2 << data2 << true; QTest::newRow("valid-2-3") << data2 << data1 << false; QNetworkCacheMetaData data3; data3.setLastModified(QDateTime::currentDateTime()); QTest::newRow("valid-3-1") << data3 << QNetworkCacheMetaData() << false; QTest::newRow("valid-3-2") << data3 << data3 << true; QTest::newRow("valid-3-3") << data3 << data1 << false; QTest::newRow("valid-3-4") << data3 << data2 << false; QNetworkCacheMetaData data4; data4.setExpirationDate(QDateTime::currentDateTime()); QTest::newRow("valid-4-1") << data4 << QNetworkCacheMetaData() << false; QTest::newRow("valid-4-2") << data4 << data4 << true; QTest::newRow("valid-4-3") << data4 << data1 << false; QTest::newRow("valid-4-4") << data4 << data2 << false; QTest::newRow("valid-4-5") << data4 << data3 << false; QNetworkCacheMetaData data5; data5.setSaveToDisk(false); QTest::newRow("valid-5-1") << data5 << QNetworkCacheMetaData() << false; QTest::newRow("valid-5-2") << data5 << data5 << true; QTest::newRow("valid-5-3") << data5 << data1 << false; QTest::newRow("valid-5-4") << data5 << data2 << false; QTest::newRow("valid-5-5") << data5 << data3 << false; QTest::newRow("valid-5-6") << data5 << data4 << false; }
/* A simple web page that can be used to test us: http://www.procata.com/cachetest/ */ bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData) { setCachingEnabled(false); if (!metaData.isValid()) return false; QAbstractNetworkCache *nc = networkCache(); Q_ASSERT(nc); QIODevice *contents = nc->data(url()); if (!contents) { #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "Can not send cache, the contents are 0" << url(); #endif return false; } contents->setParent(this); QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes(); int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (status < 100) status = 200; // fake it setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status); setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders(); QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), end = rawHeaders.constEnd(); for ( ; it != end; ++it) setRawHeader(it->first, it->second); checkForRedirect(status); writeDownstreamData(contents); #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes"; #endif if (httpReply) disconnect(httpReply, SIGNAL(finished()), this, SLOT(replyFinished())); return true; }
void tst_QNetworkCacheMetaData::stream() { QNetworkCacheMetaData data; data.setUrl(QUrl(EXAMPLE_URL)); QNetworkCacheMetaData::RawHeaderList headers; headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar")); data.setRawHeaders(headers); data.setLastModified(QDateTime::currentDateTime()); data.setExpirationDate(QDateTime::currentDateTime()); data.setSaveToDisk(false); QBuffer buffer; buffer.open(QIODevice::ReadWrite); QDataStream stream(&buffer); stream << data; buffer.seek(0); QNetworkCacheMetaData data2; stream >> data2; QCOMPARE(data2, data); }
void QgsWFSRequest::replyFinished() { if ( !mIsAborted && mReply ) { if ( mReply->error() == QNetworkReply::NoError ) { QgsDebugMsg( "reply ok" ); QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute ); if ( !redirect.isNull() ) { QgsDebugMsg( "Request redirected." ); const QUrl& toUrl = redirect.toUrl(); mReply->request(); if ( toUrl == mReply->url() ) { mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() ); QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) ); mResponse.clear(); } else { QNetworkRequest request( toUrl ); if ( !mUri.auth().setAuthorization( request ) ) { mResponse.clear(); mErrorMessage = errorMessageFailedAuth(); mErrorCode = QgsWFSRequest::NetworkError; QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) ); emit downloadFinished(); return; } request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache ); request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true ); mReply->deleteLater(); mReply = nullptr; QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) ); mReply = QgsNetworkAccessManager::instance()->get( request ); connect( mReply, SIGNAL( finished() ), this, SLOT( replyFinished() ) ); connect( mReply, SIGNAL( downloadProgress( qint64, qint64 ) ), this, SLOT( replyProgress( qint64, qint64 ) ) ); return; } } else { const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance(); if ( nam->cache() ) { QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() ); QNetworkCacheMetaData::RawHeaderList hl; Q_FOREACH ( const QNetworkCacheMetaData::RawHeader &h, cmd.rawHeaders() ) { if ( h.first != "Cache-Control" ) hl.append( h ); } cmd.setRawHeaders( hl ); QgsDebugMsg( QString( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ) ); if ( cmd.expirationDate().isNull() ) { cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) ); } nam->cache()->updateMetaData( cmd ); } else { QgsDebugMsg( "No cache!" ); } #ifdef QGISDEBUG bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool(); QgsDebugMsg( QString( "Reply was cached: %1" ).arg( fromCache ) ); #endif mResponse = mReply->readAll(); if ( mResponse.isEmpty() && !mGotNonEmptyResponse ) { mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() ); mErrorCode = QgsWFSRequest::ServerExceptionError; QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) ); } }
/** \fn MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo) * \brief Downloads a QNetworkRequest via the QNetworkAccessManager * \param dlInfo MythDownloadInfo information for download */ void MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo) { static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'"; QUrl qurl(dlInfo->m_url); QNetworkRequest request; if (dlInfo->m_request) { request = *dlInfo->m_request; delete dlInfo->m_request; dlInfo->m_request = NULL; } else request.setUrl(qurl); if (!dlInfo->m_reload) { // Prefer the in-cache item if one exists and it is less than 60 // seconds old and has not expired in the last 10 seconds. QDateTime now = QDateTime::currentDateTime(); QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl); if ((urlData.isValid()) && ((!urlData.expirationDate().isValid()) || (urlData.expirationDate().secsTo(now) < 10))) { QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders(); bool found = false; QNetworkCacheMetaData::RawHeaderList::iterator it = headers.begin(); for (; !found && it != headers.end(); ++it) { if ((*it).first == "Date") { found = true; QDateTime loadDate = QDateTime::fromString((*it).second, dateFormat); loadDate.setTimeSpec(Qt::UTC); if (loadDate.secsTo(now) < 60) dlInfo->m_preferCache = true; } } } } if (dlInfo->m_preferCache) request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); request.setRawHeader("User-Agent", "MythDownloadManager v" MYTH_BINARY_VERSION); if (dlInfo->m_post) dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data); else dlInfo->m_reply = m_manager->get(request); m_downloadReplies[dlInfo->m_reply] = dlInfo; connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(downloadError(QNetworkReply::NetworkError))); connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)), this, SLOT(downloadProgress(qint64, qint64))); }
void QgsWfsRequest::replyFinished() { if ( !mIsAborted && mReply ) { if ( mReply->error() == QNetworkReply::NoError ) { QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 4 ); QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute ); if ( !redirect.isNull() ) { QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 4 ); const QUrl &toUrl = redirect.toUrl(); mReply->request(); if ( toUrl == mReply->url() ) { mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() ); QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) ); mResponse.clear(); } else { QNetworkRequest request( toUrl ); QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsWfsRequest" ) ); if ( !mUri.auth().setAuthorization( request ) ) { mResponse.clear(); mErrorMessage = errorMessageFailedAuth(); mErrorCode = QgsWfsRequest::NetworkError; QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) ); emit downloadFinished(); return; } request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache ); request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true ); mReply->deleteLater(); mReply = nullptr; QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 4 ); mReply = QgsNetworkAccessManager::instance()->get( request ); mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT ); if ( !mUri.auth().setAuthorizationReply( mReply ) ) { mResponse.clear(); mErrorMessage = errorMessageFailedAuth(); mErrorCode = QgsWfsRequest::NetworkError; QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) ); emit downloadFinished(); return; } connect( mReply, &QNetworkReply::finished, this, &QgsWfsRequest::replyFinished, Qt::DirectConnection ); connect( mReply, &QNetworkReply::downloadProgress, this, &QgsWfsRequest::replyProgress, Qt::DirectConnection ); return; } } else { const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance(); if ( nam->cache() ) { QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() ); QNetworkCacheMetaData::RawHeaderList hl; const auto constRawHeaders = cmd.rawHeaders(); for ( const QNetworkCacheMetaData::RawHeader &h : constRawHeaders ) { if ( h.first != "Cache-Control" ) hl.append( h ); } cmd.setRawHeaders( hl ); QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 4 ); if ( cmd.expirationDate().isNull() ) { cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) ); } nam->cache()->updateMetaData( cmd ); } else { QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 4 ); } #ifdef QGISDEBUG bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool(); QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 4 ); #endif mResponse = mReply->readAll(); if ( mResponse.isEmpty() && !mGotNonEmptyResponse ) { mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() ); mErrorCode = QgsWfsRequest::ServerExceptionError; QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) ); } } } else { mErrorMessage = errorMessageWithReason( mReply->errorString() ); mErrorCode = QgsWfsRequest::ServerExceptionError; QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) ); mResponse.clear(); } } if ( mTimedout ) mErrorCode = QgsWfsRequest::TimeoutError; if ( mReply ) { mReply->deleteLater(); mReply = nullptr; } emit downloadFinished(); }