void WMSRequester::requestTile(const TileBuildOperation& op) { QUrl url(op.urlString); // Testing the 'SourceIsFromCache' attribute of replies from the OnEarth server // seems to indicate that the tiles are not being cached. However, tiles are still // appearing in the cache directory. The following code attempts to force using // the cache, but may not be effective. QNetworkCacheMetaData cacheData = m_networkManager->cache()->metaData(url); QNetworkRequest request(url); if (cacheData.isValid()) { request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysCache); } else { request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); } QNetworkReply* reply = m_networkManager->get(request); m_mutex.lock(); m_dispatchedRequestCount++; m_requestedTiles[reply] = op; m_mutex.unlock(); }
/** \fn MythDownloadManager::GetLastModified(const QString &url) * \brief Gets the Last Modified timestamp for a URI * \param url URI to test. * \return Timestamp the URI was last modified or now if an error occurred */ QDateTime MythDownloadManager::GetLastModified(const QString &url) { static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'"; VERBOSE(VB_FILE+VB_EXTRA, LOC + QString("GetLastModified('%1')").arg(url)); QDateTime result; QDateTime now = QDateTime::currentDateTime(); QNetworkCacheMetaData urlData = m_manager->cache()->metaData(QUrl(url)); if (urlData.isValid()) { if (urlData.lastModified().secsTo(now) <= 60) { result = urlData.lastModified(); } else { // If the last modification date is older than 60 seconds, and // we loaded the page over 60 seconds ago, then redownload the // page to re-verify it's last modified date. QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders(); bool found = false; QNetworkCacheMetaData::RawHeaderList::iterator it = headers.begin(); for (; !found && it != headers.end(); ++it) { if ((*it).first == "Date") { found = true; QDateTime loadDate = QDateTime::fromString((*it).second, dateFormat); loadDate.setTimeSpec(Qt::UTC); if (loadDate.secsTo(now) <= 60) result = urlData.lastModified(); } } } } if (!result.isValid()) { MythDownloadInfo *dlInfo = new MythDownloadInfo; dlInfo->m_url = url; dlInfo->m_syncMode = true; if (downloadNow(dlInfo, false) && dlInfo->m_reply) { QVariant lastMod = dlInfo->m_reply->header(QNetworkRequest::LastModifiedHeader); if (lastMod.isValid()) result = lastMod.toDateTime(); } delete dlInfo; } return result; }
bool QNetworkAccessCacheBackend::sendCacheContents() { setCachingEnabled(false); QAbstractNetworkCache *nc = networkCache(); if (!nc) return false; QNetworkCacheMetaData item = nc->metaData(url()); if (!item.isValid()) return false; QNetworkCacheMetaData::AttributesMap attributes = item.attributes(); setAttribute(QNetworkRequest::HttpStatusCodeAttribute, attributes.value(QNetworkRequest::HttpStatusCodeAttribute)); setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); // set the raw headers QNetworkCacheMetaData::RawHeaderList rawHeaders = item.rawHeaders(); QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), end = rawHeaders.constEnd(); for ( ; it != end; ++it) { if (it->first.toLower() == "cache-control" && it->second.toLower().contains("must-revalidate")) { return false; } setRawHeader(it->first, it->second); } // handle a possible redirect QVariant redirectionTarget = attributes.value(QNetworkRequest::RedirectionTargetAttribute); if (redirectionTarget.isValid()) { setAttribute(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget); redirectionRequested(redirectionTarget.toUrl()); } // signal we're open metaDataChanged(); if (operation() == QNetworkAccessManager::GetOperation) { QIODevice *contents = nc->data(url()); if (!contents) return false; contents->setParent(this); writeDownstreamData(contents); } #if defined(QNETWORKACCESSCACHEBACKEND_DEBUG) qDebug() << "Successfully sent cache:" << url(); #endif return true; }
//Times metadata as well payload lookup // i.e metaData(), rawHeaders() and data() void tst_qnetworkdiskcache::timeRead() { QFETCH(QString, cacheRootDirectory); cacheDir = QString( cacheRootDirectory + QDir::separator() + "man_qndc"); QDir d; qDebug() << "Setting cache directory to = " << d.absoluteFilePath(cacheDir); //Housekeeping cleanRecursive(cacheDir); // slow op. initCacheObject(); cache->setCacheDirectory(cacheDir); cache->setMaximumCacheSize(qint64(HugeCacheLimit)); cache->clear(); //populate some fake data to simulate partially full cache injectFakeData(); //Entries in the cache should be > what we try to remove QVERIFY(NumFakeCacheObjects > NumReadContent); //time metadata lookup of previously inserted URL. QBENCHMARK_ONCE { for (quint32 i = 0; i < NumReadContent; i++) { QString fakeURL; QTextStream stream(&fakeURL); stream << fakeURLbase << i; QUrl url(fakeURL); QNetworkCacheMetaData qndc = cache->metaData(url); QVERIFY(qndc.isValid()); // we must have read the metadata QNetworkCacheMetaData::RawHeaderList raw(qndc.rawHeaders()); QVERIFY(raw.size()); // we must have parsed the headers from the meta QIODevice *iodevice(cache->data(url)); QVERIFY(iodevice); //must not be NULL iodevice->close(); delete iodevice; } } //Cleanup (slow) cleanupCacheObject(); cleanRecursive(cacheDir); }
void tst_QNetworkCacheMetaData::qnetworkcachemetadata() { QNetworkCacheMetaData data; QCOMPARE(data.expirationDate(), QDateTime()); QCOMPARE(data.isValid(), false); QCOMPARE(data.lastModified(), QDateTime()); QCOMPARE(data.operator!=(QNetworkCacheMetaData()), false); QNetworkCacheMetaData metaData; QCOMPARE(data.operator=(metaData), QNetworkCacheMetaData()); QCOMPARE(data.operator==(QNetworkCacheMetaData()), true); QCOMPARE(data.rawHeaders(), QNetworkCacheMetaData::RawHeaderList()); QCOMPARE(data.saveToDisk(), true); QCOMPARE(data.url(), QUrl()); data.setExpirationDate(QDateTime()); data.setLastModified(QDateTime()); data.setRawHeaders(QNetworkCacheMetaData::RawHeaderList()); data.setSaveToDisk(false); data.setUrl(QUrl()); }
bool ResourceHandle::willLoadFromCache(ResourceRequest& request, Frame* frame) { if (!frame) return false; QNetworkAccessManager* manager = QWebFramePrivate::kit(frame)->page()->networkAccessManager(); QAbstractNetworkCache* cache = manager->cache(); if (!cache) return false; QNetworkCacheMetaData data = cache->metaData(request.url()); if (data.isValid()) { request.setCachePolicy(ReturnCacheDataDontLoad); return true; } return false; }
/* A simple web page that can be used to test us: http://www.procata.com/cachetest/ */ bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData) { setCachingEnabled(false); if (!metaData.isValid()) return false; QAbstractNetworkCache *nc = networkCache(); Q_ASSERT(nc); QIODevice *contents = nc->data(url()); if (!contents) { #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "Can not send cache, the contents are 0" << url(); #endif return false; } contents->setParent(this); QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes(); int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (status < 100) status = 200; // fake it setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status); setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders(); QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), end = rawHeaders.constEnd(); for ( ; it != end; ++it) setRawHeader(it->first, it->second); checkForRedirect(status); writeDownstreamData(contents); #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes"; #endif if (httpReply) disconnect(httpReply, SIGNAL(finished()), this, SLOT(replyFinished())); return true; }
bool ResourceHandle::willLoadFromCache(ResourceRequest& request, Frame* frame) { if (!frame) return false; QNetworkAccessManager* manager = 0; QAbstractNetworkCache* cache = 0; if (frame->loader()->networkingContext()) { manager = frame->loader()->networkingContext()->networkAccessManager(); cache = manager->cache(); } if (!cache) return false; QNetworkCacheMetaData data = cache->metaData(request.url()); if (data.isValid()) { request.setCachePolicy(ReturnCacheDataDontLoad); return true; } return false; }
void NetworkCache::clearCache(int period) { if (period <= 0) { clear(); emit cleared(); return; } const QDir cacheMainDirectory(cacheDirectory()); const QStringList directories(cacheMainDirectory.entryList(QDir::AllDirs | QDir::NoDotAndDotDot)); for (int i = 0; i < directories.count(); ++i) { const QDir cacheSubDirectory(cacheMainDirectory.absoluteFilePath(directories.at(i))); const QStringList subDirectories(cacheSubDirectory.entryList(QDir::AllDirs | QDir::NoDotAndDotDot)); for (int j = 0; j < subDirectories.count(); ++j) { const QDir cacheFilesDirectory(cacheSubDirectory.absoluteFilePath(subDirectories.at(j))); const QStringList files(cacheFilesDirectory.entryList(QDir::Files)); for (int k = 0; k < files.count(); ++k) { const QNetworkCacheMetaData metaData(fileMetaData(cacheFilesDirectory.absoluteFilePath(files.at(k)))); if (metaData.isValid() && metaData.lastModified().isValid() && metaData.lastModified().secsTo(QDateTime::currentDateTime()) > (period * 3600)) { remove(metaData.url()); } } } } }
/** \brief Downloads a QNetworkRequest via the QNetworkAccessManager * \param dlInfo MythDownloadInfo information for download */ void MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo) { if (!dlInfo) return; static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'"; QUrl qurl(dlInfo->m_url); QNetworkRequest request; if (dlInfo->m_request) { request = *dlInfo->m_request; delete dlInfo->m_request; dlInfo->m_request = NULL; } else request.setUrl(qurl); if (!dlInfo->m_reload) { // Prefer the in-cache item if one exists and it is less than 5 minutes // old and it will not expire in the next 10 seconds QDateTime now = MythDate::current(); // Handle redirects, we want the metadata of the file headers QString redirectLoc; int limit = 0; while (!(redirectLoc = getHeader(qurl, "Location")).isNull()) { if (limit == CACHE_REDIRECTION_LIMIT) { LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit " "reached for %1") .arg(qurl.toString())); return; } qurl.setUrl(redirectLoc); limit++; } LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1") .arg(qurl.toString())); m_infoLock->lock(); QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl); m_infoLock->unlock(); if ((urlData.isValid()) && ((!urlData.expirationDate().isValid()) || (QDateTime(urlData.expirationDate().toUTC()).secsTo(now) < 10))) { QString dateString = getHeader(urlData, "Date"); if (!dateString.isNull()) { QDateTime loadDate = MythDate::fromString(dateString, dateFormat); loadDate.setTimeSpec(Qt::UTC); if (loadDate.secsTo(now) <= 720) { dlInfo->m_preferCache = true; LOG(VB_NETWORK, LOG_DEBUG, QString("Prefering cache for %1") .arg(qurl.toString())); } } } } if (dlInfo->m_preferCache) request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); request.setRawHeader("User-Agent", "MythTV v" MYTH_BINARY_VERSION " MythDownloadManager"); if (dlInfo->m_headers) { QHash<QByteArray, QByteArray>::const_iterator it = dlInfo->m_headers->constBegin(); for ( ; it != dlInfo->m_headers->constEnd(); ++it ) { if (!it.key().isEmpty() && !it.value().isEmpty()) { request.setRawHeader(it.key(), it.value()); } } } switch (dlInfo->m_requestType) { case kRequestPost : dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data); break; case kRequestHead : dlInfo->m_reply = m_manager->head(request); break; case kRequestGet : default: dlInfo->m_reply = m_manager->get(request); break; } m_downloadReplies[dlInfo->m_reply] = dlInfo; if (dlInfo->m_authCallback) { connect(m_manager, SIGNAL(authenticationRequired(QNetworkReply *, QAuthenticator *)), this, SLOT(authCallback(QNetworkReply *, QAuthenticator *))); }
/* For a given httpRequest 1) If AlwaysNetwork, return 2) If we have a cache entry for this url populate headers so the server can return 304 3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true */ void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache) { QNetworkRequest::CacheLoadControl CacheLoadControlAttribute = (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) { // forced reload from the network // tell any caching proxy servers to reload too httpRequest.setHeaderField("Cache-Control", "no-cache"); httpRequest.setHeaderField("Pragma", "no-cache"); return; } QAbstractNetworkCache *nc = networkCache(); if (!nc) return; // no local cache QNetworkCacheMetaData metaData = nc->metaData(url()); if (!metaData.isValid()) return; // not in cache if (!metaData.saveToDisk()) return; QNetworkHeadersPrivate cacheHeaders; QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); it = cacheHeaders.findRawHeader("etag"); if (it != cacheHeaders.rawHeaders.constEnd()) httpRequest.setHeaderField("If-None-Match", it->second); QDateTime lastModified = metaData.lastModified(); if (lastModified.isValid()) httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); it = cacheHeaders.findRawHeader("Cache-Control"); if (it != cacheHeaders.rawHeaders.constEnd()) { QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second); if (cacheControl.contains("must-revalidate")) return; } /* * age_value * is the value of Age: header received by the cache with * this response. * date_value * is the value of the origin server's Date: header * request_time * is the (local) time when the cache made the request * that resulted in this cached response * response_time * is the (local) time when the cache received the * response * now * is the current (local) time */ QDateTime currentDateTime = QDateTime::currentDateTime(); int age_value = 0; it = cacheHeaders.findRawHeader("age"); if (it != cacheHeaders.rawHeaders.constEnd()) age_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t(); int date_value = 0; it = cacheHeaders.findRawHeader("date"); if (it != cacheHeaders.rawHeaders.constEnd()) date_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t(); int now = currentDateTime.toUTC().toTime_t(); int request_time = now; int response_time = now; int apparent_age = qMax(0, response_time - date_value); int corrected_received_age = qMax(apparent_age, age_value); int response_delay = response_time - request_time; int corrected_initial_age = corrected_received_age + response_delay; int resident_time = now - response_time; int current_age = corrected_initial_age + resident_time; // RFC 2616 13.2.4 Expiration Calculations QDateTime expirationDate = metaData.expirationDate(); if (!expirationDate.isValid()) { if (lastModified.isValid()) { int diff = currentDateTime.secsTo(lastModified); expirationDate = lastModified; expirationDate.addSecs(diff / 10); if (httpRequest.headerField("Warning").isEmpty()) { QDateTime dt; dt.setTime_t(current_age); if (dt.daysTo(currentDateTime) > 1) httpRequest.setHeaderField("Warning", "113"); } } } int freshness_lifetime = currentDateTime.secsTo(expirationDate); bool response_is_fresh = (freshness_lifetime > current_age); if (!response_is_fresh && CacheLoadControlAttribute == QNetworkRequest::PreferNetwork) return; loadedFromCache = true; #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "response_is_fresh" << CacheLoadControlAttribute; #endif if (!sendCacheContents(metaData)) loadedFromCache = false; }
/** \fn MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo) * \brief Downloads a QNetworkRequest via the QNetworkAccessManager * \param dlInfo MythDownloadInfo information for download */ void MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo) { static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'"; QUrl qurl(dlInfo->m_url); QNetworkRequest request; if (dlInfo->m_request) { request = *dlInfo->m_request; delete dlInfo->m_request; dlInfo->m_request = NULL; } else request.setUrl(qurl); if (!dlInfo->m_reload) { // Prefer the in-cache item if one exists and it is less than 60 // seconds old and has not expired in the last 10 seconds. QDateTime now = QDateTime::currentDateTime(); QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl); if ((urlData.isValid()) && ((!urlData.expirationDate().isValid()) || (urlData.expirationDate().secsTo(now) < 10))) { QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders(); bool found = false; QNetworkCacheMetaData::RawHeaderList::iterator it = headers.begin(); for (; !found && it != headers.end(); ++it) { if ((*it).first == "Date") { found = true; QDateTime loadDate = QDateTime::fromString((*it).second, dateFormat); loadDate.setTimeSpec(Qt::UTC); if (loadDate.secsTo(now) < 60) dlInfo->m_preferCache = true; } } } } if (dlInfo->m_preferCache) request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); request.setRawHeader("User-Agent", "MythDownloadManager v" MYTH_BINARY_VERSION); if (dlInfo->m_post) dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data); else dlInfo->m_reply = m_manager->get(request); m_downloadReplies[dlInfo->m_reply] = dlInfo; connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(downloadError(QNetworkReply::NetworkError))); connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)), this, SLOT(downloadProgress(qint64, qint64))); }
/* For a given httpRequest 1) If AlwaysNetwork, return 2) If we have a cache entry for this url populate headers so the server can return 304 3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true */ bool QNetworkAccessHttpBackend::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest) { QNetworkRequest::CacheLoadControl CacheLoadControlAttribute = (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) { // If the request does not already specify preferred cache-control // force reload from the network and tell any caching proxy servers to reload too if (!request().rawHeaderList().contains("Cache-Control")) { httpRequest.setHeaderField("Cache-Control", "no-cache"); httpRequest.setHeaderField("Pragma", "no-cache"); } return false; } // The disk cache API does not currently support partial content retrieval. // That is why we don't use the disk cache for any such requests. if (request().hasRawHeader("Range")) return false; QAbstractNetworkCache *nc = networkCache(); if (!nc) return false; // no local cache QNetworkCacheMetaData metaData = nc->metaData(url()); if (!metaData.isValid()) return false; // not in cache if (!metaData.saveToDisk()) return false; QNetworkHeadersPrivate cacheHeaders; QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); it = cacheHeaders.findRawHeader("etag"); if (it != cacheHeaders.rawHeaders.constEnd()) httpRequest.setHeaderField("If-None-Match", it->second); QDateTime lastModified = metaData.lastModified(); if (lastModified.isValid()) httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); it = cacheHeaders.findRawHeader("Cache-Control"); if (it != cacheHeaders.rawHeaders.constEnd()) { QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second); if (cacheControl.contains("must-revalidate")) return false; } QDateTime currentDateTime = QDateTime::currentDateTime(); QDateTime expirationDate = metaData.expirationDate(); #if 0 /* * age_value * is the value of Age: header received by the cache with * this response. * date_value * is the value of the origin server's Date: header * request_time * is the (local) time when the cache made the request * that resulted in this cached response * response_time * is the (local) time when the cache received the * response * now * is the current (local) time */ int age_value = 0; it = cacheHeaders.findRawHeader("age"); if (it != cacheHeaders.rawHeaders.constEnd()) age_value = it->second.toInt(); QDateTime dateHeader; int date_value = 0; it = cacheHeaders.findRawHeader("date"); if (it != cacheHeaders.rawHeaders.constEnd()) { dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second); date_value = dateHeader.toTime_t(); } int now = currentDateTime.toUTC().toTime_t(); int request_time = now; int response_time = now; // Algorithm from RFC 2616 section 13.2.3 int apparent_age = qMax(0, response_time - date_value); int corrected_received_age = qMax(apparent_age, age_value); int response_delay = response_time - request_time; int corrected_initial_age = corrected_received_age + response_delay; int resident_time = now - response_time; int current_age = corrected_initial_age + resident_time; // RFC 2616 13.2.4 Expiration Calculations if (!expirationDate.isValid()) { if (lastModified.isValid()) { int diff = currentDateTime.secsTo(lastModified); expirationDate = lastModified; expirationDate.addSecs(diff / 10); if (httpRequest.headerField("Warning").isEmpty()) { QDateTime dt; dt.setTime_t(current_age); if (dt.daysTo(currentDateTime) > 1) httpRequest.setHeaderField("Warning", "113"); } } } // the cache-saving code below sets the expirationDate with date+max_age // if "max-age" is present, or to Expires otherwise int freshness_lifetime = dateHeader.secsTo(expirationDate); bool response_is_fresh = (freshness_lifetime > current_age); #else bool response_is_fresh = currentDateTime.secsTo(expirationDate) >= 0; #endif if (!response_is_fresh) return false; #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "response_is_fresh" << CacheLoadControlAttribute; #endif return sendCacheContents(metaData); }