/** \fn MythDownloadManager::GetLastModified(const QString &url)
* \brief Gets the Last Modified timestamp for a URI
* \param url URI to test.
* \return Timestamp the URI was last modified or now if an error occurred
*/
QDateTime MythDownloadManager::GetLastModified(const QString &url)
{
static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
VERBOSE(VB_FILE+VB_EXTRA, LOC + QString("GetLastModified('%1')").arg(url));
QDateTime result;
QDateTime now = QDateTime::currentDateTime();
QNetworkCacheMetaData urlData = m_manager->cache()->metaData(QUrl(url));
if (urlData.isValid())
{
if (urlData.lastModified().secsTo(now) <= 60)
{
result = urlData.lastModified();
}
else
{
// If the last modification date is older than 60 seconds, and
// we loaded the page over 60 seconds ago, then redownload the
// page to re-verify it's last modified date.
QNetworkCacheMetaData::RawHeaderList headers =
urlData.rawHeaders();
bool found = false;
QNetworkCacheMetaData::RawHeaderList::iterator it
= headers.begin();
for (; !found && it != headers.end(); ++it)
{
if ((*it).first == "Date")
{
found = true;
QDateTime loadDate =
QDateTime::fromString((*it).second, dateFormat);
loadDate.setTimeSpec(Qt::UTC);
if (loadDate.secsTo(now) <= 60)
result = urlData.lastModified();
}
}
}
}
if (!result.isValid())
{
MythDownloadInfo *dlInfo = new MythDownloadInfo;
dlInfo->m_url = url;
dlInfo->m_syncMode = true;
if (downloadNow(dlInfo, false) && dlInfo->m_reply)
{
QVariant lastMod =
dlInfo->m_reply->header(QNetworkRequest::LastModifiedHeader);
if (lastMod.isValid())
result = lastMod.toDateTime();
}
delete dlInfo;
}
return result;
}
void QNetworkCacheMetaDataPrivate::save(QDataStream &out, const QNetworkCacheMetaData &metaData)
{
// note: if you change the contents of the meta data here
// remember to bump the cache version in qnetworkdiskcache.cpp CurrentCacheVersion
out << metaData.url();
out << metaData.expirationDate();
out << metaData.lastModified();
out << metaData.saveToDisk();
out << metaData.attributes();
out << metaData.rawHeaders();
}
bool QNetworkAccessCacheBackend::sendCacheContents()
{
setCachingEnabled(false);
QAbstractNetworkCache *nc = networkCache();
if (!nc)
return false;
QNetworkCacheMetaData item = nc->metaData(url());
if (!item.isValid())
return false;
QNetworkCacheMetaData::AttributesMap attributes = item.attributes();
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, attributes.value(QNetworkRequest::HttpStatusCodeAttribute));
setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
// set the raw headers
QNetworkCacheMetaData::RawHeaderList rawHeaders = item.rawHeaders();
QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
end = rawHeaders.constEnd();
for ( ; it != end; ++it) {
if (it->first.toLower() == "cache-control" &&
it->second.toLower().contains("must-revalidate")) {
return false;
}
setRawHeader(it->first, it->second);
}
// handle a possible redirect
QVariant redirectionTarget = attributes.value(QNetworkRequest::RedirectionTargetAttribute);
if (redirectionTarget.isValid()) {
setAttribute(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
redirectionRequested(redirectionTarget.toUrl());
}
// signal we're open
metaDataChanged();
if (operation() == QNetworkAccessManager::GetOperation) {
QIODevice *contents = nc->data(url());
if (!contents)
return false;
contents->setParent(this);
writeDownstreamData(contents);
}
#if defined(QNETWORKACCESSCACHEBACKEND_DEBUG)
qDebug() << "Successfully sent cache:" << url();
#endif
return true;
}
//Times metadata as well payload lookup
// i.e metaData(), rawHeaders() and data()
void tst_qnetworkdiskcache::timeRead()
{
QFETCH(QString, cacheRootDirectory);
cacheDir = QString( cacheRootDirectory + QDir::separator() + "man_qndc");
QDir d;
qDebug() << "Setting cache directory to = " << d.absoluteFilePath(cacheDir);
//Housekeeping
cleanRecursive(cacheDir); // slow op.
initCacheObject();
cache->setCacheDirectory(cacheDir);
cache->setMaximumCacheSize(qint64(HugeCacheLimit));
cache->clear();
//populate some fake data to simulate partially full cache
injectFakeData();
//Entries in the cache should be > what we try to remove
QVERIFY(NumFakeCacheObjects > NumReadContent);
//time metadata lookup of previously inserted URL.
QBENCHMARK_ONCE {
for (quint32 i = 0; i < NumReadContent; i++) {
QString fakeURL;
QTextStream stream(&fakeURL);
stream << fakeURLbase << i;
QUrl url(fakeURL);
QNetworkCacheMetaData qndc = cache->metaData(url);
QVERIFY(qndc.isValid()); // we must have read the metadata
QNetworkCacheMetaData::RawHeaderList raw(qndc.rawHeaders());
QVERIFY(raw.size()); // we must have parsed the headers from the meta
QIODevice *iodevice(cache->data(url));
QVERIFY(iodevice); //must not be NULL
iodevice->close();
delete iodevice;
}
}
//Cleanup (slow)
cleanupCacheObject();
cleanRecursive(cacheDir);
}
void tst_QNetworkCacheMetaData::qnetworkcachemetadata()
{
QNetworkCacheMetaData data;
QCOMPARE(data.expirationDate(), QDateTime());
QCOMPARE(data.isValid(), false);
QCOMPARE(data.lastModified(), QDateTime());
QCOMPARE(data.operator!=(QNetworkCacheMetaData()), false);
QNetworkCacheMetaData metaData;
QCOMPARE(data.operator=(metaData), QNetworkCacheMetaData());
QCOMPARE(data.operator==(QNetworkCacheMetaData()), true);
QCOMPARE(data.rawHeaders(), QNetworkCacheMetaData::RawHeaderList());
QCOMPARE(data.saveToDisk(), true);
QCOMPARE(data.url(), QUrl());
data.setExpirationDate(QDateTime());
data.setLastModified(QDateTime());
data.setRawHeaders(QNetworkCacheMetaData::RawHeaderList());
data.setSaveToDisk(false);
data.setUrl(QUrl());
}
/*
A simple web page that can be used to test us: http://www.procata.com/cachetest/
*/
bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData)
{
setCachingEnabled(false);
if (!metaData.isValid())
return false;
QAbstractNetworkCache *nc = networkCache();
Q_ASSERT(nc);
QIODevice *contents = nc->data(url());
if (!contents) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Can not send cache, the contents are 0" << url();
#endif
return false;
}
contents->setParent(this);
QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (status < 100)
status = 200; // fake it
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status);
setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true);
QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders();
QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
end = rawHeaders.constEnd();
for ( ; it != end; ++it)
setRawHeader(it->first, it->second);
checkForRedirect(status);
writeDownstreamData(contents);
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes";
#endif
if (httpReply)
disconnect(httpReply, SIGNAL(finished()), this, SLOT(replyFinished()));
return true;
}
void QgsWFSRequest::replyFinished()
{
if ( !mIsAborted && mReply )
{
if ( mReply->error() == QNetworkReply::NoError )
{
QgsDebugMsg( "reply ok" );
QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
if ( !redirect.isNull() )
{
QgsDebugMsg( "Request redirected." );
const QUrl& toUrl = redirect.toUrl();
mReply->request();
if ( toUrl == mReply->url() )
{
mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
mResponse.clear();
}
else
{
QNetworkRequest request( toUrl );
if ( !mUri.auth().setAuthorization( request ) )
{
mResponse.clear();
mErrorMessage = errorMessageFailedAuth();
mErrorCode = QgsWFSRequest::NetworkError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
emit downloadFinished();
return;
}
request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
mReply->deleteLater();
mReply = nullptr;
QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) );
mReply = QgsNetworkAccessManager::instance()->get( request );
connect( mReply, SIGNAL( finished() ), this, SLOT( replyFinished() ) );
connect( mReply, SIGNAL( downloadProgress( qint64, qint64 ) ), this, SLOT( replyProgress( qint64, qint64 ) ) );
return;
}
}
else
{
const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();
if ( nam->cache() )
{
QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );
QNetworkCacheMetaData::RawHeaderList hl;
Q_FOREACH ( const QNetworkCacheMetaData::RawHeader &h, cmd.rawHeaders() )
{
if ( h.first != "Cache-Control" )
hl.append( h );
}
cmd.setRawHeaders( hl );
QgsDebugMsg( QString( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ) );
if ( cmd.expirationDate().isNull() )
{
cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) );
}
nam->cache()->updateMetaData( cmd );
}
else
{
QgsDebugMsg( "No cache!" );
}
#ifdef QGISDEBUG
bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
QgsDebugMsg( QString( "Reply was cached: %1" ).arg( fromCache ) );
#endif
mResponse = mReply->readAll();
if ( mResponse.isEmpty() && !mGotNonEmptyResponse )
{
mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
mErrorCode = QgsWFSRequest::ServerExceptionError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
}
}
int main(int argc, char **argv)
{
QCoreApplication application(argc, argv);
QCoreApplication::setOrganizationDomain(QLatin1String("sites.google.com/site/zeromusparadoxe01"));
QCoreApplication::setApplicationName(QLatin1String("zBrowser"));
QStringList args = application.arguments();
args.takeFirst();
if (args.isEmpty()) {
QTextStream stream(stdout);
stream << "zbrowser-cacheinfo is a tool for viewing and extracting information out of zBrowser cache files." << endl;
stream << "zbrowser-cacheinfo [-o cachefile] [file | url]" << endl;
return 0;
}
NetworkDiskCache diskCache;
QString location = QDesktopServices::storageLocation(QDesktopServices::CacheLocation)
+ QLatin1String("/browser/");
diskCache.setCacheDirectory(location);
QNetworkCacheMetaData metaData;
QString last = args.takeLast();
if (QFile::exists(last)) {
qDebug() << "Reading in from a file and not a URL.";
metaData = diskCache._fileMetaData(last);
} else {
qDebug() << "Reading in from a URL and not a file.";
metaData = diskCache.metaData(last);
}
if (!args.isEmpty()
&& args.count() >= 1
&& args.first() == QLatin1String("-o")) {
QUrl url = metaData.url();
QIODevice *device = diskCache.data(url);
if (!device) {
qDebug() << "Error: data for URL is 0!";
return 1;
}
QString fileName;
if (args.count() == 2) {
fileName = args.last();
} else {
QFileInfo info(url.path());
fileName = info.fileName();
if (fileName.isEmpty()) {
qDebug() << "URL file name is empty, please specify an output file, I wont guess.";
return 1;
}
if (QFile::exists(fileName)) {
qDebug() << "File already exists, not overwriting, please specify an output file.";
return 1;
}
}
qDebug() << "Saved cache file to:" << fileName;
QFile file(fileName);
if (!file.open(QFile::ReadWrite))
qDebug() << "Unable to open the output file for writing.";
else
file.write(device->readAll());
delete device;
}
QTextStream stream(stdout);
stream << "URL: " << metaData.url().toString() << endl;
stream << "Expiration Date: " << metaData.expirationDate().toString() << endl;
stream << "Last Modified Date: " << metaData.lastModified().toString() << endl;
stream << "Save to disk: " << metaData.saveToDisk() << endl;
stream << "Headers:" << endl;
foreach (const QNetworkCacheMetaData::RawHeader &header, metaData.rawHeaders())
stream << "\t" << header.first << ": " << header.second << endl;
QIODevice *device = diskCache.data(metaData.url());
if (device) {
stream << "Data Size: " << device->size() << endl;
stream << "First line: " << device->readLine(100);
} else {
stream << "No data? Either the file is corrupt or there is an error." << endl;
}
delete device;
return 0;
}
QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const
{
QNetworkCacheMetaData metaData = oldMetaData;
QNetworkHeadersPrivate cacheHeaders;
cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
QList<QByteArray> newHeaders = rawHeaderList();
foreach (QByteArray header, newHeaders) {
QByteArray originalHeader = header;
header = header.toLower();
bool hop_by_hop =
(header == "connection"
|| header == "keep-alive"
|| header == "proxy-authenticate"
|| header == "proxy-authorization"
|| header == "te"
|| header == "trailers"
|| header == "transfer-encoding"
|| header == "upgrade");
if (hop_by_hop)
continue;
// Don't store Warning 1xx headers
if (header == "warning") {
QByteArray v = rawHeader(header);
if (v.length() == 3
&& v[0] == '1'
&& v[1] >= '0' && v[1] <= '9'
&& v[2] >= '0' && v[2] <= '9')
continue;
}
it = cacheHeaders.findRawHeader(header);
if (it != cacheHeaders.rawHeaders.constEnd()) {
// Match the behavior of Firefox and assume Cache-Control: "no-transform"
if (header == "content-encoding"
|| header == "content-range"
|| header == "content-type")
continue;
// For MS servers that send "Content-Length: 0" on 304 responses
// ignore this too
if (header == "content-length")
continue;
}
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
QByteArray n = rawHeader(header);
QByteArray o;
if (it != cacheHeaders.rawHeaders.constEnd())
o = (*it).second;
if (n != o && header != "date") {
qDebug() << "replacing" << header;
qDebug() << "new" << n;
qDebug() << "old" << o;
}
#endif
cacheHeaders.setRawHeader(originalHeader, rawHeader(header));
}
void QNetworkAccessHttpBackend::replyHeaderChanged()
{
// reconstruct the HTTP header
QList<QPair<QByteArray, QByteArray> > headerMap = httpReply->header();
QList<QPair<QByteArray, QByteArray> >::ConstIterator it = headerMap.constBegin(),
end = headerMap.constEnd();
QByteArray header;
for (; it != end; ++it) {
QByteArray value = rawHeader(it->first);
if (!value.isEmpty()) {
if (it->first.toLower() == "set-cookie")
value += "\n";
else
value += ", ";
}
value += it->second;
setRawHeader(it->first, value);
}
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, httpReply->statusCode());
setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, httpReply->reasonPhrase());
// is it a redirection?
const int statusCode = httpReply->statusCode();
checkForRedirect(statusCode);
if (statusCode >= 500 && statusCode < 600) {
QAbstractNetworkCache *nc = networkCache();
if (nc) {
QNetworkCacheMetaData metaData = nc->metaData(url());
QNetworkHeadersPrivate cacheHeaders;
cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
it = cacheHeaders.findRawHeader("Cache-Control");
bool mustReValidate = false;
if (it != cacheHeaders.rawHeaders.constEnd()) {
QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
if (cacheControl.contains("must-revalidate"))
mustReValidate = true;
}
if (!mustReValidate && sendCacheContents(metaData))
return;
}
}
if (statusCode == 304) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Received a 304 from" << url();
#endif
QAbstractNetworkCache *nc = networkCache();
if (nc) {
QNetworkCacheMetaData oldMetaData = nc->metaData(url());
QNetworkCacheMetaData metaData = fetchCacheMetaData(oldMetaData);
if (oldMetaData != metaData)
nc->updateMetaData(metaData);
if (sendCacheContents(metaData))
return;
}
}
if (statusCode != 304 && statusCode != 303) {
if (!isCachingEnabled())
setCachingEnabled(true);
}
metaDataChanged();
}
/*
For a given httpRequest
1) If AlwaysNetwork, return
2) If we have a cache entry for this url populate headers so the server can return 304
3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
*/
void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache)
{
QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
(QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
// forced reload from the network
// tell any caching proxy servers to reload too
httpRequest.setHeaderField("Cache-Control", "no-cache");
httpRequest.setHeaderField("Pragma", "no-cache");
return;
}
QAbstractNetworkCache *nc = networkCache();
if (!nc)
return; // no local cache
QNetworkCacheMetaData metaData = nc->metaData(url());
if (!metaData.isValid())
return; // not in cache
if (!metaData.saveToDisk())
return;
QNetworkHeadersPrivate cacheHeaders;
QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
it = cacheHeaders.findRawHeader("etag");
if (it != cacheHeaders.rawHeaders.constEnd())
httpRequest.setHeaderField("If-None-Match", it->second);
QDateTime lastModified = metaData.lastModified();
if (lastModified.isValid())
httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));
it = cacheHeaders.findRawHeader("Cache-Control");
if (it != cacheHeaders.rawHeaders.constEnd()) {
QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
if (cacheControl.contains("must-revalidate"))
return;
}
/*
* age_value
* is the value of Age: header received by the cache with
* this response.
* date_value
* is the value of the origin server's Date: header
* request_time
* is the (local) time when the cache made the request
* that resulted in this cached response
* response_time
* is the (local) time when the cache received the
* response
* now
* is the current (local) time
*/
QDateTime currentDateTime = QDateTime::currentDateTime();
int age_value = 0;
it = cacheHeaders.findRawHeader("age");
if (it != cacheHeaders.rawHeaders.constEnd())
age_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t();
int date_value = 0;
it = cacheHeaders.findRawHeader("date");
if (it != cacheHeaders.rawHeaders.constEnd())
date_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t();
int now = currentDateTime.toUTC().toTime_t();
int request_time = now;
int response_time = now;
int apparent_age = qMax(0, response_time - date_value);
int corrected_received_age = qMax(apparent_age, age_value);
int response_delay = response_time - request_time;
int corrected_initial_age = corrected_received_age + response_delay;
int resident_time = now - response_time;
int current_age = corrected_initial_age + resident_time;
// RFC 2616 13.2.4 Expiration Calculations
QDateTime expirationDate = metaData.expirationDate();
if (!expirationDate.isValid()) {
if (lastModified.isValid()) {
int diff = currentDateTime.secsTo(lastModified);
expirationDate = lastModified;
expirationDate.addSecs(diff / 10);
if (httpRequest.headerField("Warning").isEmpty()) {
QDateTime dt;
dt.setTime_t(current_age);
if (dt.daysTo(currentDateTime) > 1)
httpRequest.setHeaderField("Warning", "113");
}
}
}
int freshness_lifetime = currentDateTime.secsTo(expirationDate);
bool response_is_fresh = (freshness_lifetime > current_age);
if (!response_is_fresh && CacheLoadControlAttribute == QNetworkRequest::PreferNetwork)
return;
loadedFromCache = true;
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
if (!sendCacheContents(metaData))
loadedFromCache = false;
}
/** \fn MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo)
* \brief Downloads a QNetworkRequest via the QNetworkAccessManager
* \param dlInfo MythDownloadInfo information for download
*/
void MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo)
{
static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
QUrl qurl(dlInfo->m_url);
QNetworkRequest request;
if (dlInfo->m_request)
{
request = *dlInfo->m_request;
delete dlInfo->m_request;
dlInfo->m_request = NULL;
}
else
request.setUrl(qurl);
if (!dlInfo->m_reload)
{
// Prefer the in-cache item if one exists and it is less than 60
// seconds old and has not expired in the last 10 seconds.
QDateTime now = QDateTime::currentDateTime();
QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
if ((urlData.isValid()) &&
((!urlData.expirationDate().isValid()) ||
(urlData.expirationDate().secsTo(now) < 10)))
{
QNetworkCacheMetaData::RawHeaderList headers =
urlData.rawHeaders();
bool found = false;
QNetworkCacheMetaData::RawHeaderList::iterator it
= headers.begin();
for (; !found && it != headers.end(); ++it)
{
if ((*it).first == "Date")
{
found = true;
QDateTime loadDate =
QDateTime::fromString((*it).second, dateFormat);
loadDate.setTimeSpec(Qt::UTC);
if (loadDate.secsTo(now) < 60)
dlInfo->m_preferCache = true;
}
}
}
}
if (dlInfo->m_preferCache)
request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
QNetworkRequest::PreferCache);
request.setRawHeader("User-Agent",
"MythDownloadManager v" MYTH_BINARY_VERSION);
if (dlInfo->m_post)
dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
else
dlInfo->m_reply = m_manager->get(request);
m_downloadReplies[dlInfo->m_reply] = dlInfo;
connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
SLOT(downloadError(QNetworkReply::NetworkError)));
connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
this, SLOT(downloadProgress(qint64, qint64)));
}
void QgsWfsRequest::replyFinished()
{
if ( !mIsAborted && mReply )
{
if ( mReply->error() == QNetworkReply::NoError )
{
QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 4 );
QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
if ( !redirect.isNull() )
{
QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 4 );
const QUrl &toUrl = redirect.toUrl();
mReply->request();
if ( toUrl == mReply->url() )
{
mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
mResponse.clear();
}
else
{
QNetworkRequest request( toUrl );
QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsWfsRequest" ) );
if ( !mUri.auth().setAuthorization( request ) )
{
mResponse.clear();
mErrorMessage = errorMessageFailedAuth();
mErrorCode = QgsWfsRequest::NetworkError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
emit downloadFinished();
return;
}
request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
mReply->deleteLater();
mReply = nullptr;
QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 4 );
mReply = QgsNetworkAccessManager::instance()->get( request );
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mUri.auth().setAuthorizationReply( mReply ) )
{
mResponse.clear();
mErrorMessage = errorMessageFailedAuth();
mErrorCode = QgsWfsRequest::NetworkError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
emit downloadFinished();
return;
}
connect( mReply, &QNetworkReply::finished, this, &QgsWfsRequest::replyFinished, Qt::DirectConnection );
connect( mReply, &QNetworkReply::downloadProgress, this, &QgsWfsRequest::replyProgress, Qt::DirectConnection );
return;
}
}
else
{
const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();
if ( nam->cache() )
{
QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );
QNetworkCacheMetaData::RawHeaderList hl;
const auto constRawHeaders = cmd.rawHeaders();
for ( const QNetworkCacheMetaData::RawHeader &h : constRawHeaders )
{
if ( h.first != "Cache-Control" )
hl.append( h );
}
cmd.setRawHeaders( hl );
QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 4 );
if ( cmd.expirationDate().isNull() )
{
cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) );
}
nam->cache()->updateMetaData( cmd );
}
else
{
QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 4 );
}
#ifdef QGISDEBUG
bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 4 );
#endif
mResponse = mReply->readAll();
if ( mResponse.isEmpty() && !mGotNonEmptyResponse )
{
mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
mErrorCode = QgsWfsRequest::ServerExceptionError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
}
}
}
else
{
mErrorMessage = errorMessageWithReason( mReply->errorString() );
mErrorCode = QgsWfsRequest::ServerExceptionError;
QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
mResponse.clear();
}
}
if ( mTimedout )
mErrorCode = QgsWfsRequest::TimeoutError;
if ( mReply )
{
mReply->deleteLater();
mReply = nullptr;
}
emit downloadFinished();
}
/*
For a given httpRequest
1) If AlwaysNetwork, return
2) If we have a cache entry for this url populate headers so the server can return 304
3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
*/
bool QNetworkAccessHttpBackend::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest)
{
QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
(QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
// If the request does not already specify preferred cache-control
// force reload from the network and tell any caching proxy servers to reload too
if (!request().rawHeaderList().contains("Cache-Control")) {
httpRequest.setHeaderField("Cache-Control", "no-cache");
httpRequest.setHeaderField("Pragma", "no-cache");
}
return false;
}
// The disk cache API does not currently support partial content retrieval.
// That is why we don't use the disk cache for any such requests.
if (request().hasRawHeader("Range"))
return false;
QAbstractNetworkCache *nc = networkCache();
if (!nc)
return false; // no local cache
QNetworkCacheMetaData metaData = nc->metaData(url());
if (!metaData.isValid())
return false; // not in cache
if (!metaData.saveToDisk())
return false;
QNetworkHeadersPrivate cacheHeaders;
QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
it = cacheHeaders.findRawHeader("etag");
if (it != cacheHeaders.rawHeaders.constEnd())
httpRequest.setHeaderField("If-None-Match", it->second);
QDateTime lastModified = metaData.lastModified();
if (lastModified.isValid())
httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));
it = cacheHeaders.findRawHeader("Cache-Control");
if (it != cacheHeaders.rawHeaders.constEnd()) {
QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
if (cacheControl.contains("must-revalidate"))
return false;
}
QDateTime currentDateTime = QDateTime::currentDateTime();
QDateTime expirationDate = metaData.expirationDate();
#if 0
/*
* age_value
* is the value of Age: header received by the cache with
* this response.
* date_value
* is the value of the origin server's Date: header
* request_time
* is the (local) time when the cache made the request
* that resulted in this cached response
* response_time
* is the (local) time when the cache received the
* response
* now
* is the current (local) time
*/
int age_value = 0;
it = cacheHeaders.findRawHeader("age");
if (it != cacheHeaders.rawHeaders.constEnd())
age_value = it->second.toInt();
QDateTime dateHeader;
int date_value = 0;
it = cacheHeaders.findRawHeader("date");
if (it != cacheHeaders.rawHeaders.constEnd()) {
dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second);
date_value = dateHeader.toTime_t();
}
int now = currentDateTime.toUTC().toTime_t();
int request_time = now;
int response_time = now;
// Algorithm from RFC 2616 section 13.2.3
int apparent_age = qMax(0, response_time - date_value);
int corrected_received_age = qMax(apparent_age, age_value);
int response_delay = response_time - request_time;
int corrected_initial_age = corrected_received_age + response_delay;
int resident_time = now - response_time;
int current_age = corrected_initial_age + resident_time;
// RFC 2616 13.2.4 Expiration Calculations
if (!expirationDate.isValid()) {
if (lastModified.isValid()) {
int diff = currentDateTime.secsTo(lastModified);
expirationDate = lastModified;
expirationDate.addSecs(diff / 10);
if (httpRequest.headerField("Warning").isEmpty()) {
QDateTime dt;
dt.setTime_t(current_age);
if (dt.daysTo(currentDateTime) > 1)
httpRequest.setHeaderField("Warning", "113");
}
}
}
// the cache-saving code below sets the expirationDate with date+max_age
// if "max-age" is present, or to Expires otherwise
int freshness_lifetime = dateHeader.secsTo(expirationDate);
bool response_is_fresh = (freshness_lifetime > current_age);
#else
bool response_is_fresh = currentDateTime.secsTo(expirationDate) >= 0;
#endif
if (!response_is_fresh)
return false;
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
return sendCacheContents(metaData);
}
