void QNetworkReplyImplPrivate::setCachingEnabled(bool enable)
{
Q_Q(QNetworkReplyImpl);
if (!enable && !cacheEnabled)
return; // nothing to do
if (enable && cacheEnabled)
return; // nothing to do either!
if (enable) {
if (bytesDownloaded) {
// refuse to enable in this case
qCritical("QNetworkReplyImpl: backend error: caching was enabled after some bytes had been written");
return;
}
createCache();
} else {
// someone told us to turn on, then back off?
// ok... but you should make up your mind
qDebug("QNetworkReplyImpl: setCachingEnabled(true) called after setCachingEnabled(false) -- "
"backend %s probably needs to be fixed",
backend->metaObject()->className());
networkCache()->remove(url);
cacheSaveDevice = 0;
cacheEnabled = false;
QObject::disconnect(networkCache(), SIGNAL(destroyed()), q, SLOT(_q_cacheDestroyed()));
}
}
void QNetworkReplyImplPrivate::initCacheSaveDevice()
{
Q_Q(QNetworkReplyImpl);
// save the meta data
QNetworkCacheMetaData metaData;
metaData.setUrl(url);
metaData = backend->fetchCacheMetaData(metaData);
// save the redirect request also in the cache
QVariant redirectionTarget = q->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (redirectionTarget.isValid()) {
QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
attributes.insert(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
metaData.setAttributes(attributes);
}
cacheSaveDevice = networkCache()->prepare(metaData);
if (!cacheSaveDevice || (cacheSaveDevice && !cacheSaveDevice->isOpen())) {
if (cacheSaveDevice && !cacheSaveDevice->isOpen())
qCritical("QNetworkReplyImpl: network cache returned a device that is not open -- "
"class %s probably needs to be fixed",
networkCache()->metaObject()->className());
networkCache()->remove(url);
cacheSaveDevice = 0;
cacheEnabled = false;
}
}
void QNetworkReplyImplPrivate::completeCacheSave()
{
if (cacheEnabled && errorCode != QNetworkReplyImpl::NoError) {
networkCache()->remove(url);
} else if (cacheEnabled && cacheSaveDevice) {
networkCache()->insert(cacheSaveDevice);
}
cacheSaveDevice = 0;
cacheEnabled = false;
}
void QNetworkReplyImplPrivate::completeCacheSave()
{
Q_Q(QNetworkReplyImpl);
if (cacheEnabled) {
if (errorCode != QNetworkReplyImpl::NoError) {
networkCache()->remove(url);
} else if (cacheSaveDevice) {
networkCache()->insert(cacheSaveDevice);
}
QObject::disconnect(networkCache(), SIGNAL(destroyed()), q, SLOT(_q_cacheDestroyed()));
}
cacheSaveDevice = 0;
cacheEnabled = false;
}
void QNetworkReplyImplPrivate::createCache()
{
// check if we can save and if we're allowed to
if (!networkCache()
|| !request.attribute(QNetworkRequest::CacheSaveControlAttribute, true).toBool())
return;
cacheEnabled = true;
}
void QNetworkReplyImplPrivate::initCacheSaveDevice()
{
Q_Q(QNetworkReplyImpl);
// The disk cache does not support partial content, so don't even try to
// save any such content into the cache.
if (q->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 206) {
cacheEnabled = false;
return;
}
// save the meta data
QNetworkCacheMetaData metaData;
metaData.setUrl(url);
metaData = backend->fetchCacheMetaData(metaData);
// save the redirect request also in the cache
QVariant redirectionTarget = q->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (redirectionTarget.isValid()) {
QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
attributes.insert(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
metaData.setAttributes(attributes);
}
cacheSaveDevice = networkCache()->prepare(metaData);
if (!cacheSaveDevice || (cacheSaveDevice && !cacheSaveDevice->isOpen())) {
if (cacheSaveDevice && !cacheSaveDevice->isOpen())
qCritical("QNetworkReplyImpl: network cache returned a device that is not open -- "
"class %s probably needs to be fixed",
networkCache()->metaObject()->className());
networkCache()->remove(url);
cacheSaveDevice = 0;
cacheEnabled = false;
} else {
q->connect(networkCache(), SIGNAL(destroyed()), SLOT(_q_cacheDestroyed()));
}
}
bool QNetworkAccessCacheBackend::sendCacheContents()
{
setCachingEnabled(false);
QAbstractNetworkCache *nc = networkCache();
if (!nc)
return false;
QNetworkCacheMetaData item = nc->metaData(url());
if (!item.isValid())
return false;
QNetworkCacheMetaData::AttributesMap attributes = item.attributes();
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, attributes.value(QNetworkRequest::HttpStatusCodeAttribute));
setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
// set the raw headers
QNetworkCacheMetaData::RawHeaderList rawHeaders = item.rawHeaders();
QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
end = rawHeaders.constEnd();
for ( ; it != end; ++it) {
if (it->first.toLower() == "cache-control" &&
it->second.toLower().contains("must-revalidate")) {
return false;
}
setRawHeader(it->first, it->second);
}
// handle a possible redirect
QVariant redirectionTarget = attributes.value(QNetworkRequest::RedirectionTargetAttribute);
if (redirectionTarget.isValid()) {
setAttribute(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
redirectionRequested(redirectionTarget.toUrl());
}
// signal we're open
metaDataChanged();
if (operation() == QNetworkAccessManager::GetOperation) {
QIODevice *contents = nc->data(url());
if (!contents)
return false;
contents->setParent(this);
writeDownstreamData(contents);
}
#if defined(QNETWORKACCESSCACHEBACKEND_DEBUG)
qDebug() << "Successfully sent cache:" << url();
#endif
return true;
}
/*
A simple web page that can be used to test us: http://www.procata.com/cachetest/
*/
bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData)
{
setCachingEnabled(false);
if (!metaData.isValid())
return false;
QAbstractNetworkCache *nc = networkCache();
Q_ASSERT(nc);
QIODevice *contents = nc->data(url());
if (!contents) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Can not send cache, the contents are 0" << url();
#endif
return false;
}
contents->setParent(this);
QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (status < 100)
status = 200; // fake it
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status);
setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true);
QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders();
QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
end = rawHeaders.constEnd();
for ( ; it != end; ++it)
setRawHeader(it->first, it->second);
checkForRedirect(status);
writeDownstreamData(contents);
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes";
#endif
if (httpReply)
disconnect(httpReply, SIGNAL(finished()), this, SLOT(replyFinished()));
return true;
}
bool QNetworkReplyImplPrivate::isCachingEnabled() const
{
return (cacheEnabled && networkCache() != 0);
}
// we received downstream data and send this to the cache
// and to our readBuffer (which in turn gets read by the user of QNetworkReply)
void QNetworkReplyImplPrivate::appendDownstreamData(QByteDataBuffer &data)
{
Q_Q(QNetworkReplyImpl);
if (!q->isOpen())
return;
if (cacheEnabled && !cacheSaveDevice) {
// save the meta data
QNetworkCacheMetaData metaData;
metaData.setUrl(url);
metaData = backend->fetchCacheMetaData(metaData);
// save the redirect request also in the cache
QVariant redirectionTarget = q->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (redirectionTarget.isValid()) {
QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
attributes.insert(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
metaData.setAttributes(attributes);
}
cacheSaveDevice = networkCache()->prepare(metaData);
if (!cacheSaveDevice || (cacheSaveDevice && !cacheSaveDevice->isOpen())) {
if (cacheSaveDevice && !cacheSaveDevice->isOpen())
qCritical("QNetworkReplyImpl: network cache returned a device that is not open -- "
"class %s probably needs to be fixed",
networkCache()->metaObject()->className());
networkCache()->remove(url);
cacheSaveDevice = 0;
cacheEnabled = false;
}
}
qint64 bytesWritten = 0;
for (int i = 0; i < data.bufferCount(); i++) {
QByteArray item = data[i];
if (cacheSaveDevice)
cacheSaveDevice->write(item.constData(), item.size());
readBuffer.append(item);
bytesWritten += item.size();
}
data.clear();
bytesDownloaded += bytesWritten;
lastBytesDownloaded = bytesDownloaded;
QPointer<QNetworkReplyImpl> qq = q;
QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader);
if (preMigrationDownloaded != Q_INT64_C(-1))
totalSize = totalSize.toLongLong() + preMigrationDownloaded;
pauseNotificationHandling();
emit q->downloadProgress(bytesDownloaded,
totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong());
// important: At the point of this readyRead(), the data parameter list must be empty,
// else implicit sharing will trigger memcpy when the user is reading data!
emit q->readyRead();
// hopefully we haven't been deleted here
if (!qq.isNull()) {
resumeNotificationHandling();
// do we still have room in the buffer?
if (nextDownstreamBlockSize() > 0)
backendNotify(QNetworkReplyImplPrivate::NotifyDownstreamReadyWrite);
}
}
void QNetworkAccessHttpBackend::replyHeaderChanged()
{
// reconstruct the HTTP header
QList<QPair<QByteArray, QByteArray> > headerMap = httpReply->header();
QList<QPair<QByteArray, QByteArray> >::ConstIterator it = headerMap.constBegin(),
end = headerMap.constEnd();
QByteArray header;
for (; it != end; ++it) {
QByteArray value = rawHeader(it->first);
if (!value.isEmpty()) {
if (it->first.toLower() == "set-cookie")
value += "\n";
else
value += ", ";
}
value += it->second;
setRawHeader(it->first, value);
}
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, httpReply->statusCode());
setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, httpReply->reasonPhrase());
// is it a redirection?
const int statusCode = httpReply->statusCode();
checkForRedirect(statusCode);
if (statusCode >= 500 && statusCode < 600) {
QAbstractNetworkCache *nc = networkCache();
if (nc) {
QNetworkCacheMetaData metaData = nc->metaData(url());
QNetworkHeadersPrivate cacheHeaders;
cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
it = cacheHeaders.findRawHeader("Cache-Control");
bool mustReValidate = false;
if (it != cacheHeaders.rawHeaders.constEnd()) {
QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
if (cacheControl.contains("must-revalidate"))
mustReValidate = true;
}
if (!mustReValidate && sendCacheContents(metaData))
return;
}
}
if (statusCode == 304) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Received a 304 from" << url();
#endif
QAbstractNetworkCache *nc = networkCache();
if (nc) {
QNetworkCacheMetaData oldMetaData = nc->metaData(url());
QNetworkCacheMetaData metaData = fetchCacheMetaData(oldMetaData);
if (oldMetaData != metaData)
nc->updateMetaData(metaData);
if (sendCacheContents(metaData))
return;
}
}
if (statusCode != 304 && statusCode != 303) {
if (!isCachingEnabled())
setCachingEnabled(true);
}
metaDataChanged();
}
void QNetworkAccessHttpBackend::invalidateCache()
{
QAbstractNetworkCache *nc = networkCache();
if (nc)
nc->remove(url());
}
/*
For a given httpRequest
1) If AlwaysNetwork, return
2) If we have a cache entry for this url populate headers so the server can return 304
3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
*/
void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache)
{
QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
(QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
// forced reload from the network
// tell any caching proxy servers to reload too
httpRequest.setHeaderField("Cache-Control", "no-cache");
httpRequest.setHeaderField("Pragma", "no-cache");
return;
}
QAbstractNetworkCache *nc = networkCache();
if (!nc)
return; // no local cache
QNetworkCacheMetaData metaData = nc->metaData(url());
if (!metaData.isValid())
return; // not in cache
if (!metaData.saveToDisk())
return;
QNetworkHeadersPrivate cacheHeaders;
QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
it = cacheHeaders.findRawHeader("etag");
if (it != cacheHeaders.rawHeaders.constEnd())
httpRequest.setHeaderField("If-None-Match", it->second);
QDateTime lastModified = metaData.lastModified();
if (lastModified.isValid())
httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));
it = cacheHeaders.findRawHeader("Cache-Control");
if (it != cacheHeaders.rawHeaders.constEnd()) {
QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
if (cacheControl.contains("must-revalidate"))
return;
}
/*
* age_value
* is the value of Age: header received by the cache with
* this response.
* date_value
* is the value of the origin server's Date: header
* request_time
* is the (local) time when the cache made the request
* that resulted in this cached response
* response_time
* is the (local) time when the cache received the
* response
* now
* is the current (local) time
*/
QDateTime currentDateTime = QDateTime::currentDateTime();
int age_value = 0;
it = cacheHeaders.findRawHeader("age");
if (it != cacheHeaders.rawHeaders.constEnd())
age_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t();
int date_value = 0;
it = cacheHeaders.findRawHeader("date");
if (it != cacheHeaders.rawHeaders.constEnd())
date_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t();
int now = currentDateTime.toUTC().toTime_t();
int request_time = now;
int response_time = now;
int apparent_age = qMax(0, response_time - date_value);
int corrected_received_age = qMax(apparent_age, age_value);
int response_delay = response_time - request_time;
int corrected_initial_age = corrected_received_age + response_delay;
int resident_time = now - response_time;
int current_age = corrected_initial_age + resident_time;
// RFC 2616 13.2.4 Expiration Calculations
QDateTime expirationDate = metaData.expirationDate();
if (!expirationDate.isValid()) {
if (lastModified.isValid()) {
int diff = currentDateTime.secsTo(lastModified);
expirationDate = lastModified;
expirationDate.addSecs(diff / 10);
if (httpRequest.headerField("Warning").isEmpty()) {
QDateTime dt;
dt.setTime_t(current_age);
if (dt.daysTo(currentDateTime) > 1)
httpRequest.setHeaderField("Warning", "113");
}
}
}
int freshness_lifetime = currentDateTime.secsTo(expirationDate);
bool response_is_fresh = (freshness_lifetime > current_age);
if (!response_is_fresh && CacheLoadControlAttribute == QNetworkRequest::PreferNetwork)
return;
loadedFromCache = true;
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
if (!sendCacheContents(metaData))
loadedFromCache = false;
}
/*
For a given httpRequest
1) If AlwaysNetwork, return
2) If we have a cache entry for this url populate headers so the server can return 304
3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
*/
bool QNetworkAccessHttpBackend::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest)
{
QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
(QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
// If the request does not already specify preferred cache-control
// force reload from the network and tell any caching proxy servers to reload too
if (!request().rawHeaderList().contains("Cache-Control")) {
httpRequest.setHeaderField("Cache-Control", "no-cache");
httpRequest.setHeaderField("Pragma", "no-cache");
}
return false;
}
// The disk cache API does not currently support partial content retrieval.
// That is why we don't use the disk cache for any such requests.
if (request().hasRawHeader("Range"))
return false;
QAbstractNetworkCache *nc = networkCache();
if (!nc)
return false; // no local cache
QNetworkCacheMetaData metaData = nc->metaData(url());
if (!metaData.isValid())
return false; // not in cache
if (!metaData.saveToDisk())
return false;
QNetworkHeadersPrivate cacheHeaders;
QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
it = cacheHeaders.findRawHeader("etag");
if (it != cacheHeaders.rawHeaders.constEnd())
httpRequest.setHeaderField("If-None-Match", it->second);
QDateTime lastModified = metaData.lastModified();
if (lastModified.isValid())
httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));
it = cacheHeaders.findRawHeader("Cache-Control");
if (it != cacheHeaders.rawHeaders.constEnd()) {
QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
if (cacheControl.contains("must-revalidate"))
return false;
}
QDateTime currentDateTime = QDateTime::currentDateTime();
QDateTime expirationDate = metaData.expirationDate();
#if 0
/*
* age_value
* is the value of Age: header received by the cache with
* this response.
* date_value
* is the value of the origin server's Date: header
* request_time
* is the (local) time when the cache made the request
* that resulted in this cached response
* response_time
* is the (local) time when the cache received the
* response
* now
* is the current (local) time
*/
int age_value = 0;
it = cacheHeaders.findRawHeader("age");
if (it != cacheHeaders.rawHeaders.constEnd())
age_value = it->second.toInt();
QDateTime dateHeader;
int date_value = 0;
it = cacheHeaders.findRawHeader("date");
if (it != cacheHeaders.rawHeaders.constEnd()) {
dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second);
date_value = dateHeader.toTime_t();
}
int now = currentDateTime.toUTC().toTime_t();
int request_time = now;
int response_time = now;
// Algorithm from RFC 2616 section 13.2.3
int apparent_age = qMax(0, response_time - date_value);
int corrected_received_age = qMax(apparent_age, age_value);
int response_delay = response_time - request_time;
int corrected_initial_age = corrected_received_age + response_delay;
int resident_time = now - response_time;
int current_age = corrected_initial_age + resident_time;
// RFC 2616 13.2.4 Expiration Calculations
if (!expirationDate.isValid()) {
if (lastModified.isValid()) {
int diff = currentDateTime.secsTo(lastModified);
expirationDate = lastModified;
expirationDate.addSecs(diff / 10);
if (httpRequest.headerField("Warning").isEmpty()) {
QDateTime dt;
dt.setTime_t(current_age);
if (dt.daysTo(currentDateTime) > 1)
httpRequest.setHeaderField("Warning", "113");
}
}
}
// the cache-saving code below sets the expirationDate with date+max_age
// if "max-age" is present, or to Expires otherwise
int freshness_lifetime = dateHeader.secsTo(expirationDate);
bool response_is_fresh = (freshness_lifetime > current_age);
#else
bool response_is_fresh = currentDateTime.secsTo(expirationDate) >= 0;
#endif
if (!response_is_fresh)
return false;
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
return sendCacheContents(metaData);
}
