Beispiel #1
0
/** \fn MythDownloadManager::GetLastModified(const QString &url)
 *  \brief Gets the Last Modified timestamp for a URI
 *  \param url    URI to test.
 *  \return Timestamp the URI was last modified or now if an error occurred
 */
QDateTime MythDownloadManager::GetLastModified(const QString &url)
{
    static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
    VERBOSE(VB_FILE+VB_EXTRA, LOC + QString("GetLastModified('%1')").arg(url));
    QDateTime result;

    QDateTime now = QDateTime::currentDateTime();
    QNetworkCacheMetaData urlData = m_manager->cache()->metaData(QUrl(url));

    if (urlData.isValid())
    {
        if (urlData.lastModified().secsTo(now) <= 60)
        {
            result = urlData.lastModified();
        }
        else
        {
            // If the last modification date is older than 60 seconds, and
            // we loaded the page over 60 seconds ago, then redownload the
            // page to re-verify it's last modified date.
            QNetworkCacheMetaData::RawHeaderList headers =
                urlData.rawHeaders();
            bool found = false;
            QNetworkCacheMetaData::RawHeaderList::iterator it
                = headers.begin();
            for (; !found && it != headers.end(); ++it)
            {
                if ((*it).first == "Date")
                {
                    found = true;
                    QDateTime loadDate =
                       QDateTime::fromString((*it).second, dateFormat);
                    loadDate.setTimeSpec(Qt::UTC);
                    if (loadDate.secsTo(now) <= 60)
                        result = urlData.lastModified();
                }
            }
        }
    }

    if (!result.isValid())
    {
        MythDownloadInfo *dlInfo = new MythDownloadInfo;
        dlInfo->m_url      = url;
        dlInfo->m_syncMode = true;

        if (downloadNow(dlInfo, false) && dlInfo->m_reply)
        {
            QVariant lastMod =
                dlInfo->m_reply->header(QNetworkRequest::LastModifiedHeader);
            if (lastMod.isValid())
                result = lastMod.toDateTime();
        }

        delete dlInfo;
    }

    return result;
}
void QNetworkCacheMetaDataPrivate::save(QDataStream &out, const QNetworkCacheMetaData &metaData)
{
    // note: if you change the contents of the meta data here
    // remember to bump the cache version in qnetworkdiskcache.cpp CurrentCacheVersion
    out << metaData.url();
    out << metaData.expirationDate();
    out << metaData.lastModified();
    out << metaData.saveToDisk();
    out << metaData.attributes();
    out << metaData.rawHeaders();
}
bool QNetworkAccessCacheBackend::sendCacheContents()
{
    setCachingEnabled(false);
    QAbstractNetworkCache *nc = networkCache();
    if (!nc)
        return false;

    QNetworkCacheMetaData item = nc->metaData(url());
    if (!item.isValid())
        return false;

    QNetworkCacheMetaData::AttributesMap attributes = item.attributes();
    setAttribute(QNetworkRequest::HttpStatusCodeAttribute, attributes.value(QNetworkRequest::HttpStatusCodeAttribute));
    setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));

    // set the raw headers
    QNetworkCacheMetaData::RawHeaderList rawHeaders = item.rawHeaders();
    QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
                                                       end = rawHeaders.constEnd();
    for ( ; it != end; ++it) {
        if (it->first.toLower() == "cache-control" &&
            it->second.toLower().contains("must-revalidate")) {
            return false;
        }
        setRawHeader(it->first, it->second);
    }

    // handle a possible redirect
    QVariant redirectionTarget = attributes.value(QNetworkRequest::RedirectionTargetAttribute);
    if (redirectionTarget.isValid()) {
        setAttribute(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
        redirectionRequested(redirectionTarget.toUrl());
    }

    // signal we're open
    metaDataChanged();

    if (operation() == QNetworkAccessManager::GetOperation) {
        QIODevice *contents = nc->data(url());
        if (!contents)
            return false;
        contents->setParent(this);
        writeDownstreamData(contents);
    }

#if defined(QNETWORKACCESSCACHEBACKEND_DEBUG)
    qDebug() << "Successfully sent cache:" << url();
#endif
    return true;
}
//Times metadata as well payload lookup
// i.e metaData(), rawHeaders() and data()
void tst_qnetworkdiskcache::timeRead()
{

    QFETCH(QString, cacheRootDirectory);

    cacheDir = QString( cacheRootDirectory + QDir::separator() + "man_qndc");
    QDir d;
    qDebug() << "Setting cache directory to = " << d.absoluteFilePath(cacheDir);

    //Housekeeping
    cleanRecursive(cacheDir); // slow op.
    initCacheObject();
    cache->setCacheDirectory(cacheDir);
    cache->setMaximumCacheSize(qint64(HugeCacheLimit));
    cache->clear();

    //populate some fake data to simulate partially full cache
    injectFakeData();

    //Entries in the cache should be > what we try to remove
    QVERIFY(NumFakeCacheObjects > NumReadContent);

    //time metadata lookup of previously inserted URL.
    QBENCHMARK_ONCE {
        for (quint32 i = 0; i < NumReadContent; i++) {
            QString fakeURL;
            QTextStream stream(&fakeURL);
            stream << fakeURLbase << i;
            QUrl url(fakeURL);

            QNetworkCacheMetaData qndc = cache->metaData(url);
            QVERIFY(qndc.isValid()); // we must have read the metadata

            QNetworkCacheMetaData::RawHeaderList raw(qndc.rawHeaders());
            QVERIFY(raw.size()); // we must have parsed the headers from the meta

            QIODevice *iodevice(cache->data(url));
            QVERIFY(iodevice);    //must not be NULL
            iodevice->close();
            delete iodevice;
        }
    }

    //Cleanup (slow)
    cleanupCacheObject();
    cleanRecursive(cacheDir);

}
Beispiel #5
0
void tst_QNetworkCacheMetaData::qnetworkcachemetadata()
{
    QNetworkCacheMetaData data;
    QCOMPARE(data.expirationDate(), QDateTime());
    QCOMPARE(data.isValid(), false);
    QCOMPARE(data.lastModified(), QDateTime());
    QCOMPARE(data.operator!=(QNetworkCacheMetaData()), false);
    QNetworkCacheMetaData metaData;
    QCOMPARE(data.operator=(metaData), QNetworkCacheMetaData());
    QCOMPARE(data.operator==(QNetworkCacheMetaData()), true);
    QCOMPARE(data.rawHeaders(), QNetworkCacheMetaData::RawHeaderList());
    QCOMPARE(data.saveToDisk(), true);
    QCOMPARE(data.url(), QUrl());
    data.setExpirationDate(QDateTime());
    data.setLastModified(QDateTime());
    data.setRawHeaders(QNetworkCacheMetaData::RawHeaderList());
    data.setSaveToDisk(false);
    data.setUrl(QUrl());
}
Beispiel #6
0
/*
    A simple web page that can be used to test us: http://www.procata.com/cachetest/
 */
bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData)
{
    setCachingEnabled(false);
    if (!metaData.isValid())
        return false;

    QAbstractNetworkCache *nc = networkCache();
    Q_ASSERT(nc);
    QIODevice *contents = nc->data(url());
    if (!contents) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
        qDebug() << "Can not send cache, the contents are 0" << url();
#endif
        return false;
    }
    contents->setParent(this);

    QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
    int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt();
    if (status < 100)
        status = 200;           // fake it

    setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status);
    setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
    setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true);

    QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders();
    QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
                                                       end = rawHeaders.constEnd();
    for ( ; it != end; ++it)
        setRawHeader(it->first, it->second);

    checkForRedirect(status);

    writeDownstreamData(contents);
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
    qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes";
#endif
    if (httpReply)
        disconnect(httpReply, SIGNAL(finished()), this, SLOT(replyFinished()));
    return true;
}
Beispiel #7
0
void QgsWFSRequest::replyFinished()
{
    if ( !mIsAborted && mReply )
    {
        if ( mReply->error() == QNetworkReply::NoError )
        {
            QgsDebugMsg( "reply ok" );
            QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
            if ( !redirect.isNull() )
            {
                QgsDebugMsg( "Request redirected." );

                const QUrl& toUrl = redirect.toUrl();
                mReply->request();
                if ( toUrl == mReply->url() )
                {
                    mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
                    QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
                    mResponse.clear();
                }
                else
                {
                    QNetworkRequest request( toUrl );
                    if ( !mUri.auth().setAuthorization( request ) )
                    {
                        mResponse.clear();
                        mErrorMessage = errorMessageFailedAuth();
                        mErrorCode = QgsWFSRequest::NetworkError;
                        QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
                        emit downloadFinished();
                        return;
                    }
                    request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
                    request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );

                    mReply->deleteLater();
                    mReply = nullptr;

                    QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) );
                    mReply = QgsNetworkAccessManager::instance()->get( request );
                    connect( mReply, SIGNAL( finished() ), this, SLOT( replyFinished() ) );
                    connect( mReply, SIGNAL( downloadProgress( qint64, qint64 ) ), this, SLOT( replyProgress( qint64, qint64 ) ) );
                    return;
                }
            }
            else
            {
                const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();

                if ( nam->cache() )
                {
                    QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );

                    QNetworkCacheMetaData::RawHeaderList hl;
                    Q_FOREACH ( const QNetworkCacheMetaData::RawHeader &h, cmd.rawHeaders() )
                    {
                        if ( h.first != "Cache-Control" )
                            hl.append( h );
                    }
                    cmd.setRawHeaders( hl );

                    QgsDebugMsg( QString( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ) );
                    if ( cmd.expirationDate().isNull() )
                    {
                        cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) );
                    }

                    nam->cache()->updateMetaData( cmd );
                }
                else
                {
                    QgsDebugMsg( "No cache!" );
                }

#ifdef QGISDEBUG
                bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
                QgsDebugMsg( QString( "Reply was cached: %1" ).arg( fromCache ) );
#endif

                mResponse = mReply->readAll();

                if ( mResponse.isEmpty() && !mGotNonEmptyResponse )
                {
                    mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
                    mErrorCode = QgsWFSRequest::ServerExceptionError;
                    QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
                }
            }
Beispiel #8
0
int main(int argc, char **argv)
{
    QCoreApplication application(argc, argv);
    QCoreApplication::setOrganizationDomain(QLatin1String("sites.google.com/site/zeromusparadoxe01"));
    QCoreApplication::setApplicationName(QLatin1String("zBrowser"));

    QStringList args = application.arguments();
    args.takeFirst();
    if (args.isEmpty()) {
        QTextStream stream(stdout);
        stream << "zbrowser-cacheinfo is a tool for viewing and extracting information out of zBrowser cache files." << endl;
        stream << "zbrowser-cacheinfo [-o cachefile] [file | url]" << endl;
        return 0;
    }

    NetworkDiskCache diskCache;
    QString location = QDesktopServices::storageLocation(QDesktopServices::CacheLocation)
            + QLatin1String("/browser/");
    diskCache.setCacheDirectory(location);

    QNetworkCacheMetaData metaData;
    QString last = args.takeLast();
    if (QFile::exists(last)) {
        qDebug() << "Reading in from a file and not a URL.";
        metaData = diskCache._fileMetaData(last);
    } else {
        qDebug() << "Reading in from a URL and not a file.";
        metaData = diskCache.metaData(last);
    }

    if (!args.isEmpty()
        && args.count() >= 1
        && args.first() == QLatin1String("-o")) {
        QUrl url = metaData.url();
        QIODevice *device = diskCache.data(url);
        if (!device) {
            qDebug() << "Error: data for URL is 0!";
            return 1;
        }
        QString fileName;
        if (args.count() == 2) {
            fileName = args.last();
        } else {
            QFileInfo info(url.path());
            fileName = info.fileName();
            if (fileName.isEmpty()) {
                qDebug() << "URL file name is empty, please specify an output file, I wont guess.";
                return 1;
            }
            if (QFile::exists(fileName)) {
                qDebug() << "File already exists, not overwriting, please specify an output file.";
                return 1;
            }
        }
        qDebug() << "Saved cache file to:" << fileName;
        QFile file(fileName);
        if (!file.open(QFile::ReadWrite))
            qDebug() << "Unable to open the output file for writing.";
        else
            file.write(device->readAll());
        delete device;
    }

    QTextStream stream(stdout);
    stream << "URL: " << metaData.url().toString() << endl;
    stream << "Expiration Date: " << metaData.expirationDate().toString() << endl;
    stream << "Last Modified Date: " << metaData.lastModified().toString() << endl;
    stream << "Save to disk: " << metaData.saveToDisk() << endl;
    stream << "Headers:" << endl;
    foreach (const QNetworkCacheMetaData::RawHeader &header, metaData.rawHeaders())
        stream << "\t" << header.first << ": " << header.second << endl;
    QIODevice *device = diskCache.data(metaData.url());
    if (device) {
        stream << "Data Size: " << device->size() << endl;
        stream << "First line: " << device->readLine(100);
    } else {
        stream << "No data? Either the file is corrupt or there is an error." << endl;
    }

    delete device;
    return 0;
}
Beispiel #9
0
QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const
{
    QNetworkCacheMetaData metaData = oldMetaData;

    QNetworkHeadersPrivate cacheHeaders;
    cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
    QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;

    QList<QByteArray> newHeaders = rawHeaderList();
    foreach (QByteArray header, newHeaders) {
        QByteArray originalHeader = header;
        header = header.toLower();
        bool hop_by_hop =
            (header == "connection"
             || header == "keep-alive"
             || header == "proxy-authenticate"
             || header == "proxy-authorization"
             || header == "te"
             || header == "trailers"
             || header == "transfer-encoding"
             || header ==  "upgrade");
        if (hop_by_hop)
            continue;

        // Don't store Warning 1xx headers
        if (header == "warning") {
            QByteArray v = rawHeader(header);
            if (v.length() == 3
                && v[0] == '1'
                && v[1] >= '0' && v[1] <= '9'
                && v[2] >= '0' && v[2] <= '9')
                continue;
        }

        it = cacheHeaders.findRawHeader(header);
        if (it != cacheHeaders.rawHeaders.constEnd()) {
            // Match the behavior of Firefox and assume Cache-Control: "no-transform"
            if (header == "content-encoding"
                || header == "content-range"
                || header == "content-type")
                continue;

            // For MS servers that send "Content-Length: 0" on 304 responses
            // ignore this too
            if (header == "content-length")
                continue;
        }

#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
        QByteArray n = rawHeader(header);
        QByteArray o;
        if (it != cacheHeaders.rawHeaders.constEnd())
            o = (*it).second;
        if (n != o && header != "date") {
            qDebug() << "replacing" << header;
            qDebug() << "new" << n;
            qDebug() << "old" << o;
        }
#endif
        cacheHeaders.setRawHeader(originalHeader, rawHeader(header));
    }
Beispiel #10
0
void QNetworkAccessHttpBackend::replyHeaderChanged()
{
    // reconstruct the HTTP header
    QList<QPair<QByteArray, QByteArray> > headerMap = httpReply->header();
    QList<QPair<QByteArray, QByteArray> >::ConstIterator it = headerMap.constBegin(),
                                                        end = headerMap.constEnd();
    QByteArray header;

    for (; it != end; ++it) {
        QByteArray value = rawHeader(it->first);
        if (!value.isEmpty()) {
            if (it->first.toLower() == "set-cookie")
                value += "\n";
            else
                value += ", ";
        }
        value += it->second;
        setRawHeader(it->first, value);
    }

    setAttribute(QNetworkRequest::HttpStatusCodeAttribute, httpReply->statusCode());
    setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, httpReply->reasonPhrase());

    // is it a redirection?
    const int statusCode = httpReply->statusCode();
    checkForRedirect(statusCode);

    if (statusCode >= 500 && statusCode < 600) {
        QAbstractNetworkCache *nc = networkCache();
        if (nc) {
            QNetworkCacheMetaData metaData = nc->metaData(url());
            QNetworkHeadersPrivate cacheHeaders;
            cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
            QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
            it = cacheHeaders.findRawHeader("Cache-Control");
            bool mustReValidate = false;
            if (it != cacheHeaders.rawHeaders.constEnd()) {
                QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
                if (cacheControl.contains("must-revalidate"))
                    mustReValidate = true;
            }
            if (!mustReValidate && sendCacheContents(metaData))
                return;
        }
    }

    if (statusCode == 304) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
        qDebug() << "Received a 304 from" << url();
#endif
        QAbstractNetworkCache *nc = networkCache();
        if (nc) {
            QNetworkCacheMetaData oldMetaData = nc->metaData(url());
            QNetworkCacheMetaData metaData = fetchCacheMetaData(oldMetaData);
            if (oldMetaData != metaData)
                nc->updateMetaData(metaData);
            if (sendCacheContents(metaData))
                return;
        }
    }


    if (statusCode != 304 && statusCode != 303) {
        if (!isCachingEnabled())
            setCachingEnabled(true);
    }
    metaDataChanged();
}
Beispiel #11
0
/*
    For a given httpRequest
    1) If AlwaysNetwork, return
    2) If we have a cache entry for this url populate headers so the server can return 304
    3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
 */
void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache)
{
    QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
        (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
    if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
        // forced reload from the network
        // tell any caching proxy servers to reload too
        httpRequest.setHeaderField("Cache-Control", "no-cache");
        httpRequest.setHeaderField("Pragma", "no-cache");
        return;
    }

    QAbstractNetworkCache *nc = networkCache();
    if (!nc)
        return;                 // no local cache

    QNetworkCacheMetaData metaData = nc->metaData(url());
    if (!metaData.isValid())
        return;                 // not in cache

    if (!metaData.saveToDisk())
        return;

    QNetworkHeadersPrivate cacheHeaders;
    QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
    cacheHeaders.setAllRawHeaders(metaData.rawHeaders());

    it = cacheHeaders.findRawHeader("etag");
    if (it != cacheHeaders.rawHeaders.constEnd())
        httpRequest.setHeaderField("If-None-Match", it->second);

    QDateTime lastModified = metaData.lastModified();
    if (lastModified.isValid())
        httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));

    it = cacheHeaders.findRawHeader("Cache-Control");
    if (it != cacheHeaders.rawHeaders.constEnd()) {
        QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
        if (cacheControl.contains("must-revalidate"))
            return;
    }

    /*
     * age_value
     *      is the value of Age: header received by the cache with
     *              this response.
     * date_value
     *      is the value of the origin server's Date: header
     * request_time
     *      is the (local) time when the cache made the request
     *              that resulted in this cached response
     * response_time
     *      is the (local) time when the cache received the
     *              response
     * now
     *      is the current (local) time
     */
    QDateTime currentDateTime = QDateTime::currentDateTime();
    int age_value = 0;
    it = cacheHeaders.findRawHeader("age");
    if (it != cacheHeaders.rawHeaders.constEnd())
        age_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t();

    int date_value = 0;
    it = cacheHeaders.findRawHeader("date");
    if (it != cacheHeaders.rawHeaders.constEnd())
        date_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t();

    int now = currentDateTime.toUTC().toTime_t();
    int request_time = now;
    int response_time = now;

    int apparent_age = qMax(0, response_time - date_value);
    int corrected_received_age = qMax(apparent_age, age_value);
    int response_delay = response_time - request_time;
    int corrected_initial_age = corrected_received_age + response_delay;
    int resident_time = now - response_time;
    int current_age   = corrected_initial_age + resident_time;

    // RFC 2616 13.2.4 Expiration Calculations
    QDateTime expirationDate = metaData.expirationDate();
    if (!expirationDate.isValid()) {
        if (lastModified.isValid()) {
            int diff = currentDateTime.secsTo(lastModified);
            expirationDate = lastModified;
            expirationDate.addSecs(diff / 10);
            if (httpRequest.headerField("Warning").isEmpty()) {
                QDateTime dt;
                dt.setTime_t(current_age);
                if (dt.daysTo(currentDateTime) > 1)
                    httpRequest.setHeaderField("Warning", "113");
            }
        }
    }

    int freshness_lifetime = currentDateTime.secsTo(expirationDate);
    bool response_is_fresh = (freshness_lifetime > current_age);

    if (!response_is_fresh && CacheLoadControlAttribute == QNetworkRequest::PreferNetwork)
        return;

    loadedFromCache = true;
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
    qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
    if (!sendCacheContents(metaData))
        loadedFromCache = false;
}
Beispiel #12
0
/** \fn MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo)
 *  \brief Downloads a QNetworkRequest via the QNetworkAccessManager
 *  \param dlInfo   MythDownloadInfo information for download
 */
void MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo)
{
    static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
    QUrl qurl(dlInfo->m_url);
    QNetworkRequest request;
    
    if (dlInfo->m_request)
    {
        request = *dlInfo->m_request;
        delete dlInfo->m_request;
        dlInfo->m_request = NULL;
    }
    else
        request.setUrl(qurl);

    if (!dlInfo->m_reload)
    {
        // Prefer the in-cache item if one exists and it is less than 60
        // seconds old and has not expired in the last 10 seconds.
        QDateTime now = QDateTime::currentDateTime();
        QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
        if ((urlData.isValid()) &&
            ((!urlData.expirationDate().isValid()) ||
             (urlData.expirationDate().secsTo(now) < 10)))
        {
            QNetworkCacheMetaData::RawHeaderList headers =
                urlData.rawHeaders();
            bool found = false;
            QNetworkCacheMetaData::RawHeaderList::iterator it
                = headers.begin();
            for (; !found && it != headers.end(); ++it)
            {
                if ((*it).first == "Date")
                {
                    found = true;
                    QDateTime loadDate =
                       QDateTime::fromString((*it).second, dateFormat);
                    loadDate.setTimeSpec(Qt::UTC);
                    if (loadDate.secsTo(now) < 60)
                        dlInfo->m_preferCache = true;
                }
            }
        }
    }

    if (dlInfo->m_preferCache)
        request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
                             QNetworkRequest::PreferCache);

    request.setRawHeader("User-Agent",
                         "MythDownloadManager v" MYTH_BINARY_VERSION);

    if (dlInfo->m_post)
        dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
    else
        dlInfo->m_reply = m_manager->get(request);

    m_downloadReplies[dlInfo->m_reply] = dlInfo;

    connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
            SLOT(downloadError(QNetworkReply::NetworkError)));
    connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
            this, SLOT(downloadProgress(qint64, qint64))); 
}
Beispiel #13
0
void QgsWfsRequest::replyFinished()
{
  if ( !mIsAborted && mReply )
  {
    if ( mReply->error() == QNetworkReply::NoError )
    {
      QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 4 );
      QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
      if ( !redirect.isNull() )
      {
        QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 4 );

        const QUrl &toUrl = redirect.toUrl();
        mReply->request();
        if ( toUrl == mReply->url() )
        {
          mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
          QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
          mResponse.clear();
        }
        else
        {
          QNetworkRequest request( toUrl );
          QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsWfsRequest" ) );
          if ( !mUri.auth().setAuthorization( request ) )
          {
            mResponse.clear();
            mErrorMessage = errorMessageFailedAuth();
            mErrorCode = QgsWfsRequest::NetworkError;
            QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
            emit downloadFinished();
            return;
          }
          request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
          request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );

          mReply->deleteLater();
          mReply = nullptr;

          QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 4 );
          mReply = QgsNetworkAccessManager::instance()->get( request );
          mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
          if ( !mUri.auth().setAuthorizationReply( mReply ) )
          {
            mResponse.clear();
            mErrorMessage = errorMessageFailedAuth();
            mErrorCode = QgsWfsRequest::NetworkError;
            QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
            emit downloadFinished();
            return;
          }
          connect( mReply, &QNetworkReply::finished, this, &QgsWfsRequest::replyFinished, Qt::DirectConnection );
          connect( mReply, &QNetworkReply::downloadProgress, this, &QgsWfsRequest::replyProgress, Qt::DirectConnection );
          return;
        }
      }
      else
      {
        const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();

        if ( nam->cache() )
        {
          QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );

          QNetworkCacheMetaData::RawHeaderList hl;
          const auto constRawHeaders = cmd.rawHeaders();
          for ( const QNetworkCacheMetaData::RawHeader &h : constRawHeaders )
          {
            if ( h.first != "Cache-Control" )
              hl.append( h );
          }
          cmd.setRawHeaders( hl );

          QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 4 );
          if ( cmd.expirationDate().isNull() )
          {
            cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) );
          }

          nam->cache()->updateMetaData( cmd );
        }
        else
        {
          QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 4 );
        }

#ifdef QGISDEBUG
        bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
        QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 4 );
#endif

        mResponse = mReply->readAll();

        if ( mResponse.isEmpty() && !mGotNonEmptyResponse )
        {
          mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
          mErrorCode = QgsWfsRequest::ServerExceptionError;
          QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
        }
      }
    }
    else
    {
      mErrorMessage = errorMessageWithReason( mReply->errorString() );
      mErrorCode = QgsWfsRequest::ServerExceptionError;
      QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
      mResponse.clear();
    }
  }
  if ( mTimedout )
    mErrorCode = QgsWfsRequest::TimeoutError;

  if ( mReply )
  {
    mReply->deleteLater();
    mReply = nullptr;
  }

  emit downloadFinished();
}
Beispiel #14
0
/*
    For a given httpRequest
    1) If AlwaysNetwork, return
    2) If we have a cache entry for this url populate headers so the server can return 304
    3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
 */
bool QNetworkAccessHttpBackend::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest)
{
    QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
        (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
    if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
        // If the request does not already specify preferred cache-control
        // force reload from the network and tell any caching proxy servers to reload too
        if (!request().rawHeaderList().contains("Cache-Control")) {
            httpRequest.setHeaderField("Cache-Control", "no-cache");
            httpRequest.setHeaderField("Pragma", "no-cache");
        }
        return false;
    }

    // The disk cache API does not currently support partial content retrieval.
    // That is why we don't use the disk cache for any such requests.
    if (request().hasRawHeader("Range"))
        return false;

    QAbstractNetworkCache *nc = networkCache();
    if (!nc)
        return false;                 // no local cache

    QNetworkCacheMetaData metaData = nc->metaData(url());
    if (!metaData.isValid())
        return false;                 // not in cache

    if (!metaData.saveToDisk())
        return false;

    QNetworkHeadersPrivate cacheHeaders;
    QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
    cacheHeaders.setAllRawHeaders(metaData.rawHeaders());

    it = cacheHeaders.findRawHeader("etag");
    if (it != cacheHeaders.rawHeaders.constEnd())
        httpRequest.setHeaderField("If-None-Match", it->second);

    QDateTime lastModified = metaData.lastModified();
    if (lastModified.isValid())
        httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));

    it = cacheHeaders.findRawHeader("Cache-Control");
    if (it != cacheHeaders.rawHeaders.constEnd()) {
        QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
        if (cacheControl.contains("must-revalidate"))
            return false;
    }

    QDateTime currentDateTime = QDateTime::currentDateTime();
    QDateTime expirationDate = metaData.expirationDate();

#if 0
    /*
     * age_value
     *      is the value of Age: header received by the cache with
     *              this response.
     * date_value
     *      is the value of the origin server's Date: header
     * request_time
     *      is the (local) time when the cache made the request
     *              that resulted in this cached response
     * response_time
     *      is the (local) time when the cache received the
     *              response
     * now
     *      is the current (local) time
     */
    int age_value = 0;
    it = cacheHeaders.findRawHeader("age");
    if (it != cacheHeaders.rawHeaders.constEnd())
        age_value = it->second.toInt();

    QDateTime dateHeader;
    int date_value = 0;
    it = cacheHeaders.findRawHeader("date");
    if (it != cacheHeaders.rawHeaders.constEnd()) {
        dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second);
        date_value = dateHeader.toTime_t();
    }

    int now = currentDateTime.toUTC().toTime_t();
    int request_time = now;
    int response_time = now;

    // Algorithm from RFC 2616 section 13.2.3
    int apparent_age = qMax(0, response_time - date_value);
    int corrected_received_age = qMax(apparent_age, age_value);
    int response_delay = response_time - request_time;
    int corrected_initial_age = corrected_received_age + response_delay;
    int resident_time = now - response_time;
    int current_age   = corrected_initial_age + resident_time;

    // RFC 2616 13.2.4 Expiration Calculations
    if (!expirationDate.isValid()) {
        if (lastModified.isValid()) {
            int diff = currentDateTime.secsTo(lastModified);
            expirationDate = lastModified;
            expirationDate.addSecs(diff / 10);
            if (httpRequest.headerField("Warning").isEmpty()) {
                QDateTime dt;
                dt.setTime_t(current_age);
                if (dt.daysTo(currentDateTime) > 1)
                    httpRequest.setHeaderField("Warning", "113");
            }
        }
    }

    // the cache-saving code below sets the expirationDate with date+max_age
    // if "max-age" is present, or to Expires otherwise
    int freshness_lifetime = dateHeader.secsTo(expirationDate);
    bool response_is_fresh = (freshness_lifetime > current_age);
#else
    bool response_is_fresh = currentDateTime.secsTo(expirationDate) >= 0;
#endif

    if (!response_is_fresh)
        return false;

#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
    qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
    return sendCacheContents(metaData);
}