示例#1
0
void tst_QNetworkCacheMetaData::isValid_data()
{
    QTest::addColumn<QNetworkCacheMetaData>("data");
    QTest::addColumn<bool>("isValid");

    QNetworkCacheMetaData metaData;
    QTest::newRow("null") << metaData << false;

    QNetworkCacheMetaData data1;
    data1.setUrl(QUrl(EXAMPLE_URL));
    QTest::newRow("valid-1") << data1 << true;

    QNetworkCacheMetaData data2;
    QNetworkCacheMetaData::RawHeaderList headers;
    headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
    data2.setRawHeaders(headers);
    QTest::newRow("valid-2") << data2 << true;

    QNetworkCacheMetaData data3;
    data3.setLastModified(QDateTime::currentDateTime());
    QTest::newRow("valid-3") << data3 << true;

    QNetworkCacheMetaData data4;
    data4.setExpirationDate(QDateTime::currentDateTime());
    QTest::newRow("valid-4") << data4 << true;

    QNetworkCacheMetaData data5;
    data5.setSaveToDisk(false);
    QTest::newRow("valid-5") << data5 << true;
}
// This function simulates a partially or fully occupied disk cache
// like a normal user of a cache might encounter is real-life browsing.
// The point of this is to trigger degradation in file-system and media performance
// that occur due to the quantity and layout of data.
void tst_qnetworkdiskcache::injectFakeData()
{

    QNetworkCacheMetaData::RawHeaderList headers;
    headers.append(qMakePair(QByteArray("X-TestHeader"),QByteArray("HeaderValue")));


    //Prep cache dir with fake data using QNetworkDiskCache APIs
    for (quint32 i = 0; i < NumFakeCacheObjects; i++) {

        //prepare metata for url
        QNetworkCacheMetaData meta;
        QString fakeURL;
        QTextStream stream(&fakeURL);
        stream << fakeURLbase << i;
        QUrl url(fakeURL);
        meta.setUrl(url);
        meta.setRawHeaders(headers);
        meta.setSaveToDisk(true);

        //commit payload and metadata to disk
        QIODevice *device = cache->prepare(meta);
        device->write(payload);
        cache->insert(device);
    }

}
示例#3
0
QIODevice *TabCache::prepare(const QNetworkCacheMetaData &metaData) {
    QLOG_DEBUG() << "TabCache::prepare";
    QNetworkCacheMetaData local{metaData};

    //Default policy based on received HTTP headers is to not save to disk.
    //Override here, and set proper expiration so items are put in cache
    //(setSaveToDisk) and valid when retrieved using metaData call -
    //(setExpirationDate)

    local.setSaveToDisk(true);

    // Need to set some reasonable length of time in which our cache entries
    // will expire.  It's possible we'll want to allow users to customize this.
    local.setExpirationDate(QDateTime().currentDateTime().addDays(kCacheExpireInDays));

    QNetworkCacheMetaData::RawHeaderList headers;

    for (auto &header: local.rawHeaders()) {
        // Modify Cache-Control headers - basically need to drop 'no-store'
        // as we want to store to cache and 'must-revalidate' as we don't have
        // ETag or last modified headers available to attempt re-validation. To
        // be on the safe side though just drop Cache-Control and Pragma headers.

        if (header.first == "Cache-Control") continue;
        if (header.first == "Pragma") continue;

        headers.push_back(header);
    }
    local.setRawHeaders(headers);

    return QNetworkDiskCache::prepare(local);
}
示例#4
0
/** \fn MythDownloadManager::GetLastModified(const QString &url)
 *  \brief Gets the Last Modified timestamp for a URI
 *  \param url    URI to test.
 *  \return Timestamp the URI was last modified or now if an error occurred
 */
QDateTime MythDownloadManager::GetLastModified(const QString &url)
{
    static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
    VERBOSE(VB_FILE+VB_EXTRA, LOC + QString("GetLastModified('%1')").arg(url));
    QDateTime result;

    QDateTime now = QDateTime::currentDateTime();
    QNetworkCacheMetaData urlData = m_manager->cache()->metaData(QUrl(url));

    if (urlData.isValid())
    {
        if (urlData.lastModified().secsTo(now) <= 60)
        {
            result = urlData.lastModified();
        }
        else
        {
            // If the last modification date is older than 60 seconds, and
            // we loaded the page over 60 seconds ago, then redownload the
            // page to re-verify it's last modified date.
            QNetworkCacheMetaData::RawHeaderList headers =
                urlData.rawHeaders();
            bool found = false;
            QNetworkCacheMetaData::RawHeaderList::iterator it
                = headers.begin();
            for (; !found && it != headers.end(); ++it)
            {
                if ((*it).first == "Date")
                {
                    found = true;
                    QDateTime loadDate =
                       QDateTime::fromString((*it).second, dateFormat);
                    loadDate.setTimeSpec(Qt::UTC);
                    if (loadDate.secsTo(now) <= 60)
                        result = urlData.lastModified();
                }
            }
        }
    }

    if (!result.isValid())
    {
        MythDownloadInfo *dlInfo = new MythDownloadInfo;
        dlInfo->m_url      = url;
        dlInfo->m_syncMode = true;

        if (downloadNow(dlInfo, false) && dlInfo->m_reply)
        {
            QVariant lastMod =
                dlInfo->m_reply->header(QNetworkRequest::LastModifiedHeader);
            if (lastMod.isValid())
                result = lastMod.toDateTime();
        }

        delete dlInfo;
    }

    return result;
}
示例#5
0
void tst_QNetworkCacheMetaData::rawHeaders_data()
{
    QTest::addColumn<QNetworkCacheMetaData::RawHeaderList>("rawHeaders");
    QTest::newRow("null") << QNetworkCacheMetaData::RawHeaderList();
    QNetworkCacheMetaData::RawHeaderList headers;
    headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
    QTest::newRow("valie") << headers;
}
bool QNetworkAccessCacheBackend::sendCacheContents()
{
    setCachingEnabled(false);
    QAbstractNetworkCache *nc = networkCache();
    if (!nc)
        return false;

    QNetworkCacheMetaData item = nc->metaData(url());
    if (!item.isValid())
        return false;

    QNetworkCacheMetaData::AttributesMap attributes = item.attributes();
    setAttribute(QNetworkRequest::HttpStatusCodeAttribute, attributes.value(QNetworkRequest::HttpStatusCodeAttribute));
    setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));

    // set the raw headers
    QNetworkCacheMetaData::RawHeaderList rawHeaders = item.rawHeaders();
    QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
                                                       end = rawHeaders.constEnd();
    for ( ; it != end; ++it) {
        if (it->first.toLower() == "cache-control" &&
            it->second.toLower().contains("must-revalidate")) {
            return false;
        }
        setRawHeader(it->first, it->second);
    }

    // handle a possible redirect
    QVariant redirectionTarget = attributes.value(QNetworkRequest::RedirectionTargetAttribute);
    if (redirectionTarget.isValid()) {
        setAttribute(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
        redirectionRequested(redirectionTarget.toUrl());
    }

    // signal we're open
    metaDataChanged();

    if (operation() == QNetworkAccessManager::GetOperation) {
        QIODevice *contents = nc->data(url());
        if (!contents)
            return false;
        contents->setParent(this);
        writeDownstreamData(contents);
    }

#if defined(QNETWORKACCESSCACHEBACKEND_DEBUG)
    qDebug() << "Successfully sent cache:" << url();
#endif
    return true;
}
示例#7
0
void tst_QNetworkCacheMetaData::operatorEqual_data()
{
    QTest::addColumn<QNetworkCacheMetaData>("other");
    QTest::newRow("null") << QNetworkCacheMetaData();

    QNetworkCacheMetaData data;
    data.setUrl(QUrl(EXAMPLE_URL));
    QNetworkCacheMetaData::RawHeaderList headers;
    headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
    data.setRawHeaders(headers);
    data.setLastModified(QDateTime::currentDateTime());
    data.setExpirationDate(QDateTime::currentDateTime());
    data.setSaveToDisk(false);
    QTest::newRow("valid") << data;
}
示例#8
0
void tst_QNetworkCacheMetaData::operatorEqualEqual_data()
{
    QTest::addColumn<QNetworkCacheMetaData>("a");
    QTest::addColumn<QNetworkCacheMetaData>("b");
    QTest::addColumn<bool>("operatorEqualEqual");
    QTest::newRow("null") << QNetworkCacheMetaData() << QNetworkCacheMetaData() << true;

    QNetworkCacheMetaData data1;
    data1.setUrl(QUrl(EXAMPLE_URL));
    QTest::newRow("valid-1-1") << data1 << QNetworkCacheMetaData() << false;
    QTest::newRow("valid-1-2") << data1 << data1 << true;

    QNetworkCacheMetaData data2;
    QNetworkCacheMetaData::RawHeaderList headers;
    headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
    data2.setRawHeaders(headers);
    QTest::newRow("valid-2-1") << data2 << QNetworkCacheMetaData() << false;
    QTest::newRow("valid-2-2") << data2 << data2 << true;
    QTest::newRow("valid-2-3") << data2 << data1 << false;

    QNetworkCacheMetaData data3;
    data3.setLastModified(QDateTime::currentDateTime());
    QTest::newRow("valid-3-1") << data3 << QNetworkCacheMetaData() << false;
    QTest::newRow("valid-3-2") << data3 << data3 << true;
    QTest::newRow("valid-3-3") << data3 << data1 << false;
    QTest::newRow("valid-3-4") << data3 << data2 << false;

    QNetworkCacheMetaData data4;
    data4.setExpirationDate(QDateTime::currentDateTime());
    QTest::newRow("valid-4-1") << data4 << QNetworkCacheMetaData() << false;
    QTest::newRow("valid-4-2") << data4 << data4 << true;
    QTest::newRow("valid-4-3") << data4 << data1 << false;
    QTest::newRow("valid-4-4") << data4 << data2 << false;
    QTest::newRow("valid-4-5") << data4 << data3 << false;

    QNetworkCacheMetaData data5;
    data5.setSaveToDisk(false);
    QTest::newRow("valid-5-1") << data5 << QNetworkCacheMetaData() << false;
    QTest::newRow("valid-5-2") << data5 << data5 << true;
    QTest::newRow("valid-5-3") << data5 << data1 << false;
    QTest::newRow("valid-5-4") << data5 << data2 << false;
    QTest::newRow("valid-5-5") << data5 << data3 << false;
    QTest::newRow("valid-5-6") << data5 << data4 << false;
}
示例#9
0
/*
    A simple web page that can be used to test us: http://www.procata.com/cachetest/
 */
bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData)
{
    setCachingEnabled(false);
    if (!metaData.isValid())
        return false;

    QAbstractNetworkCache *nc = networkCache();
    Q_ASSERT(nc);
    QIODevice *contents = nc->data(url());
    if (!contents) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
        qDebug() << "Can not send cache, the contents are 0" << url();
#endif
        return false;
    }
    contents->setParent(this);

    QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
    int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt();
    if (status < 100)
        status = 200;           // fake it

    setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status);
    setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
    setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true);

    QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders();
    QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
                                                       end = rawHeaders.constEnd();
    for ( ; it != end; ++it)
        setRawHeader(it->first, it->second);

    checkForRedirect(status);

    writeDownstreamData(contents);
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
    qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes";
#endif
    if (httpReply)
        disconnect(httpReply, SIGNAL(finished()), this, SLOT(replyFinished()));
    return true;
}
示例#10
0
void tst_QNetworkCacheMetaData::stream()
{
    QNetworkCacheMetaData data;
    data.setUrl(QUrl(EXAMPLE_URL));
    QNetworkCacheMetaData::RawHeaderList headers;
    headers.append(QNetworkCacheMetaData::RawHeader("foo", "Bar"));
    data.setRawHeaders(headers);
    data.setLastModified(QDateTime::currentDateTime());
    data.setExpirationDate(QDateTime::currentDateTime());
    data.setSaveToDisk(false);

    QBuffer buffer;
    buffer.open(QIODevice::ReadWrite);
    QDataStream stream(&buffer);
    stream << data;

    buffer.seek(0);
    QNetworkCacheMetaData data2;
    stream >> data2;
    QCOMPARE(data2, data);
}
示例#11
0
void QgsWFSRequest::replyFinished()
{
    if ( !mIsAborted && mReply )
    {
        if ( mReply->error() == QNetworkReply::NoError )
        {
            QgsDebugMsg( "reply ok" );
            QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
            if ( !redirect.isNull() )
            {
                QgsDebugMsg( "Request redirected." );

                const QUrl& toUrl = redirect.toUrl();
                mReply->request();
                if ( toUrl == mReply->url() )
                {
                    mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
                    QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
                    mResponse.clear();
                }
                else
                {
                    QNetworkRequest request( toUrl );
                    if ( !mUri.auth().setAuthorization( request ) )
                    {
                        mResponse.clear();
                        mErrorMessage = errorMessageFailedAuth();
                        mErrorCode = QgsWFSRequest::NetworkError;
                        QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
                        emit downloadFinished();
                        return;
                    }
                    request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
                    request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );

                    mReply->deleteLater();
                    mReply = nullptr;

                    QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) );
                    mReply = QgsNetworkAccessManager::instance()->get( request );
                    connect( mReply, SIGNAL( finished() ), this, SLOT( replyFinished() ) );
                    connect( mReply, SIGNAL( downloadProgress( qint64, qint64 ) ), this, SLOT( replyProgress( qint64, qint64 ) ) );
                    return;
                }
            }
            else
            {
                const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();

                if ( nam->cache() )
                {
                    QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );

                    QNetworkCacheMetaData::RawHeaderList hl;
                    Q_FOREACH ( const QNetworkCacheMetaData::RawHeader &h, cmd.rawHeaders() )
                    {
                        if ( h.first != "Cache-Control" )
                            hl.append( h );
                    }
                    cmd.setRawHeaders( hl );

                    QgsDebugMsg( QString( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ) );
                    if ( cmd.expirationDate().isNull() )
                    {
                        cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) );
                    }

                    nam->cache()->updateMetaData( cmd );
                }
                else
                {
                    QgsDebugMsg( "No cache!" );
                }

#ifdef QGISDEBUG
                bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
                QgsDebugMsg( QString( "Reply was cached: %1" ).arg( fromCache ) );
#endif

                mResponse = mReply->readAll();

                if ( mResponse.isEmpty() && !mGotNonEmptyResponse )
                {
                    mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
                    mErrorCode = QgsWFSRequest::ServerExceptionError;
                    QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
                }
            }
示例#12
0
/** \fn MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo)
 *  \brief Downloads a QNetworkRequest via the QNetworkAccessManager
 *  \param dlInfo   MythDownloadInfo information for download
 */
void MythDownloadManager::downloadQNetworkRequest(MythDownloadInfo *dlInfo)
{
    static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
    QUrl qurl(dlInfo->m_url);
    QNetworkRequest request;
    
    if (dlInfo->m_request)
    {
        request = *dlInfo->m_request;
        delete dlInfo->m_request;
        dlInfo->m_request = NULL;
    }
    else
        request.setUrl(qurl);

    if (!dlInfo->m_reload)
    {
        // Prefer the in-cache item if one exists and it is less than 60
        // seconds old and has not expired in the last 10 seconds.
        QDateTime now = QDateTime::currentDateTime();
        QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
        if ((urlData.isValid()) &&
            ((!urlData.expirationDate().isValid()) ||
             (urlData.expirationDate().secsTo(now) < 10)))
        {
            QNetworkCacheMetaData::RawHeaderList headers =
                urlData.rawHeaders();
            bool found = false;
            QNetworkCacheMetaData::RawHeaderList::iterator it
                = headers.begin();
            for (; !found && it != headers.end(); ++it)
            {
                if ((*it).first == "Date")
                {
                    found = true;
                    QDateTime loadDate =
                       QDateTime::fromString((*it).second, dateFormat);
                    loadDate.setTimeSpec(Qt::UTC);
                    if (loadDate.secsTo(now) < 60)
                        dlInfo->m_preferCache = true;
                }
            }
        }
    }

    if (dlInfo->m_preferCache)
        request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
                             QNetworkRequest::PreferCache);

    request.setRawHeader("User-Agent",
                         "MythDownloadManager v" MYTH_BINARY_VERSION);

    if (dlInfo->m_post)
        dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
    else
        dlInfo->m_reply = m_manager->get(request);

    m_downloadReplies[dlInfo->m_reply] = dlInfo;

    connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
            SLOT(downloadError(QNetworkReply::NetworkError)));
    connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
            this, SLOT(downloadProgress(qint64, qint64))); 
}
示例#13
0
void QgsWfsRequest::replyFinished()
{
  if ( !mIsAborted && mReply )
  {
    if ( mReply->error() == QNetworkReply::NoError )
    {
      QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 4 );
      QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
      if ( !redirect.isNull() )
      {
        QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 4 );

        const QUrl &toUrl = redirect.toUrl();
        mReply->request();
        if ( toUrl == mReply->url() )
        {
          mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
          QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
          mResponse.clear();
        }
        else
        {
          QNetworkRequest request( toUrl );
          QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsWfsRequest" ) );
          if ( !mUri.auth().setAuthorization( request ) )
          {
            mResponse.clear();
            mErrorMessage = errorMessageFailedAuth();
            mErrorCode = QgsWfsRequest::NetworkError;
            QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
            emit downloadFinished();
            return;
          }
          request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
          request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );

          mReply->deleteLater();
          mReply = nullptr;

          QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 4 );
          mReply = QgsNetworkAccessManager::instance()->get( request );
          mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
          if ( !mUri.auth().setAuthorizationReply( mReply ) )
          {
            mResponse.clear();
            mErrorMessage = errorMessageFailedAuth();
            mErrorCode = QgsWfsRequest::NetworkError;
            QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
            emit downloadFinished();
            return;
          }
          connect( mReply, &QNetworkReply::finished, this, &QgsWfsRequest::replyFinished, Qt::DirectConnection );
          connect( mReply, &QNetworkReply::downloadProgress, this, &QgsWfsRequest::replyProgress, Qt::DirectConnection );
          return;
        }
      }
      else
      {
        const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();

        if ( nam->cache() )
        {
          QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );

          QNetworkCacheMetaData::RawHeaderList hl;
          const auto constRawHeaders = cmd.rawHeaders();
          for ( const QNetworkCacheMetaData::RawHeader &h : constRawHeaders )
          {
            if ( h.first != "Cache-Control" )
              hl.append( h );
          }
          cmd.setRawHeaders( hl );

          QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 4 );
          if ( cmd.expirationDate().isNull() )
          {
            cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( defaultExpirationInSec() ) );
          }

          nam->cache()->updateMetaData( cmd );
        }
        else
        {
          QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 4 );
        }

#ifdef QGISDEBUG
        bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
        QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 4 );
#endif

        mResponse = mReply->readAll();

        if ( mResponse.isEmpty() && !mGotNonEmptyResponse )
        {
          mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
          mErrorCode = QgsWfsRequest::ServerExceptionError;
          QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
        }
      }
    }
    else
    {
      mErrorMessage = errorMessageWithReason( mReply->errorString() );
      mErrorCode = QgsWfsRequest::ServerExceptionError;
      QgsMessageLog::logMessage( mErrorMessage, tr( "WFS" ) );
      mResponse.clear();
    }
  }
  if ( mTimedout )
    mErrorCode = QgsWfsRequest::TimeoutError;

  if ( mReply )
  {
    mReply->deleteLater();
    mReply = nullptr;
  }

  emit downloadFinished();
}