Пример #1
0
/** \fn MythDownloadManager::GetLastModified(const QString &url)
 *  \brief Gets the Last Modified timestamp for a URI
 *  \param url    URI to test.
 *  \return Timestamp the URI was last modified or now if an error occurred
 */
QDateTime MythDownloadManager::GetLastModified(const QString &url)
{
    static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
    VERBOSE(VB_FILE+VB_EXTRA, LOC + QString("GetLastModified('%1')").arg(url));
    QDateTime result;

    QDateTime now = QDateTime::currentDateTime();
    QNetworkCacheMetaData urlData = m_manager->cache()->metaData(QUrl(url));

    if (urlData.isValid())
    {
        if (urlData.lastModified().secsTo(now) <= 60)
        {
            result = urlData.lastModified();
        }
        else
        {
            // If the last modification date is older than 60 seconds, and
            // we loaded the page over 60 seconds ago, then redownload the
            // page to re-verify it's last modified date.
            QNetworkCacheMetaData::RawHeaderList headers =
                urlData.rawHeaders();
            bool found = false;
            QNetworkCacheMetaData::RawHeaderList::iterator it
                = headers.begin();
            for (; !found && it != headers.end(); ++it)
            {
                if ((*it).first == "Date")
                {
                    found = true;
                    QDateTime loadDate =
                       QDateTime::fromString((*it).second, dateFormat);
                    loadDate.setTimeSpec(Qt::UTC);
                    if (loadDate.secsTo(now) <= 60)
                        result = urlData.lastModified();
                }
            }
        }
    }

    if (!result.isValid())
    {
        MythDownloadInfo *dlInfo = new MythDownloadInfo;
        dlInfo->m_url      = url;
        dlInfo->m_syncMode = true;

        if (downloadNow(dlInfo, false) && dlInfo->m_reply)
        {
            QVariant lastMod =
                dlInfo->m_reply->header(QNetworkRequest::LastModifiedHeader);
            if (lastMod.isValid())
                result = lastMod.toDateTime();
        }

        delete dlInfo;
    }

    return result;
}
void QNetworkCacheMetaDataPrivate::save(QDataStream &out, const QNetworkCacheMetaData &metaData)
{
    // note: if you change the contents of the meta data here
    // remember to bump the cache version in qnetworkdiskcache.cpp CurrentCacheVersion
    out << metaData.url();
    out << metaData.expirationDate();
    out << metaData.lastModified();
    out << metaData.saveToDisk();
    out << metaData.attributes();
    out << metaData.rawHeaders();
}
Пример #3
0
void NetworkCache::clearCache(int period)
{
	if (period <= 0)
	{
		clear();

		emit cleared();

		return;
	}

	const QDir cacheMainDirectory(cacheDirectory());
	const QStringList directories(cacheMainDirectory.entryList(QDir::AllDirs | QDir::NoDotAndDotDot));

	for (int i = 0; i < directories.count(); ++i)
	{
		const QDir cacheSubDirectory(cacheMainDirectory.absoluteFilePath(directories.at(i)));
		const QStringList subDirectories(cacheSubDirectory.entryList(QDir::AllDirs | QDir::NoDotAndDotDot));

		for (int j = 0; j < subDirectories.count(); ++j)
		{
			const QDir cacheFilesDirectory(cacheSubDirectory.absoluteFilePath(subDirectories.at(j)));
			const QStringList files(cacheFilesDirectory.entryList(QDir::Files));

			for (int k = 0; k < files.count(); ++k)
			{
				const QNetworkCacheMetaData metaData(fileMetaData(cacheFilesDirectory.absoluteFilePath(files.at(k))));

				if (metaData.isValid() && metaData.lastModified().isValid() && metaData.lastModified().secsTo(QDateTime::currentDateTime()) > (period * 3600))
				{
					remove(metaData.url());
				}
			}
		}
	}
}
Пример #4
0
void tst_QNetworkCacheMetaData::qnetworkcachemetadata()
{
    QNetworkCacheMetaData data;
    QCOMPARE(data.expirationDate(), QDateTime());
    QCOMPARE(data.isValid(), false);
    QCOMPARE(data.lastModified(), QDateTime());
    QCOMPARE(data.operator!=(QNetworkCacheMetaData()), false);
    QNetworkCacheMetaData metaData;
    QCOMPARE(data.operator=(metaData), QNetworkCacheMetaData());
    QCOMPARE(data.operator==(QNetworkCacheMetaData()), true);
    QCOMPARE(data.rawHeaders(), QNetworkCacheMetaData::RawHeaderList());
    QCOMPARE(data.saveToDisk(), true);
    QCOMPARE(data.url(), QUrl());
    data.setExpirationDate(QDateTime());
    data.setLastModified(QDateTime());
    data.setRawHeaders(QNetworkCacheMetaData::RawHeaderList());
    data.setSaveToDisk(false);
    data.setUrl(QUrl());
}
Пример #5
0
int main(int argc, char **argv)
{
    QCoreApplication application(argc, argv);
    QCoreApplication::setOrganizationDomain(QLatin1String("sites.google.com/site/zeromusparadoxe01"));
    QCoreApplication::setApplicationName(QLatin1String("zBrowser"));

    QStringList args = application.arguments();
    args.takeFirst();
    if (args.isEmpty()) {
        QTextStream stream(stdout);
        stream << "zbrowser-cacheinfo is a tool for viewing and extracting information out of zBrowser cache files." << endl;
        stream << "zbrowser-cacheinfo [-o cachefile] [file | url]" << endl;
        return 0;
    }

    NetworkDiskCache diskCache;
    QString location = QDesktopServices::storageLocation(QDesktopServices::CacheLocation)
            + QLatin1String("/browser/");
    diskCache.setCacheDirectory(location);

    QNetworkCacheMetaData metaData;
    QString last = args.takeLast();
    if (QFile::exists(last)) {
        qDebug() << "Reading in from a file and not a URL.";
        metaData = diskCache._fileMetaData(last);
    } else {
        qDebug() << "Reading in from a URL and not a file.";
        metaData = diskCache.metaData(last);
    }

    if (!args.isEmpty()
        && args.count() >= 1
        && args.first() == QLatin1String("-o")) {
        QUrl url = metaData.url();
        QIODevice *device = diskCache.data(url);
        if (!device) {
            qDebug() << "Error: data for URL is 0!";
            return 1;
        }
        QString fileName;
        if (args.count() == 2) {
            fileName = args.last();
        } else {
            QFileInfo info(url.path());
            fileName = info.fileName();
            if (fileName.isEmpty()) {
                qDebug() << "URL file name is empty, please specify an output file, I wont guess.";
                return 1;
            }
            if (QFile::exists(fileName)) {
                qDebug() << "File already exists, not overwriting, please specify an output file.";
                return 1;
            }
        }
        qDebug() << "Saved cache file to:" << fileName;
        QFile file(fileName);
        if (!file.open(QFile::ReadWrite))
            qDebug() << "Unable to open the output file for writing.";
        else
            file.write(device->readAll());
        delete device;
    }

    QTextStream stream(stdout);
    stream << "URL: " << metaData.url().toString() << endl;
    stream << "Expiration Date: " << metaData.expirationDate().toString() << endl;
    stream << "Last Modified Date: " << metaData.lastModified().toString() << endl;
    stream << "Save to disk: " << metaData.saveToDisk() << endl;
    stream << "Headers:" << endl;
    foreach (const QNetworkCacheMetaData::RawHeader &header, metaData.rawHeaders())
        stream << "\t" << header.first << ": " << header.second << endl;
    QIODevice *device = diskCache.data(metaData.url());
    if (device) {
        stream << "Data Size: " << device->size() << endl;
        stream << "First line: " << device->readLine(100);
    } else {
        stream << "No data? Either the file is corrupt or there is an error." << endl;
    }

    delete device;
    return 0;
}
Пример #6
0
/*
    For a given httpRequest
    1) If AlwaysNetwork, return
    2) If we have a cache entry for this url populate headers so the server can return 304
    3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
 */
void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache)
{
    QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
        (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
    if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
        // forced reload from the network
        // tell any caching proxy servers to reload too
        httpRequest.setHeaderField("Cache-Control", "no-cache");
        httpRequest.setHeaderField("Pragma", "no-cache");
        return;
    }

    QAbstractNetworkCache *nc = networkCache();
    if (!nc)
        return;                 // no local cache

    QNetworkCacheMetaData metaData = nc->metaData(url());
    if (!metaData.isValid())
        return;                 // not in cache

    if (!metaData.saveToDisk())
        return;

    QNetworkHeadersPrivate cacheHeaders;
    QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
    cacheHeaders.setAllRawHeaders(metaData.rawHeaders());

    it = cacheHeaders.findRawHeader("etag");
    if (it != cacheHeaders.rawHeaders.constEnd())
        httpRequest.setHeaderField("If-None-Match", it->second);

    QDateTime lastModified = metaData.lastModified();
    if (lastModified.isValid())
        httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));

    it = cacheHeaders.findRawHeader("Cache-Control");
    if (it != cacheHeaders.rawHeaders.constEnd()) {
        QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
        if (cacheControl.contains("must-revalidate"))
            return;
    }

    /*
     * age_value
     *      is the value of Age: header received by the cache with
     *              this response.
     * date_value
     *      is the value of the origin server's Date: header
     * request_time
     *      is the (local) time when the cache made the request
     *              that resulted in this cached response
     * response_time
     *      is the (local) time when the cache received the
     *              response
     * now
     *      is the current (local) time
     */
    QDateTime currentDateTime = QDateTime::currentDateTime();
    int age_value = 0;
    it = cacheHeaders.findRawHeader("age");
    if (it != cacheHeaders.rawHeaders.constEnd())
        age_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t();

    int date_value = 0;
    it = cacheHeaders.findRawHeader("date");
    if (it != cacheHeaders.rawHeaders.constEnd())
        date_value = QNetworkHeadersPrivate::fromHttpDate(it->second).toTime_t();

    int now = currentDateTime.toUTC().toTime_t();
    int request_time = now;
    int response_time = now;

    int apparent_age = qMax(0, response_time - date_value);
    int corrected_received_age = qMax(apparent_age, age_value);
    int response_delay = response_time - request_time;
    int corrected_initial_age = corrected_received_age + response_delay;
    int resident_time = now - response_time;
    int current_age   = corrected_initial_age + resident_time;

    // RFC 2616 13.2.4 Expiration Calculations
    QDateTime expirationDate = metaData.expirationDate();
    if (!expirationDate.isValid()) {
        if (lastModified.isValid()) {
            int diff = currentDateTime.secsTo(lastModified);
            expirationDate = lastModified;
            expirationDate.addSecs(diff / 10);
            if (httpRequest.headerField("Warning").isEmpty()) {
                QDateTime dt;
                dt.setTime_t(current_age);
                if (dt.daysTo(currentDateTime) > 1)
                    httpRequest.setHeaderField("Warning", "113");
            }
        }
    }

    int freshness_lifetime = currentDateTime.secsTo(expirationDate);
    bool response_is_fresh = (freshness_lifetime > current_age);

    if (!response_is_fresh && CacheLoadControlAttribute == QNetworkRequest::PreferNetwork)
        return;

    loadedFromCache = true;
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
    qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
    if (!sendCacheContents(metaData))
        loadedFromCache = false;
}
Пример #7
0
/*
    For a given httpRequest
    1) If AlwaysNetwork, return
    2) If we have a cache entry for this url populate headers so the server can return 304
    3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
 */
bool QNetworkAccessHttpBackend::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest)
{
    QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
        (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
    if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
        // If the request does not already specify preferred cache-control
        // force reload from the network and tell any caching proxy servers to reload too
        if (!request().rawHeaderList().contains("Cache-Control")) {
            httpRequest.setHeaderField("Cache-Control", "no-cache");
            httpRequest.setHeaderField("Pragma", "no-cache");
        }
        return false;
    }

    // The disk cache API does not currently support partial content retrieval.
    // That is why we don't use the disk cache for any such requests.
    if (request().hasRawHeader("Range"))
        return false;

    QAbstractNetworkCache *nc = networkCache();
    if (!nc)
        return false;                 // no local cache

    QNetworkCacheMetaData metaData = nc->metaData(url());
    if (!metaData.isValid())
        return false;                 // not in cache

    if (!metaData.saveToDisk())
        return false;

    QNetworkHeadersPrivate cacheHeaders;
    QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
    cacheHeaders.setAllRawHeaders(metaData.rawHeaders());

    it = cacheHeaders.findRawHeader("etag");
    if (it != cacheHeaders.rawHeaders.constEnd())
        httpRequest.setHeaderField("If-None-Match", it->second);

    QDateTime lastModified = metaData.lastModified();
    if (lastModified.isValid())
        httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));

    it = cacheHeaders.findRawHeader("Cache-Control");
    if (it != cacheHeaders.rawHeaders.constEnd()) {
        QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
        if (cacheControl.contains("must-revalidate"))
            return false;
    }

    QDateTime currentDateTime = QDateTime::currentDateTime();
    QDateTime expirationDate = metaData.expirationDate();

#if 0
    /*
     * age_value
     *      is the value of Age: header received by the cache with
     *              this response.
     * date_value
     *      is the value of the origin server's Date: header
     * request_time
     *      is the (local) time when the cache made the request
     *              that resulted in this cached response
     * response_time
     *      is the (local) time when the cache received the
     *              response
     * now
     *      is the current (local) time
     */
    int age_value = 0;
    it = cacheHeaders.findRawHeader("age");
    if (it != cacheHeaders.rawHeaders.constEnd())
        age_value = it->second.toInt();

    QDateTime dateHeader;
    int date_value = 0;
    it = cacheHeaders.findRawHeader("date");
    if (it != cacheHeaders.rawHeaders.constEnd()) {
        dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second);
        date_value = dateHeader.toTime_t();
    }

    int now = currentDateTime.toUTC().toTime_t();
    int request_time = now;
    int response_time = now;

    // Algorithm from RFC 2616 section 13.2.3
    int apparent_age = qMax(0, response_time - date_value);
    int corrected_received_age = qMax(apparent_age, age_value);
    int response_delay = response_time - request_time;
    int corrected_initial_age = corrected_received_age + response_delay;
    int resident_time = now - response_time;
    int current_age   = corrected_initial_age + resident_time;

    // RFC 2616 13.2.4 Expiration Calculations
    if (!expirationDate.isValid()) {
        if (lastModified.isValid()) {
            int diff = currentDateTime.secsTo(lastModified);
            expirationDate = lastModified;
            expirationDate.addSecs(diff / 10);
            if (httpRequest.headerField("Warning").isEmpty()) {
                QDateTime dt;
                dt.setTime_t(current_age);
                if (dt.daysTo(currentDateTime) > 1)
                    httpRequest.setHeaderField("Warning", "113");
            }
        }
    }

    // the cache-saving code below sets the expirationDate with date+max_age
    // if "max-age" is present, or to Expires otherwise
    int freshness_lifetime = dateHeader.secsTo(expirationDate);
    bool response_is_fresh = (freshness_lifetime > current_age);
#else
    bool response_is_fresh = currentDateTime.secsTo(expirationDate) >= 0;
#endif

    if (!response_is_fresh)
        return false;

#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
    qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
    return sendCacheContents(metaData);
}