Beispiel #1
0
RefPtr<FetchRequest> FetchRequest::create(ScriptExecutionContext& context, FetchRequest& input, const Dictionary& init, ExceptionCode& ec)
{
    if (input.isDisturbed()) {
        ec = TypeError;
        return nullptr;
    }

    FetchRequest::InternalRequest internalRequest(input.m_internalRequest);

    if (!buildOptions(internalRequest, context, init)) {
        ec = TypeError;
        return nullptr;
    }

    RefPtr<FetchHeaders> headers = buildHeaders(init, internalRequest, input.m_headers.ptr());
    if (!headers) {
        ec = TypeError;
        return nullptr;
    }

    FetchBody body = buildBody(init, *headers, &input.m_body);
    if (!validateBodyAndMethod(body, internalRequest)) {
        ec = TypeError;
        return nullptr;
    }

    if (!input.m_body.isEmpty())
        input.setDisturbed();

    return adoptRef(*new FetchRequest(context, WTFMove(body), headers.releaseNonNull(), WTFMove(internalRequest)));
}
Beispiel #2
0
static ResourceLoadPriority loadPriority(Resource::Type type, const FetchRequest& request)
{
    if (request.priority() != ResourceLoadPriorityUnresolved)
        return request.priority();

    switch (type) {
    case Resource::MainResource:
        return ResourceLoadPriorityVeryHigh;
    case Resource::Raw:
    case Resource::Font:
    case Resource::ImportResource:
        return ResourceLoadPriorityMedium;
    case Resource::Image:
        // We'll default images to VeryLow, and promote whatever is visible. This improves
        // speed-index by ~5% on average, ~14% at the 99th percentile.
        return ResourceLoadPriorityVeryLow;
    case Resource::Media:
        return ResourceLoadPriorityLow;
    case Resource::LinkPrefetch:
        return ResourceLoadPriorityVeryLow;
    case Resource::LinkSubresource:
        return ResourceLoadPriorityLow;
    }
    ASSERT_NOT_REACHED();
    return ResourceLoadPriorityUnresolved;
}
bool ScriptResource::mustRefetchDueToIntegrityMetadata(const FetchRequest& request) const
{
    if (request.integrityMetadata().isEmpty())
        return false;

    return !IntegrityMetadata::setsEqual(m_integrityMetadata, request.integrityMetadata());
}
Beispiel #4
0
void FetchLoader::start(ScriptExecutionContext& context, const FetchRequest& request)
{
    ThreadableLoaderOptions options(request.fetchOptions(), ConsiderPreflight,
        context.shouldBypassMainWorldContentSecurityPolicy() ? ContentSecurityPolicyEnforcement::DoNotEnforce : ContentSecurityPolicyEnforcement::EnforceConnectSrcDirective,
        String(cachedResourceRequestInitiators().fetch),
        OpaqueResponseBodyPolicy::DoNotReceive);
    options.sendLoadCallbacks = SendCallbacks;
    options.dataBufferingPolicy = DoNotBufferData;
    options.sameOriginDataURLFlag = SameOriginDataURLFlag::Set;

    ResourceRequest fetchRequest = request.internalRequest();

    ASSERT(context.contentSecurityPolicy());
    auto& contentSecurityPolicy = *context.contentSecurityPolicy();

    contentSecurityPolicy.upgradeInsecureRequestIfNeeded(fetchRequest, ContentSecurityPolicy::InsecureRequestType::Load);

    if (!context.shouldBypassMainWorldContentSecurityPolicy() && !contentSecurityPolicy.allowConnectToSource(fetchRequest.url())) {
        m_client.didFail();
        return;
    }

    String referrer = request.internalRequestReferrer();
    if (referrer == "no-referrer") {
        options.referrerPolicy = FetchOptions::ReferrerPolicy::NoReferrer;
        referrer = String();
    } else
        referrer = (referrer == "client") ? context.url().strippedForUseAsReferrer() : URL(context.url(), referrer).strippedForUseAsReferrer();

    m_loader = ThreadableLoader::create(context, *this, WTFMove(fetchRequest), options, WTFMove(referrer));
    m_isStarted = m_loader;
}
void FrameFetchContext::upgradeInsecureRequest(FetchRequest& fetchRequest)
{
    KURL url = fetchRequest.resourceRequest().url();

    // Tack an 'Upgrade-Insecure-Requests' header to outgoing navigational requests, as described in
    // https://w3c.github.io/webappsec/specs/upgrade/#feature-detect
    if (fetchRequest.resourceRequest().frameType() != WebURLRequest::FrameTypeNone)
        fetchRequest.mutableResourceRequest().addHTTPHeaderField("Upgrade-Insecure-Requests", "1");

    if (m_document && m_document->insecureRequestsPolicy() == SecurityContext::InsecureRequestsUpgrade && url.protocolIs("http")) {
        ASSERT(m_document->insecureNavigationsToUpgrade());

        // We always upgrade requests that meet any of the following criteria:
        //
        // 1. Are for subresources (including nested frames).
        // 2. Are form submissions.
        // 3. Whose hosts are contained in the document's InsecureNavigationSet.
        const ResourceRequest& request = fetchRequest.resourceRequest();
        if (request.frameType() == WebURLRequest::FrameTypeNone
            || request.frameType() == WebURLRequest::FrameTypeNested
            || request.requestContext() == WebURLRequest::RequestContextForm
            || (!url.host().isNull() && m_document->insecureNavigationsToUpgrade()->contains(url.host().impl()->hash())))
        {
            UseCounter::count(m_document, UseCounter::UpgradeInsecureRequestsUpgradedRequest);
            url.setProtocol("https");
            if (url.port() == 80)
                url.setPort(443);
            fetchRequest.mutableResourceRequest().setURL(url);
        }
    }
}
ResourcePtr<Resource> LinkFetchResource::fetch(Resource::Type type, FetchRequest& request, ResourceFetcher* fetcher)
{
    ASSERT(type == LinkPrefetch);
    ASSERT(request.resourceRequest().frameType() == WebURLRequest::FrameTypeNone);
    fetcher->determineRequestContext(request.mutableResourceRequest(), type);
    return fetcher->requestResource(request, LinkResourceFactory(type));
}
void FrameFetchContext::upgradeInsecureRequest(FetchRequest& fetchRequest)
{
    KURL url = fetchRequest.resourceRequest().url();

    // Tack an 'Upgrade-Insecure-Requests' header to outgoing navigational requests, as described in
    // https://w3c.github.io/webappsec/specs/upgrade/#feature-detect
    if (fetchRequest.resourceRequest().frameType() != WebURLRequest::FrameTypeNone)
        fetchRequest.mutableResourceRequest().addHTTPHeaderField("Upgrade-Insecure-Requests", "1");

    // If we don't yet have an |m_document| (because we're loading an iframe, for instance), check the FrameLoader's policy.
    WebInsecureRequestPolicy relevantPolicy = m_document ? m_document->getInsecureRequestPolicy() : frame()->loader().getInsecureRequestPolicy();
    SecurityContext::InsecureNavigationsSet* relevantNavigationSet = m_document ? m_document->insecureNavigationsToUpgrade() : frame()->loader().insecureNavigationsToUpgrade();

    if (url.protocolIs("http") && relevantPolicy & kUpgradeInsecureRequests) {
        // We always upgrade requests that meet any of the following criteria:
        //
        // 1. Are for subresources (including nested frames).
        // 2. Are form submissions.
        // 3. Whose hosts are contained in the document's InsecureNavigationSet.
        const ResourceRequest& request = fetchRequest.resourceRequest();
        if (request.frameType() == WebURLRequest::FrameTypeNone
            || request.frameType() == WebURLRequest::FrameTypeNested
            || request.requestContext() == WebURLRequest::RequestContextForm
            || (!url.host().isNull() && relevantNavigationSet->contains(url.host().impl()->hash())))
        {
            UseCounter::count(m_document, UseCounter::UpgradeInsecureRequestsUpgradedRequest);
            url.setProtocol("https");
            if (url.port() == 80)
                url.setPort(443);
            fetchRequest.mutableResourceRequest().setURL(url);
        }
    }
}
ResourcePtr<ScriptResource> ScriptResource::fetch(FetchRequest& request, ResourceFetcher* fetcher)
{
    ASSERT(request.resourceRequest().frameType() == WebURLRequest::FrameTypeNone);
    request.mutableResourceRequest().setRequestContext(WebURLRequest::RequestContextScript);
    ResourcePtr<ScriptResource> resource = toScriptResource(fetcher->requestResource(request, ScriptResourceFactory()));
    if (resource && !request.integrityMetadata().isEmpty())
        resource->setIntegrityMetadata(request.integrityMetadata());
    return resource;
}
ResourceLoadPriority FrameFetchContext::modifyPriorityForExperiments(ResourceLoadPriority priority, Resource::Type type, const FetchRequest& request, ResourcePriority::VisibilityStatus visibility)
{
    // An image fetch is used to distinguish between "early" and "late" scripts in a document
    if (type == Resource::Image)
        m_imageFetched = true;

    // If Settings is null, we can't verify any experiments are in force.
    if (!frame()->settings())
        return priority;

    // If enabled, drop the priority of all resources in a subframe.
    if (frame()->settings()->lowPriorityIframes() && !frame()->isMainFrame())
        return ResourceLoadPriorityVeryLow;

    // Async/Defer scripts.
    if (type == Resource::Script && FetchRequest::LazyLoad == request.defer())
        return frame()->settings()->fetchIncreaseAsyncScriptPriority() ? ResourceLoadPriorityMedium : ResourceLoadPriorityLow;

    // Runtime experiment that change how we prioritize resources.
    // The toggles do not depend on each other and can be flipped individually
    // though the cumulative result will depend on the interaction between them.
    // Background doc: https://docs.google.com/document/d/1bCDuq9H1ih9iNjgzyAL0gpwNFiEP4TZS-YLRp_RuMlc/edit?usp=sharing

    // Increases the priorities for CSS, Scripts, Fonts and Images all by one level
    // and parser-blocking scripts and visible images by 2.
    // This is used in conjunction with logic on the Chrome side to raise the threshold
    // of "layout-blocking" resources and provide a boost to resources that are needed
    // as soon as possible for something currently on the screen.
    int modifiedPriority = static_cast<int>(priority);
    if (frame()->settings()->fetchIncreasePriorities()) {
        if (type == Resource::CSSStyleSheet || type == Resource::Script || type == Resource::Font || type == Resource::Image)
            modifiedPriority++;
    }

    // Always give visible resources a bump, and an additional bump if generally increasing priorities.
    if (visibility == ResourcePriority::Visible) {
        modifiedPriority++;
        if (frame()->settings()->fetchIncreasePriorities())
            modifiedPriority++;
    }

    if (frame()->settings()->fetchIncreaseFontPriority() && type == Resource::Font)
        modifiedPriority++;

    if (type == Resource::Script) {
        // Reduce the priority of late-body scripts.
        if (frame()->settings()->fetchDeferLateScripts() && request.forPreload() && m_imageFetched)
            modifiedPriority--;
        // Parser-blocking scripts.
        if (frame()->settings()->fetchIncreasePriorities() && !request.forPreload())
            modifiedPriority++;
    }

    // Clamp priority
    modifiedPriority = std::min(static_cast<int>(ResourceLoadPriorityHighest), std::max(static_cast<int>(ResourceLoadPriorityLowest), modifiedPriority));
    return static_cast<ResourceLoadPriority>(modifiedPriority);
}
    void expectHeader(const char* input, const char* headerName, bool isPresent, const char* headerValue)
    {
        KURL inputURL(ParsedURLString, input);
        FetchRequest fetchRequest = FetchRequest(ResourceRequest(inputURL), FetchInitiatorInfo());
        fetchContext->addClientHintsIfNecessary(fetchRequest);

        EXPECT_STREQ(isPresent ? headerValue : "",
                     fetchRequest.resourceRequest().httpHeaderField(headerName).utf8().data());
    }
Beispiel #11
0
void ResourceFetcher::requestLoadStarted(Resource* resource, const FetchRequest& request, ResourceLoadStartType type)
{
    if (type == ResourceLoadingFromCache)
        notifyLoadedFromMemoryCache(resource);

    if (request.resourceRequest().url().protocolIsData())
        return;

    m_validatedURLs.add(request.resourceRequest().url());
}
int main(int argc, char **argv)
{
  Client::Configuration configuration;
  configuration.auto_connect = true;
  configuration.client_id = "libkafka_asio_example";
  configuration.socket_timeout = 10000;
  configuration.AddBrokerFromString("192.168.15.137:49162");

  boost::asio::io_service ios;
  Client client(ios, configuration);

  // Create a 'Fetch' request and try to get data for partition 0 of topic
  // 'mytopic', starting with offset 1
  FetchRequest request;
  request.FetchTopic("mytopic", 0, 1);

  // Helper to interpret the received bytes as string
  auto BytesToString = [](const libkafka_asio::Bytes& bytes) -> std::string
  {
    if (!bytes || bytes->empty())
    {
      return "";
    }
    return std::string((const char*) &(*bytes)[0], bytes->size());
  };

  // Send the prepared fetch request.
  // The client will attempt to automatically connect to the broker, specified
  // in the configuration.
  client.AsyncRequest(
    request,
    [&](const Client::ErrorCodeType& err,
        const FetchResponse::OptionalType& response)
  {
    if (err)
    {
      std::cerr
        << "Error: " << boost::system::system_error(err).what()
        << std::endl;
      return;
    }

    // Loop through the received messages.
    // A range based for loop might also work.
    std::for_each(response->begin(), response->end(),
      [&](const MessageAndOffset& message)
    {
      std::cout << BytesToString(message.value()) << std::endl;
    });
  });

  // Let's go!
  ios.run();
  return 0;
}
PassRefPtrWillBeRawPtr<XSLStyleSheetResource> XSLStyleSheetResource::fetchSynchronously(FetchRequest& request, ResourceFetcher* fetcher)
{
    request.mutableResourceRequest().setTimeoutInterval(10);
    request.mutableResourceRequest().setRequestContext(WebURLRequest::RequestContextXSLT);
    ResourceLoaderOptions options(request.options());
    options.synchronousPolicy = RequestSynchronously;
    request.setOptions(options);
    RefPtrWillBeRawPtr<XSLStyleSheetResource> resource = toXSLStyleSheetResource(fetcher->requestResource(request, XSLStyleSheetResourceFactory()));
    if (resource && resource->m_data)
        resource->m_sheet = resource->decodedText();
    return resource;
}
Beispiel #14
0
ResourcePtr<Resource> ResourceFetcher::createResourceForLoading(Resource::Type type, FetchRequest& request, const String& charset)
{
    ASSERT(!memoryCache()->resourceForURL(request.resourceRequest().url()));

    WTF_LOG(ResourceLoading, "Loading Resource for '%s'.", request.resourceRequest().url().elidedString().latin1().data());

    addAdditionalRequestHeaders(request.mutableResourceRequest(), type);
    ResourcePtr<Resource> resource = createResource(type, request.resourceRequest(), charset);

    memoryCache()->add(resource.get());
    return resource;
}
static void configureRequest(FetchRequest& request, ImageLoader::BypassMainWorldBehavior bypassBehavior, Element& element, const ClientHintsPreferences& clientHintsPreferences)
{
    if (bypassBehavior == ImageLoader::BypassMainWorldCSP)
        request.setContentSecurityCheck(DoNotCheckContentSecurityPolicy);

    AtomicString crossOriginMode = element.fastGetAttribute(HTMLNames::crossoriginAttr);
    if (!crossOriginMode.isNull())
        request.setCrossOriginAccessControl(element.document().securityOrigin(), crossOriginMode);

    if (clientHintsPreferences.shouldSendResourceWidth() && isHTMLImageElement(element))
        request.setResourceWidth(toHTMLImageElement(element).resourceWidth());
}
    void expectHTTPSHeader(const char* input, WebURLRequest::FrameType frameType, bool shouldPrefer)
    {
        KURL inputURL(ParsedURLString, input);

        FetchRequest fetchRequest = FetchRequest(ResourceRequest(inputURL), FetchInitiatorInfo());
        fetchRequest.mutableResourceRequest().setRequestContext(WebURLRequest::RequestContextScript);
        fetchRequest.mutableResourceRequest().setFrameType(frameType);

        fetchContext->upgradeInsecureRequest(fetchRequest);

        EXPECT_STREQ(shouldPrefer ? "1" : "",
            fetchRequest.resourceRequest().httpHeaderField(HTTPNames::Upgrade_Insecure_Requests).utf8().data());
    }
    void expectHeader(const char* input, const char* headerName, bool isPresent, const char* headerValue, float width = 0)
    {
        KURL inputURL(ParsedURLString, input);
        FetchRequest fetchRequest = FetchRequest(ResourceRequest(inputURL), FetchInitiatorInfo());
        if (width > 0) {
            FetchRequest::ResourceWidth resourceWidth;
            resourceWidth.width = width;
            resourceWidth.isSet = true;
            fetchRequest.setResourceWidth(resourceWidth);
        }
        fetchContext->addClientHintsIfNecessary(fetchRequest);

        EXPECT_STREQ(isPresent ? headerValue : "",
            fetchRequest.resourceRequest().httpHeaderField(headerName).utf8().data());
    }
Beispiel #18
0
bool ResourceFetcher::resourceNeedsLoad(Resource* resource, const FetchRequest& request, RevalidationPolicy policy)
{
    if (FetchRequest::DeferredByClient == request.defer())
        return false;
    if (policy != Use)
        return true;
    return resource->stillNeedsLoad();
}
void FrameFetchContext::addCSPHeaderIfNecessary(Resource::Type type, FetchRequest& fetchRequest)
{
    if (!m_document)
        return;

    const ContentSecurityPolicy* csp = m_document->contentSecurityPolicy();
    if (csp->shouldSendCSPHeader(type))
        fetchRequest.mutableResourceRequest().addHTTPHeaderField("CSP", "active");
}
void HTMLResourcePreloader::preload(PassOwnPtr<PreloadRequest> preload)
{
    if (preload->isPreconnect()) {
        preconnectHost(preload.get());
        return;
    }
    // TODO(yoichio): Should preload if document is imported.
    if (!m_document->loader())
        return;
    FetchRequest request = preload->resourceRequest(m_document);
    // TODO(dgozman): This check should go to HTMLPreloadScanner, but this requires
    // making Document::completeURLWithOverride logic to be statically accessible.
    if (request.url().protocolIsData())
        return;
    if (preload->resourceType() == Resource::Script || preload->resourceType() == Resource::CSSStyleSheet || preload->resourceType() == Resource::ImportResource)
        request.setCharset(preload->charset().isEmpty() ? m_document->charset().string() : preload->charset());
    request.setForPreload(true);
    Platform::current()->histogramCustomCounts("WebCore.PreloadDelayMs", static_cast<int>(1000 * (monotonicallyIncreasingTime() - preload->discoveryTime())), 0, 2000, 20);
    m_document->loader()->startPreload(preload->resourceType(), request);
}
void HTMLResourcePreloader::preload(PassOwnPtr<PreloadRequest> preload, const NetworkHintsInterface& networkHintsInterface)
{
    if (preload->isPreconnect()) {
        preconnectHost(preload.get(), networkHintsInterface);
        return;
    }
    // TODO(yoichio): Should preload if document is imported.
    if (!m_document->loader())
        return;
    FetchRequest request = preload->resourceRequest(m_document);
    // TODO(dgozman): This check should go to HTMLPreloadScanner, but this requires
    // making Document::completeURLWithOverride logic to be statically accessible.
    if (request.url().protocolIsData())
        return;
    if (preload->resourceType() == Resource::Script || preload->resourceType() == Resource::CSSStyleSheet || preload->resourceType() == Resource::ImportResource)
        request.setCharset(preload->charset().isEmpty() ? m_document->characterSet().getString() : preload->charset());
    request.setForPreload(true);
    int duration = static_cast<int>(1000 * (monotonicallyIncreasingTime() - preload->discoveryTime()));
    DEFINE_STATIC_LOCAL(CustomCountHistogram, preloadDelayHistogram, ("WebCore.PreloadDelayMs", 0, 2000, 20));
    preloadDelayHistogram.count(duration);
    m_document->loader()->startPreload(preload->resourceType(), request);
}
Beispiel #22
0
TEST_F(ResourceFetcherTest, LinkPreloadImageAndUse) {
    ResourceFetcher* fetcher =
        ResourceFetcher::create(ResourceFetcherTestMockFetchContext::create());

    KURL url(ParsedURLString, "http://127.0.0.1:8000/foo.png");
    URLTestHelpers::registerMockedURLLoad(url, testImageFilename, "image/png");

    // Link preload preload scanner
    FetchRequest fetchRequestOriginal = FetchRequest(url, FetchInitiatorInfo());
    fetchRequestOriginal.setLinkPreload(true);
    Resource* resource = ImageResource::fetch(fetchRequestOriginal, fetcher);
    ASSERT_TRUE(resource);
    EXPECT_TRUE(resource->isLinkPreload());
    Platform::current()->getURLLoaderMockFactory()->serveAsynchronousRequests();
    fetcher->preloadStarted(resource);

    // Image preload scanner
    FetchRequest fetchRequestPreloadScanner =
        FetchRequest(url, FetchInitiatorInfo());
    Resource* imgPreloadScannerResource =
        ImageResource::fetch(fetchRequestPreloadScanner, fetcher);
    EXPECT_EQ(resource, imgPreloadScannerResource);
    EXPECT_FALSE(resource->isLinkPreload());
    fetcher->preloadStarted(resource);

    // Image created by parser
    FetchRequest fetchRequest = FetchRequest(url, FetchInitiatorInfo());
    Resource* newResource = ImageResource::fetch(fetchRequest, fetcher);
    Persistent<MockResourceClient> client = new MockResourceClient(newResource);
    EXPECT_EQ(resource, newResource);
    EXPECT_FALSE(resource->isLinkPreload());

    // DCL reached
    fetcher->clearPreloads(ResourceFetcher::ClearSpeculativeMarkupPreloads);
    Platform::current()->getURLLoaderMockFactory()->unregisterURL(url);
    EXPECT_TRUE(memoryCache()->contains(resource));
    EXPECT_FALSE(resource->isPreloaded());
}
Beispiel #23
0
void FetchLoader::start(ScriptExecutionContext& context, const FetchRequest& request)
{
    // FIXME: Compute loading options according fetch options.
    ThreadableLoaderOptions options;
    options.setSendLoadCallbacks(SendCallbacks);
    options.setSniffContent(DoNotSniffContent);
    options.setDataBufferingPolicy(DoNotBufferData);
    options.preflightPolicy = ConsiderPreflight;
    options.setAllowCredentials(AllowStoredCredentials);
    options.crossOriginRequestPolicy = DenyCrossOriginRequests;
    options.contentSecurityPolicyEnforcement = ContentSecurityPolicyEnforcement::DoNotEnforce;

    m_loader = ThreadableLoader::create(&context, this, request.internalRequest(), options);
}
void FrameFetchContext::addClientHintsIfNecessary(FetchRequest& fetchRequest)
{
    if (!RuntimeEnabledFeatures::clientHintsEnabled() || !m_document)
        return;

    bool shouldSendDPR = m_document->clientHintsPreferences().shouldSendDPR() || fetchRequest.clientHintsPreferences().shouldSendDPR();
    bool shouldSendResourceWidth = m_document->clientHintsPreferences().shouldSendResourceWidth() || fetchRequest.clientHintsPreferences().shouldSendResourceWidth();
    bool shouldSendViewportWidth = m_document->clientHintsPreferences().shouldSendViewportWidth() || fetchRequest.clientHintsPreferences().shouldSendViewportWidth();

    if (shouldSendDPR)
        fetchRequest.mutableResourceRequest().addHTTPHeaderField("DPR", AtomicString(String::number(m_document->devicePixelRatio())));

    if (shouldSendResourceWidth) {
        FetchRequest::ResourceWidth resourceWidth = fetchRequest.getResourceWidth();
        if (resourceWidth.isSet) {
            float physicalWidth = resourceWidth.width * m_document->devicePixelRatio();
            fetchRequest.mutableResourceRequest().addHTTPHeaderField("Width", AtomicString(String::number(ceil(physicalWidth))));
        }
    }

    if (shouldSendViewportWidth && frame()->view())
        fetchRequest.mutableResourceRequest().addHTTPHeaderField("Viewport-Width", AtomicString(String::number(frame()->view()->viewportWidth())));
}
    void expectUpgrade(const char* input, WebURLRequest::RequestContext requestContext, WebURLRequest::FrameType frameType, const char* expected)
    {
        KURL inputURL(ParsedURLString, input);
        KURL expectedURL(ParsedURLString, expected);

        FetchRequest fetchRequest = FetchRequest(ResourceRequest(inputURL), FetchInitiatorInfo());
        fetchRequest.mutableResourceRequest().setRequestContext(requestContext);
        fetchRequest.mutableResourceRequest().setFrameType(frameType);

        fetchContext->upgradeInsecureRequest(fetchRequest);

        EXPECT_STREQ(expectedURL.getString().utf8().data(), fetchRequest.resourceRequest().url().getString().utf8().data());
        EXPECT_EQ(expectedURL.protocol(), fetchRequest.resourceRequest().url().protocol());
        EXPECT_EQ(expectedURL.host(), fetchRequest.resourceRequest().url().host());
        EXPECT_EQ(expectedURL.port(), fetchRequest.resourceRequest().url().port());
        EXPECT_EQ(expectedURL.hasPort(), fetchRequest.resourceRequest().url().hasPort());
        EXPECT_EQ(expectedURL.path(), fetchRequest.resourceRequest().url().path());
    }
Beispiel #26
0
/** Send a Fetch task to a target worker.
 * @param to a worker that will receive a split
 * @param from a worker that will send a split
 * @param name a name of a split that will be sent
 * @param size the size of a split that will be sent
 * @param id id of this task (generally 0)
 * @param uid id of this fetch task.
 * @param parentid parent id of this task
 * @return NULL
 */
static void dispatch_fetch(WorkerInfo *to, const ServerInfo &from,
                           const string &name, size_t size, 
                           ::uint64_t id, ::uint64_t uid) {
  FetchRequest req;
  req.set_name(name);
  req.mutable_location()->CopyFrom(from);
  req.set_size(size);
  req.set_id(id);
  req.set_uid(uid);

  to->Fetch(req);
  LOG_DEBUG("FETCH TaskID %16d - Sent to Worker %s", static_cast<int>(uid), to->hostname().c_str());
}
ResourcePtr<ImageResource> ImageResource::fetch(FetchRequest& request, ResourceFetcher* fetcher)
{
    if (request.resourceRequest().requestContext() == WebURLRequest::RequestContextUnspecified)
        request.mutableResourceRequest().setRequestContext(WebURLRequest::RequestContextImage);
    if (fetcher->context().pageDismissalEventBeingDispatched()) {
        KURL requestURL = request.resourceRequest().url();
        if (requestURL.isValid() && fetcher->context().canRequest(Resource::Image, request.resourceRequest(), requestURL, request.options(), request.forPreload(), request.originRestriction()))
            fetcher->context().sendImagePing(requestURL);
        return 0;
    }

    if (fetcher->clientDefersImage(request.resourceRequest().url()))
        request.setDefer(FetchRequest::DeferredByClient);

    return toImageResource(fetcher->requestResource(request, ImageResourceFactory()));
}
Beispiel #28
0
ResourceFetcher::RevalidationPolicy ResourceFetcher::determineRevalidationPolicy(Resource::Type type, const FetchRequest& fetchRequest, Resource* existingResource) const
{
    const ResourceRequest& request = fetchRequest.resourceRequest();

    if (!existingResource)
        return Load;

    // If the same URL has been loaded as a different type, we need to reload.
    if (existingResource->type() != type) {
        WTF_LOG(ResourceLoading, "ResourceFetcher::determineRevalidationPolicy reloading due to type mismatch.");
        return Reload;
    }

    // Do not load from cache if images are not enabled. The load for this image will be blocked
    // in ImageResource::load.
    if (FetchRequest::DeferredByClient == fetchRequest.defer())
        return Reload;

    // Always use data uris.
    // FIXME: Extend this to non-images.
    if (type == Resource::Image && request.url().protocolIsData())
        return Use;

    if (!existingResource->canReuse(request))
        return Reload;

    // Never use cache entries for downloadToFile requests. The caller expects the resource in a file.
    if (request.downloadToFile())
        return Reload;

    // Certain requests (e.g., XHRs) might have manually set headers that require revalidation.
    // FIXME: In theory, this should be a Revalidate case. In practice, the MemoryCache revalidation path assumes a whole bunch
    // of things about how revalidation works that manual headers violate, so punt to Reload instead.
    if (request.isConditional())
        return Reload;

    // Don't reload resources while pasting.
    if (m_allowStaleResources)
        return Use;

    if (!fetchRequest.options().canReuseRequest(existingResource->options()))
        return Reload;

    // CachePolicyHistoryBuffer uses the cache no matter what.
    CachePolicy cachePolicy = context().cachePolicy(document());
    if (cachePolicy == CachePolicyHistoryBuffer)
        return Use;

    // Don't reuse resources with Cache-control: no-store.
    if (existingResource->hasCacheControlNoStoreHeader()) {
        WTF_LOG(ResourceLoading, "ResourceFetcher::determineRevalidationPolicy reloading due to Cache-control: no-store.");
        return Reload;
    }

    // If credentials were sent with the previous request and won't be
    // with this one, or vice versa, re-fetch the resource.
    //
    // This helps with the case where the server sends back
    // "Access-Control-Allow-Origin: *" all the time, but some of the
    // client's requests are made without CORS and some with.
    if (existingResource->resourceRequest().allowStoredCredentials() != request.allowStoredCredentials()) {
        WTF_LOG(ResourceLoading, "ResourceFetcher::determineRevalidationPolicy reloading due to difference in credentials settings.");
        return Reload;
    }

    // During the initial load, avoid loading the same resource multiple times for a single document,
    // even if the cache policies would tell us to.
    // We also group loads of the same resource together.
    // Raw resources are exempted, as XHRs fall into this category and may have user-set Cache-Control:
    // headers or other factors that require separate requests.
    if (type != Resource::Raw) {
        if (document() && !document()->loadEventFinished() && m_validatedURLs.contains(existingResource->url()))
            return Use;
        if (existingResource->isLoading())
            return Use;
    }

    // CachePolicyReload always reloads
    if (cachePolicy == CachePolicyReload) {
        WTF_LOG(ResourceLoading, "ResourceFetcher::determineRevalidationPolicy reloading due to CachePolicyReload.");
        return Reload;
    }

    // We'll try to reload the resource if it failed last time.
    if (existingResource->errorOccurred()) {
        WTF_LOG(ResourceLoading, "ResourceFetcher::determineRevalidationPolicye reloading due to resource being in the error state");
        return Reload;
    }

    // List of available images logic allows images to be re-used without cache validation. We restrict this only to images
    // from memory cache which are the same as the version in the current document.
    if (type == Resource::Image && existingResource == cachedResource(request.url()))
        return Use;

    // Check if the cache headers requires us to revalidate (cache expiration for example).
    if (cachePolicy == CachePolicyRevalidate || existingResource->mustRevalidateDueToCacheHeaders()
        || request.cacheControlContainsNoCache()) {
        // See if the resource has usable ETag or Last-modified headers.
        if (existingResource->canUseCacheValidator())
            return Revalidate;

        // No, must reload.
        WTF_LOG(ResourceLoading, "ResourceFetcher::determineRevalidationPolicy reloading due to missing cache validators.");
        return Reload;
    }

    return Use;
}
Beispiel #29
0
ResourcePtr<Resource> ResourceFetcher::requestResource(Resource::Type type, FetchRequest& request)
{
    ASSERT(request.options().synchronousPolicy == RequestAsynchronously || type == Resource::Raw);

    TRACE_EVENT0("blink", "ResourceFetcher::requestResource");

    KURL url = request.resourceRequest().url();

    WTF_LOG(ResourceLoading, "ResourceFetcher::requestResource '%s', charset '%s', priority=%d, type=%s", url.elidedString().latin1().data(), request.charset().latin1().data(), request.priority(), ResourceTypeName(type));

    // If only the fragment identifiers differ, it is the same resource.
    url = MemoryCache::removeFragmentIdentifierIfNeeded(url);

    if (!url.isValid())
        return 0;

    if (!canRequest(type, url, request.options(), request.originRestriction()))
        return 0;

    if (LocalFrame* f = frame())
        f->loaderClient()->dispatchWillRequestResource(&request);

    // See if we can use an existing resource from the cache.
    ResourcePtr<Resource> resource = memoryCache()->resourceForURL(url);

    const RevalidationPolicy policy = determineRevalidationPolicy(type, request, resource.get());
    switch (policy) {
    case Reload:
        memoryCache()->remove(resource.get());
        // Fall through
    case Load:
        resource = createResourceForLoading(type, request, request.charset());
        break;
    case Revalidate:
        resource = createResourceForRevalidation(request, resource.get());
        break;
    case Use:
        memoryCache()->updateForAccess(resource.get());
        break;
    }

    if (!resource)
        return 0;

    if (!resource->hasClients())
        m_deadStatsRecorder.update(policy);

    if (policy != Use)
        resource->setIdentifier(createUniqueIdentifier());

    ResourceLoadPriority priority = loadPriority(type, request);
    if (priority != resource->resourceRequest().priority()) {
        resource->mutableResourceRequest().setPriority(priority);
        resource->didChangePriority(priority, 0);
    }

    if (resourceNeedsLoad(resource.get(), request, policy)) {
        if (!shouldLoadNewResource(type)) {
            if (memoryCache()->contains(resource.get()))
                memoryCache()->remove(resource.get());
            return 0;
        }

        resource->load(this, request.options());

        // For asynchronous loads that immediately fail, it's sufficient to return a
        // null Resource, as it indicates that something prevented the load from starting.
        // If there's a network error, that failure will happen asynchronously. However, if
        // a sync load receives a network error, it will have already happened by this point.
        // In that case, the requester should have access to the relevant ResourceError, so
        // we need to return a non-null Resource.
        if (resource->errorOccurred()) {
            if (memoryCache()->contains(resource.get()))
                memoryCache()->remove(resource.get());
            return 0;
        }
    }

    requestLoadStarted(resource.get(), request, policy == Use ? ResourceLoadingFromCache : ResourceLoadingFromNetwork);

    ASSERT(resource->url() == url.string());
    m_documentResources.set(resource->url(), resource);
    return resource;
}
Beispiel #30
0
ResourcePtr<FontResource> ResourceFetcher::fetchFont(FetchRequest& request)
{
    ASSERT(request.resourceRequest().frameType() == WebURLRequest::FrameTypeNone);
    request.mutableResourceRequest().setRequestContext(WebURLRequest::RequestContextFont);
    return toFontResource(requestResource(Resource::Font, request));
}