// called with data after all headers have been processed via headerCallback
static size_t writeCallback(void* ptr, size_t size, size_t nmemb, void* data)
{
ResourceHandle* job = static_cast<ResourceHandle*>(data);
ResourceHandleInternal* d = job->getInternal();
if (d->m_cancelled)
return 0;
size_t totalSize = size * nmemb;
// this shouldn't be necessary but apparently is. CURL writes the data
// of html page even if it is a redirect that was handled internally
// can be observed e.g. on gmail.com
CURL* h = d->m_handle;
long httpCode = 0;
CURLcode err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
if (CURLE_OK == err && httpCode >= 300 && httpCode < 400)
return totalSize;
// since the code in headerCallback will not have run for local files
// the code to set the URL and fire didReceiveResponse is never run,
// which means the ResourceLoader's response does not contain the URL.
// Run the code here for local files to resolve the issue.
// TODO: See if there is a better approach for handling this.
if (!d->m_response.responseFired()) {
const char* hdr;
err = curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &hdr);
d->m_response.setUrl(KURL(hdr));
if (d->client())
d->client()->didReceiveResponse(job, d->m_response);
d->m_response.setResponseFired(true);
}
if (d->client())
d->client()->didReceiveData(job, static_cast<char*>(ptr), totalSize, 0);
return totalSize;
}
// Called each time the message is going to be sent again except the first time.
// It's used mostly to let webkit know about redirects.
static void restartedCallback(SoupMessage* msg, gpointer data)
{
ResourceHandle* handle = static_cast<ResourceHandle*>(data);
if (!handle)
return;
ResourceHandleInternal* d = handle->getInternal();
if (d->m_cancelled)
return;
char* uri = soup_uri_to_string(soup_message_get_uri(msg), false);
String location = String(uri);
g_free(uri);
KURL newURL = KURL(handle->request().url(), location);
// FIXME: This is needed because some servers use broken URIs in
// their Location header, when redirecting, such as URIs with
// white spaces instead of %20; this should be fixed in soup, in
// the future, and this work-around removed.
// See http://bugzilla.gnome.org/show_bug.cgi?id=575378.
SoupURI* soup_uri = soup_uri_new(newURL.string().utf8().data());
soup_message_set_uri(msg, soup_uri);
soup_uri_free(soup_uri);
ResourceRequest request = handle->request();
ResourceResponse response;
request.setURL(newURL);
fillResponseFromMessage(msg, &response);
if (d->client())
d->client()->willSendRequest(handle, request, response);
}
// called with data after all headers have been processed via headerCallback
static size_t writeCallback(void* ptr, size_t size, size_t nmemb, void* obj)
{
ResourceHandle* job = static_cast<ResourceHandle*>(obj);
ResourceHandleInternal* d = job->getInternal();
int totalSize = size * nmemb;
if (d->client())
d->client()->didReceiveData(job, static_cast<char*>(ptr), totalSize, 0);
return totalSize;
}
void MultipartHandle::didReceiveData(size_t length)
{
ResourceHandleInternal* d = m_resourceHandle->getInternal();
if (d->m_cancelled) {
// Request has been canceled, so we'll go to the end state.
m_state = End;
return;
}
if (d->client()) {
const char* data = m_buffer.data();
d->client()->didReceiveData(m_resourceHandle, data, length, length);
}
}
void ResourceHandleManager::dispatchSynchronousJob(ResourceHandle* job)
{
KURL kurl = job->request().url();
if (kurl.protocolIs("data")) {
parseDataUrl(job);
return;
}
ResourceHandleInternal* handle = job->getInternal();
#if LIBCURL_VERSION_NUM > 0x071200
// If defersLoading is true and we call curl_easy_perform
// on a paused handle, libcURL would do the transfert anyway
// and we would assert so force defersLoading to be false.
handle->m_defersLoading = false;
#endif
initializeHandle(job);
// curl_easy_perform blocks until the transfert is finished.
CURLcode ret = curl_easy_perform(handle->m_handle);
if (ret != 0) {
ResourceError error(String(handle->m_url), ret, String(handle->m_url), String(curl_easy_strerror(ret)));
handle->client()->didFail(job, error);
}
curl_easy_cleanup(handle->m_handle);
}
// called with data after all headers have been processed via headerCallback
static size_t writeCallback(void* ptr, size_t size, size_t nmemb, void* data)
{
ResourceHandle* job = static_cast<ResourceHandle*>(data);
ResourceHandleInternal* d = job->getInternal();
int totalSize = size * nmemb;
// this shouldn't be necessary but apparently is. CURL writes the data
// of html page even if it is a redirect that was handled internally
// can be observed e.g. on gmail.com
CURL* h = d->m_handle;
long httpCode = 0;
CURLcode err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
if (CURLE_OK == err && httpCode >= 300 && httpCode < 400)
return totalSize;
if (d->client())
d->client()->didReceiveData(job, static_cast<char*>(ptr), totalSize, 0);
return totalSize;
}
void MultipartHandle::didReceiveResponse()
{
ResourceHandleInternal* d = m_resourceHandle->getInternal();
if (d->client()) {
auto response = d->m_response;
HTTPHeaderMap::const_iterator end = m_headers.end();
for (HTTPHeaderMap::const_iterator it = m_headers.begin(); it != end; ++it)
response.setHTTPHeaderField(it->key, it->value);
String contentType = m_headers.get(HTTPHeaderName::ContentType);
String mimeType = extractMIMETypeFromMediaType(contentType);
response.setMimeType(mimeType.convertToASCIILowercase());
response.setTextEncodingName(extractCharsetFromMediaType(contentType));
d->client()->didReceiveResponse(m_resourceHandle, WTFMove(response));
}
}
// Called each time the message is going to be sent again except the first time.
// It's used mostly to let webkit know about redirects.
static void restartedCallback(SoupMessage* msg, gpointer data)
{
ResourceHandle* handle = static_cast<ResourceHandle*>(data);
if (!handle)
return;
ResourceHandleInternal* d = handle->getInternal();
if (d->m_cancelled)
return;
char* uri = soup_uri_to_string(soup_message_get_uri(msg), false);
String location = String(uri);
g_free(uri);
KURL newURL = KURL(handle->request().url(), location);
ResourceRequest request = handle->request();
ResourceResponse response;
request.setURL(newURL);
request.setHTTPMethod(msg->method);
fillResponseFromMessage(msg, &response);
// Should not set Referer after a redirect from a secure resource to non-secure one.
if (!request.url().protocolIs("https") && protocolIs(request.httpReferrer(), "https")) {
request.clearHTTPReferrer();
soup_message_headers_remove(msg->request_headers, "Referer");
}
if (d->client())
d->client()->willSendRequest(handle, request, response);
if (d->m_cancelled)
return;
#ifdef HAVE_LIBSOUP_2_29_90
// Update the first party in case the base URL changed with the redirect
String firstPartyString = request.firstPartyForCookies().string();
if (!firstPartyString.isEmpty()) {
GOwnPtr<SoupURI> firstParty(soup_uri_new(firstPartyString.utf8().data()));
soup_message_set_first_party(d->m_msg, firstParty.get());
}
#endif
}
void MultipartHandle::didReceiveResponse()
{
ResourceHandleInternal* d = m_resourceHandle->getInternal();
if (d->client()) {
OwnPtr<ResourceResponse> response = ResourceResponseBase::adopt(d->m_response.copyData());
HTTPHeaderMap::const_iterator end = m_headers.end();
for (HTTPHeaderMap::const_iterator it = m_headers.begin(); it != end; ++it)
response->setHTTPHeaderField(it->key, it->value);
String contentType = m_headers.get("Content-Type");
String mimeType = extractMIMETypeFromMediaType(contentType);
response->setMimeType(mimeType.lower());
response->setTextEncodingName(extractCharsetFromMediaType(contentType));
response->setSuggestedFilename(filenameFromHTTPContentDisposition(response->httpHeaderField("Content-Disposition")));
d->client()->didReceiveResponse(m_resourceHandle, *response);
response->setResponseFired(true);
}
}
// Called each time the message is going to be sent again except the first time.
// It's used mostly to let webkit know about redirects.
static void restartedCallback(SoupMessage* msg, gpointer data)
{
ResourceHandle* handle = static_cast<ResourceHandle*>(data);
if (!handle)
return;
ResourceHandleInternal* d = handle->getInternal();
if (d->m_cancelled)
return;
char* uri = soup_uri_to_string(soup_message_get_uri(msg), false);
String location = String(uri);
g_free(uri);
KURL newURL = KURL(handle->request().url(), location);
ResourceRequest request = handle->request();
ResourceResponse response;
request.setURL(newURL);
fillResponseFromMessage(msg, &response);
if (d->client())
d->client()->willSendRequest(handle, request, response);
}
/*
* This is being called for each HTTP header in the response. This includes '\r\n'
* for the last line of the header.
*
* We will add each HTTP Header to the ResourceResponse and on the termination
* of the header (\r\n) we will parse Content-Type and Content-Disposition and
* update the ResourceResponse and then send it away.
*
*/
static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
{
ResourceHandle* job = static_cast<ResourceHandle*>(data);
ResourceHandleInternal* d = job->getInternal();
unsigned int totalSize = size * nmemb;
ResourceHandleClient* client = d->client();
if (!client) {
return totalSize;
}
String header(static_cast<const char*>(ptr), totalSize);
/*
* a) We can finish and send the ResourceResponse
* b) We will add the current header to the HTTPHeaderMap of the ResourceResponse
*/
if (header == String("\r\n")) {
CURL* h = d->m_handle;
CURLcode err;
double contentLength = 0;
err = curl_easy_getinfo(h, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength);
d->m_response.setExpectedContentLength(static_cast<long long int>(contentLength));
const char* hdr;
err = curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &hdr);
d->m_response.setUrl(KURL(hdr));
long httpCode = 0;
err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
d->m_response.setHTTPStatusCode(httpCode);
d->m_response.setMimeType(extractMIMETypeFromMediaType(d->m_response.httpHeaderField("Content-Type")));
d->m_response.setTextEncodingName(extractCharsetFromMediaType(d->m_response.httpHeaderField("Content-Type")));
d->m_response.setSuggestedFilename(filenameFromHTTPContentDisposition(d->m_response.httpHeaderField("Content-Disposition")));
client->didReceiveResponse(job, d->m_response);
} else {
int splitPos = header.find(":");
if (splitPos != -1)
d->m_response.setHTTPHeaderField(header.left(splitPos), header.substring(splitPos+1).stripWhiteSpace());
}
return totalSize;
}
void ResourceHandleManager::dispatchSynchronousJob(ResourceHandle* job)
{
KURL kurl = job->request().url();
if (kurl.protocolIs("data")) {
parseDataUrl(job);
return;
}
initializeHandle(job);
ResourceHandleInternal* handle = job->getInternal();
// curl_easy_perform blocks until the transfert is finished.
CURLcode ret = curl_easy_perform(handle->m_handle);
if (ret != 0) {
ResourceError error(String(handle->m_url), ret, String(handle->m_url), String(curl_easy_strerror(ret)));
handle->client()->didFail(job, error);
}
curl_easy_cleanup(handle->m_handle);
}
void ResourceHandleManager::downloadTimerCallback(Timer<ResourceHandleManager>* timer)
{
startScheduledJobs();
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = 0;
struct timeval timeout;
timeout.tv_sec = 0;
timeout.tv_usec = selectTimeoutMS * 1000; // select waits microseconds
// Temporarily disable timers since signals may interrupt select(), raising EINTR errors on some platforms
setDeferringTimers(true);
int rc = 0;
do {
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
curl_multi_fdset(m_curlMultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
// When the 3 file descriptors are empty, winsock will return -1
// and bail out, stopping the file download. So make sure we
// have valid file descriptors before calling select.
if (maxfd >= 0)
rc = ::select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
} while (rc == -1 && errno == EINTR);
setDeferringTimers(false);
if (-1 == rc) {
#ifndef NDEBUG
perror("bad: select() returned -1: ");
#endif
return;
}
int runningHandles = 0;
while (curl_multi_perform(m_curlMultiHandle, &runningHandles) == CURLM_CALL_MULTI_PERFORM) { }
// check the curl messages indicating completed transfers
// and free their resources
while (true) {
int messagesInQueue;
CURLMsg* msg = curl_multi_info_read(m_curlMultiHandle, &messagesInQueue);
if (!msg)
break;
// find the node which has same d->m_handle as completed transfer
CURL* handle = msg->easy_handle;
ASSERT(handle);
ResourceHandle* job = 0;
CURLcode err = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &job);
ASSERT(CURLE_OK == err);
ASSERT(job);
if (!job)
continue;
ResourceHandleInternal* d = job->getInternal();
ASSERT(d->m_handle == handle);
if (d->m_cancelled) {
removeFromCurl(job);
continue;
}
if (CURLMSG_DONE != msg->msg)
continue;
if (CURLE_OK == msg->data.result) {
if (d->client())
d->client()->didFinishLoading(job);
} else {
#ifndef NDEBUG
char* url = 0;
curl_easy_getinfo(d->m_handle, CURLINFO_EFFECTIVE_URL, &url);
printf("Curl ERROR for url='%s', error: '%s'\n", url, curl_easy_strerror(msg->data.result));
#endif
if (d->client())
d->client()->didFail(job, ResourceError());
}
removeFromCurl(job);
}
bool started = startScheduledJobs(); // new jobs might have been added in the meantime
if (!m_downloadTimer.isActive() && (started || (runningHandles > 0)))
m_downloadTimer.startOneShot(pollTimeSeconds);
}
/*
* This is being called for each HTTP header in the response. This includes '\r\n'
* for the last line of the header.
*
* We will add each HTTP Header to the ResourceResponse and on the termination
* of the header (\r\n) we will parse Content-Type and Content-Disposition and
* update the ResourceResponse and then send it away.
*
*/
static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
{
ResourceHandle* job = static_cast<ResourceHandle*>(data);
ResourceHandleInternal* d = job->getInternal();
if (d->m_cancelled)
return 0;
size_t totalSize = size * nmemb;
ResourceHandleClient* client = d->client();
String header(static_cast<const char*>(ptr), totalSize);
/*
* a) We can finish and send the ResourceResponse
* b) We will add the current header to the HTTPHeaderMap of the ResourceResponse
*
* The HTTP standard requires to use \r\n but for compatibility it recommends to
* accept also \n.
*/
if (header == String("\r\n") || header == String("\n")) {
CURL* h = d->m_handle;
CURLcode err;
double contentLength = 0;
err = curl_easy_getinfo(h, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength);
d->m_response.setExpectedContentLength(static_cast<long long int>(contentLength));
const char* hdr;
err = curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &hdr);
d->m_response.setUrl(KURL(hdr));
long httpCode = 0;
err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
d->m_response.setHTTPStatusCode(httpCode);
d->m_response.setMimeType(extractMIMETypeFromMediaType(d->m_response.httpHeaderField("Content-Type")));
d->m_response.setTextEncodingName(extractCharsetFromMediaType(d->m_response.httpHeaderField("Content-Type")));
d->m_response.setSuggestedFilename(filenameFromHTTPContentDisposition(d->m_response.httpHeaderField("Content-Disposition")));
// HTTP redirection
if (httpCode >= 300 && httpCode < 400) {
String location = d->m_response.httpHeaderField("location");
if (!location.isEmpty()) {
KURL newURL = KURL(job->request().url(), location);
ResourceRequest redirectedRequest = job->request();
redirectedRequest.setURL(newURL);
if (client)
client->willSendRequest(job, redirectedRequest, d->m_response);
d->m_request.setURL(newURL);
return totalSize;
}
}
if (client)
client->didReceiveResponse(job, d->m_response);
d->m_response.setResponseFired(true);
} else {
int splitPos = header.find(":");
if (splitPos != -1)
d->m_response.setHTTPHeaderField(header.left(splitPos), header.substring(splitPos+1).stripWhiteSpace());
}
return totalSize;
}
void ResourceHandleManager::downloadTimerCallback(Timer<ResourceHandleManager>* timer)
{
startScheduledJobs();
fd_set fdread;
FD_ZERO(&fdread);
fd_set fdwrite;
FD_ZERO(&fdwrite);
fd_set fdexcep;
FD_ZERO(&fdexcep);
int maxfd = 0;
curl_multi_fdset(m_curlMultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
struct timeval timeout;
timeout.tv_sec = 0;
timeout.tv_usec = selectTimeoutMS * 1000; // select waits microseconds
int rc = ::select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
if (-1 == rc) {
#ifndef NDEBUG
printf("bad: select() returned -1\n");
#endif
return;
}
int runningHandles = 0;
CURLMcode curlCode = CURLM_CALL_MULTI_PERFORM;
while (CURLM_CALL_MULTI_PERFORM == curlCode) {
curlCode = curl_multi_perform(m_curlMultiHandle, &runningHandles);
}
// check the curl messages indicating completed transfers
// and free their resources
while (true) {
int messagesInQueue;
CURLMsg* msg = curl_multi_info_read(m_curlMultiHandle, &messagesInQueue);
if (!msg)
break;
if (CURLMSG_DONE != msg->msg)
continue;
// find the node which has same d->m_handle as completed transfer
CURL* handle = msg->easy_handle;
ASSERT(handle);
ResourceHandle* job = 0;
CURLcode err = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &job);
ASSERT(CURLE_OK == err);
ASSERT(job);
if (!job)
continue;
ResourceHandleInternal* d = job->getInternal();
ASSERT(d->m_handle == handle);
if (CURLE_OK == msg->data.result) {
if (d->client())
d->client()->didFinishLoading(job);
} else {
#ifndef NDEBUG
char* url = 0;
curl_easy_getinfo(d->m_handle, CURLINFO_EFFECTIVE_URL, &url);
printf("Curl ERROR for url='%s', error: '%s'\n", url, curl_easy_strerror(msg->data.result));
#endif
if (d->client())
d->client()->didFail(job, ResourceError());
}
removeFromCurl(job);
}
bool started = startScheduledJobs(); // new jobs might have been added in the meantime
if (!m_downloadTimer.isActive() && (started || (runningHandles > 0)))
m_downloadTimer.startOneShot(pollTimeSeconds);
}
static bool startHttp(ResourceHandle* handle, String urlString)
{
ASSERT(handle);
SoupSession* session = handle->defaultSession();
ensureSessionIsInitialized(session);
ResourceHandleInternal* d = handle->getInternal();
d->m_msg = handle->request().toSoupMessage();
if (!d->m_msg) {
ResourceError resourceError(g_quark_to_string(SOUP_HTTP_ERROR),
SOUP_STATUS_MALFORMED,
urlString,
handle->request().httpMethod());
d->client()->didFail(handle, resourceError);
return false;
}
g_signal_connect(d->m_msg, "restarted", G_CALLBACK(restartedCallback), handle);
g_signal_connect(d->m_msg, "got-headers", G_CALLBACK(gotHeadersCallback), handle);
g_signal_connect(d->m_msg, "content-sniffed", G_CALLBACK(contentSniffedCallback), handle);
g_signal_connect(d->m_msg, "got-chunk", G_CALLBACK(gotChunkCallback), handle);
g_object_set_data(G_OBJECT(d->m_msg), "resourceHandle", reinterpret_cast<void*>(handle));
FormData* httpBody = d->m_request.httpBody();
if (httpBody && !httpBody->isEmpty()) {
size_t numElements = httpBody->elements().size();
// handle the most common case (i.e. no file upload)
if (numElements < 2) {
Vector<char> body;
httpBody->flatten(body);
soup_message_set_request(d->m_msg, d->m_request.httpContentType().utf8().data(),
SOUP_MEMORY_COPY, body.data(), body.size());
} else {
/*
* we have more than one element to upload, and some may
* be (big) files, which we will want to mmap instead of
* copying into memory; TODO: support upload of non-local
* (think sftp://) files by using GIO?
*/
soup_message_body_set_accumulate(d->m_msg->request_body, FALSE);
for (size_t i = 0; i < numElements; i++) {
const FormDataElement& element = httpBody->elements()[i];
if (element.m_type == FormDataElement::data)
soup_message_body_append(d->m_msg->request_body, SOUP_MEMORY_TEMPORARY, element.m_data.data(), element.m_data.size());
else {
/*
* mapping for uploaded files code inspired by technique used in
* libsoup's simple-httpd test
*/
GError* error = 0;
gchar* fileName = filenameFromString(element.m_filename);
GMappedFile* fileMapping = g_mapped_file_new(fileName, false, &error);
g_free(fileName);
if (error) {
ResourceError resourceError(g_quark_to_string(SOUP_HTTP_ERROR),
d->m_msg->status_code,
urlString,
String::fromUTF8(error->message));
g_error_free(error);
d->client()->didFail(handle, resourceError);
g_signal_handlers_disconnect_matched(d->m_msg, G_SIGNAL_MATCH_DATA,
0, 0, 0, 0, handle);
g_object_unref(d->m_msg);
d->m_msg = 0;
return false;
}
SoupBuffer* soupBuffer = soup_buffer_new_with_owner(g_mapped_file_get_contents(fileMapping),
g_mapped_file_get_length(fileMapping),
fileMapping,
#if GLIB_CHECK_VERSION(2, 21, 3)
reinterpret_cast<GDestroyNotify>(g_mapped_file_unref));
#else
reinterpret_cast<GDestroyNotify>(g_mapped_file_free));
#endif
soup_message_body_append_buffer(d->m_msg->request_body, soupBuffer);
soup_buffer_free(soupBuffer);
}
}
}
}