HttpStatus HttpOpRequest::cancel()
{
mStatus = HttpStatus(HttpStatus::LLCORE, HE_OP_CANCELED);
addAsReply();
return HttpStatus();
}
void HttpOpRequest::stageFromActive(HttpService * service)
{
if (mReplyLength)
{
// If non-zero, we received and processed a Content-Range
// header with the response. If there is received data
// (and there may not be due to protocol violations,
// HEAD requests, etc., see BUG-2295) Verify that what it
// says is consistent with the received data.
if (mReplyBody && mReplyBody->size() && mReplyLength != mReplyBody->size())
{
// Not as expected, fail the request
mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
}
}
if (mCurlHeaders)
{
// We take these headers out of the request now as they were
// allocated originally in this thread and the notifier doesn't
// need them. This eliminates one source of heap moving across
// threads.
curl_slist_free_all(mCurlHeaders);
mCurlHeaders = NULL;
}
// Also not needed on the other side
delete [] mCurlTemp;
mCurlTemp = NULL;
mCurlTempLen = 0;
addAsReply();
}
// Immediately search for the request on various queues
// and cancel operations if found. Return the status of
// the search and cancel as the status of this request.
// The canceled request will return a canceled status to
// its handler.
void HttpOpCancel::stageFromRequest(HttpService * service)
{
if (! service->cancel(mHandle))
{
mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND);
}
addAsReply();
}
HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id,
HttpRequest::priority_t priority,
const std::string & url,
HttpOptions * options,
HttpHeaders * headers)
{
setupCommon(policy_id, priority, url, NULL, options, headers);
mReqMethod = HOR_GET;
return HttpStatus();
}
HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value)
{
HttpStatus status;
if (HttpService::sOptionDesc[opt].mIsLong)
{
return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);
}
if (! HttpService::sOptionDesc[opt].mIsDynamic)
{
return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);
}
mReqOption = opt;
mReqClass = pclass;
mReqDoSet = true;
mReqStrValue = value;
return status;
}
HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
HttpRequest::priority_t priority,
const std::string & url,
BufferArray * body,
HttpOptions * options,
HttpHeaders * headers)
{
setupCommon(policy_id, priority, url, body, options, headers);
mReqMethod = HOR_PUT;
return HttpStatus();
}
bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)
{
// Retry or finalize
if (! op->mStatus)
{
// *DEBUG: For "[curl:bugs] #1420" tests. This will interfere
// with unit tests due to allocation retention by logging code.
// But you won't be checking this in enabled.
#if 0
if (op->mStatus == HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT))
{
LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
<< " timed out."
<< LL_ENDL;
}
#endif
// If this failed, we might want to retry.
if (op->mPolicyRetries < op->mPolicyRetryLimit && op->mStatus.isRetryable())
{
// Okay, worth a retry.
retryOp(op);
return true; // still active/ready
}
}
// This op is done, finalize it delivering it to the reply queue...
if (! op->mStatus)
{
LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
<< " failed after " << op->mPolicyRetries
<< " retries. Reason: " << op->mStatus.toString()
<< " (" << op->mStatus.toTerseString() << ")"
<< LL_ENDL;
}
else if (op->mPolicyRetries)
{
LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op)
<< " succeeded on retry " << op->mPolicyRetries << "."
<< LL_ENDL;
}
op->stageFromActive(mService);
op->release();
return false; // not active
}
HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
HttpRequest::priority_t priority,
const std::string & url,
size_t offset,
size_t len,
HttpOptions * options,
HttpHeaders * headers)
{
setupCommon(policy_id, priority, url, NULL, options, headers);
mReqMethod = HOR_GET;
mReqOffset = offset;
mReqLength = len;
if (offset || len)
{
mProcFlags |= PF_SCAN_RANGE_HEADER;
}
return HttpStatus();
}
size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata)
{
static const char status_line[] = "HTTP/";
static const size_t status_line_len = sizeof(status_line) - 1;
static const char con_ran_line[] = "content-range";
static const char con_retry_line[] = "retry-after";
HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));
const size_t hdr_size(size * nmemb);
const char * hdr_data(static_cast<const char *>(data)); // Not null terminated
bool is_header(true);
if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len))
{
// One of possibly several status lines. Reset what we know and start over
// taking results from the last header stanza we receive.
op->mReplyOffset = 0;
op->mReplyLength = 0;
op->mReplyFullLength = 0;
op->mReplyRetryAfter = 0;
op->mStatus = HttpStatus();
if (op->mReplyHeaders)
{
op->mReplyHeaders->clear();
}
is_header = false;
}
// Nothing in here wants a final CR/LF combination. Remove
// it as much as possible.
size_t wanted_hdr_size(hdr_size);
if (wanted_hdr_size && '\n' == hdr_data[wanted_hdr_size - 1])
{
if (--wanted_hdr_size && '\r' == hdr_data[wanted_hdr_size - 1])
{
--wanted_hdr_size;
}
}
// Copy and normalize header fragments for the following
// stages. Would like to modify the data in-place but that
// may not be allowed and we need one byte extra for NUL.
// At the end of this we will have:
//
// If ':' present in header:
// 1. name points to text to left of colon which
// will be ascii lower-cased and left and right
// trimmed of whitespace.
// 2. value points to text to right of colon which
// will be left trimmed of whitespace.
// Otherwise:
// 1. name points to header which will be left
// trimmed of whitespace.
// 2. value is NULL
// Any non-NULL pointer may point to a zero-length string.
//
if (wanted_hdr_size >= op->mCurlTempLen)
{
delete [] op->mCurlTemp;
op->mCurlTempLen = 2 * wanted_hdr_size + 1;
op->mCurlTemp = new char [op->mCurlTempLen];
}
memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size);
op->mCurlTemp[wanted_hdr_size] = '\0';
char * name(op->mCurlTemp);
char * value(strchr(name, ':'));
if (value)
{
*value++ = '\0';
os_strlower(name);
name = os_strtrim(name);
value = os_strltrim(value);
}
else
{
// Doesn't look well-formed, do minimal normalization on it
name = os_strltrim(name);
}
// Normalized, now reject headers with empty names.
if (! *name)
{
// No use continuing
return hdr_size;
}
// Save header if caller wants them in the response
if (is_header && op->mProcFlags & PF_SAVE_HEADERS)
{
// Save headers in response
if (! op->mReplyHeaders)
{
op->mReplyHeaders = new HttpHeaders;
}
op->mReplyHeaders->append(name, value ? value : "");
}
// From this point, header-specific processors are free to
// modify the header value.
// Detect and parse 'Content-Range' headers
if (is_header
&& op->mProcFlags & PF_SCAN_RANGE_HEADER
&& value && *value
&& ! strcmp(name, con_ran_line))
{
unsigned int first(0), last(0), length(0);
int status;
if (! (status = parse_content_range_header(value, &first, &last, &length)))
{
// Success, record the fragment position
op->mReplyOffset = first;
op->mReplyLength = last - first + 1;
op->mReplyFullLength = length;
}
else if (-1 == status)
{
// Response is badly formed and shouldn't be accepted
op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
}
else
{
// Ignore the unparsable.
LL_INFOS_ONCE(LOG_CORE) << "Problem parsing odd Content-Range header: '"
<< std::string(hdr_data, wanted_hdr_size)
<< "'. Ignoring."
<< LL_ENDL;
}
}
// Detect and parse 'Retry-After' headers
if (is_header
&& op->mProcFlags & PF_USE_RETRY_AFTER
&& value && *value
&& ! strcmp(name, con_retry_line))
{
int time(0);
if (! parse_retry_after_header(value, &time))
{
op->mReplyRetryAfter = time;
}
}
return hdr_size;
}
// Sets all libcurl options and data for a request.
//
// Used both for initial requests and to 'reload' for
// a retry, generally with a different CURL handle.
// Junk may be left around from a failed request and that
// needs to be cleaned out.
//
// *TODO: Move this to _httplibcurl where it belongs.
HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
{
CURLcode code;
// Scrub transport and result data for retried op case
mCurlActive = false;
mCurlHandle = NULL;
mCurlService = NULL;
if (mCurlHeaders)
{
curl_slist_free_all(mCurlHeaders);
mCurlHeaders = NULL;
}
mCurlBodyPos = 0;
if (mReplyBody)
{
mReplyBody->release();
mReplyBody = NULL;
}
mReplyOffset = 0;
mReplyLength = 0;
mReplyFullLength = 0;
if (mReplyHeaders)
{
mReplyHeaders->release();
mReplyHeaders = NULL;
}
mReplyConType.clear();
// *FIXME: better error handling later
HttpStatus status;
// Get global and class policy options
HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions());
HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy));
mCurlHandle = service->getTransport().getHandle();
if (! mCurlHandle)
{
// We're in trouble. We'll continue but it won't go well.
LL_WARNS(LOG_CORE) << "Failed to allocate libcurl easy handle. Continuing."
<< LL_ENDL;
return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC);
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
check_curl_easy_code(code, CURLOPT_IPRESOLVE);
code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
check_curl_easy_code(code, CURLOPT_NOSIGNAL);
code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
check_curl_easy_code(code, CURLOPT_NOPROGRESS);
code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
check_curl_easy_code(code, CURLOPT_URL);
code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
check_curl_easy_code(code, CURLOPT_PRIVATE);
code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
check_curl_easy_code(code, CURLOPT_ENCODING);
// The Linksys WRT54G V5 router has an issue with frequent
// DNS lookups from LAN machines. If they happen too often,
// like for every HTTP request, the router gets annoyed after
// about 700 or so requests and starts issuing TCP RSTs to
// new connections. Reuse the DNS lookups for even a few
// seconds and no RSTs.
code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15);
check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT);
code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
check_curl_easy_code(code, CURLOPT_AUTOREFERER);
code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION);
code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT);
check_curl_easy_code(code, CURLOPT_MAXREDIRS);
code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
check_curl_easy_code(code, CURLOPT_WRITEFUNCTION);
code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this);
check_curl_easy_code(code, CURLOPT_WRITEDATA);
code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
check_curl_easy_code(code, CURLOPT_READFUNCTION);
code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this);
check_curl_easy_code(code, CURLOPT_READDATA);
code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1);
check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER);
code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST);
if (gpolicy.mUseLLProxy)
{
// Use the viewer-based thread-safe API which has a
// fast/safe check for proxy enable. Would like to
// encapsulate this someway...
LLProxy::getInstance()->applyProxySettings(mCurlHandle);
}
else if (gpolicy.mHttpProxy.size())
{
// *TODO: This is fine for now but get fuller socks5/
// authentication thing going later....
code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, gpolicy.mHttpProxy.c_str());
check_curl_easy_code(code, CURLOPT_PROXY);
code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
check_curl_easy_code(code, CURLOPT_PROXYTYPE);
}
if (gpolicy.mCAPath.size())
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, gpolicy.mCAPath.c_str());
check_curl_easy_code(code, CURLOPT_CAPATH);
}
if (gpolicy.mCAFile.size())
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, gpolicy.mCAFile.c_str());
check_curl_easy_code(code, CURLOPT_CAINFO);
}
switch (mReqMethod)
{
case HOR_GET:
code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
check_curl_easy_code(code, CURLOPT_HTTPGET);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
break;
case HOR_POST:
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
check_curl_easy_code(code, CURLOPT_POST);
code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
check_curl_easy_code(code, CURLOPT_ENCODING);
long data_size(0);
if (mReqBody)
{
data_size = mReqBody->size();
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
check_curl_easy_code(code, CURLOPT_POSTFIELDS);
code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size);
check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
}
break;
case HOR_PUT:
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1);
check_curl_easy_code(code, CURLOPT_UPLOAD);
long data_size(0);
if (mReqBody)
{
data_size = mReqBody->size();
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
check_curl_easy_code(code, CURLOPT_INFILESIZE);
code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
check_curl_easy_code(code, CURLOPT_POSTFIELDS);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
// *TODO: Should this be 'Keep-Alive' ?
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
}
break;
default:
LL_ERRS(LOG_CORE) << "Invalid HTTP method in request: "
<< int(mReqMethod) << ". Can't recover."
<< LL_ENDL;
break;
}
// Tracing
if (mTracing >= HTTP_TRACE_CURL_HEADERS)
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
check_curl_easy_code(code, CURLOPT_VERBOSE);
code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this);
check_curl_easy_code(code, CURLOPT_DEBUGDATA);
code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION);
}
// There's a CURLOPT for this now...
if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
{
static const char * const fmt1("Range: bytes=%lu-%lu");
static const char * const fmt2("Range: bytes=%lu-");
char range_line[64];
#if LL_WINDOWS
_snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1,
(mReqLength ? fmt1 : fmt2),
(unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
#else
if ( mReqLength )
{
snprintf(range_line, sizeof(range_line),
fmt1,
(unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
}
else
{
snprintf(range_line, sizeof(range_line),
fmt2,
(unsigned long) mReqOffset);
}
#endif // LL_WINDOWS
range_line[sizeof(range_line) - 1] = '\0';
mCurlHeaders = curl_slist_append(mCurlHeaders, range_line);
}
mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
// Request options
long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT);
long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT);
if (mReqOptions)
{
timeout = mReqOptions->getTimeout();
timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
xfer_timeout = mReqOptions->getTransferTimeout();
xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
}
if (xfer_timeout == 0L)
{
xfer_timeout = timeout;
}
if (cpolicy.mPipelining > 1L)
{
// Pipelining affects both connection and transfer timeout values.
// Requests that are added to a pipeling immediately have completed
// their connection so the connection delay tends to be less than
// the non-pipelined value. Transfers are the opposite. Transfer
// timeout starts once the connection is established and completion
// can be delayed due to the pipelined requests ahead. So, it's
// a handwave but bump the transfer timeout up by the pipelining
// depth to give some room.
//
// BUG-7698, BUG-7688, BUG-7694 (others). Scylla and Charybdis
// situation. Operating against a CDN having service issues may
// lead to requests stalling for an arbitrarily long time with only
// the CURLOPT_TIMEOUT value leading to a closed connection. Sadly
// for pipelining, libcurl (7.39.0 and earlier, at minimum) starts
// the clock on this value as soon as a request is started down
// the wire. We want a short value to recover and retry from the
// CDN. We need a long value to safely deal with a succession of
// piled-up pipelined requests.
//
// *TODO: Find a better scheme than timeouts to guarantee liveness.
// Progress on the connection is what we really want, not timeouts.
// But we don't have access to that and the request progress indicators
// (various libcurl callbacks) have the same problem TIMEOUT does.
//
// xfer_timeout *= cpolicy.mPipelining;
xfer_timeout *= 2L;
}
// *DEBUG: Enable following override for timeout handling and "[curl:bugs] #1420" tests
// xfer_timeout = 1L;
code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout);
check_curl_easy_code(code, CURLOPT_TIMEOUT);
code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT);
// Request headers
if (mReqHeaders)
{
// Caller's headers last to override
mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
check_curl_easy_code(code, CURLOPT_HTTPHEADER);
if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER))
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
check_curl_easy_code(code, CURLOPT_HEADERFUNCTION);
code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
check_curl_easy_code(code, CURLOPT_HEADERDATA);
}
if (status)
{
mCurlService = service;
}
return status;
}
// *NOTE: cancelRequest logic parallels completeRequest logic.
// Keep them synchronized as necessary.
bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status)
{
HttpOpRequest * op(NULL);
curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op);
if (handle != op->mCurlHandle || ! op->mCurlActive)
{
LL_WARNS(LOG_CORE) << "libcurl handle and HttpOpRequest handle in disagreement or inactive request."
<< " Handle: " << static_cast<HttpHandle>(handle)
<< LL_ENDL;
return false;
}
active_set_t::iterator it(mActiveOps.find(op));
if (mActiveOps.end() == it)
{
LL_WARNS(LOG_CORE) << "libcurl completion for request not on active list. Continuing."
<< " Handle: " << static_cast<HttpHandle>(handle)
<< LL_ENDL;
return false;
}
// Deactivate request
mActiveOps.erase(it);
--mActiveHandles[op->mReqPolicy];
op->mCurlActive = false;
// Set final status of request if it hasn't failed by other mechanisms yet
if (op->mStatus)
{
op->mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, status);
}
if (op->mStatus)
{
int http_status(HTTP_OK);
curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status);
if (http_status >= 100 && http_status <= 999)
{
char * cont_type(NULL);
curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &cont_type);
if (cont_type)
{
op->mReplyConType = cont_type;
}
op->mStatus = HttpStatus(http_status);
}
else
{
LL_WARNS(LOG_CORE) << "Invalid HTTP response code ("
<< http_status << ") received from server."
<< LL_ENDL;
op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS);
}
}
// Detach from multi and recycle handle
curl_multi_remove_handle(multi_handle, handle);
mHandleCache.freeHandle(op->mCurlHandle);
op->mCurlHandle = NULL;
// Tracing
if (op->mTracing > HTTP_TRACE_OFF)
{
LL_INFOS(LOG_CORE) << "TRACE, RequestComplete, Handle: "
<< static_cast<HttpHandle>(op)
<< ", Status: " << op->mStatus.toTerseString()
<< LL_ENDL;
}
// Dispatch to next stage
HttpPolicy & policy(mService->getPolicy());
bool still_active(policy.stageAfterCompletion(op));
return still_active;
}
/* * HttpStatus.cpp * * Created on: Apr 10, 2014 * Author: enrico */ #include <utility> #include "HttpStatus.h" const HttpStatus HttpStatus::status_100_Continue = HttpStatus(100, "Continue"); const HttpStatus HttpStatus::status_101_Switching_Protocols = HttpStatus(101, "Switching Protocols"); const HttpStatus HttpStatus::status_200_OK = HttpStatus(200, "OK"); const HttpStatus HttpStatus::status_201_Created = HttpStatus(201, "Created"); const HttpStatus HttpStatus::status_202_Accepted = HttpStatus(202, "Accepted"); const HttpStatus HttpStatus::status_203_Non_Authoritative_Information = HttpStatus(203, "Non-Authoritative Information"); const HttpStatus HttpStatus::status_204_No_Content = HttpStatus(204, "No Content"); const HttpStatus HttpStatus::status_205_Reset_Content = HttpStatus(205, "Reset Content"); const HttpStatus HttpStatus::status_206_Partial_Content = HttpStatus(206, "Partial Content"); const HttpStatus HttpStatus::status_300_Multiple_Choices = HttpStatus(300, "Multiple Choices"); const HttpStatus HttpStatus::status_301_Moved_Permanently = HttpStatus(301, "Moved Permanently"); const HttpStatus HttpStatus::status_302_Found = HttpStatus(302, "Found"); const HttpStatus HttpStatus::status_303_See_Other = HttpStatus(303, "See Other"); const HttpStatus HttpStatus::status_304_Not_Modified = HttpStatus(304, "Not Modified"); const HttpStatus HttpStatus::status_305_Use_Proxy = HttpStatus(305, "Use Proxy"); const HttpStatus HttpStatus::status_307_Temporary_Redirect = HttpStatus(307, "Temporary Redirect"); const HttpStatus HttpStatus::status_400_Bad_Request = HttpStatus(400, "Bad Request"); const HttpStatus HttpStatus::status_401_Unauthorized = HttpStatus(401, "Unauthorized"); const HttpStatus HttpStatus::status_402_Payment_Required = HttpStatus(402, "Payment Required"); const HttpStatus HttpStatus::status_403_Forbidden = HttpStatus(403, "Forbidden"); const HttpStatus HttpStatus::status_404_Not_Found = HttpStatus(404, "Not Found");
// Sets all libcurl options and data for a request.
//
// Used both for initial requests and to 'reload' for
// a retry, generally with a different CURL handle.
// Junk may be left around from a failed request and that
// needs to be cleaned out.
//
HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
{
CURLcode code;
// Scrub transport and result data for retried op case
mCurlActive = false;
mCurlHandle = NULL;
mCurlService = NULL;
if (mCurlHeaders)
{
curl_slist_free_all(mCurlHeaders);
mCurlHeaders = NULL;
}
mCurlBodyPos = 0;
if (mReplyBody)
{
mReplyBody->release();
mReplyBody = NULL;
}
mReplyOffset = 0;
mReplyLength = 0;
mReplyFullLength = 0;
if (mReplyHeaders)
{
mReplyHeaders->release();
mReplyHeaders = NULL;
}
mReplyConType.clear();
// *FIXME: better error handling later
HttpStatus status;
// Get global policy options
HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions());
mCurlHandle = LLCurl::createStandardCurlHandle();
if (! mCurlHandle)
{
// We're in trouble. We'll continue but it won't go well.
LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing."
<< LL_ENDL;
return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC);
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
check_curl_easy_code(code, CURLOPT_IPRESOLVE);
code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
check_curl_easy_code(code, CURLOPT_NOSIGNAL);
code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
check_curl_easy_code(code, CURLOPT_NOPROGRESS);
code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
check_curl_easy_code(code, CURLOPT_URL);
code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
check_curl_easy_code(code, CURLOPT_PRIVATE);
code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
check_curl_easy_code(code, CURLOPT_ENCODING);
// The Linksys WRT54G V5 router has an issue with frequent
// DNS lookups from LAN machines. If they happen too often,
// like for every HTTP request, the router gets annoyed after
// about 700 or so requests and starts issuing TCP RSTs to
// new connections. Reuse the DNS lookups for even a few
// seconds and no RSTs.
code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15);
check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT);
code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
check_curl_easy_code(code, CURLOPT_AUTOREFERER);
code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION);
code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT);
check_curl_easy_code(code, CURLOPT_MAXREDIRS);
code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
check_curl_easy_code(code, CURLOPT_WRITEFUNCTION);
code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this);
check_curl_easy_code(code, CURLOPT_WRITEDATA);
code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
check_curl_easy_code(code, CURLOPT_READFUNCTION);
code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this);
check_curl_easy_code(code, CURLOPT_READDATA);
code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1);
check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER);
code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0);
check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST);
if (policy.mUseLLProxy)
{
// Use the viewer-based thread-safe API which has a
// fast/safe check for proxy enable. Would like to
// encapsulate this someway...
LLProxy::getInstance()->applyProxySettings(mCurlHandle);
}
else if (policy.mHttpProxy.size())
{
// *TODO: This is fine for now but get fuller socks5/
// authentication thing going later....
code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str());
check_curl_easy_code(code, CURLOPT_PROXY);
code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
check_curl_easy_code(code, CURLOPT_PROXYTYPE);
}
if (policy.mCAPath.size())
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str());
check_curl_easy_code(code, CURLOPT_CAPATH);
}
if (policy.mCAFile.size())
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str());
check_curl_easy_code(code, CURLOPT_CAINFO);
}
switch (mReqMethod)
{
case HOR_GET:
code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
check_curl_easy_code(code, CURLOPT_HTTPGET);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
break;
case HOR_POST:
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
check_curl_easy_code(code, CURLOPT_POST);
code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
check_curl_easy_code(code, CURLOPT_ENCODING);
long data_size(0);
if (mReqBody)
{
data_size = mReqBody->size();
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
check_curl_easy_code(code, CURLOPT_POSTFIELDS);
code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size);
check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
}
break;
case HOR_PUT:
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1);
check_curl_easy_code(code, CURLOPT_UPLOAD);
long data_size(0);
if (mReqBody)
{
data_size = mReqBody->size();
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
check_curl_easy_code(code, CURLOPT_INFILESIZE);
code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL);
check_curl_easy_code(code, CURLOPT_POSTFIELDS);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
}
break;
default:
LL_ERRS("CoreHttp") << "Invalid HTTP method in request: "
<< int(mReqMethod) << ". Can't recover."
<< LL_ENDL;
break;
}
// Tracing
if (mTracing >= HTTP_TRACE_CURL_HEADERS)
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
check_curl_easy_code(code, CURLOPT_VERBOSE);
code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this);
check_curl_easy_code(code, CURLOPT_DEBUGDATA);
code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION);
}
// There's a CURLOPT for this now...
if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
{
static const char * const fmt1("Range: bytes=%lu-%lu");
static const char * const fmt2("Range: bytes=%lu-");
char range_line[64];
#if LL_WINDOWS
_snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1,
(mReqLength ? fmt1 : fmt2),
(unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
#else
snprintf(range_line, sizeof(range_line),
(mReqLength ? fmt1 : fmt2),
(unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
#endif // LL_WINDOWS
range_line[sizeof(range_line) - 1] = '\0';
mCurlHeaders = curl_slist_append(mCurlHeaders, range_line);
}
mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
// Request options
long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT);
long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT);
if (mReqOptions)
{
timeout = mReqOptions->getTimeout();
timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
xfer_timeout = mReqOptions->getTransferTimeout();
xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
}
if (xfer_timeout == 0L)
{
xfer_timeout = timeout;
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout);
check_curl_easy_code(code, CURLOPT_TIMEOUT);
code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT);
// Request headers
if (mReqHeaders)
{
// Caller's headers last to override
mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
}
code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
check_curl_easy_code(code, CURLOPT_HTTPHEADER);
if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER))
{
code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
check_curl_easy_code(code, CURLOPT_HEADERFUNCTION);
code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
check_curl_easy_code(code, CURLOPT_HEADERDATA);
}
if (status)
{
mCurlService = service;
}
return status;
}
