// Sets all libcurl options and data for a request. // // Used both for initial requests and to 'reload' for // a retry, generally with a different CURL handle. // Junk may be left around from a failed request and that // needs to be cleaned out. // // *TODO: Move this to _httplibcurl where it belongs. HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { CURLcode code; // Scrub transport and result data for retried op case mCurlActive = false; mCurlHandle = NULL; mCurlService = NULL; if (mCurlHeaders) { curl_slist_free_all(mCurlHeaders); mCurlHeaders = NULL; } mCurlBodyPos = 0; if (mReplyBody) { mReplyBody->release(); mReplyBody = NULL; } mReplyOffset = 0; mReplyLength = 0; mReplyFullLength = 0; if (mReplyHeaders) { mReplyHeaders->release(); mReplyHeaders = NULL; } mReplyConType.clear(); // *FIXME: better error handling later HttpStatus status; // Get global and class policy options HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions()); HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy)); mCurlHandle = service->getTransport().getHandle(); if (! mCurlHandle) { // We're in trouble. We'll continue but it won't go well. LL_WARNS(LOG_CORE) << "Failed to allocate libcurl easy handle. Continuing." << LL_ENDL; return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); } code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); check_curl_easy_code(code, CURLOPT_IPRESOLVE); code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); check_curl_easy_code(code, CURLOPT_NOSIGNAL); code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); check_curl_easy_code(code, CURLOPT_NOPROGRESS); code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); check_curl_easy_code(code, CURLOPT_URL); code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); check_curl_easy_code(code, CURLOPT_PRIVATE); code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); check_curl_easy_code(code, CURLOPT_ENCODING); // The Linksys WRT54G V5 router has an issue with frequent // DNS lookups from LAN machines. If they happen too often, // like for every HTTP request, the router gets annoyed after // about 700 or so requests and starts issuing TCP RSTs to // new connections. Reuse the DNS lookups for even a few // seconds and no RSTs. code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); check_curl_easy_code(code, CURLOPT_AUTOREFERER); code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION); code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); check_curl_easy_code(code, CURLOPT_MAXREDIRS); code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); check_curl_easy_code(code, CURLOPT_WRITEFUNCTION); code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); check_curl_easy_code(code, CURLOPT_WRITEDATA); code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); check_curl_easy_code(code, CURLOPT_READFUNCTION); code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); check_curl_easy_code(code, CURLOPT_READDATA); code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER); code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); if (gpolicy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } else if (gpolicy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, gpolicy.mHttpProxy.c_str()); check_curl_easy_code(code, CURLOPT_PROXY); code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); check_curl_easy_code(code, CURLOPT_PROXYTYPE); } if (gpolicy.mCAPath.size()) { code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, gpolicy.mCAPath.c_str()); check_curl_easy_code(code, CURLOPT_CAPATH); } if (gpolicy.mCAFile.size()) { code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, gpolicy.mCAFile.c_str()); check_curl_easy_code(code, CURLOPT_CAINFO); } switch (mReqMethod) { case HOR_GET: code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); check_curl_easy_code(code, CURLOPT_HTTPGET); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); break; case HOR_POST: { code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); check_curl_easy_code(code, CURLOPT_POST); code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); check_curl_easy_code(code, CURLOPT_ENCODING); long data_size(0); if (mReqBody) { data_size = mReqBody->size(); } code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); check_curl_easy_code(code, CURLOPT_POSTFIELDS); code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); } break; case HOR_PUT: { code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); check_curl_easy_code(code, CURLOPT_UPLOAD); long data_size(0); if (mReqBody) { data_size = mReqBody->size(); } code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); check_curl_easy_code(code, CURLOPT_INFILESIZE); code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); check_curl_easy_code(code, CURLOPT_POSTFIELDS); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); // *TODO: Should this be 'Keep-Alive' ? mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); } break; default: LL_ERRS(LOG_CORE) << "Invalid HTTP method in request: " << int(mReqMethod) << ". Can't recover." << LL_ENDL; break; } // Tracing if (mTracing >= HTTP_TRACE_CURL_HEADERS) { code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); check_curl_easy_code(code, CURLOPT_VERBOSE); code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); check_curl_easy_code(code, CURLOPT_DEBUGDATA); code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION); } // There's a CURLOPT for this now... if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) { static const char * const fmt1("Range: bytes=%lu-%lu"); static const char * const fmt2("Range: bytes=%lu-"); char range_line[64]; #if LL_WINDOWS _snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1, (mReqLength ? fmt1 : fmt2), (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); #else if ( mReqLength ) { snprintf(range_line, sizeof(range_line), fmt1, (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); } else { snprintf(range_line, sizeof(range_line), fmt2, (unsigned long) mReqOffset); } #endif // LL_WINDOWS range_line[sizeof(range_line) - 1] = '\0'; mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); } mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); // Request options long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); if (mReqOptions) { timeout = mReqOptions->getTimeout(); timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); xfer_timeout = mReqOptions->getTransferTimeout(); xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); } if (xfer_timeout == 0L) { xfer_timeout = timeout; } if (cpolicy.mPipelining > 1L) { // Pipelining affects both connection and transfer timeout values. // Requests that are added to a pipeling immediately have completed // their connection so the connection delay tends to be less than // the non-pipelined value. Transfers are the opposite. Transfer // timeout starts once the connection is established and completion // can be delayed due to the pipelined requests ahead. So, it's // a handwave but bump the transfer timeout up by the pipelining // depth to give some room. // // BUG-7698, BUG-7688, BUG-7694 (others). Scylla and Charybdis // situation. Operating against a CDN having service issues may // lead to requests stalling for an arbitrarily long time with only // the CURLOPT_TIMEOUT value leading to a closed connection. Sadly // for pipelining, libcurl (7.39.0 and earlier, at minimum) starts // the clock on this value as soon as a request is started down // the wire. We want a short value to recover and retry from the // CDN. We need a long value to safely deal with a succession of // piled-up pipelined requests. // // *TODO: Find a better scheme than timeouts to guarantee liveness. // Progress on the connection is what we really want, not timeouts. // But we don't have access to that and the request progress indicators // (various libcurl callbacks) have the same problem TIMEOUT does. // // xfer_timeout *= cpolicy.mPipelining; xfer_timeout *= 2L; } // *DEBUG: Enable following override for timeout handling and "[curl:bugs] #1420" tests // xfer_timeout = 1L; code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT); // Request headers if (mReqHeaders) { // Caller's headers last to override mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); } code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); check_curl_easy_code(code, CURLOPT_HTTPHEADER); if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) { code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); check_curl_easy_code(code, CURLOPT_HEADERFUNCTION); code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); check_curl_easy_code(code, CURLOPT_HEADERDATA); } if (status) { mCurlService = service; } return status; }
// Sets all libcurl options and data for a request. // // Used both for initial requests and to 'reload' for // a retry, generally with a different CURL handle. // Junk may be left around from a failed request and that // needs to be cleaned out. // HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { CURLcode code; // Scrub transport and result data for retried op case mCurlActive = false; mCurlHandle = NULL; mCurlService = NULL; if (mCurlHeaders) { curl_slist_free_all(mCurlHeaders); mCurlHeaders = NULL; } mCurlBodyPos = 0; if (mReplyBody) { mReplyBody->release(); mReplyBody = NULL; } mReplyOffset = 0; mReplyLength = 0; mReplyFullLength = 0; if (mReplyHeaders) { mReplyHeaders->release(); mReplyHeaders = NULL; } mReplyConType.clear(); // *FIXME: better error handling later HttpStatus status; // Get global policy options HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); mCurlHandle = LLCurl::createStandardCurlHandle(); if (! mCurlHandle) { // We're in trouble. We'll continue but it won't go well. LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing." << LL_ENDL; return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); } code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); check_curl_easy_code(code, CURLOPT_IPRESOLVE); code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); check_curl_easy_code(code, CURLOPT_NOSIGNAL); code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); check_curl_easy_code(code, CURLOPT_NOPROGRESS); code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); check_curl_easy_code(code, CURLOPT_URL); code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); check_curl_easy_code(code, CURLOPT_PRIVATE); code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); check_curl_easy_code(code, CURLOPT_ENCODING); // The Linksys WRT54G V5 router has an issue with frequent // DNS lookups from LAN machines. If they happen too often, // like for every HTTP request, the router gets annoyed after // about 700 or so requests and starts issuing TCP RSTs to // new connections. Reuse the DNS lookups for even a few // seconds and no RSTs. code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); check_curl_easy_code(code, CURLOPT_AUTOREFERER); code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION); code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); check_curl_easy_code(code, CURLOPT_MAXREDIRS); code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); check_curl_easy_code(code, CURLOPT_WRITEFUNCTION); code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); check_curl_easy_code(code, CURLOPT_WRITEDATA); code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); check_curl_easy_code(code, CURLOPT_READFUNCTION); code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); check_curl_easy_code(code, CURLOPT_READDATA); code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER); code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); if (policy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } else if (policy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); check_curl_easy_code(code, CURLOPT_PROXY); code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); check_curl_easy_code(code, CURLOPT_PROXYTYPE); } if (policy.mCAPath.size()) { code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); check_curl_easy_code(code, CURLOPT_CAPATH); } if (policy.mCAFile.size()) { code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); check_curl_easy_code(code, CURLOPT_CAINFO); } switch (mReqMethod) { case HOR_GET: code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); check_curl_easy_code(code, CURLOPT_HTTPGET); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); break; case HOR_POST: { code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); check_curl_easy_code(code, CURLOPT_POST); code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); check_curl_easy_code(code, CURLOPT_ENCODING); long data_size(0); if (mReqBody) { data_size = mReqBody->size(); } code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); check_curl_easy_code(code, CURLOPT_POSTFIELDS); code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); } break; case HOR_PUT: { code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); check_curl_easy_code(code, CURLOPT_UPLOAD); long data_size(0); if (mReqBody) { data_size = mReqBody->size(); } code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); check_curl_easy_code(code, CURLOPT_INFILESIZE); code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); check_curl_easy_code(code, CURLOPT_POSTFIELDS); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); } break; default: LL_ERRS("CoreHttp") << "Invalid HTTP method in request: " << int(mReqMethod) << ". Can't recover." << LL_ENDL; break; } // Tracing if (mTracing >= HTTP_TRACE_CURL_HEADERS) { code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); check_curl_easy_code(code, CURLOPT_VERBOSE); code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); check_curl_easy_code(code, CURLOPT_DEBUGDATA); code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION); } // There's a CURLOPT for this now... if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) { static const char * const fmt1("Range: bytes=%lu-%lu"); static const char * const fmt2("Range: bytes=%lu-"); char range_line[64]; #if LL_WINDOWS _snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1, (mReqLength ? fmt1 : fmt2), (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); #else snprintf(range_line, sizeof(range_line), (mReqLength ? fmt1 : fmt2), (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); #endif // LL_WINDOWS range_line[sizeof(range_line) - 1] = '\0'; mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); } mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); // Request options long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); if (mReqOptions) { timeout = mReqOptions->getTimeout(); timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); xfer_timeout = mReqOptions->getTransferTimeout(); xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); } if (xfer_timeout == 0L) { xfer_timeout = timeout; } code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); check_curl_easy_code(code, CURLOPT_TIMEOUT); code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT); // Request headers if (mReqHeaders) { // Caller's headers last to override mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); } code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); check_curl_easy_code(code, CURLOPT_HTTPHEADER); if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) { code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); check_curl_easy_code(code, CURLOPT_HEADERFUNCTION); code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); check_curl_easy_code(code, CURLOPT_HEADERDATA); } if (status) { mCurlService = service; } return status; }