/**
* Find the status code and content type and inform the caller.
*
* Return true if the fetch is being aborted.
*/
static bool fetch_curl_process_headers(struct curl_fetch_info *f)
{
long http_code;
CURLcode code;
fetch_msg msg;
f->had_headers = true;
if (!f->http_code) {
code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE,
&f->http_code);
fetch_set_http_code(f->fetch_handle, f->http_code);
assert(code == CURLE_OK);
}
http_code = f->http_code;
LOG("HTTP status code %li", http_code);
if (http_code == 304 && !f->post_urlenc && !f->post_multipart) {
/* Not Modified && GET request */
msg.type = FETCH_NOTMODIFIED;
fetch_send_callback(&msg, f->fetch_handle);
return true;
}
/* handle HTTP redirects (3xx response codes) */
if (300 <= http_code && http_code < 400 && f->location != 0) {
LOG("FETCH_REDIRECT, '%s'", f->location);
msg.type = FETCH_REDIRECT;
msg.data.redirect = f->location;
fetch_send_callback(&msg, f->fetch_handle);
return true;
}
/* handle HTTP 401 (Authentication errors) */
if (http_code == 401) {
msg.type = FETCH_AUTH;
msg.data.auth.realm = f->realm;
fetch_send_callback(&msg, f->fetch_handle);
return true;
}
/* handle HTTP errors (non 2xx response codes) */
if (f->only_2xx && strncmp(nsurl_access(f->url), "http", 4) == 0 &&
(http_code < 200 || 299 < http_code)) {
msg.type = FETCH_ERROR;
msg.data.error = messages_get("Not2xx");
fetch_send_callback(&msg, f->fetch_handle);
return true;
}
if (f->abort)
return true;
return false;
}
static void fetch_rsrc_send_callback(const fetch_msg *msg,
struct fetch_rsrc_context *c)
{
c->locked = true;
fetch_send_callback(msg, c->parent_fetch);
c->locked = false;
}
/** issue fetch callbacks with locking */
static inline bool fetch_javascript_send_callback(const fetch_msg *msg,
struct fetch_javascript_context *ctx)
{
ctx->locked = true;
fetch_send_callback(msg, ctx->fetchh);
ctx->locked = false;
return ctx->aborted;
}
static void ami_fetch_file_send_callback(fetch_msg msg,
struct ami_file_fetch_info *fetch, const void *data,
unsigned long size, fetch_error_code errorcode)
{
fetch->locked = true;
/* LOG(("ami file fetcher callback %ld",msg)); */
fetch_send_callback(msg,fetch->fetch_handle,data,size,errorcode);
fetch->locked = false;
}
/**
* Callback function for fetch progress.
*/
static int
fetch_curl_progress(void *clientp,
double dltotal,
double dlnow,
double ultotal,
double ulnow)
{
static char fetch_progress_buffer[256]; /**< Progress buffer for cURL */
struct curl_fetch_info *f = (struct curl_fetch_info *) clientp;
uint64_t time_now_ms;
fetch_msg msg;
if (f->abort) {
return 0;
}
msg.type = FETCH_PROGRESS;
msg.data.progress = fetch_progress_buffer;
/* Rate limit each fetch's progress notifications */
nsu_getmonotonic_ms(&time_now_ms);
#define UPDATE_DELAY_MS (1000 / UPDATES_PER_SECOND)
if (time_now_ms - f->last_progress_update < UPDATE_DELAY_MS) {
return 0;
}
#undef UPDATE_DELAY_MS
f->last_progress_update = time_now_ms;
if (dltotal > 0) {
snprintf(fetch_progress_buffer, 255,
messages_get("Progress"),
human_friendly_bytesize(dlnow),
human_friendly_bytesize(dltotal));
fetch_send_callback(&msg, f->fetch_handle);
} else {
snprintf(fetch_progress_buffer, 255,
messages_get("ProgressU"),
human_friendly_bytesize(dlnow));
fetch_send_callback(&msg, f->fetch_handle);
}
return 0;
}
/**
* Callback function for cURL.
*/
static size_t fetch_curl_data(char *data, size_t size, size_t nmemb, void *_f)
{
struct curl_fetch_info *f = _f;
CURLcode code;
fetch_msg msg;
/* ensure we only have to get this information once */
if (!f->http_code)
{
code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE,
&f->http_code);
fetch_set_http_code(f->fetch_handle, f->http_code);
assert(code == CURLE_OK);
}
/* ignore body if this is a 401 reply by skipping it and reset
* the HTTP response code to enable follow up fetches.
*/
if (f->http_code == 401)
{
f->http_code = 0;
return size * nmemb;
}
if (f->abort || (!f->had_headers && fetch_curl_process_headers(f))) {
f->stopped = true;
return 0;
}
/* send data to the caller */
msg.type = FETCH_DATA;
msg.data.header_or_data.buf = (const uint8_t *) data;
msg.data.header_or_data.len = size * nmemb;
fetch_send_callback(&msg, f->fetch_handle);
if (f->abort) {
f->stopped = true;
return 0;
}
return size * nmemb;
}
/**
* setup callback to allow the user to examine certificates which have
* failed to validate during fetch.
*/
static void
curl_start_cert_validate(struct curl_fetch_info *f,
struct cert_info *certs)
{
int depth;
BIO *mem;
BUF_MEM *buf;
struct ssl_cert_info ssl_certs[MAX_CERTS];
fetch_msg msg;
for (depth = 0; depth <= f->cert_depth; depth++) {
assert(certs[depth].cert != NULL);
/* get certificate version */
ssl_certs[depth].version = X509_get_version(certs[depth].cert);
/* not before date */
mem = BIO_new(BIO_s_mem());
ASN1_TIME_print(mem, X509_get_notBefore(certs[depth].cert));
BIO_get_mem_ptr(mem, &buf);
(void) BIO_set_close(mem, BIO_NOCLOSE);
BIO_free(mem);
memcpy(ssl_certs[depth].not_before,
buf->data,
min(sizeof(ssl_certs[depth].not_before) - 1,
(unsigned)buf->length));
ssl_certs[depth].not_before[min(sizeof(ssl_certs[depth].not_before) - 1,
(unsigned)buf->length)] = 0;
BUF_MEM_free(buf);
/* not after date */
mem = BIO_new(BIO_s_mem());
ASN1_TIME_print(mem,
X509_get_notAfter(certs[depth].cert));
BIO_get_mem_ptr(mem, &buf);
(void) BIO_set_close(mem, BIO_NOCLOSE);
BIO_free(mem);
memcpy(ssl_certs[depth].not_after,
buf->data,
min(sizeof(ssl_certs[depth].not_after) - 1,
(unsigned)buf->length));
ssl_certs[depth].not_after[min(sizeof(ssl_certs[depth].not_after) - 1,
(unsigned)buf->length)] = 0;
BUF_MEM_free(buf);
/* signature type */
ssl_certs[depth].sig_type =
X509_get_signature_type(certs[depth].cert);
/* serial number */
ssl_certs[depth].serial =
ASN1_INTEGER_get(
X509_get_serialNumber(certs[depth].cert));
/* issuer name */
mem = BIO_new(BIO_s_mem());
X509_NAME_print_ex(mem,
X509_get_issuer_name(certs[depth].cert),
0, XN_FLAG_SEP_CPLUS_SPC |
XN_FLAG_DN_REV | XN_FLAG_FN_NONE);
BIO_get_mem_ptr(mem, &buf);
(void) BIO_set_close(mem, BIO_NOCLOSE);
BIO_free(mem);
memcpy(ssl_certs[depth].issuer,
buf->data,
min(sizeof(ssl_certs[depth].issuer) - 1,
(unsigned) buf->length));
ssl_certs[depth].issuer[min(sizeof(ssl_certs[depth].issuer) - 1,
(unsigned) buf->length)] = 0;
BUF_MEM_free(buf);
/* subject */
mem = BIO_new(BIO_s_mem());
X509_NAME_print_ex(mem,
X509_get_subject_name(certs[depth].cert),
0,
XN_FLAG_SEP_CPLUS_SPC |
XN_FLAG_DN_REV |
XN_FLAG_FN_NONE);
BIO_get_mem_ptr(mem, &buf);
(void) BIO_set_close(mem, BIO_NOCLOSE);
BIO_free(mem);
memcpy(ssl_certs[depth].subject,
buf->data,
min(sizeof(ssl_certs[depth].subject) - 1,
(unsigned)buf->length));
ssl_certs[depth].subject[min(sizeof(ssl_certs[depth].subject) - 1,
(unsigned) buf->length)] = 0;
BUF_MEM_free(buf);
/* type of certificate */
ssl_certs[depth].cert_type =
X509_certificate_type(certs[depth].cert,
X509_get_pubkey(certs[depth].cert));
/* and clean up */
certs[depth].cert->references--;
if (certs[depth].cert->references == 0) {
X509_free(certs[depth].cert);
}
}
msg.type = FETCH_CERT_ERR;
msg.data.cert_err.certs = ssl_certs;
msg.data.cert_err.num_certs = depth;
fetch_send_callback(&msg, f->fetch_handle);
}
/**
* Callback function for headers.
*
* See RFC 2616 4.2.
*/
static size_t
fetch_curl_header(char *data, size_t size, size_t nmemb, void *_f)
{
struct curl_fetch_info *f = _f;
int i;
fetch_msg msg;
size *= nmemb;
if (f->abort) {
f->stopped = true;
return 0;
}
msg.type = FETCH_HEADER;
msg.data.header_or_data.buf = (const uint8_t *) data;
msg.data.header_or_data.len = size;
fetch_send_callback(&msg, f->fetch_handle);
#define SKIP_ST(o) for (i = (o); i < (int) size && (data[i] == ' ' || data[i] == '\t'); i++)
if (12 < size && strncasecmp(data, "Location:", 9) == 0) {
/* extract Location header */
free(f->location);
f->location = malloc(size);
if (!f->location) {
LOG("malloc failed");
return size;
}
SKIP_ST(9);
strncpy(f->location, data + i, size - i);
f->location[size - i] = '\0';
for (i = size - i - 1; i >= 0 &&
(f->location[i] == ' ' ||
f->location[i] == '\t' ||
f->location[i] == '\r' ||
f->location[i] == '\n'); i--)
f->location[i] = '\0';
} else if (15 < size && strncasecmp(data, "Content-Length:", 15) == 0) {
/* extract Content-Length header */
SKIP_ST(15);
if (i < (int)size && '0' <= data[i] && data[i] <= '9')
f->content_length = atol(data + i);
} else if (17 < size && strncasecmp(data, "WWW-Authenticate:", 17) == 0) {
/* extract the first Realm from WWW-Authenticate header */
SKIP_ST(17);
while (i < (int) size - 5 &&
strncasecmp(data + i, "realm", 5))
i++;
while (i < (int) size - 1 && data[++i] != '"')
/* */;
i++;
if (i < (int) size) {
size_t end = i;
while (end < size && data[end] != '"')
++end;
if (end < size) {
free(f->realm);
f->realm = malloc(end - i + 1);
if (f->realm != NULL) {
strncpy(f->realm, data + i, end - i);
f->realm[end - i] = '\0';
}
}
}
} else if (11 < size && strncasecmp(data, "Set-Cookie:", 11) == 0) {
/* extract Set-Cookie header */
SKIP_ST(11);
fetch_set_cookie(f->fetch_handle, &data[i]);
}
return size;
#undef SKIP_ST
}
/**
* Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()).
*
* \param curl_handle curl easy handle of fetch
* \param result The result code of the completed fetch.
*/
static void fetch_curl_done(CURL *curl_handle, CURLcode result)
{
bool finished = false;
bool error = false;
bool cert = false;
bool abort_fetch;
struct curl_fetch_info *f;
char **_hideous_hack = (char **) (void *) &f;
CURLcode code;
struct cert_info certs[MAX_CERTS];
/* find the structure associated with this fetch */
/* For some reason, cURL thinks CURLINFO_PRIVATE should be a string?! */
code = curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, _hideous_hack);
assert(code == CURLE_OK);
abort_fetch = f->abort;
LOG("done %s", nsurl_access(f->url));
if ((abort_fetch == false) &&
(result == CURLE_OK ||
((result == CURLE_WRITE_ERROR) && (f->stopped == false)))) {
/* fetch completed normally or the server fed us a junk gzip
* stream (usually in the form of garbage at the end of the
* stream). Curl will have fed us all but the last chunk of
* decoded data, which is sad as, if we'd received the last
* chunk, too, we'd be able to render the whole object.
* As is, we'll just have to accept that the end of the
* object will be truncated in this case and leave it to
* the content handlers to cope.
*/
if (f->stopped ||
(!f->had_headers && fetch_curl_process_headers(f))) {
; /* redirect with no body or similar */
} else {
finished = true;
}
} else if (result == CURLE_PARTIAL_FILE) {
/* CURLE_PARTIAL_FILE occurs if the received body of a
* response is smaller than that specified in the
* Content-Length header.
*/
if (!f->had_headers && fetch_curl_process_headers(f))
; /* redirect with partial body, or similar */
else {
finished = true;
}
} else if (result == CURLE_WRITE_ERROR && f->stopped) {
/* CURLE_WRITE_ERROR occurs when fetch_curl_data
* returns 0, which we use to abort intentionally
*/
;
} else if (result == CURLE_SSL_PEER_CERTIFICATE ||
result == CURLE_SSL_CACERT) {
/* CURLE_SSL_PEER_CERTIFICATE renamed to
* CURLE_PEER_FAILED_VERIFICATION
*/
memset(certs, 0, sizeof(certs));
memcpy(certs, f->cert_data, sizeof(certs));
memset(f->cert_data, 0, sizeof(f->cert_data));
cert = true;
} else {
LOG("Unknown cURL response code %d", result);
error = true;
}
fetch_curl_stop(f);
if (abort_fetch) {
; /* fetch was aborted: no callback */
} else if (finished) {
fetch_msg msg;
msg.type = FETCH_FINISHED;
fetch_send_callback(&msg, f->fetch_handle);
} else if (cert) {
/* user needs to validate certificate with issue */
curl_start_cert_validate(f, certs);
} else if (error) {
fetch_msg msg;
switch (result) {
case CURLE_SSL_CONNECT_ERROR:
msg.type = FETCH_SSL_ERR;
break;
case CURLE_OPERATION_TIMEDOUT:
msg.type = FETCH_TIMEDOUT;
msg.data.error = curl_easy_strerror(result);
break;
default:
msg.type = FETCH_ERROR;
msg.data.error = curl_easy_strerror(result);
}
fetch_send_callback(&msg, f->fetch_handle);
}
fetch_free(f->fetch_handle);
}
/**
* Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()).
*
* \param curl_handle curl easy handle of fetch
* \param result The result code of the completed fetch.
*/
static void fetch_curl_done(CURL *curl_handle, CURLcode result)
{
fetch_msg msg;
bool finished = false;
bool error = false;
bool cert = false;
bool abort_fetch;
struct curl_fetch_info *f;
char **_hideous_hack = (char **) (void *) &f;
CURLcode code;
struct cert_info certs[MAX_CERTS];
memset(certs, 0, sizeof(certs));
/* find the structure associated with this fetch */
/* For some reason, cURL thinks CURLINFO_PRIVATE should be a string?! */
code = curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, _hideous_hack);
assert(code == CURLE_OK);
abort_fetch = f->abort;
LOG("done %s", nsurl_access(f->url));
if (abort_fetch == false && (result == CURLE_OK ||
(result == CURLE_WRITE_ERROR && f->stopped == false))) {
/* fetch completed normally or the server fed us a junk gzip
* stream (usually in the form of garbage at the end of the
* stream). Curl will have fed us all but the last chunk of
* decoded data, which is sad as, if we'd received the last
* chunk, too, we'd be able to render the whole object.
* As is, we'll just have to accept that the end of the
* object will be truncated in this case and leave it to
* the content handlers to cope.
*/
if (f->stopped ||
(!f->had_headers &&
fetch_curl_process_headers(f)))
; /* redirect with no body or similar */
else
finished = true;
} else if (result == CURLE_PARTIAL_FILE) {
/* CURLE_PARTIAL_FILE occurs if the received body of a
* response is smaller than that specified in the
* Content-Length header. */
if (!f->had_headers && fetch_curl_process_headers(f))
; /* redirect with partial body, or similar */
else {
finished = true;
}
} else if (result == CURLE_WRITE_ERROR && f->stopped) {
/* CURLE_WRITE_ERROR occurs when fetch_curl_data
* returns 0, which we use to abort intentionally */
;
} else if (result == CURLE_SSL_PEER_CERTIFICATE ||
result == CURLE_SSL_CACERT) {
memcpy(certs, f->cert_data, sizeof(certs));
memset(f->cert_data, 0, sizeof(f->cert_data));
cert = true;
} else {
LOG("Unknown cURL response code %d", result);
error = true;
}
fetch_curl_stop(f);
if (abort_fetch)
; /* fetch was aborted: no callback */
else if (finished) {
msg.type = FETCH_FINISHED;
fetch_send_callback(&msg, f->fetch_handle);
} else if (cert) {
int i;
BIO *mem;
BUF_MEM *buf;
struct ssl_cert_info ssl_certs[MAX_CERTS];
for (i = 0; i < MAX_CERTS && certs[i].cert; i++) {
ssl_certs[i].version =
X509_get_version(certs[i].cert);
mem = BIO_new(BIO_s_mem());
ASN1_TIME_print(mem,
X509_get_notBefore(certs[i].cert));
BIO_get_mem_ptr(mem, &buf);
(void) BIO_set_close(mem, BIO_NOCLOSE);
BIO_free(mem);
memcpy(ssl_certs[i].not_before,
buf->data,
min(sizeof(ssl_certs[i].not_before) - 1,
(unsigned)buf->length));
ssl_certs[i].not_before[min(sizeof(ssl_certs[i].not_before) - 1,
(unsigned)buf->length)] = 0;
BUF_MEM_free(buf);
mem = BIO_new(BIO_s_mem());
ASN1_TIME_print(mem,
X509_get_notAfter(certs[i].cert));
BIO_get_mem_ptr(mem, &buf);
(void) BIO_set_close(mem, BIO_NOCLOSE);
BIO_free(mem);
memcpy(ssl_certs[i].not_after,
buf->data,
min(sizeof(ssl_certs[i].not_after) - 1,
(unsigned)buf->length));
ssl_certs[i].not_after[min(sizeof(ssl_certs[i].not_after) - 1,
(unsigned)buf->length)] = 0;
BUF_MEM_free(buf);
ssl_certs[i].sig_type =
X509_get_signature_type(certs[i].cert);
ssl_certs[i].serial =
ASN1_INTEGER_get(
X509_get_serialNumber(certs[i].cert));
mem = BIO_new(BIO_s_mem());
X509_NAME_print_ex(mem,
X509_get_issuer_name(certs[i].cert),
0, XN_FLAG_SEP_CPLUS_SPC |
XN_FLAG_DN_REV | XN_FLAG_FN_NONE);
BIO_get_mem_ptr(mem, &buf);
(void) BIO_set_close(mem, BIO_NOCLOSE);
BIO_free(mem);
memcpy(ssl_certs[i].issuer,
buf->data,
min(sizeof(ssl_certs[i].issuer) - 1,
(unsigned) buf->length));
ssl_certs[i].issuer[min(sizeof(ssl_certs[i].issuer) - 1,
(unsigned) buf->length)] = 0;
BUF_MEM_free(buf);
mem = BIO_new(BIO_s_mem());
X509_NAME_print_ex(mem,
X509_get_subject_name(certs[i].cert),
0,
XN_FLAG_SEP_CPLUS_SPC |
XN_FLAG_DN_REV |
XN_FLAG_FN_NONE);
BIO_get_mem_ptr(mem, &buf);
(void) BIO_set_close(mem, BIO_NOCLOSE);
BIO_free(mem);
memcpy(ssl_certs[i].subject,
buf->data,
min(sizeof(ssl_certs[i].subject) - 1,
(unsigned)buf->length));
ssl_certs[i].subject[min(sizeof(ssl_certs[i].subject) - 1,
(unsigned) buf->length)] = 0;
BUF_MEM_free(buf);
ssl_certs[i].cert_type =
X509_certificate_type(certs[i].cert,
X509_get_pubkey(certs[i].cert));
/* and clean up */
certs[i].cert->references--;
if (certs[i].cert->references == 0)
X509_free(certs[i].cert);
}
msg.type = FETCH_CERT_ERR;
msg.data.cert_err.certs = ssl_certs;
msg.data.cert_err.num_certs = i;
fetch_send_callback(&msg, f->fetch_handle);
} else if (error) {
switch (result) {
case CURLE_SSL_CONNECT_ERROR:
msg.type = FETCH_SSL_ERR;
break;
case CURLE_OPERATION_TIMEDOUT:
msg.type = FETCH_TIMEDOUT;
msg.data.error = curl_easy_strerror(result);
break;
default:
msg.type = FETCH_ERROR;
msg.data.error = curl_easy_strerror(result);
}
fetch_send_callback(&msg, f->fetch_handle);
}
fetch_free(f->fetch_handle);
}
