/* downloads a decrypted image from Walrus based on the manifest URL,
* saves it to outfile */
static int walrus_request (const char * walrus_op, const char * verb, const char * requested_url, const char * outfile, const int do_compress)
{
int code = ERROR;
char url [BUFSIZE];
strncpy (url, requested_url, BUFSIZE);
#if defined(CAN_GZIP)
if (do_compress)
snprintf (url, BUFSIZE, "%s%s", requested_url, "?IsCompressed=true");
#endif
logprintfl (EUCAINFO, "walrus_request(): downloading %s\n", outfile);
logprintfl (EUCAINFO, " from %s\n", url);
/* isolate the PATH in the URL as it will be needed for signing */
char * url_path;
if (strncasecmp (url, "http://", 7)!=0) {
logprintfl (EUCAERROR, "walrus_request(): URL must start with http://...\n");
return code;
}
if ((url_path=strchr(url+7, '/'))==NULL) { /* find first '/' after hostname */
logprintfl (EUCAERROR, "walrus_request(): URL has no path\n");
return code;
}
if (euca_init_cert()) {
logprintfl (EUCAERROR, "walrus_request(): failed to initialize certificate\n");
return code;
}
FILE * fp = fopen64 (outfile, "w");
if (fp==NULL) {
logprintfl (EUCAERROR, "walrus_request(): failed to open %s for writing\n", outfile);
return code;
}
CURL * curl;
CURLcode result;
curl = curl_easy_init ();
if (curl==NULL) {
logprintfl (EUCAERROR, "walrus_request(): could not initialize libcurl\n");
fclose(fp);
return code;
}
char error_msg [CURL_ERROR_SIZE];
curl_easy_setopt (curl, CURLOPT_ERRORBUFFER, error_msg);
curl_easy_setopt (curl, CURLOPT_URL, url);
curl_easy_setopt (curl, CURLOPT_HEADERFUNCTION, write_header);
if (strncmp (verb, "GET", 4)==0) {
curl_easy_setopt (curl, CURLOPT_HTTPGET, 1L);
} else if (strncmp (verb, "HEAD", 5)==0) {
/* TODO: HEAD isn't very useful atm since we don't look at headers */
curl_easy_setopt (curl, CURLOPT_NOBODY, 1L);
} else {
fclose(fp);
logprintfl (EUCAERROR, "walrus_request(): invalid HTTP verb %s\n", verb);
return ERROR; /* TODO: dealloc structs before returning! */
}
/* set up the default write function, but possibly override
* it below, if compression is desired and possible */
struct request params;
params.fp = fp;
curl_easy_setopt (curl, CURLOPT_WRITEDATA, ¶ms);
curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_data);
#if defined(CAN_GZIP)
if (do_compress) {
curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_data_zlib);
}
#endif
struct curl_slist * headers = NULL; /* beginning of a DLL with headers */
headers = curl_slist_append (headers, "Authorization: Euca");
char op_hdr [STRSIZE];
if(walrus_op != NULL) {
snprintf (op_hdr, STRSIZE, "EucaOperation: %s", walrus_op);
headers = curl_slist_append (headers, op_hdr);
}
time_t t = time(NULL);
char * date_str = asctime(localtime(&t)); /* points to a static area */
if (date_str==NULL) {
fclose(fp);
return ERROR;
}
assert (strlen(date_str)+7<=STRSIZE);
date_str [strlen(date_str)-1] = '\0'; /* trim off the newline */
char date_hdr [STRSIZE];
snprintf (date_hdr, STRSIZE, "Date: %s", date_str);
headers = curl_slist_append (headers, date_hdr);
char * cert_str = euca_get_cert (0); /* read the cloud-wide cert */
if (cert_str==NULL) {
fclose(fp);
return ERROR;
}
char * cert64_str = base64_enc ((unsigned char *)cert_str, strlen(cert_str));
assert (strlen(cert64_str)+11<=BUFSIZE);
char cert_hdr [BUFSIZE];
snprintf (cert_hdr, BUFSIZE, "EucaCert: %s", cert64_str);
logprintfl (EUCADEBUG2, "walrus_request(): base64 certificate, %s\n", get_string_stats(cert64_str));
headers = curl_slist_append (headers, cert_hdr);
free (cert64_str);
free (cert_str);
char * sig_str = euca_sign_url (verb, date_str, url_path); /* create Walrus-compliant sig */
if (sig_str==NULL) {
fclose(fp);
return ERROR;
}
assert (strlen(sig_str)+16<=BUFSIZE);
char sig_hdr [BUFSIZE];
snprintf (sig_hdr, BUFSIZE, "EucaSignature: %s", sig_str);
headers = curl_slist_append (headers, sig_hdr);
curl_easy_setopt (curl, CURLOPT_HTTPHEADER, headers); /* register headers */
if (walrus_op) {
logprintfl (EUCADEBUG, "walrus_request(): writing %s/%s output to %s\n", verb, walrus_op, outfile);
} else {
logprintfl (EUCADEBUG, "walrus_request(): writing %s output to %s\n", verb, outfile);
}
int retries = TOTAL_RETRIES;
int timeout = FIRST_TIMEOUT;
do {
params.total_wrote = 0L;
params.total_calls = 0L;
#if defined(CAN_GZIP)
if (do_compress) {
/* allocate zlib inflate state */
params.strm.zalloc = Z_NULL;
params.strm.zfree = Z_NULL;
params.strm.opaque = Z_NULL;
params.strm.avail_in = 0;
params.strm.next_in = Z_NULL;
params.ret = inflateInit2 (&(params.strm), 31);
if (params.ret != Z_OK) {
zerr (params.ret, "walrus_request");
break;
}
}
#endif
result = curl_easy_perform (curl); /* do it */
logprintfl (EUCADEBUG, "walrus_request(): wrote %ld bytes in %ld writes\n", params.total_wrote, params.total_calls);
#if defined(CAN_GZIP)
if (do_compress) {
inflateEnd(&(params.strm));
if (params.ret != Z_STREAM_END) {
zerr (params.ret, "walrus_request");
}
}
#endif
if (result) { // curl error (connection or transfer failed)
logprintfl (EUCAERROR, "walrus_request(): %s (%d)\n", error_msg, result);
if (retries > 0) {
logprintfl (EUCAERROR, " download retry %d of %d will commence in %d seconds\n", retries, TOTAL_RETRIES, timeout);
}
sleep (timeout);
fseek (fp, 0L, SEEK_SET);
timeout <<= 1;
retries--;
} else {
long httpcode;
curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &httpcode);
/* TODO: pull out response message, too */
switch (httpcode) {
case 200L: /* all good */
logprintfl (EUCAINFO, "walrus_request(): saved image in %s\n", outfile);
code = OK;
retries = 0;
break;
case 408L: /* timeout, retry */
logprintfl (EUCAWARN, "walrus_request(): server responded with HTTP code %ld (timeout), retrying\n", httpcode);
logcat (EUCADEBUG, outfile); /* dump the error from outfile into the log */
break;
default: /* some kind of error */
logprintfl (EUCAERROR, "walrus_request(): server responded with HTTP code %ld, retrying\n", httpcode);
logcat (EUCADEBUG, outfile); /* dump the error from outfile into the log */
}
}
} while (code!=OK && retries>0);
fclose (fp);
if ( code != OK ) {
logprintfl (EUCAINFO, "walrus_request(): due to error, removing %s\n", outfile);
remove (outfile);
}
free (sig_str);
curl_slist_free_all (headers);
curl_easy_cleanup (curl);
return code;
}
//!
//! downloads a decrypted image from Walrus based on the manifest URL,
//! saves it to outfile
//!
//! @param[in] walrus_op
//! @param[in] verb
//! @param[in] requested_url
//! @param[in] outfile
//! @param[in] do_compress
//! @param[in] connect_timeout
//! @param[in] total_timeout
//!
//! @return EUCA_OK on success or proper error code. Known error code returned include: EUCA_ERROR.
//!
static int walrus_request_timeout(const char *walrus_op, const char *verb, const char *requested_url, const char *outfile, const int do_compress,
int connect_timeout, int total_timeout)
{
int code = EUCA_ERROR;
char url[BUFSIZE];
pthread_mutex_lock(&wreq_mutex); /* lock for curl construction */
euca_strncpy(url, requested_url, BUFSIZE);
#if defined(CAN_GZIP)
if (do_compress)
snprintf(url, BUFSIZE, "%s%s", requested_url, "?IsCompressed=true");
#endif /* CAN_GZIP */
/* isolate the PATH in the URL as it will be needed for signing */
char *url_path;
if (strncasecmp(url, "http://", 7) != 0 && strncasecmp(url, "https://", 8) != 0) {
logprintfl(EUCAERROR, "Walrus URL must start with http(s)://...\n");
pthread_mutex_unlock(&wreq_mutex);
return code;
}
if ((url_path = strchr(url + 8, '/')) == NULL) { /* find first '/' after hostname */
logprintfl(EUCAERROR, "Walrus URL has no path\n");
pthread_mutex_unlock(&wreq_mutex);
return code;
}
if (euca_init_cert()) {
logprintfl(EUCAERROR, "failed to initialize certificate for Walrus request\n");
pthread_mutex_unlock(&wreq_mutex);
return code;
}
int fd = open(outfile, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); // we do not truncate the file
if (fd == -1 || lseek(fd, 0, SEEK_SET) == -1) {
logprintfl(EUCAERROR, "failed to open %s for writing Walrus request\n", outfile);
pthread_mutex_unlock(&wreq_mutex);
if (fd >= 0)
close(fd);
return code;
}
logprintfl(EUCADEBUG, "will use URL: %s\n", url);
CURL *curl;
CURLcode result;
curl = curl_easy_init();
if (curl == NULL) {
logprintfl(EUCAERROR, "could not initialize libcurl for Walrus request\n");
close(fd);
pthread_mutex_unlock(&wreq_mutex);
return code;
}
char error_msg[CURL_ERROR_SIZE];
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error_msg);
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, write_header);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); //! @todo make this optional?
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 360L); // must have at least a 360 baud modem
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 10L); // abort if below speed limit for this many seconds
// curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1); //! @todo remove the comment once we want to follow redirects (e.g., on HTTP 407)
if (strncmp(verb, "GET", 4) == 0) {
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
} else if (strncmp(verb, "HEAD", 5) == 0) {
//! @todo HEAD isn't very useful atm since we don't look at headers
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
} else {
close(fd);
logprintfl(EUCAERROR, "invalid HTTP verb %s in Walrus request\n", verb);
pthread_mutex_unlock(&wreq_mutex);
return EUCA_ERROR; //! @todo dealloc structs before returning!
}
if (connect_timeout > 0) {
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, connect_timeout);
}
if (total_timeout > 0) {
curl_easy_setopt(curl, CURLOPT_TIMEOUT, total_timeout);
}
/* set up the default write function, but possibly override
* it below, if compression is desired and possible */
struct request params;
params.fd = fd;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, ¶ms);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
#if defined(CAN_GZIP)
if (do_compress) {
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_zlib);
}
#endif /* CAN_GZIP */
struct curl_slist *headers = NULL; /* beginning of a DLL with headers */
headers = curl_slist_append(headers, "Authorization: Euca");
char op_hdr[STRSIZE];
if (walrus_op != NULL) {
snprintf(op_hdr, STRSIZE, "EucaOperation: %s", walrus_op);
headers = curl_slist_append(headers, op_hdr);
}
time_t t = time(NULL);
char date_str[26];
if (ctime_r(&t, date_str) == NULL) {
close(fd);
pthread_mutex_unlock(&wreq_mutex);
return EUCA_ERROR;
}
assert(strlen(date_str) + 7 <= STRSIZE);
char *newline = strchr(date_str, '\n');
if (newline != NULL) {
*newline = '\0';
} // remove newline that terminates asctime() output
char date_hdr[STRSIZE];
snprintf(date_hdr, STRSIZE, "Date: %s", date_str);
headers = curl_slist_append(headers, date_hdr);
char *cert_str = euca_get_cert(0); /* read the cloud-wide cert */
if (cert_str == NULL) {
close(fd);
pthread_mutex_unlock(&wreq_mutex);
return EUCA_ERROR;
}
char *cert64_str = base64_enc((unsigned char *)cert_str, strlen(cert_str));
assert(strlen(cert64_str) + 11 <= BUFSIZE);
char cert_hdr[BUFSIZE];
snprintf(cert_hdr, BUFSIZE, "EucaCert: %s", cert64_str);
logprintfl(EUCATRACE, "base64 certificate: %s\n", get_string_stats(cert64_str));
headers = curl_slist_append(headers, cert_hdr);
EUCA_FREE(cert64_str);
EUCA_FREE(cert_str);
char *sig_str = euca_sign_url(verb, date_str, url_path); /* create Walrus-compliant sig */
if (sig_str == NULL) {
close(fd);
pthread_mutex_unlock(&wreq_mutex);
return EUCA_ERROR;
}
assert(strlen(sig_str) + 16 <= BUFSIZE);
char sig_hdr[BUFSIZE];
snprintf(sig_hdr, BUFSIZE, "EucaSignature: %s", sig_str);
headers = curl_slist_append(headers, sig_hdr);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); /* register headers */
if (walrus_op) {
logprintfl(EUCADEBUG, "writing %s/%s output\n", verb, walrus_op);
logprintfl(EUCADEBUG, " from %s\n", url);
logprintfl(EUCADEBUG, " to %s\n", outfile);
} else {
logprintfl(EUCADEBUG, "writing %s output to %s\n", verb, outfile);
}
int retries = TOTAL_RETRIES;
int timeout = FIRST_TIMEOUT;
do {
params.total_wrote = 0L;
params.total_calls = 0L;
#if defined(CAN_GZIP)
if (do_compress) {
/* allocate zlib inflate state */
params.strm.zalloc = Z_NULL;
params.strm.zfree = Z_NULL;
params.strm.opaque = Z_NULL;
params.strm.avail_in = 0;
params.strm.next_in = Z_NULL;
params.ret = inflateInit2(&(params.strm), 31);
if (params.ret != Z_OK) {
zerr(params.ret, "walrus_request");
break;
}
}
#endif /* CAN_GZIP */
//! @todo There used to be a 'pthread_mutex_unlock(&wreq_mutex)' before curl invocation
//! and a 'lock' after it, but under heavy load we were seeing failures inside
//! libcurl code that would propagate to NC, implying lack of thread safety in
//! the library. For now, we will serialize all curl operations, but in the future
//! an approach to parallelizing Walrus downloads is necessary
result = curl_easy_perform(curl); /* do it */
logprintfl(EUCADEBUG, "wrote %lld byte(s) in %lld write(s)\n", params.total_wrote, params.total_calls);
#if defined(CAN_GZIP)
if (do_compress) {
inflateEnd(&(params.strm));
if (params.ret != Z_STREAM_END) {
zerr(params.ret, "walrus_request");
}
}
#endif /* CAN_GZIP */
if (result) { // curl error (connection or transfer failed)
logprintfl(EUCAERROR, "curl error: %s (%d)\n", error_msg, result);
} else {
long httpcode;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpcode);
//! @todo pull out response message, too
switch (httpcode) {
case 200L: /* all good */
logprintfl(EUCAINFO, "downloaded %s\n", outfile);
code = EUCA_OK;
break;
case 408L: /* timeout, retry */
logprintfl(EUCAWARN, "server responded with HTTP code %ld (timeout) for %s\n", httpcode, url);
//logcat (EUCADEBUG, outfile); /* dump the error from outfile into the log */
break;
default: /* some kind of error */
logprintfl(EUCAERROR, "server responded with HTTP code %ld for %s\n", httpcode, url);
//logcat (EUCADEBUG, outfile); /* dump the error from outfile into the log */
retries = 0;
break;
}
}
if (code != EUCA_OK && retries > 0) {
logprintfl(EUCAWARN, "download retry %d of %d will commence in %d sec for %s\n", retries, TOTAL_RETRIES, timeout, url);
sleep(timeout);
lseek(fd, 0L, SEEK_SET);
timeout <<= 1;
if (timeout > MAX_TIMEOUT)
timeout = MAX_TIMEOUT;
}
retries--;
} while (code != EUCA_OK && retries > 0);
close(fd);
if (code != EUCA_OK) {
logprintfl(EUCAWARN, "removing %s\n", outfile);
remove(outfile);
}
EUCA_FREE(sig_str);
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
pthread_mutex_unlock(&wreq_mutex);
return code;
}
