void *htext_get_method(void *h)
{
htext_handle *handle = (htext_handle*)h;
CURL *curl;
htext_partial *partial_array, head;
size_t unused, fsize, stream_size, last_size;
char err_buffer[CURL_ERROR_SIZE];
void *f;
int i, npartials;
/* Create the file */
f = GETIO(handle)->open(GETSTR(handle, HTEXTOP_DESTINATIONURL), "w",
GETPTR(handle, HTEXTOP_IO_HANDLER_DATA));
if (!f) {
handle->status = HTEXTS_FAILED;
strerror_r(errno, err_buffer, sizeof(err_buffer));
htext_error(handle, err_buffer);
return NULL;
}
GETIO(handle)->close(f);
/* Create and initialize CURL handle with common stuff */
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, htext_header_callback);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curl, CURLOPT_URL, GETSTR(handle, HTEXTOP_SOURCEURL));
curl_easy_setopt(curl, CURLOPT_USERAGENT, GETSTR(handle, HTEXTOP_CLIENTID));
curl_easy_setopt(curl, CURLOPT_CAPATH, GETSTR(handle, HTEXTOP_CAPATH));
curl_easy_setopt(curl, CURLOPT_CAINFO, GETSTR(handle, HTEXTOP_CAFILE));
curl_easy_setopt(curl, CURLOPT_SSLCERT, GETSTR(handle, HTEXTOP_USERCERTIFICATE));
curl_easy_setopt(curl, CURLOPT_SSLKEY, GETSTR(handle, HTEXTOP_USERPRIVKEY));
curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, GETSTR(handle, HTEXTOP_USERPRIVKEYPASS));
curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, htext_cipher_suite());
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
if (GETINT(handle, HTEXTOP_VERBOSITY) > 1) {
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, htext_debug_callback);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
}
if (GETINT(handle, HTEXTOP_BUFFERSIZE) > 0) {
curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, GETINT(handle, HTEXTOP_BUFFERSIZE));
}
/* Shared */
curl_easy_setopt(curl, CURLOPT_SHARE, handle->curl_share);
/* Do a HEAD to get the size */
htext_partial_init(&head);
fsize = 0;
head.partial_total = &fsize;
head.partial_done = &unused;
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &head);
handle->status = HTEXTS_WAITING;
htext_log(handle, "Asking for the size");
if (curl_easy_perform(curl) != CURLE_OK) {
htext_error(handle, err_buffer);
}
if (head.http_status >= 400 || handle->status == HTEXTS_FAILED) {
handle->http_status = head.http_status;
handle->status = HTEXTS_FAILED;
htext_partial_clean(&head);
curl_easy_cleanup(curl);
return NULL;
}
/* Did we get the location, and is it cacheable? */
if (head.location && head.redirect_is_cacheable) {
curl_easy_setopt(curl, CURLOPT_URL, head.location);
}
/* Calculate stream size and final stream size */
if (fsize) {
npartials = GETINT(handle, HTEXTOP_NUMBEROFSTREAMS);
if (fsize % npartials == 0) {
stream_size = last_size = fsize / npartials;
}
else {
stream_size = fsize / npartials;
last_size = stream_size + (fsize % npartials);
}
}
else {
/* No size? Too bad, won't be able to stream */
npartials = 1;
stream_size = last_size = 0;
htext_log(handle, "No Content-Length, so no multistream possible");
}
/* Set progress function */
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, htext_progress_callback);
/* Set up partials */
handle->partial_total = calloc(sizeof(size_t), npartials);
handle->partial_done = calloc(sizeof(size_t), npartials);
handle->partials = npartials;
/* Spawn a thread per partial */
htext_log(handle, "Downloading");
handle->status = HTEXTS_RUNNING;
partial_array = calloc(sizeof(htext_partial), npartials);
for (i = 0; i < npartials; ++i) {
htext_partial_init(&(partial_array[i]));
partial_array[i].index = i;
partial_array[i].handle = handle;
partial_array[i].curl = curl_easy_duphandle(curl);
partial_array[i].partial_total = &(handle->partial_total[i]);
partial_array[i].partial_done = &(handle->partial_done[i]);
/* duphandle doesn't duplicate this :( */
curl_easy_setopt(partial_array[i].curl, CURLOPT_SHARE, handle->curl_share);
/* Open */
partial_array[i].fd = GETIO(handle)->open(GETSTR(handle, HTEXTOP_DESTINATIONURL),
"w",
GETPTR(handle, HTEXTOP_IO_HANDLER_DATA));
if (!partial_array[i].fd) {
handle->status = HTEXTS_FAILED;
break;
}
/* Range and seek */
partial_array[i].start = i * stream_size;
if (i < handle->partials - 1)
partial_array[i].end = partial_array[i].start + stream_size - 1;
else
partial_array[i].end = partial_array[i].start + last_size - 1;
*(partial_array[i].partial_total) = partial_array[i].end - partial_array[i].start;
if (GETIO(handle)->seek(partial_array[i].fd, partial_array[i].start, SEEK_SET) < 0) {
handle->status = HTEXTS_FAILED;
break;
}
/* Launch */
pthread_create(&(partial_array[i].thread), NULL, htext_get_subthread, &(partial_array[i]));
}
/* Exited on error? */
if (handle->status == HTEXTS_FAILED) {
strerror_r(errno, err_buffer, sizeof(err_buffer));
htext_error(handle, err_buffer);
}
/* Wait for all of them */
for (i = 0; i < npartials; ++i) {
/* Wait, or kill if failed */
if (handle->status == HTEXTS_FAILED)
pthread_cancel(partial_array[i].thread);
pthread_join(partial_array[i].thread, NULL);
if (handle->status != HTEXTS_FAILED)
handle->http_status = partial_array[i].http_status;
/* Clean partial */
htext_partial_clean(&(partial_array[i]));
/* Check code */
if (handle->http_status < 200 || handle->http_status >= 300) {
handle->status = HTEXTS_FAILED;
}
}
/* Done */
if (handle->status == HTEXTS_RUNNING) {
handle->status = HTEXTS_SUCCEEDED;
}
curl_easy_cleanup(curl);
htext_partial_clean(&head);
free(partial_array);
return NULL;
}
void *htext_copy_method(void *h)
{
htext_handle *handle = (htext_handle*)h;
CURL *curl;
char *delegation_id = NULL, *location;
htext_partial control;
char buffer[CURL_MAX_WRITE_SIZE], err_buffer[CURL_ERROR_SIZE];
char host[64];
/* Create and initialize CURL handle */
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, htext_header_callback);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, htext_write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &control);
curl_easy_setopt(curl, CURLOPT_URL, GETSTR(handle, HTEXTOP_SOURCEURL));
curl_easy_setopt(curl, CURLOPT_USERAGENT, GETSTR(handle, HTEXTOP_CLIENTID));
curl_easy_setopt(curl, CURLOPT_CAPATH, GETSTR(handle, HTEXTOP_CAPATH));
curl_easy_setopt(curl, CURLOPT_CAINFO, GETSTR(handle, HTEXTOP_CAFILE));
curl_easy_setopt(curl, CURLOPT_SSLCERT, GETSTR(handle, HTEXTOP_USERCERTIFICATE));
curl_easy_setopt(curl, CURLOPT_SSLKEY, GETSTR(handle, HTEXTOP_USERPRIVKEY));
curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, GETSTR(handle, HTEXTOP_USERPRIVKEYPASS));
if (GETINT(handle, HTEXTOP_VERBOSITY) > 1) {
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, htext_debug_callback);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
}
curl_easy_setopt(curl, CURLOPT_SHARE, handle->curl_share);
/* Reserve space for the partials */
handle->partial_done = calloc(sizeof(size_t), 1);
handle->partial_total = calloc(sizeof(size_t), 1);
handle->partials = 1;
/* Do a HEAD to know where do we have to go */
htext_partial_init(&control);
control.partial_total = handle->partial_total;
control.partial_done = handle->partial_done;
control.handle = handle;
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &control);
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, &control);
handle->status = HTEXTS_WAITING;
htext_log(handle, "Retrieving the actual location");
if (curl_easy_perform(curl) != CURLE_OK)
htext_error(handle, buffer);
/* If we got it, delegate */
if (handle->status != HTEXTS_FAILED && control.location)
location = control.location;
else
htext_error(handle, "Could not get the location");
/* Delegate */
if (handle->status != HTEXTS_FAILED) {
/* Pick from configuration if not available */
if (control.delegation_service == NULL) {
control.delegation_service = GETSTR(handle, HTEXTOP_DELEGATION_URL);
}
/* Not yet? */
if (control.delegation_service == NULL) {
htext_error(handle, "Could not get the delegation endpoint");
}
/* Build the full URL */
else {
snprintf(buffer, sizeof(buffer), "https://%s%s",
get_host(location, host),
control.delegation_service);
htext_log(handle, "Delegation endpoint: %s", buffer);
delegation_id = htext_delegate(handle, buffer, err_buffer);
htext_log(handle, "Got delegation id %s", delegation_id);
}
}
/* Do the COPY */
if (handle->status != HTEXTS_FAILED) {
char *full_url;
htext_log(handle, "Running the remote copy");
handle->status = HTEXTS_RUNNING;
full_url = malloc(strlen(location) + strlen(delegation_id) + 14);
/* Beware: when copying, the user cert is needed, so ignore http:// prefixes,
* and assume https:// */
if (strncmp(location, "http:", 5) == 0) {
sprintf(full_url, "https:%s&delegation=%s", location + 5, delegation_id);
}
else {
sprintf(full_url, "%s&delegation=%s", location, delegation_id);
}
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "COPY");
curl_easy_setopt(curl, CURLOPT_URL, full_url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, htext_copy_write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &control);
/* Need to flush often to get progress */
curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, 64);
snprintf(buffer, sizeof(buffer),
"Destination: %s", GETSTR(handle, HTEXTOP_DESTINATIONURL));
control.headers = curl_slist_append(control.headers, buffer);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, control.headers);
control.http_status = 0; /* Reset this */
control.extra = buffer;
buffer[0] = '\0';
if (curl_easy_perform(curl) != CURLE_OK) {
htext_error(handle, err_buffer);
}
else if (control.http_status >= 400 || handle->status == HTEXTS_FAILED) {
htext_error(handle, NULL);
}
else {
handle->status = HTEXTS_SUCCEEDED;
}
handle->http_status = control.http_status;
free(full_url);
}
/* Clean up */
free(delegation_id);
htext_partial_clean(&control);
curl_easy_cleanup(curl);
return NULL;
}
