void *htext_get_method(void *h)
{
htext_handle *handle = (htext_handle*)h;
CURL *curl;
htext_partial *partial_array, head;
size_t unused, fsize, stream_size, last_size;
char err_buffer[CURL_ERROR_SIZE];
void *f;
int i, npartials;
/* Create the file */
f = GETIO(handle)->open(GETSTR(handle, HTEXTOP_DESTINATIONURL), "w",
GETPTR(handle, HTEXTOP_IO_HANDLER_DATA));
if (!f) {
handle->status = HTEXTS_FAILED;
strerror_r(errno, err_buffer, sizeof(err_buffer));
htext_error(handle, err_buffer);
return NULL;
}
GETIO(handle)->close(f);
/* Create and initialize CURL handle with common stuff */
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, htext_header_callback);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curl, CURLOPT_URL, GETSTR(handle, HTEXTOP_SOURCEURL));
curl_easy_setopt(curl, CURLOPT_USERAGENT, GETSTR(handle, HTEXTOP_CLIENTID));
curl_easy_setopt(curl, CURLOPT_CAPATH, GETSTR(handle, HTEXTOP_CAPATH));
curl_easy_setopt(curl, CURLOPT_CAINFO, GETSTR(handle, HTEXTOP_CAFILE));
curl_easy_setopt(curl, CURLOPT_SSLCERT, GETSTR(handle, HTEXTOP_USERCERTIFICATE));
curl_easy_setopt(curl, CURLOPT_SSLKEY, GETSTR(handle, HTEXTOP_USERPRIVKEY));
curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, GETSTR(handle, HTEXTOP_USERPRIVKEYPASS));
curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, htext_cipher_suite());
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
if (GETINT(handle, HTEXTOP_VERBOSITY) > 1) {
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, htext_debug_callback);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
}
if (GETINT(handle, HTEXTOP_BUFFERSIZE) > 0) {
curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, GETINT(handle, HTEXTOP_BUFFERSIZE));
}
/* Shared */
curl_easy_setopt(curl, CURLOPT_SHARE, handle->curl_share);
/* Do a HEAD to get the size */
htext_partial_init(&head);
fsize = 0;
head.partial_total = &fsize;
head.partial_done = &unused;
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &head);
handle->status = HTEXTS_WAITING;
htext_log(handle, "Asking for the size");
if (curl_easy_perform(curl) != CURLE_OK) {
htext_error(handle, err_buffer);
}
if (head.http_status >= 400 || handle->status == HTEXTS_FAILED) {
handle->http_status = head.http_status;
handle->status = HTEXTS_FAILED;
htext_partial_clean(&head);
curl_easy_cleanup(curl);
return NULL;
}
/* Did we get the location, and is it cacheable? */
if (head.location && head.redirect_is_cacheable) {
curl_easy_setopt(curl, CURLOPT_URL, head.location);
}
/* Calculate stream size and final stream size */
if (fsize) {
npartials = GETINT(handle, HTEXTOP_NUMBEROFSTREAMS);
if (fsize % npartials == 0) {
stream_size = last_size = fsize / npartials;
}
else {
stream_size = fsize / npartials;
last_size = stream_size + (fsize % npartials);
}
}
else {
/* No size? Too bad, won't be able to stream */
npartials = 1;
stream_size = last_size = 0;
htext_log(handle, "No Content-Length, so no multistream possible");
}
/* Set progress function */
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, htext_progress_callback);
/* Set up partials */
handle->partial_total = calloc(sizeof(size_t), npartials);
handle->partial_done = calloc(sizeof(size_t), npartials);
handle->partials = npartials;
/* Spawn a thread per partial */
htext_log(handle, "Downloading");
handle->status = HTEXTS_RUNNING;
partial_array = calloc(sizeof(htext_partial), npartials);
for (i = 0; i < npartials; ++i) {
htext_partial_init(&(partial_array[i]));
partial_array[i].index = i;
partial_array[i].handle = handle;
partial_array[i].curl = curl_easy_duphandle(curl);
partial_array[i].partial_total = &(handle->partial_total[i]);
partial_array[i].partial_done = &(handle->partial_done[i]);
/* duphandle doesn't duplicate this :( */
curl_easy_setopt(partial_array[i].curl, CURLOPT_SHARE, handle->curl_share);
/* Open */
partial_array[i].fd = GETIO(handle)->open(GETSTR(handle, HTEXTOP_DESTINATIONURL),
"w",
GETPTR(handle, HTEXTOP_IO_HANDLER_DATA));
if (!partial_array[i].fd) {
handle->status = HTEXTS_FAILED;
break;
}
/* Range and seek */
partial_array[i].start = i * stream_size;
if (i < handle->partials - 1)
partial_array[i].end = partial_array[i].start + stream_size - 1;
else
partial_array[i].end = partial_array[i].start + last_size - 1;
*(partial_array[i].partial_total) = partial_array[i].end - partial_array[i].start;
if (GETIO(handle)->seek(partial_array[i].fd, partial_array[i].start, SEEK_SET) < 0) {
handle->status = HTEXTS_FAILED;
break;
}
/* Launch */
pthread_create(&(partial_array[i].thread), NULL, htext_get_subthread, &(partial_array[i]));
}
/* Exited on error? */
if (handle->status == HTEXTS_FAILED) {
strerror_r(errno, err_buffer, sizeof(err_buffer));
htext_error(handle, err_buffer);
}
/* Wait for all of them */
for (i = 0; i < npartials; ++i) {
/* Wait, or kill if failed */
if (handle->status == HTEXTS_FAILED)
pthread_cancel(partial_array[i].thread);
pthread_join(partial_array[i].thread, NULL);
if (handle->status != HTEXTS_FAILED)
handle->http_status = partial_array[i].http_status;
/* Clean partial */
htext_partial_clean(&(partial_array[i]));
/* Check code */
if (handle->http_status < 200 || handle->http_status >= 300) {
handle->status = HTEXTS_FAILED;
}
}
/* Done */
if (handle->status == HTEXTS_RUNNING) {
handle->status = HTEXTS_SUCCEEDED;
}
curl_easy_cleanup(curl);
htext_partial_clean(&head);
free(partial_array);
return NULL;
}
/**
* Do the delegation
* @param handle The handle
* @param url The delegation endpoint
* @param err_buffer Used to build the error string
* @return NULL on failure, or an allocated string with the delegation ID
*/
static char *htext_delegate(htext_handle *handle, char *url, char *err_buffer)
{
char *delegation_id = NULL;
char *reqtxt = NULL, *certtxt = NULL;
char *keycert = NULL;
struct soap *soap_get = NULL, *soap_put = NULL;
struct tns__getNewProxyReqResponse getNewProxyReqResponse;
char *ucert = NULL, *ukey = NULL, *capath = NULL;
int lifetime;
/* Get from the handle */
ucert = GETSTR(handle, HTEXTOP_USERCERTIFICATE);
ukey = GETSTR(handle, HTEXTOP_USERPRIVKEY);
capath = GETSTR(handle, HTEXTOP_CAPATH);
lifetime = GETINT(handle, HTEXTOP_PROXYLIFE);
/* Only one is needed if they are the same */
if (ucert && !ukey) ukey = ucert;
if (!ucert && ukey) ucert = ukey;
/* Cert and key need to be in the same file */
if (strcmp(ucert, ukey) == 0) {
keycert = strdup(ucert);
}
else {
FILE *ifp, *ofp;
int fd;
char c;
keycert = strdup("/tmp/.XXXXXX");
fd = mkstemp(keycert);
ofp = fdopen(fd, "w");
ifp = fopen(ukey, "r");
while ((c = fgetc(ifp)) != EOF) fputc(c, ofp);
fclose(ifp);
ifp = fopen(ukey, "r");
while ((c = fgetc(ifp)) != EOF) fputc(c, ofp);
fclose(ifp);
fclose(ofp);
}
/* Initialize SSL */
ERR_load_crypto_strings ();
OpenSSL_add_all_algorithms();
/* Request a new delegation ID */
soap_get = soap_new();
if (soap_ssl_client_context(soap_get, SOAP_SSL_DEFAULT, keycert, "",
NULL, capath, NULL) == 0) {
soap_call_tns__getNewProxyReq(soap_get,
url,
"http://www.gridsite.org/namespaces/delegation-1",
&getNewProxyReqResponse);
if(soap_get->error == 0) {
reqtxt = getNewProxyReqResponse.getNewProxyReqReturn->proxyRequest;
delegation_id = strdup(getNewProxyReqResponse.getNewProxyReqReturn->delegationID);
/* Generate proxy */
if (GRSTx509MakeProxyCert(&certtxt, stderr, reqtxt,
ucert, ukey, lifetime) == GRST_RET_OK) {
/* Submit the proxy */
soap_put = soap_new();
if (soap_ssl_client_context(soap_put, SOAP_SSL_DEFAULT, keycert, "",
NULL, capath, NULL) == 0) {
soap_call_tns__putProxy(soap_put,
url,
"http://www.gridsite.org/namespaces/delegation-1",
delegation_id, certtxt, NULL);
if (soap_put->error) {
/* Could not PUT */
#ifndef NO_SOAP_SPRINT
soap_sprint_fault(soap_put, err_buffer, sizeof(err_buffer));
htext_error(handle, err_buffer);
#else
soap_print_fault(soap_put, stderr);
handle->status = HTEXTS_FAILED;
#endif
}
}
else { /* soap_put ssl error */
#ifndef NO_SOAP_SPRINT
soap_sprint_fault(soap_put, err_buffer, sizeof(err_buffer));
htext_error(handle, err_buffer);
#else
soap_print_fault(soap_put, stderr);
handle->status = HTEXTS_FAILED;
#endif
}
soap_free(soap_put);
}
else {
htext_error(handle, "Could not generate the proxy");
}
}
else { /* Could not get ID */
#ifndef NO_SOAP_SPRINT
soap_sprint_fault(soap_get, err_buffer, sizeof(err_buffer));
htext_error(handle, err_buffer);
#else
soap_print_fault(soap_get, stderr);
handle->status = HTEXTS_FAILED;
#endif
}
}
else { /* soap_get ssl error */
#ifndef NO_SOAP_SPRINT
soap_sprint_fault(soap_get, err_buffer, sizeof(err_buffer));
htext_error(handle, err_buffer);
#else
soap_print_fault(soap_put, stderr);
handle->status = HTEXTS_FAILED;
#endif
}
/* Clean soap_get */
soap_free(soap_get);
free(keycert);
free(certtxt);
/* Return delegation ID */
return delegation_id;
}
void *htext_copy_method(void *h)
{
htext_handle *handle = (htext_handle*)h;
CURL *curl;
char *delegation_id = NULL, *location;
htext_partial control;
char buffer[CURL_MAX_WRITE_SIZE], err_buffer[CURL_ERROR_SIZE];
char host[64];
/* Create and initialize CURL handle */
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, htext_header_callback);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, htext_write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &control);
curl_easy_setopt(curl, CURLOPT_URL, GETSTR(handle, HTEXTOP_SOURCEURL));
curl_easy_setopt(curl, CURLOPT_USERAGENT, GETSTR(handle, HTEXTOP_CLIENTID));
curl_easy_setopt(curl, CURLOPT_CAPATH, GETSTR(handle, HTEXTOP_CAPATH));
curl_easy_setopt(curl, CURLOPT_CAINFO, GETSTR(handle, HTEXTOP_CAFILE));
curl_easy_setopt(curl, CURLOPT_SSLCERT, GETSTR(handle, HTEXTOP_USERCERTIFICATE));
curl_easy_setopt(curl, CURLOPT_SSLKEY, GETSTR(handle, HTEXTOP_USERPRIVKEY));
curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, GETSTR(handle, HTEXTOP_USERPRIVKEYPASS));
if (GETINT(handle, HTEXTOP_VERBOSITY) > 1) {
curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, htext_debug_callback);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
}
curl_easy_setopt(curl, CURLOPT_SHARE, handle->curl_share);
/* Reserve space for the partials */
handle->partial_done = calloc(sizeof(size_t), 1);
handle->partial_total = calloc(sizeof(size_t), 1);
handle->partials = 1;
/* Do a HEAD to know where do we have to go */
htext_partial_init(&control);
control.partial_total = handle->partial_total;
control.partial_done = handle->partial_done;
control.handle = handle;
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &control);
curl_easy_setopt(curl, CURLOPT_DEBUGDATA, &control);
handle->status = HTEXTS_WAITING;
htext_log(handle, "Retrieving the actual location");
if (curl_easy_perform(curl) != CURLE_OK)
htext_error(handle, buffer);
/* If we got it, delegate */
if (handle->status != HTEXTS_FAILED && control.location)
location = control.location;
else
htext_error(handle, "Could not get the location");
/* Delegate */
if (handle->status != HTEXTS_FAILED) {
/* Pick from configuration if not available */
if (control.delegation_service == NULL) {
control.delegation_service = GETSTR(handle, HTEXTOP_DELEGATION_URL);
}
/* Not yet? */
if (control.delegation_service == NULL) {
htext_error(handle, "Could not get the delegation endpoint");
}
/* Build the full URL */
else {
snprintf(buffer, sizeof(buffer), "https://%s%s",
get_host(location, host),
control.delegation_service);
htext_log(handle, "Delegation endpoint: %s", buffer);
delegation_id = htext_delegate(handle, buffer, err_buffer);
htext_log(handle, "Got delegation id %s", delegation_id);
}
}
/* Do the COPY */
if (handle->status != HTEXTS_FAILED) {
char *full_url;
htext_log(handle, "Running the remote copy");
handle->status = HTEXTS_RUNNING;
full_url = malloc(strlen(location) + strlen(delegation_id) + 14);
/* Beware: when copying, the user cert is needed, so ignore http:// prefixes,
* and assume https:// */
if (strncmp(location, "http:", 5) == 0) {
sprintf(full_url, "https:%s&delegation=%s", location + 5, delegation_id);
}
else {
sprintf(full_url, "%s&delegation=%s", location, delegation_id);
}
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "COPY");
curl_easy_setopt(curl, CURLOPT_URL, full_url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, htext_copy_write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &control);
/* Need to flush often to get progress */
curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, 64);
snprintf(buffer, sizeof(buffer),
"Destination: %s", GETSTR(handle, HTEXTOP_DESTINATIONURL));
control.headers = curl_slist_append(control.headers, buffer);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, control.headers);
control.http_status = 0; /* Reset this */
control.extra = buffer;
buffer[0] = '\0';
if (curl_easy_perform(curl) != CURLE_OK) {
htext_error(handle, err_buffer);
}
else if (control.http_status >= 400 || handle->status == HTEXTS_FAILED) {
htext_error(handle, NULL);
}
else {
handle->status = HTEXTS_SUCCEEDED;
}
handle->http_status = control.http_status;
free(full_url);
}
/* Clean up */
free(delegation_id);
htext_partial_clean(&control);
curl_easy_cleanup(curl);
return NULL;
}
/**
* Used to parse the output from the server to know the progress
*/
static size_t htext_copy_write_callback(char *buffer, size_t size, size_t nmemb, void *pp)
{
htext_partial *partial = (htext_partial*)pp;
size_t received = size * nmemb;
size_t bsize, parsed;
size_t aux_done, aux_total;
char *next, *prev;
char *remaining = (char*)partial->extra;
htext_write_callback(buffer, size, nmemb, pp);
if (remaining[0] != '\0') {
size_t r_size = strlen(remaining);
memcpy(remaining + r_size, buffer, received);
bsize = received + r_size;
memcpy(buffer, remaining, bsize);
remaining[0] = '\0';
}
else
bsize = received;
buffer[bsize] = '\0'; /* To prevent strchr from going beyond */
next = buffer;
parsed = 0;
while (parsed < bsize) {
if (sscanf(next, "%zu/%zu\n", &aux_done, &aux_total) == 2) {
prev = next;
next = strchr(next, '\n');
/* Maybe sscanf got both numbers, but the line was incomplete!*/
if (next == NULL) {
memcpy(remaining, prev, bsize - parsed);
remaining[bsize - parsed] = '\0';
parsed = bsize;
}
else {
*partial->partial_done = aux_done;
*partial->partial_total = aux_total;
++next;
parsed = next - buffer;
}
}
else {
/* If it doesn't follow, it may be a message */
if (strncasecmp("success", next, 7) == 0) {
// Leave things going
}
else if (strncasecmp("aborted", next, 7) == 0) {
partial->handle->status = HTEXTS_ABORTED;
}
else if (strncasecmp("failed", next, 6) == 0) {
htext_error(partial->handle, next);
}
else {
memcpy(remaining, next, bsize - parsed);
remaining[bsize - parsed] = '\0';
}
parsed = bsize;
}
}
return received;
}
