void *htext_get_method(void *h) { htext_handle *handle = (htext_handle*)h; CURL *curl; htext_partial *partial_array, head; size_t unused, fsize, stream_size, last_size; char err_buffer[CURL_ERROR_SIZE]; void *f; int i, npartials; /* Create the file */ f = GETIO(handle)->open(GETSTR(handle, HTEXTOP_DESTINATIONURL), "w", GETPTR(handle, HTEXTOP_IO_HANDLER_DATA)); if (!f) { handle->status = HTEXTS_FAILED; strerror_r(errno, err_buffer, sizeof(err_buffer)); htext_error(handle, err_buffer); return NULL; } GETIO(handle)->close(f); /* Create and initialize CURL handle with common stuff */ curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, htext_header_callback); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curl, CURLOPT_URL, GETSTR(handle, HTEXTOP_SOURCEURL)); curl_easy_setopt(curl, CURLOPT_USERAGENT, GETSTR(handle, HTEXTOP_CLIENTID)); curl_easy_setopt(curl, CURLOPT_CAPATH, GETSTR(handle, HTEXTOP_CAPATH)); curl_easy_setopt(curl, CURLOPT_CAINFO, GETSTR(handle, HTEXTOP_CAFILE)); curl_easy_setopt(curl, CURLOPT_SSLCERT, GETSTR(handle, HTEXTOP_USERCERTIFICATE)); curl_easy_setopt(curl, CURLOPT_SSLKEY, GETSTR(handle, HTEXTOP_USERPRIVKEY)); curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, GETSTR(handle, HTEXTOP_USERPRIVKEYPASS)); curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, htext_cipher_suite()); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); if (GETINT(handle, HTEXTOP_VERBOSITY) > 1) { curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, htext_debug_callback); curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); } if (GETINT(handle, HTEXTOP_BUFFERSIZE) > 0) { curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, GETINT(handle, HTEXTOP_BUFFERSIZE)); } /* Shared */ curl_easy_setopt(curl, CURLOPT_SHARE, handle->curl_share); /* Do a HEAD to get the size */ htext_partial_init(&head); fsize = 0; head.partial_total = &fsize; head.partial_done = &unused; curl_easy_setopt(curl, CURLOPT_NOBODY, 1); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_HEADERDATA, &head); handle->status = HTEXTS_WAITING; htext_log(handle, "Asking for the size"); if (curl_easy_perform(curl) != CURLE_OK) { htext_error(handle, err_buffer); } if (head.http_status >= 400 || handle->status == HTEXTS_FAILED) { handle->http_status = head.http_status; handle->status = HTEXTS_FAILED; htext_partial_clean(&head); curl_easy_cleanup(curl); return NULL; } /* Did we get the location, and is it cacheable? */ if (head.location && head.redirect_is_cacheable) { curl_easy_setopt(curl, CURLOPT_URL, head.location); } /* Calculate stream size and final stream size */ if (fsize) { npartials = GETINT(handle, HTEXTOP_NUMBEROFSTREAMS); if (fsize % npartials == 0) { stream_size = last_size = fsize / npartials; } else { stream_size = fsize / npartials; last_size = stream_size + (fsize % npartials); } } else { /* No size? Too bad, won't be able to stream */ npartials = 1; stream_size = last_size = 0; htext_log(handle, "No Content-Length, so no multistream possible"); } /* Set progress function */ curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, htext_progress_callback); /* Set up partials */ handle->partial_total = calloc(sizeof(size_t), npartials); handle->partial_done = calloc(sizeof(size_t), npartials); handle->partials = npartials; /* Spawn a thread per partial */ htext_log(handle, "Downloading"); handle->status = HTEXTS_RUNNING; partial_array = calloc(sizeof(htext_partial), npartials); for (i = 0; i < npartials; ++i) { htext_partial_init(&(partial_array[i])); partial_array[i].index = i; partial_array[i].handle = handle; partial_array[i].curl = curl_easy_duphandle(curl); partial_array[i].partial_total = &(handle->partial_total[i]); partial_array[i].partial_done = &(handle->partial_done[i]); /* duphandle doesn't duplicate this :( */ curl_easy_setopt(partial_array[i].curl, CURLOPT_SHARE, handle->curl_share); /* Open */ partial_array[i].fd = GETIO(handle)->open(GETSTR(handle, HTEXTOP_DESTINATIONURL), "w", GETPTR(handle, HTEXTOP_IO_HANDLER_DATA)); if (!partial_array[i].fd) { handle->status = HTEXTS_FAILED; break; } /* Range and seek */ partial_array[i].start = i * stream_size; if (i < handle->partials - 1) partial_array[i].end = partial_array[i].start + stream_size - 1; else partial_array[i].end = partial_array[i].start + last_size - 1; *(partial_array[i].partial_total) = partial_array[i].end - partial_array[i].start; if (GETIO(handle)->seek(partial_array[i].fd, partial_array[i].start, SEEK_SET) < 0) { handle->status = HTEXTS_FAILED; break; } /* Launch */ pthread_create(&(partial_array[i].thread), NULL, htext_get_subthread, &(partial_array[i])); } /* Exited on error? */ if (handle->status == HTEXTS_FAILED) { strerror_r(errno, err_buffer, sizeof(err_buffer)); htext_error(handle, err_buffer); } /* Wait for all of them */ for (i = 0; i < npartials; ++i) { /* Wait, or kill if failed */ if (handle->status == HTEXTS_FAILED) pthread_cancel(partial_array[i].thread); pthread_join(partial_array[i].thread, NULL); if (handle->status != HTEXTS_FAILED) handle->http_status = partial_array[i].http_status; /* Clean partial */ htext_partial_clean(&(partial_array[i])); /* Check code */ if (handle->http_status < 200 || handle->http_status >= 300) { handle->status = HTEXTS_FAILED; } } /* Done */ if (handle->status == HTEXTS_RUNNING) { handle->status = HTEXTS_SUCCEEDED; } curl_easy_cleanup(curl); htext_partial_clean(&head); free(partial_array); return NULL; }
void *htext_copy_method(void *h) { htext_handle *handle = (htext_handle*)h; CURL *curl; char *delegation_id = NULL, *location; htext_partial control; char buffer[CURL_MAX_WRITE_SIZE], err_buffer[CURL_ERROR_SIZE]; char host[64]; /* Create and initialize CURL handle */ curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, htext_header_callback); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, htext_write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &control); curl_easy_setopt(curl, CURLOPT_URL, GETSTR(handle, HTEXTOP_SOURCEURL)); curl_easy_setopt(curl, CURLOPT_USERAGENT, GETSTR(handle, HTEXTOP_CLIENTID)); curl_easy_setopt(curl, CURLOPT_CAPATH, GETSTR(handle, HTEXTOP_CAPATH)); curl_easy_setopt(curl, CURLOPT_CAINFO, GETSTR(handle, HTEXTOP_CAFILE)); curl_easy_setopt(curl, CURLOPT_SSLCERT, GETSTR(handle, HTEXTOP_USERCERTIFICATE)); curl_easy_setopt(curl, CURLOPT_SSLKEY, GETSTR(handle, HTEXTOP_USERPRIVKEY)); curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, GETSTR(handle, HTEXTOP_USERPRIVKEYPASS)); if (GETINT(handle, HTEXTOP_VERBOSITY) > 1) { curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, htext_debug_callback); curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); } curl_easy_setopt(curl, CURLOPT_SHARE, handle->curl_share); /* Reserve space for the partials */ handle->partial_done = calloc(sizeof(size_t), 1); handle->partial_total = calloc(sizeof(size_t), 1); handle->partials = 1; /* Do a HEAD to know where do we have to go */ htext_partial_init(&control); control.partial_total = handle->partial_total; control.partial_done = handle->partial_done; control.handle = handle; curl_easy_setopt(curl, CURLOPT_NOBODY, 1); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_HEADERDATA, &control); curl_easy_setopt(curl, CURLOPT_DEBUGDATA, &control); handle->status = HTEXTS_WAITING; htext_log(handle, "Retrieving the actual location"); if (curl_easy_perform(curl) != CURLE_OK) htext_error(handle, buffer); /* If we got it, delegate */ if (handle->status != HTEXTS_FAILED && control.location) location = control.location; else htext_error(handle, "Could not get the location"); /* Delegate */ if (handle->status != HTEXTS_FAILED) { /* Pick from configuration if not available */ if (control.delegation_service == NULL) { control.delegation_service = GETSTR(handle, HTEXTOP_DELEGATION_URL); } /* Not yet? */ if (control.delegation_service == NULL) { htext_error(handle, "Could not get the delegation endpoint"); } /* Build the full URL */ else { snprintf(buffer, sizeof(buffer), "https://%s%s", get_host(location, host), control.delegation_service); htext_log(handle, "Delegation endpoint: %s", buffer); delegation_id = htext_delegate(handle, buffer, err_buffer); htext_log(handle, "Got delegation id %s", delegation_id); } } /* Do the COPY */ if (handle->status != HTEXTS_FAILED) { char *full_url; htext_log(handle, "Running the remote copy"); handle->status = HTEXTS_RUNNING; full_url = malloc(strlen(location) + strlen(delegation_id) + 14); /* Beware: when copying, the user cert is needed, so ignore http:// prefixes, * and assume https:// */ if (strncmp(location, "http:", 5) == 0) { sprintf(full_url, "https:%s&delegation=%s", location + 5, delegation_id); } else { sprintf(full_url, "%s&delegation=%s", location, delegation_id); } curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "COPY"); curl_easy_setopt(curl, CURLOPT_URL, full_url); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, htext_copy_write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &control); /* Need to flush often to get progress */ curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, 64); snprintf(buffer, sizeof(buffer), "Destination: %s", GETSTR(handle, HTEXTOP_DESTINATIONURL)); control.headers = curl_slist_append(control.headers, buffer); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, control.headers); control.http_status = 0; /* Reset this */ control.extra = buffer; buffer[0] = '\0'; if (curl_easy_perform(curl) != CURLE_OK) { htext_error(handle, err_buffer); } else if (control.http_status >= 400 || handle->status == HTEXTS_FAILED) { htext_error(handle, NULL); } else { handle->status = HTEXTS_SUCCEEDED; } handle->http_status = control.http_status; free(full_url); } /* Clean up */ free(delegation_id); htext_partial_clean(&control); curl_easy_cleanup(curl); return NULL; }