/** * Implements a partial download * @param pp A pointer to a htext_partial structure * @return NULL on return */ static void* htext_get_subthread(void *pp) { htext_partial *partial = (htext_partial*)pp; char err_buffer[CURL_ERROR_SIZE]; char range_buffer[128]; /* Set the function's user data */ curl_easy_setopt(partial->curl, CURLOPT_HEADERDATA, partial); curl_easy_setopt(partial->curl, CURLOPT_PROGRESSDATA, partial); curl_easy_setopt(partial->curl, CURLOPT_ERRORBUFFER, err_buffer); curl_easy_setopt(partial->curl, CURLOPT_WRITEFUNCTION, GETIO(partial->handle)->write); curl_easy_setopt(partial->curl, CURLOPT_WRITEDATA, partial->fd); curl_easy_setopt(partial->curl, CURLOPT_DEBUGDATA, partial); /* Range */ sprintf(range_buffer, "Range: bytes=%zu-%zu", partial->start, partial->end); partial->headers = curl_slist_append(partial->headers, range_buffer); curl_easy_setopt(partial->curl, CURLOPT_HTTPHEADER, partial->headers); /* Perform */ if (curl_easy_perform(partial->curl) != CURLE_OK) { partial->handle->error_string = strdup(err_buffer); partial->handle->status = HTEXTS_FAILED; } /* Curl blocks, so here we are done */ return NULL; }
void htext_partial_clean(htext_partial *p) { if (p->curl) curl_easy_cleanup(p->curl); if (p->fd) GETIO(p->handle)->close(p->fd); free(p->location); if (p->headers) curl_slist_free_all(p->headers); sem_destroy(&(p->final)); free(p->delegation_service); }
void *htext_get_method(void *h) { htext_handle *handle = (htext_handle*)h; CURL *curl; htext_partial *partial_array, head; size_t unused, fsize, stream_size, last_size; char err_buffer[CURL_ERROR_SIZE]; void *f; int i, npartials; /* Create the file */ f = GETIO(handle)->open(GETSTR(handle, HTEXTOP_DESTINATIONURL), "w", GETPTR(handle, HTEXTOP_IO_HANDLER_DATA)); if (!f) { handle->status = HTEXTS_FAILED; strerror_r(errno, err_buffer, sizeof(err_buffer)); htext_error(handle, err_buffer); return NULL; } GETIO(handle)->close(f); /* Create and initialize CURL handle with common stuff */ curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, htext_header_callback); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curl, CURLOPT_URL, GETSTR(handle, HTEXTOP_SOURCEURL)); curl_easy_setopt(curl, CURLOPT_USERAGENT, GETSTR(handle, HTEXTOP_CLIENTID)); curl_easy_setopt(curl, CURLOPT_CAPATH, GETSTR(handle, HTEXTOP_CAPATH)); curl_easy_setopt(curl, CURLOPT_CAINFO, GETSTR(handle, HTEXTOP_CAFILE)); curl_easy_setopt(curl, CURLOPT_SSLCERT, GETSTR(handle, HTEXTOP_USERCERTIFICATE)); curl_easy_setopt(curl, CURLOPT_SSLKEY, GETSTR(handle, HTEXTOP_USERPRIVKEY)); curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, GETSTR(handle, HTEXTOP_USERPRIVKEYPASS)); curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, htext_cipher_suite()); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); if (GETINT(handle, HTEXTOP_VERBOSITY) > 1) { curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, htext_debug_callback); curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); } if (GETINT(handle, HTEXTOP_BUFFERSIZE) > 0) { curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, GETINT(handle, HTEXTOP_BUFFERSIZE)); } /* Shared */ curl_easy_setopt(curl, CURLOPT_SHARE, handle->curl_share); /* Do a HEAD to get the size */ htext_partial_init(&head); fsize = 0; head.partial_total = &fsize; head.partial_done = &unused; curl_easy_setopt(curl, CURLOPT_NOBODY, 1); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_HEADERDATA, &head); handle->status = HTEXTS_WAITING; htext_log(handle, "Asking for the size"); if (curl_easy_perform(curl) != CURLE_OK) { htext_error(handle, err_buffer); } if (head.http_status >= 400 || handle->status == HTEXTS_FAILED) { handle->http_status = head.http_status; handle->status = HTEXTS_FAILED; htext_partial_clean(&head); curl_easy_cleanup(curl); return NULL; } /* Did we get the location, and is it cacheable? */ if (head.location && head.redirect_is_cacheable) { curl_easy_setopt(curl, CURLOPT_URL, head.location); } /* Calculate stream size and final stream size */ if (fsize) { npartials = GETINT(handle, HTEXTOP_NUMBEROFSTREAMS); if (fsize % npartials == 0) { stream_size = last_size = fsize / npartials; } else { stream_size = fsize / npartials; last_size = stream_size + (fsize % npartials); } } else { /* No size? Too bad, won't be able to stream */ npartials = 1; stream_size = last_size = 0; htext_log(handle, "No Content-Length, so no multistream possible"); } /* Set progress function */ curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, htext_progress_callback); /* Set up partials */ handle->partial_total = calloc(sizeof(size_t), npartials); handle->partial_done = calloc(sizeof(size_t), npartials); handle->partials = npartials; /* Spawn a thread per partial */ htext_log(handle, "Downloading"); handle->status = HTEXTS_RUNNING; partial_array = calloc(sizeof(htext_partial), npartials); for (i = 0; i < npartials; ++i) { htext_partial_init(&(partial_array[i])); partial_array[i].index = i; partial_array[i].handle = handle; partial_array[i].curl = curl_easy_duphandle(curl); partial_array[i].partial_total = &(handle->partial_total[i]); partial_array[i].partial_done = &(handle->partial_done[i]); /* duphandle doesn't duplicate this :( */ curl_easy_setopt(partial_array[i].curl, CURLOPT_SHARE, handle->curl_share); /* Open */ partial_array[i].fd = GETIO(handle)->open(GETSTR(handle, HTEXTOP_DESTINATIONURL), "w", GETPTR(handle, HTEXTOP_IO_HANDLER_DATA)); if (!partial_array[i].fd) { handle->status = HTEXTS_FAILED; break; } /* Range and seek */ partial_array[i].start = i * stream_size; if (i < handle->partials - 1) partial_array[i].end = partial_array[i].start + stream_size - 1; else partial_array[i].end = partial_array[i].start + last_size - 1; *(partial_array[i].partial_total) = partial_array[i].end - partial_array[i].start; if (GETIO(handle)->seek(partial_array[i].fd, partial_array[i].start, SEEK_SET) < 0) { handle->status = HTEXTS_FAILED; break; } /* Launch */ pthread_create(&(partial_array[i].thread), NULL, htext_get_subthread, &(partial_array[i])); } /* Exited on error? */ if (handle->status == HTEXTS_FAILED) { strerror_r(errno, err_buffer, sizeof(err_buffer)); htext_error(handle, err_buffer); } /* Wait for all of them */ for (i = 0; i < npartials; ++i) { /* Wait, or kill if failed */ if (handle->status == HTEXTS_FAILED) pthread_cancel(partial_array[i].thread); pthread_join(partial_array[i].thread, NULL); if (handle->status != HTEXTS_FAILED) handle->http_status = partial_array[i].http_status; /* Clean partial */ htext_partial_clean(&(partial_array[i])); /* Check code */ if (handle->http_status < 200 || handle->http_status >= 300) { handle->status = HTEXTS_FAILED; } } /* Done */ if (handle->status == HTEXTS_RUNNING) { handle->status = HTEXTS_SUCCEEDED; } curl_easy_cleanup(curl); htext_partial_clean(&head); free(partial_array); return NULL; }