void main_loop(void)
{
int i, running, msgs_left;
CURLMsg *msg;
/* For now do not enable SSL - make valgrind easier */
if (curl_global_init(0) || !(curlm = curl_multi_init())) {
printf("Unable to initialize curl\n");
exit(1);
}
/* Setup for the first comics */
for (i = 0; i < thread_limit; ++i)
start_next_comic();
curl_multi_perform(curlm, &running);
while (start_next_comic() || outstanding) {
int numfds=0;
curl_multi_wait(curlm, NULL, 0, MAX_WAIT_MSECS, &numfds);
curl_multi_perform(curlm, &running);
while ((msg = curl_multi_info_read(curlm, &msgs_left)))
if (msg->msg == CURLMSG_DONE)
msg_done(msg->easy_handle);
}
curl_multi_cleanup(curlm);
}
static void download_wait_loop(CURLM *multi) {
int active_handles;
do {
int nfd, rc = curl_multi_wait(multi, NULL, 0, 1000, &nfd);
if (rc != CURLM_OK) {
fprintf(stderr, "error: curl_multi_wait failed (%d)\n", rc);
break;
}
if (nfd < 0) {
fprintf(stderr, "error: poll error, possible network problem\n");
break;
}
rc = curl_multi_perform(multi, &active_handles);
if (rc != CURLM_OK) {
fprintf(stderr, "error: curl_multi_perform failed (%d)\n", rc);
break;
}
while (download_check_complete(multi, active_handles) == 0)
;
} while (active_handles > 0);
}
void curl_perform_wait()
{
long timeout_ms = -1;
CURLMcode result = curl_multi_timeout(curl_handle, &timeout_ms);
if (result != CURLM_OK) log_error("curl_multi_timeout error %d", result);
if (timeout_ms < 1) timeout_ms = 1;
int numfds = 0;
result = curl_multi_wait(curl_handle, NULL, 0, (int)timeout_ms, &numfds);
if (result != CURLM_OK) log_error("curl_multi_wait error %d", result);
int numrunning = 0;
result = curl_multi_perform(curl_handle, &numrunning);
if (result != CURLM_OK) log_error("curl_multi_perform error %d", result);
int pending = 0;
CURLMsg *message;
while ((message = curl_multi_info_read(curl_handle, &pending))) {
switch (message->msg) {
case CURLMSG_DONE:
{
const char* done_url;
CURL* easy = message->easy_handle;
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &done_url);
CURLcode code = message->data.result;
printf("%s DONE\ncode:%d - %s\n", done_url, code,
curl_easy_strerror(code));
struct curl_slist* list;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &list);
--remaining;
if (--repeats)
{
add_download(done_url);
}
curl_multi_remove_handle(curl_handle, easy);
curl_easy_cleanup(easy);
curl_slist_free_all(list);
break;
}
default:
log_error("CURLMSG default\n");
abort();
}
}
if (remaining == 0)
{
curl_multi_cleanup(curl_handle);
exit(0);
}
}
static CURLcode easy_transfer(CURLM *multi)
{
bool done = FALSE;
CURLMcode mcode = CURLM_OK;
CURLcode code = CURLE_OK;
struct timeval before;
int without_fds = 0; /* count number of consecutive returns from
curl_multi_wait() without any filedescriptors */
while(!done && !mcode) {
int still_running;
int ret;
before = curlx_tvnow();
mcode = curl_multi_wait(multi, NULL, 0, 1000, &ret);
if(mcode == CURLM_OK) {
if(ret == -1) {
/* poll() failed not on EINTR, indicate a network problem */
code = CURLE_RECV_ERROR;
break;
}
else if(ret == 0) {
struct timeval after = curlx_tvnow();
/* If it returns without any filedescriptor instantly, we need to
avoid busy-looping during periods where it has nothing particular
to wait for */
if(curlx_tvdiff(after, before) <= 10) {
without_fds++;
if(without_fds > 2) {
int sleep_ms = without_fds < 10 ? (1 << (without_fds-1)): 1000;
Curl_wait_ms(sleep_ms);
}
}
else
/* it wasn't "instant", restart counter */
without_fds = 0;
}
else
/* got file descriptor, restart counter */
without_fds = 0;
mcode = curl_multi_perform(multi, &still_running);
}
/* only read 'still_running' if curl_multi_perform() return OK */
if((mcode == CURLM_OK) && !still_running) {
int rc;
CURLMsg *msg = curl_multi_info_read(multi, &rc);
if(msg) {
code = msg->data.result;
done = TRUE;
}
}
}
return code;
}
std::streamsize download_source::read(char* s, std::streamsize n) {
dgd_scopef(trace_download);
dgd_echo(m_url);
dgd_echo(n);
if( m_mcurl == NULL ) {
initialize();
}
std::streamsize total_bytes = 0;
std::streamsize bytes_to_read = 0;
int handles_changed = 0;
curl_multi_perform(m_mcurl, &handles_changed);
dgd_echo(handles_changed);
do {
bytes_to_read = std::min(m_tail-m_head, n - total_bytes);
dgd_echo(bytes_to_read);
if( bytes_to_read > 0 ) {
std::copy(m_head, m_head + bytes_to_read, s + total_bytes);
m_head += bytes_to_read;
if( m_tail == m_head ) {
curl_easy_pause(m_curl, CURLPAUSE_CONT);
dgd_logger << "resume curl" << std::endl;
}
total_bytes += bytes_to_read;
dgd_echo(total_bytes);
m_total_bytes += bytes_to_read;
if(report_progress() != 0) {
m_eof = true;
dgd_logger << "download aborted" << std::endl;
return -1;
}
} else if( is_eof() ) {
return ( total_bytes == 0 ) ? -1 : total_bytes;
} else {
handles_changed = 0;
curl_multi_wait(m_mcurl, NULL, 0, 15000, &handles_changed);
if( handles_changed == 0 )
{
m_eof = true;
dgd_logger << "download timed out" << std::endl;
throw download_exception("Download timed out");
}
}
curl_multi_perform(m_mcurl, &handles_changed);
} while( total_bytes < n);
return total_bytes;
}
int curl_multi_done_2(CURL* curl_e, int& reDirectCount)
{
int iRetcode = -1;
CURLM * curl_m = curl_multi_init();
curl_multi_add_handle(curl_m, curl_e);
/*
* 调用curl_multi_perform函数执行curl请求
* url_multi_perform返回CURLM_CALL_MULTI_PERFORM时，表示需要继续调用该函数直到返回值不是CURLM_CALL_MULTI_PERFORM为止
* running_handles变量返回正在处理的easy curl数量，running_handles为0表示当前没有正在执行的curl请求
*/
int running_handles;
//while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(curl_m, &running_handles));
curl_multi_perform(curl_m, &running_handles);
do
{
int numfds = 0;
int res = curl_multi_wait(curl_m, NULL, 0, MAX_WAIT_MSECS, &numfds);
if (res != CURLM_OK) {
break;
}
if (!numfds) {
break;
}
//while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(curl_m, &running_handles));
curl_multi_perform(curl_m, &running_handles);
} while (running_handles);
// 输出执行结果 //
int msgs_left;
CURLMsg * msg;
CURLcode code = CURL_LAST;
while ((msg = curl_multi_info_read(curl_m, &msgs_left)))
{
if (CURLMSG_DONE == msg->msg)
{
code = msg->data.result;
if (code == CURLE_OK)
{
curl_easy_getinfo(curl_e, CURLINFO_HTTP_CODE, &iRetcode);
curl_easy_getinfo(curl_e, CURLINFO_REDIRECT_COUNT, &reDirectCount);
}
}
}
curl_multi_remove_handle(curl_m, curl_e);
curl_easy_cleanup(curl_e);
curl_multi_cleanup(curl_m);
return iRetcode;
}
void processTasks()
{
cocos2d::CCLog("==HTTP Started");
std::unique_lock<std::mutex>
lock(_cond_mutex);
while(true)
{
startTasks();
int still_running = 0;
curl_multi_perform(cm, &still_running);
int msgs_left = 0;
CURLMsg *msg=NULL;
while ((msg = curl_multi_info_read(cm, &msgs_left))) {
if (msg->msg == CURLMSG_DONE) {
CURL* curl = msg->easy_handle;
CURLcode return_code = msg->data.result;
onEnd(curl, return_code);
}
else {
fprintf(stderr, "error: after curl_multi_info_read(), CURLMsg=%d\n", msg->msg);
}
}
int old_running = getRunning();
startTasks();
int new_running = getRunning();
if(!still_running && new_running == 0)
{
cocos2d::CCLog("==HTTP sleep");
_cond.wait(lock);
cocos2d::CCLog("==HTTP wakeup");
if(stop)
return;
}
if(still_running && old_running == new_running)
{
int numfds=0;
curl_multi_wait(cm, NULL, 0, 1000, &numfds);
}
}
}
bool rest_multi_is_available(MultiRestState *state) {
CURLM *multi_handle = state->multi_handle;
int still_running, numfds = 0;
/* Has something finished? */
curl_multi_perform(multi_handle, &still_running);
if (still_running < state->nhandles)
return true;
/* Not yet, so wait for some action to be performed by curl */
curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
if (numfds == 0) /* no action, so get out now */
return false;
/* see if something has finished */
curl_multi_perform(multi_handle, &still_running);
return still_running < state->nhandles;
}
static void *pull_one_url(void *arg)
{
thread_info_t *thread_info = (thread_info_t *)arg;
global_info_t *global_info = thread_info->global_info;
work_info_t *work_info = NULL;
int still_running;
int ignore_sig[] = {SIGINT, SIGTERM};
pthread_set_ignore_sig(ignore_sig, sizeof(ignore_sig) / sizeof(int));
curl_easy_setopt(thread_info->curl, CURLOPT_WRITEFUNCTION, write_data);
int repeats = 0;
while (global_info->read_work_idx < global_info->work_num && !(global_info->do_exit)) {
repeats = 0;
DUMP("[%u:%lu]enter, read_work_idx: %u\n", thread_info->idx, time(NULL), global_info->read_work_idx);
mutex_lock(global_info->rmtx);
work_info = global_info->work_list[global_info->read_work_idx];
global_info->work_list[global_info->read_work_idx] = NULL;
global_info->read_work_idx++;
mutex_unlock(global_info->rmtx);
DUMP("[%u:%lu]work %u, read_work_idx: %u\n", thread_info->idx, time(NULL), work_info->idx, global_info->read_work_idx);
curl_easy_setopt(thread_info->curl, CURLOPT_WRITEDATA, work_info);
curl_easy_setopt(thread_info->curl, CURLOPT_URL, work_info->url);
int idx = 0;
for (idx = 0; idx < global_info->round; ++idx) {
curl_multi_add_handle(thread_info->multi_handle, thread_info->curl);
curl_multi_perform(thread_info->multi_handle, &still_running);
do {
CURLMcode mc; /* curl_multi_wait() return code */
int numfds;
/* wait for activity, timeout or "nothing" */
mc = curl_multi_wait(thread_info->multi_handle, NULL, 0, 1000, &numfds);
if(mc != CURLM_OK)
{
fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
break;
}
/* 'numfds' being zero means either a timeout or no file descriptors to
wait for. Try timeout on first occurance, then assume no file
descriptors and no file descriptors to wait for means wait for 100
milliseconds. */
if(!numfds) {
repeats++; /* count number of repeated zero numfds */
if(repeats > 1) {
WAITMS(100); /* sleep 100 milliseconds */
}
}
else
repeats = 0;
curl_multi_perform(thread_info->multi_handle, &still_running);
} while(still_running);
curl_multi_remove_handle(thread_info->multi_handle, thread_info->curl);
}
DUMP("[%u:%lu]done %u, write_work_idx: %u\n", thread_info->idx, time(NULL), work_info->idx, global_info->write_work_idx);
work_info->status = STAT_DONE;
mutex_lock(global_info->wmtx);
global_info->work_list[global_info->write_work_idx++] = work_info;
mutex_unlock(global_info->wmtx);
DUMP("[%u:%lu]out, write_work_idx: %u\n", thread_info->idx, time(NULL), global_info->write_work_idx);
}
return NULL;
}
static CURLcode easy_transfer(struct Curl_multi *multi)
{
bool done = FALSE;
CURLMcode mcode = CURLM_OK;
CURLcode result = CURLE_OK;
struct timeval before;
int without_fds = 0; /* count number of consecutive returns from
curl_multi_wait() without any filedescriptors */
while(!done && !mcode) {
int still_running = 0;
int rc;
before = curlx_tvnow();
mcode = curl_multi_wait(multi, NULL, 0, 1000, &rc);
if(!mcode) {
if(!rc) {
struct timeval after = curlx_tvnow();
/* If it returns without any filedescriptor instantly, we need to
avoid busy-looping during periods where it has nothing particular
to wait for */
if(curlx_tvdiff(after, before) <= 10) {
without_fds++;
if(without_fds > 2) {
int sleep_ms = without_fds < 10 ? (1 << (without_fds - 1)) : 1000;
Curl_wait_ms(sleep_ms);
}
}
else
/* it wasn't "instant", restart counter */
without_fds = 0;
}
else
/* got file descriptor, restart counter */
without_fds = 0;
mcode = curl_multi_perform(multi, &still_running);
}
/* only read 'still_running' if curl_multi_perform() return OK */
if(!mcode && !still_running) {
CURLMsg *msg = curl_multi_info_read(multi, &rc);
if(msg) {
result = msg->data.result;
done = TRUE;
}
}
}
/* Make sure to return some kind of error if there was a multi problem */
if(mcode) {
result = (mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY :
/* The other multi errors should never happen, so return
something suitably generic */
CURLE_BAD_FUNCTION_ARGUMENT;
}
return result;
}
void RequestManager::Worker()
{
bool shutting_down = false;
while (!shutting_down)
{
{
std::unique_lock<std::mutex> l(rt_mutex);
if (!requests_added_to_multi)
{
while (!rt_shutting_down && requests_to_add.empty() && !requests_to_start && !requests_to_remove)
{
rt_cv.wait(l);
}
}
shutting_down = rt_shutting_down;
requests_to_remove = false;
requests_to_start = false;
for (Request *request : requests_to_add)
{
request->status = 0;
requests.insert(request);
}
requests_to_add.clear();
}
if (multi && requests_added_to_multi)
{
int dontcare;
struct CURLMsg *msg;
curl_multi_wait(multi, nullptr, 0, curl_multi_wait_timeout_ms, &dontcare);
curl_multi_perform(multi, &dontcare);
while ((msg = curl_multi_info_read(multi, &dontcare)))
{
if (msg->msg == CURLMSG_DONE)
{
Request *request;
curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &request);
int finish_with = 600;
switch (msg->data.result)
{
case CURLE_OK:
long code;
curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
finish_with = (int)code;
break;
case CURLE_UNSUPPORTED_PROTOCOL: finish_with = 601; break;
case CURLE_COULDNT_RESOLVE_HOST: finish_with = 602; break;
case CURLE_OPERATION_TIMEDOUT: finish_with = 605; break;
case CURLE_URL_MALFORMAT: finish_with = 606; break;
case CURLE_COULDNT_CONNECT: finish_with = 607; break;
case CURLE_COULDNT_RESOLVE_PROXY: finish_with = 608; break;
case CURLE_TOO_MANY_REDIRECTS: finish_with = 611; break;
case CURLE_SSL_CONNECT_ERROR: finish_with = 612; break;
case CURLE_SSL_ENGINE_NOTFOUND: finish_with = 613; break;
case CURLE_SSL_ENGINE_SETFAILED: finish_with = 614; break;
case CURLE_SSL_CERTPROBLEM: finish_with = 615; break;
case CURLE_SSL_CIPHER: finish_with = 616; break;
case CURLE_SSL_ENGINE_INITFAILED: finish_with = 617; break;
case CURLE_SSL_CACERT_BADFILE: finish_with = 618; break;
case CURLE_SSL_CRL_BADFILE: finish_with = 619; break;
case CURLE_SSL_ISSUER_ERROR: finish_with = 620; break;
case CURLE_SSL_PINNEDPUBKEYNOTMATCH: finish_with = 621; break;
case CURLE_SSL_INVALIDCERTSTATUS: finish_with = 609; break;
case CURLE_HTTP2:
case CURLE_HTTP2_STREAM:
case CURLE_FAILED_INIT:
case CURLE_NOT_BUILT_IN:
default:
break;
}
if (finish_with >= 600)
{
std::cerr << request->error_buffer << std::endl;
}
request->status = finish_with;
}
};
}
std::set<Request *> requests_to_remove;
for (Request *request : requests)
{
bool signal_done = false;
{
std::lock_guard<std::mutex> g(request->rm_mutex);
if (shutting_down)
{
// In the weird case that a http::Request::Simple* call is
// waiting on this Request, we should fail the request
// instead of cancelling it ourselves.
request->status = 610;
}
if (!request->rm_canceled && request->rm_started && !request->added_to_multi && !request->status)
{
if (multi && request->easy)
{
MultiAdd(request);
}
else
{
request->status = 604;
}
}
if (!request->rm_canceled && request->rm_started && !request->rm_finished)
{
if (multi && request->easy)
{
#ifdef REQUEST_USE_CURL_OFFSET_T
curl_easy_getinfo(request->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &request->rm_total);
curl_easy_getinfo(request->easy, CURLINFO_SIZE_DOWNLOAD_T, &request->rm_done);
#else
double total, done;
curl_easy_getinfo(request->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &total);
curl_easy_getinfo(request->easy, CURLINFO_SIZE_DOWNLOAD, &done);
request->rm_total = (curl_off_t)total;
request->rm_done = (curl_off_t)done;
#endif
}
if (request->status)
{
request->rm_finished = true;
MultiRemove(request);
signal_done = true;
}
}
if (request->rm_canceled)
{
requests_to_remove.insert(request);
}
}
if (signal_done)
{
request->done_cv.notify_one();
}
}
for (Request *request : requests_to_remove)
{
requests.erase(request);
MultiRemove(request);
delete request;
}
}
}
int main(void)
{
signal(SIGINT, sighandler);
LIBXML_TEST_VERSION;
curl_global_init(CURL_GLOBAL_DEFAULT);
CURLM *multi_handle = curl_multi_init();
curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_con);
curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 6L);
/* enables http/2 if available */
#ifdef CURLPIPE_MULTIPLEX
curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
#endif
/* sets html start page */
curl_multi_add_handle(multi_handle, make_handle(start_page));
int msgs_left;
int pending = 0;
int complete = 0;
int still_running = 1;
while(still_running && !pending_interrupt) {
int numfds;
curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
curl_multi_perform(multi_handle, &still_running);
/* See how the transfers went */
CURLMsg *m = NULL;
while((m = curl_multi_info_read(multi_handle, &msgs_left))) {
if(m->msg == CURLMSG_DONE) {
CURL *handle = m->easy_handle;
char *url;
memory *mem;
curl_easy_getinfo(handle, CURLINFO_PRIVATE, &mem);
curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &url);
if(m->data.result == CURLE_OK) {
long res_status;
curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &res_status);
if(res_status == 200) {
char *ctype;
curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &ctype);
printf("[%d] HTTP 200 (%s): %s\n", complete, ctype, url);
if(is_html(ctype) && mem->size > 100) {
if(pending < max_requests && (complete + pending) < max_total) {
pending += follow_links(multi_handle, mem, url);
still_running = 1;
}
}
}
else {
printf("[%d] HTTP %d: %s\n", complete, (int) res_status, url);
}
}
else {
printf("[%d] Connection failure: %s\n", complete, url);
}
curl_multi_remove_handle(multi_handle, handle);
curl_easy_cleanup(handle);
free(mem->buf);
free(mem);
complete++;
pending--;
}
}
}
curl_multi_cleanup(multi_handle);
curl_global_cleanup();
return 0;
}
int main(int argc, char **argv)
{
int c;
int num_handle = 4;
int img = 1;
bool received_all_fragments = false;
bool * received_fragments = calloc(N, sizeof(bool));
bool * tempBoolCheck = calloc(N, sizeof(bool));
while ((c = getopt (argc, argv, "t:i:")) != -1) {
switch (c) {
case 't':
num_handle = strtoul(optarg, NULL, 10);
if (num_handle == 0) {
printf("%s: option requires an argument > 0 -- 't'\n", argv[0]);
return -1;
}
break;
case 'i':
img = strtoul(optarg, NULL, 10);
if (img == 0) {
printf("%s: option requires an argument > 0 -- 'i'\n", argv[0]);
return -1;
}
break;
default:
return -1;
}
}
png_structp png_ptr[num_handle];
png_infop info_ptr[num_handle];
png_byte * output_buffer = calloc(WIDTH*HEIGHT*4, sizeof(png_byte));
//Lock for pthread
// pthread_mutex_init(&lock, NULL);
pthread_mutex_init(&lock_hd, NULL);
// pthread_mutex_init(&lock_read_cb, NULL);
char * url = malloc(sizeof(char)*strlen(BASE_URL)+4*5);
char * url_2 = malloc(sizeof(char)*strlen(BASE_URL_2)+4*5);
char * url_3 = malloc(sizeof(char)*strlen(BASE_URL_3)+4*5);
png_bytep input_buffer[num_handle];
struct bufdata bd[num_handle];
struct headerdata hd[num_handle];
for(int i = 0; i< num_handle; i++)
{
input_buffer[i] = malloc(sizeof(png_byte)*BUF_SIZE);
bd[i].buf = input_buffer[i];
hd[i].received_fragments = received_fragments;
}
// request appropriate URL
sprintf(url, BASE_URL, img);
sprintf(url_2, BASE_URL_2, img);
sprintf(url_3, BASE_URL_3, img);
printf("requesting URL %s\n", url);
CURL *handles[num_handle];
CURLM *multi_handle;
CURLcode res = 0;
int running_handles;
CURLMsg *msg; /* for picking up messages with the transfer status */
int msgs_left; /* how many messages are left */
for (int i = 0; i < num_handle; i++)
{
handles[i] = curl_easy_init();
if(!handles[i])
{
abort_("[main] could not initialize curl");
}
curl_easy_setopt(handles[i], CURLOPT_WRITEFUNCTION, write_cb);
curl_easy_setopt(handles[i], CURLOPT_WRITEDATA, &(bd[i]));
curl_easy_setopt(handles[i], CURLOPT_HEADERDATA, &hd[i]);
curl_easy_setopt(handles[i], CURLOPT_HEADERFUNCTION, header_cb);
if(i%3 == 0)
{
curl_easy_setopt(handles[i], CURLOPT_URL, url);
}
else if(i%3 == 1)
{
curl_easy_setopt(handles[i], CURLOPT_URL, url_2);
}
else
{
curl_easy_setopt(handles[i], CURLOPT_URL, url_3);
}
//keep pointer to array index
//curl_easy_setopt(handles[i], CURLOPT_PRIVATE, url);
// reset input buffer
bd[i].len = bd[i].pos = 0; bd[i].max_size = BUF_SIZE;
}
/* init a multi stack */
multi_handle = curl_multi_init();
/* add the individual transfers */
for (int i = 0; i < num_handle; i++)
{
curl_multi_add_handle(multi_handle, handles[i]);
}
// do curl request; check for errors
// res = curl_easy_perform(curl);
// if(res != CURLE_OK)
// abort_("[main] curl_easy_perform() failed: %s\n",
// curl_easy_strerror(res));
//curl_multi_perform(multi_handle, &running_handles);
do
{
int numfds=0;
res = curl_multi_wait(multi_handle, NULL, 0, MAX_WAIT_MSECS, &numfds);
if(res != CURLM_OK) {
fprintf(stderr, "error: curl_multi_wait() returned %d\n", res);
return EXIT_FAILURE;
}
curl_multi_perform(multi_handle, &running_handles);
#ifdef _DEBUG_1_
//printf("\n## numfds: %d, num_handle: %d\n", numfds, num_handle);
fflush(stdout);
#endif
/* If the amount of running_handles is changed from the previous call (or is less than the amount of easy handles you've added to the multi handle), you know that there is one or more transfers less "running". You can then call curl_multi_info_read to get information about each individual completed transfer, and that returned info includes CURLcode and more. If an added handle fails very quickly, it may never be counted as a running_handle.*/
if(running_handles < num_handle)
{
#ifdef _DEBUG_1_
printf("\nIn IF:, running_handles: %d\n", running_handles);
fflush(stdout);
#endif
while((msg = curl_multi_info_read(multi_handle, &msgs_left)))
{
#ifdef _DEBUG_1_
printf("\nmsgs_left:%d\n", msgs_left);
fflush(stdout);
#endif
if (msg->msg == CURLMSG_DONE)
{
CURL* replaceHandle = msg->easy_handle;
curl_multi_remove_handle(multi_handle, replaceHandle);
curl_multi_add_handle(multi_handle, replaceHandle);
int iBdIndex = -1;
for (int i = 0; i < num_handle; i++)
{
if (handles[i] == replaceHandle)
{
iBdIndex = i;
#ifdef _DEBUG_1_
printf("\nfound handle, i:%d", i);
fflush(stdout);
#endif
break;
}
}
res = msg->data.result;
if(res != CURLE_OK)
{
#ifdef _DEBUG_1_
printf("\nContinued in the while loop, res is %d", res);
fflush(stdout);
#endif
continue;
//abort_("[main] curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
}
//check for dup
// if (received_fragments[hd[iBdIndex].n])
// {
// #ifdef _DEBUG_1_
// printf("\nContinued in the while loop, dup for %d", hd[iBdIndex].n);
// fflush(stdout);
// #endif
// continue;
// }
//pthread_mutex_lock(&lock);
png_ptr[iBdIndex] = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
if (!png_ptr[iBdIndex])
{
abort_("[main] png_create_read_struct failed");
}
// Write to the shared resource .
// read PNG (as downloaded from network) and copy it to output buffer
#ifdef _DEBUG_1_
printf("\nindex: %d, bd->pos: %d, bd->len: %d, hd.n: %d\n",iBdIndex, bd[iBdIndex].pos, bd[iBdIndex].len,hd[iBdIndex].n);
fflush(stdout);
#endif
png_bytep* row_pointers = read_png_file(png_ptr[iBdIndex], &info_ptr[iBdIndex], &bd[iBdIndex]);
paint_destination(png_ptr[iBdIndex], row_pointers, hd[iBdIndex].n*BUF_WIDTH, 0, output_buffer);
// reset input buffer
bd[iBdIndex].len = bd[iBdIndex].pos = 0; bd[iBdIndex].max_size = BUF_SIZE;
//bd[1].len = bd[1].pos = 0; bd[1].max_size = BUF_SIZE;
// free allocated memory
for (int y=0; y<BUF_HEIGHT; y++)
{
free(row_pointers[y]);
}
free(row_pointers);
png_destroy_read_struct(&png_ptr[iBdIndex], &info_ptr[iBdIndex], NULL);
// check for unreceived fragments
received_all_fragments = true;
for (int i = 0; i < N; i++)
{
if (!received_fragments[i])
{
received_all_fragments = false;
}
}
//tempBoolCheck
tempBoolCheck[hd[iBdIndex].n]=true;
for(int i = 0; i < N; i++)
{
if(!tempBoolCheck[i])
{
received_all_fragments=false;
}
}
//pthread_mutex_unlock(&lock);
}
else
{
//Error, replace the handle
#ifdef _DEBUG_1_
printf("\nError, replace the handle\n");
fflush(stdout);
#endif
CURL* replaceHandle = msg->easy_handle;
curl_multi_remove_handle(multi_handle, replaceHandle);
curl_multi_add_handle(multi_handle, replaceHandle);
for (int i = 0; i < num_handle; i++)
{
if (handles[i] == replaceHandle)
{
bd[i].len = bd[i].pos = 0; bd[i].max_size = BUF_SIZE;
break;
}
}
}
//break;
}
}
} while(!received_all_fragments);
free(url);
free(url_2);
free(url_3);
//free(input_buffer);
//curl_easy_cleanup(curl);
curl_multi_cleanup(multi_handle);
/* Free the CURL handles */
for (int i=0; i<num_handle; i++)
{
free(input_buffer[i]);
curl_easy_cleanup(handles[i]);
}
// now, write the array back to disk using write_png_file
png_bytep * output_row_pointers = (png_bytep*) malloc(sizeof(png_bytep) * HEIGHT);
for (int i = 0; i < HEIGHT; i++)
output_row_pointers[i] = &output_buffer[i*WIDTH*4];
write_png_file("output.png", output_row_pointers);
free(output_row_pointers);
free(output_buffer);
free(received_fragments);
//destory lock
// pthread_mutex_destroy(&lock);
pthread_mutex_destroy(&lock_hd);
// pthread_mutex_destroy(&lock_read_cb);
return 0;
}
static void handler (zsock_t *pipe, void *args) {
curl_global_init(CURL_GLOBAL_ALL);
CURLM *multi = curl_multi_init ();
CURLSH *share = curl_share_init ();
curl_share_setopt (share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
curl_share_setopt (share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
curl_share_setopt (share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
long verbose = (*(bool *) args) ? 1L : 0L;
long timeout = 30;
CURLMcode code;
SOCKET pipefd = zsock_fd (pipe);
struct curl_waitfd waitfd = {pipefd, CURL_WAIT_POLLIN};
// List to hold pending curl handles, in case we are destroy the client
// while request are inprogress
zlistx_t *pending_handles = zlistx_new ();
zlistx_set_destructor (pending_handles, (zlistx_destructor_fn *) curl_destructor);
zsock_signal (pipe, 0);
bool terminated = false;
while (!terminated) {
int events = zsock_events (pipe);
if ((events & ZMQ_POLLIN) == 0) {
code = curl_multi_wait (multi, &waitfd, 1, 1000, NULL);
assert (code == CURLM_OK);
}
events = zsock_events (pipe);
if (events & ZMQ_POLLIN) {
char* command = zstr_recv (pipe);
if (!command)
break; // Interrupted
// All actors must handle $TERM in this way
if (streq (command, "$TERM"))
terminated = true;
else if (streq (command, "GET")) {
char *url;
zlistx_t *headers;
void *userp;
int rc = zsock_recv (pipe, "slp", &url, &headers, &userp);
assert (rc == 0);
zchunk_t *data = zchunk_new (NULL, 100);
assert (data);
struct curl_slist *curl_headers = zlistx_to_slist (headers);
CURL *curl = curl_easy_init ();
zlistx_add_end (pending_handles, curl);
http_request *request = (http_request *) zmalloc (sizeof (http_request));
assert (request);
request->userp = userp;
request->curl = curl;
request->data = data;
request->headers = curl_headers;
curl_easy_setopt (curl, CURLOPT_SHARE, share);
curl_easy_setopt (curl, CURLOPT_TIMEOUT, timeout);
curl_easy_setopt (curl, CURLOPT_VERBOSE, verbose);
curl_easy_setopt (curl, CURLOPT_HTTPHEADER, curl_headers);
curl_easy_setopt (curl, CURLOPT_URL, url);
curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt (curl, CURLOPT_WRITEDATA, data);
curl_easy_setopt (curl, CURLOPT_PRIVATE, request);
code = curl_multi_add_handle (multi, curl);
assert (code == CURLM_OK);
zlistx_destroy (&headers);
zstr_free (&url);
}
else {
puts ("E: invalid message to actor");
assert (false);
}
zstr_free (&command);
}
int still_running;
code = curl_multi_perform (multi, &still_running);
assert (code == CURLM_OK);
int msgq = 0;
struct CURLMsg *msg = curl_multi_info_read(multi, &msgq);
while (msg) {
if(msg->msg == CURLMSG_DONE) {
CURL *curl = msg->easy_handle;
http_request *request;
curl_easy_getinfo(curl, CURLINFO_PRIVATE, &request);
long response_code_long;
curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &response_code_long);
int response_code = (int)response_code_long;
int rc = zsock_send (pipe, "icp", response_code, request->data, request->userp);
assert (rc == 0);
curl_multi_remove_handle (multi, curl);
// Remove curl from the pending handles and delete it
void *handle = zlistx_find (pending_handles, curl);
assert (handle);
rc = zlistx_delete (pending_handles, handle);
assert (rc == 0);
}
msg = curl_multi_info_read(multi, &msgq);
}
}
zlistx_destroy (&pending_handles);
curl_share_cleanup (share);
curl_multi_cleanup (multi);
curl_global_cleanup ();
}
/*
* curl_easy_perform() is the external interface that performs a blocking
* transfer as previously setup.
*
* CONCEPT: This function creates a multi handle, adds the easy handle to it,
* runs curl_multi_perform() until the transfer is done, then detaches the
* easy handle, destroys the multi handle and returns the easy handle's return
* code.
*
* REALITY: it can't just create and destroy the multi handle that easily. It
* needs to keep it around since if this easy handle is used again by this
* function, the same multi handle must be re-used so that the same pools and
* caches can be used.
*/
CURLcode curl_easy_perform(CURL *easy)
{
CURLM *multi;
CURLMcode mcode;
CURLcode code = CURLE_OK;
CURLMsg *msg;
bool done = FALSE;
int rc;
struct SessionHandle *data = easy;
if(!easy)
return CURLE_BAD_FUNCTION_ARGUMENT;
if(data->multi) {
failf(data, "easy handled already used in multi handle");
return CURLE_FAILED_INIT;
}
if(data->multi_easy)
multi = data->multi_easy;
else {
/* this multi handle will only ever have a single easy handled attached
to it, so make it use minimal hashes */
multi = Curl_multi_handle(1, 3);
if(!multi)
return CURLE_OUT_OF_MEMORY;
data->multi_easy = multi;
}
/* Copy the MAXCONNECTS option to the multi handle */
curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, data->set.maxconnects);
mcode = curl_multi_add_handle(multi, easy);
if(mcode) {
curl_multi_cleanup(multi);
if(mcode == CURLM_OUT_OF_MEMORY)
return CURLE_OUT_OF_MEMORY;
else
return CURLE_FAILED_INIT;
}
/* assign this after curl_multi_add_handle() since that function checks for
it and rejects this handle otherwise */
data->multi = multi;
while(!done && !mcode) {
int still_running;
int ret;
mcode = curl_multi_wait(multi, NULL, 0, 1000, &ret);
if(mcode == CURLM_OK) {
if(ret == -1) {
/* poll() failed not on EINTR, indicate a network problem */
code = CURLE_RECV_ERROR;
break;
}
mcode = curl_multi_perform(multi, &still_running);
}
/* only read 'still_running' if curl_multi_perform() return OK */
if((mcode == CURLM_OK) && !still_running) {
msg = curl_multi_info_read(multi, &rc);
if(msg) {
code = msg->data.result;
done = TRUE;
}
}
}
/* ignoring the return code isn't nice, but atm we can't really handle
a failure here, room for future improvement! */
(void)curl_multi_remove_handle(multi, easy);
/* The multi handle is kept alive, owned by the easy handle */
return code;
}
extern "C" int32_t HttpNative_MultiWait(CURLM* multiHandle,
intptr_t extraFileDescriptor,
int32_t* isExtraFileDescriptorActive,
int32_t* isTimeout)
{
assert(isExtraFileDescriptorActive != nullptr);
assert(isTimeout != nullptr);
curl_waitfd extraFds = {.fd = ToFileDescriptor(extraFileDescriptor), .events = CURL_WAIT_POLLIN, .revents = 0};
// Even with our cancellation mechanism, we specify a timeout so that
// just in case something goes wrong we can recover gracefully. This timeout is relatively long.
// Note, though, that libcurl has its own internal timeout, which can be requested separately
// via curl_multi_timeout, but which is used implicitly by curl_multi_wait if it's shorter
// than the value we provide.
const int FailsafeTimeoutMilliseconds = 1000;
int numFds;
CURLMcode result = curl_multi_wait(multiHandle, &extraFds, 1, FailsafeTimeoutMilliseconds, &numFds);
if (numFds == 0)
{
*isTimeout = true;
*isExtraFileDescriptorActive = false;
}
else
{
*isTimeout = false;
//
// Prior to libcurl version 7.32.0, the revents field was not returned properly for "extra" file descriptors
// passed to curl_multi_wait. See https://github.com/dotnet/corefx/issues/9751. So if we have a libcurl
// prior to that version, we need to do our own poll to get the status of the extra file descriptor.
//
if (curl_version_info(CURLVERSION_NOW)->version_num >= 0x072000)
{
*isExtraFileDescriptorActive = (extraFds.revents & CURL_WAIT_POLLIN) != 0;
}
else
{
pollfd pfd = { .fd = ToFileDescriptor(extraFileDescriptor),.events = POLLIN,.revents = 0 };
poll(&pfd, 1, 0);
//
// We ignore any failure in poll(), to preserve the result from curl_multi_wait. If poll() fails, it should
// leave revents cleared.
//
*isExtraFileDescriptorActive = (pfd.revents & POLLIN) != 0;
}
}
return result;
}
extern "C" int32_t HttpNative_MultiPerform(CURLM* multiHandle)
{
int running_handles;
return curl_multi_perform(multiHandle, &running_handles);
}
extern "C" int32_t HttpNative_MultiInfoRead(CURLM* multiHandle, int32_t* message, CURL** easyHandle, int32_t* result)
{
assert(message != nullptr);
assert(easyHandle != nullptr);
assert(result != nullptr);
int msgs_in_queue;
CURLMsg* curlMessage = curl_multi_info_read(multiHandle, &msgs_in_queue);
if (curlMessage == nullptr)
{
*message = 0;
*easyHandle = nullptr;
*result = 0;
return 0;
}
*message = curlMessage->msg;
*easyHandle = curlMessage->easy_handle;
*result = curlMessage->data.result;
return 1;
}
void Downloader::groupBatchDownload(const DownloadUnits &units)
{
CURLM* multi_handle = curl_multi_init();
int still_running = 0;
for (auto it = units.cbegin(); it != units.cend(); ++it)
{
DownloadUnit unit = it->second;
std::string srcUrl = unit.srcUrl;
std::string storagePath = unit.storagePath;
std::string customId = unit.customId;
FileDescriptor *fDesc = new FileDescriptor();
ProgressData *data = new ProgressData();
prepareDownload(srcUrl, storagePath, customId, unit.resumeDownload, fDesc, data);
if (fDesc->fp != NULL)
{
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, srcUrl.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fileWriteFunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fDesc->fp);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, batchDownloadProgressFunc);
curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, data);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
if (_connectionTimeout) curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, _connectionTimeout);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, LOW_SPEED_LIMIT);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, LOW_SPEED_TIME);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, MAX_REDIRS);
// Resuming download support
if (_supportResuming && unit.resumeDownload)
{
// Check already downloaded size for current download unit
long size = _fileUtils->getFileSize(storagePath + TEMP_EXT);
if (size != -1)
{
curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, size);
}
}
fDesc->curl = curl;
CURLMcode code = curl_multi_add_handle(multi_handle, curl);
if (code != CURLM_OK)
{
// Avoid memory leak
fclose(fDesc->fp);
delete data;
delete fDesc;
std::string msg = StringUtils::format("Unable to add curl handler for %s: [curl error]%s", customId.c_str(), curl_multi_strerror(code));
this->notifyError(msg, code, customId);
}
else
{
// Add to list for tracking
_progDatas.push_back(data);
_files.push_back(fDesc);
}
}
}
// Query multi perform
CURLMcode curlm_code = CURLM_CALL_MULTI_PERFORM;
while(CURLM_CALL_MULTI_PERFORM == curlm_code) {
curlm_code = curl_multi_perform(multi_handle, &still_running);
}
if (curlm_code != CURLM_OK) {
std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code));
this->notifyError(msg, curlm_code);
}
else
{
bool failed = false;
while (still_running > 0 && !failed)
{
// set a suitable timeout to play around with
struct timeval select_tv;
long curl_timeo = -1;
select_tv.tv_sec = 1;
select_tv.tv_usec = 0;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo >= 0) {
select_tv.tv_sec = curl_timeo / 1000;
if(select_tv.tv_sec > 1)
select_tv.tv_sec = 1;
else
select_tv.tv_usec = (curl_timeo % 1000) * 1000;
}
int rc;
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
// FIXME: when jenkins migrate to ubuntu, we should remove this hack code
#if (CC_TARGET_PLATFORM == CC_PLATFORM_LINUX)
curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &select_tv);
#else
rc = curl_multi_wait(multi_handle, nullptr, 0, MAX_WAIT_MSECS, &maxfd);
#endif
switch(rc)
{
case -1:
failed = true;
break;
case 0:
default:
curlm_code = CURLM_CALL_MULTI_PERFORM;
while(CURLM_CALL_MULTI_PERFORM == curlm_code) {
curlm_code = curl_multi_perform(multi_handle, &still_running);
}
if (curlm_code != CURLM_OK) {
std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code));
this->notifyError(msg, curlm_code);
}
break;
}
}
}
// Clean up and close files
curl_multi_cleanup(multi_handle);
for (auto it = _files.begin(); it != _files.end(); ++it)
{
FILE *f = (*it)->fp;
fclose(f);
CURL *single = (CURL *)((*it)->curl);
curl_multi_remove_handle(multi_handle, single);
curl_easy_cleanup(single);
}
// Check unfinished files and notify errors, succeed files will be renamed from temporary file name to real name
for (auto it = _progDatas.begin(); it != _progDatas.end(); ++it) {
ProgressData *data = *it;
if (data->downloaded < data->totalToDownload || data->totalToDownload == 0)
{
this->notifyError(ErrorCode::NETWORK, "Unable to download file", data->customId);
}
else
{
_fileUtils->renameFile(data->path, data->name + TEMP_EXT, data->name);
}
}
clearBatchDownloadData();
}
int test(char *URL)
{
CURL* curls = NULL;
CURLM* multi = NULL;
int still_running;
int i = TEST_ERR_FAILURE;
int res = 0;
CURLMsg *msg;
start_test_timing();
global_init(CURL_GLOBAL_ALL);
multi_init(multi);
easy_init(curls);
easy_setopt(curls, CURLOPT_URL, URL);
easy_setopt(curls, CURLOPT_HEADER, 1L);
multi_add_handle(multi, curls);
multi_perform(multi, &still_running);
abort_on_test_timeout();
while(still_running) {
int num;
res = curl_multi_wait(multi, NULL, 0, TEST_HANG_TIMEOUT, &num);
if (res != CURLM_OK) {
printf("curl_multi_wait() returned %d\n", res);
res = TEST_ERR_MAJOR_BAD;
goto test_cleanup;
}
abort_on_test_timeout();
multi_perform(multi, &still_running);
abort_on_test_timeout();
}
msg = curl_multi_info_read(multi, &still_running);
if(msg)
/* this should now contain a result code from the easy handle,
get it */
i = msg->data.result;
test_cleanup:
/* undocumented cleanup sequence - type UA */
curl_multi_cleanup(multi);
curl_easy_cleanup(curls);
curl_global_cleanup();
if(res)
i = res;
return i; /* return the final return code */
}
/*
* Simply download a HTTP file.
*/
int main(int argc, char **argv)
{
CURL *http_handle;
CURLM *multi_handle;
int still_running; /* keep number of running handles */
int repeats = 0;
CURLMsg *message;
curl_global_init(CURL_GLOBAL_DEFAULT);
http_handle = curl_easy_init();
/* set the options (I left out a few, you'll get the point anyway) */
curl_easy_setopt(http_handle, CURLOPT_URL, "http://www.example.com/");
multi_handle = curl_multi_init();
curl_multi_add_handle(multi_handle, http_handle);
int i = curl_multi_perform(multi_handle, &still_running);
do {
CURLMcode mc; /* curl_multi_wait() return code */
int numfds;
int pending;
message = curl_multi_info_read(multi_handle, &pending);
while (message != NULL) {
printf ("%d is the pending\n", pending);
message = curl_multi_info_read(multi_handle, &pending);
}
/* wait for activity, timeout or "nothing" */
mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
printf ("%d - %d\n", mc, numfds);
if(mc != CURLM_OK)
{
fprintf(stderr, "curl_multi_wait() failed, code %d.\n", mc);
break;
}
/* 'numfds' being zero means either a timeout or no file descriptors to
wait for. Try timeout on first occurrence, then assume no file
descriptors and no file descriptors to wait for means wait for 100
milliseconds. */
if(!numfds) {
repeats++; /* count number of repeated zero numfds */
if(repeats > 100) {
WAITMS(100); /* sleep 100 milliseconds */
}
}
else
repeats = 0;
curl_multi_perform(multi_handle, &still_running);
} while(still_running);
curl_multi_remove_handle(multi_handle, http_handle);
curl_easy_cleanup(http_handle);
curl_multi_cleanup(multi_handle);
curl_global_cleanup();
return 0;
}
static void* ov_http_client_wait_task(void* data) {
CURLMsg* message;
int count;
int pending;
long timeout;
ov_http_client_wait_context* context_ptr;
#if LIBCURL_VERSION_NUM < 0x071c00
fd_set fd_read;
fd_set fd_write;
fd_set fd_error;
int fd_count;
struct timeval tv;
#endif
/* The passed data is the wait context: */
context_ptr = data;
/* Get the timeout preferred by libcurl, or one 100 ms by default: */
curl_multi_timeout(context_ptr->handle, &timeout);
if (timeout < 0) {
timeout = 100;
}
#if LIBCURL_VERSION_NUM >= 0x071c00
/* Wait till there is activity: */
context_ptr->code = curl_multi_wait(context_ptr->handle, NULL, 0, timeout, NULL);
if (context_ptr->code != CURLE_OK) {
return NULL;
}
#else
/* Versions of libcurl older than 7.28.0 don't provide the 'curl_multi_wait' function, so we need to get the file
descriptors used by libcurl, and explicitly use the 'select' system call: */
FD_ZERO(&fd_read);
FD_ZERO(&fd_write);
FD_ZERO(&fd_error);
context_ptr->code = curl_multi_fdset(context_ptr->handle, &fd_read, &fd_write, &fd_error, &fd_count);
if (context_ptr->code != CURLE_OK) {
return NULL;
}
tv.tv_sec = timeout / 1000;
tv.tv_usec = (timeout % 1000) * 1000;
select(fd_count + 1, &fd_read, &fd_write, &fd_error, &tv);
#endif
/* Let libcurl do its work, even if no file descriptor needs attention. This is necessary because some of its
activities can't be monitored using file descriptors. */
context_ptr->code = curl_multi_perform(context_ptr->handle, &pending);
if (context_ptr->code != CURLE_OK) {
return NULL;
}
/* Check if there are finished transfers. For each of them call the function that completes them, with the global
interpreter lock acquired, as it will call Ruby code. */
while ((message = curl_multi_info_read(context_ptr->handle, &count)) != NULL) {
if (message->msg == CURLMSG_DONE) {
/* Call the Ruby code that completes the transfer: */
rb_thread_call_with_gvl(ov_http_client_complete_task, message);
/* Remove the easy handle from the multi handle and discard it: */
curl_multi_remove_handle(context_ptr->handle, message->easy_handle);
curl_easy_cleanup(message->easy_handle);
}
}
/* Everything worked correctly: */
context_ptr->code = CURLE_OK;
return NULL;
}
/*
* curl_easy_perform() is the external interface that performs a blocking
* transfer as previously setup.
*
* CONCEPT: This function creates a multi handle, adds the easy handle to it,
* runs curl_multi_perform() until the transfer is done, then detaches the
* easy handle, destroys the multi handle and returns the easy handle's return
* code.
*
* REALITY: it can't just create and destroy the multi handle that easily. It
* needs to keep it around since if this easy handle is used again by this
* function, the same multi handle must be re-used so that the same pools and
* caches can be used.
*/
CURLcode curl_easy_perform(CURL *easy)
{
CURLM *multi;
CURLMcode mcode;
CURLcode code = CURLE_OK;
CURLMsg *msg;
bool done = FALSE;
int rc;
struct SessionHandle *data = easy;
int without_fds = 0; /* count number of consecutive returns from
curl_multi_wait() without any filedescriptors */
struct timeval before;
if(!easy)
return CURLE_BAD_FUNCTION_ARGUMENT;
if(data->multi) {
failf(data, "easy handled already used in multi handle");
return CURLE_FAILED_INIT;
}
if(data->multi_easy)
multi = data->multi_easy;
else {
/* this multi handle will only ever have a single easy handled attached
to it, so make it use minimal hashes */
multi = Curl_multi_handle(1, 3);
if(!multi)
return CURLE_OUT_OF_MEMORY;
data->multi_easy = multi;
}
/* Copy the MAXCONNECTS option to the multi handle */
curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, data->set.maxconnects);
mcode = curl_multi_add_handle(multi, easy);
if(mcode) {
curl_multi_cleanup(multi);
if(mcode == CURLM_OUT_OF_MEMORY)
return CURLE_OUT_OF_MEMORY;
else
return CURLE_FAILED_INIT;
}
/* assign this after curl_multi_add_handle() since that function checks for
it and rejects this handle otherwise */
data->multi = multi;
while(!done && !mcode) {
int still_running;
int ret;
before = curlx_tvnow();
mcode = curl_multi_wait(multi, NULL, 0, 1000, &ret);
if(mcode == CURLM_OK) {
if(ret == -1) {
/* poll() failed not on EINTR, indicate a network problem */
code = CURLE_RECV_ERROR;
break;
}
else if(ret == 0) {
struct timeval after = curlx_tvnow();
/* If it returns without any filedescriptor instantly, we need to
avoid busy-looping during periods where it has nothing particular
to wait for */
if(curlx_tvdiff(after, before) <= 10) {
without_fds++;
if(without_fds > 2) {
int sleep_ms = without_fds * 50;
if(sleep_ms > 1000)
sleep_ms = 1000;
Curl_wait_ms(sleep_ms);
}
}
else
/* it wasn't "instant", restart counter */
without_fds = 0;
}
else
/* got file descriptor, restart counter */
without_fds = 0;
mcode = curl_multi_perform(multi, &still_running);
}
/* only read 'still_running' if curl_multi_perform() return OK */
if((mcode == CURLM_OK) && !still_running) {
msg = curl_multi_info_read(multi, &rc);
if(msg) {
code = msg->data.result;
done = TRUE;
}
}
}
/* ignoring the return code isn't nice, but atm we can't really handle
a failure here, room for future improvement! */
(void)curl_multi_remove_handle(multi, easy);
/* The multi handle is kept alive, owned by the easy handle */
return code;
}