static void http_put_dest_destroy(gpointer destination) { struct http_put_dest_s *dest = destination; g_assert(dest != NULL); if (dest->url) g_free(dest->url); if (dest->handle) { CURLMcode rc; rc = curl_multi_remove_handle(dest->http_put->mhandle, dest->handle); g_assert(rc == CURLM_OK); curl_easy_cleanup(dest->handle); } if (dest->headers) g_slist_free_full(dest->headers, g_free); if (dest->curl_headers) curl_slist_free_all(dest->curl_headers); if (dest->response_headers) g_hash_table_destroy(dest->response_headers); g_free(dest); }
~HTTPFetchOngoing() { if (multi != NULL) { CURLMcode mres = curl_multi_remove_handle(multi, curl); if (mres != CURLM_OK) { errorstream<<"curl_multi_remove_handle" <<" returned error code "<<mres <<std::endl; } } // Set safe options for the reusable cURL handle curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, httpfetch_discardfunction); curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL); if (httpheader != NULL) { curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL); curl_slist_free_all(httpheader); } // Store the cURL handle for reuse pool->free(curl); }
/* Check for completed transfers, and remove their easy handles */ static void check_multi_info(GlobalInfo *g) { char *eff_url; CURLMsg *msg; int msgs_left; ConnInfo *conn; CURL *easy; CURLcode res; fprintf(MSG_OUT, "REMAINING: %d\n", g->still_running); while ((msg = curl_multi_info_read(g->multi, &msgs_left))) { if (msg->msg == CURLMSG_DONE) { easy = msg->easy_handle; res = msg->data.result; curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn); curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url); fprintf(MSG_OUT, "DONE: %s => (%d) %s\n", eff_url, res, conn->error); curl_multi_remove_handle(g->multi, easy); free(conn->url); curl_easy_cleanup(easy); free(conn); } } }
void check_multi_info(Mathilda *m) { int32_t msgs_left; uint32_t response_code; CURLMsg *msg = NULL; CURLcode cc = CURLE_OK; Instruction *i = NULL; while((msg = curl_multi_info_read(m->multi_handle, &msgs_left))) { if(msg->msg == CURLMSG_DONE) { cc = curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &i); if(cc != CURLE_OK || i == NULL) { continue; } curl_easy_getinfo(i->easy, CURLINFO_RESPONSE_CODE, &response_code); i->curl_code = msg->data.result; if((response_code == i->response_code || i->response_code == 0) && i->after) { i->after(i, i->easy, &i->response); } if(i->response.text) { free(i->response.text); i->response.text = NULL; } curl_multi_remove_handle(m->multi_handle, i->easy); curl_easy_reset(i->easy); m->easy_handles.push_back(i->easy); delete i; i = NULL; } } }
tbool CXloader_MultiWrapper::Remove(CXloader* pXloader, std::string* psError) { GetLockForMultiInstance(); *psError = ""; if (gpCXloader_MultiWrapper) { std::list<CXloader*>& rlist = gpCXloader_MultiWrapper->mlistXloaders; std::list<CXloader*>::iterator it; if (Find(pXloader, &it)) { // Was added - now remove it rlist.erase(it); // Stop any ongoing transactions CURLMcode rc = curl_multi_remove_handle( gpCXloader_MultiWrapper->mpCURLMulti, pXloader->mpCURLEasyHandle ); if (rc != 0) { tchar pszErr[512]; sprintf(pszErr, "curl_multi_remove_handle(..) returned %d", rc); *psError = pszErr; } } if (rlist.size() == 0) { // All IUploader and IDownloader instances were removed - kill multi handler delete gpCXloader_MultiWrapper; gpCXloader_MultiWrapper = NULL; } } ReleaseLockForMultiInstance(); return (psError->size() == 0); } // Remove
static void curl_main () { fd_set rs; fd_set ws; fd_set es; int max; struct GNUNET_NETWORK_FDSet nrs; struct GNUNET_NETWORK_FDSet nws; struct GNUNET_TIME_Relative delay; long timeout; int running; struct CURLMsg *msg; max = 0; FD_ZERO (&rs); FD_ZERO (&ws); FD_ZERO (&es); curl_multi_perform (multi, &running); if (running == 0) { GNUNET_assert (NULL != (msg = curl_multi_info_read (multi, &running))); if (msg->msg == CURLMSG_DONE) { if (msg->data.result != CURLE_OK) { fprintf (stderr, "%s failed at %s:%d: `%s'\n", "curl_multi_perform", __FILE__, __LINE__, curl_easy_strerror (msg->data.result)); global_ret = 1; } } curl_multi_remove_handle (multi, curl); curl_multi_cleanup (multi); curl_easy_cleanup (curl); curl = NULL; multi = NULL; if (cbc.pos != strlen ("/hello_world")) { GNUNET_break (0); global_ret = 2; } if (0 != strncmp ("/hello_world", cbc.buf, strlen ("/hello_world"))) { GNUNET_break (0); global_ret = 3; } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download complete, shutting down!\n"); do_shutdown (); return; } GNUNET_assert (CURLM_OK == curl_multi_fdset (multi, &rs, &ws, &es, &max)); if ( (CURLM_OK != curl_multi_timeout (multi, &timeout)) || (-1 == timeout) ) delay = GNUNET_TIME_UNIT_SECONDS; else delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, (unsigned int) timeout); GNUNET_NETWORK_fdset_copy_native (&nrs, &rs, max + 1); GNUNET_NETWORK_fdset_copy_native (&nws, &ws, max + 1); curl_task_id = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT, delay, &nrs, &nws, &curl_task, NULL); }
static int perform_curl_io_and_complete(int *left) { CURLMsg *msg; long ret; CURLMcode curlm_ret; CURLcode curl_ret; curlm_ret = curl_multi_perform(mcurl, left); if (curlm_ret != CURLM_OK) { return -1; } while (true) { CURL *handle; struct file *file; msg = curl_multi_info_read(mcurl, left); if (!msg) { break; } if (msg->msg != CURLMSG_DONE) { continue; } handle = msg->easy_handle; ret = 404; curl_ret = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &ret); if (curl_ret != CURLE_OK) { continue; } curl_ret = curl_easy_getinfo(handle, CURLINFO_PRIVATE, (char **)&file); if (curl_ret != CURLE_OK) { curl_easy_cleanup(handle); continue; } /* The easy handle may have an error set, even if the server returns * HTTP 200, so retry the download for this case. */ if (ret == 200 && msg->data.result != CURLE_OK) { printf("Error for %s download: %s\n", file->hash, curl_easy_strerror(msg->data.result)); failed = list_prepend_data(failed, file); } else if (ret == 200) { /* When both web server and CURL report success, only then * proceed to uncompress. */ if (untar_full_download(file)) { printf("Error for %s tarfile extraction, (check free space for %s?)\n", file->hash, state_dir); failed = list_prepend_data(failed, file); } } else if (ret == 0) { /* When using the FILE:// protocol, 0 indicates success. * Otherwise, it means the web server hasn't responded yet. */ if (local_download) { if (untar_full_download(file)) { printf("Error for %s tarfile extraction, (check free space for %s?)\n", file->hash, state_dir); failed = list_prepend_data(failed, file); } } else { printf("Error for %s download: No response received\n", file->hash); failed = list_prepend_data(failed, file); } } else { printf("Error for %s download: Received %ld response\n", file->hash, ret); failed = list_prepend_data(failed, file); unlink_all_staged_content(file); } if (file->staging) { free(file->staging); file->staging = NULL; } /* NOTE: Intentionally no removal of file from hashmap. All * needed files need determined and queued in one complete * preparation phase. Once all needed files are all present, * they can be staged. Otherwise a complex datastructure and * retries are needed to insure only one download of a file * happens fully to success AND a HASH.tar is uncompressed to * and HASH and staged to the _multiple_ filenames with that * hash. */ curl_multi_remove_handle(mcurl, handle); curl_easy_cleanup(handle); file->curl = NULL; } curlm_ret = curl_multi_perform(mcurl, left); if (curlm_ret != CURLM_OK) { return -1; } return 0; }
/*************************************************************************** * This function is still only for testing purposes. It makes a great way * to run the full test suite on the multi interface instead of the easy one. *************************************************************************** * * The *new* curl_easy_perform() is the external interface that performs a * transfer previously setup. * * Wrapper-function that: creates a multi handle, adds the easy handle to it, * runs curl_multi_perform() until the transfer is done, then detaches the * easy handle, destroys the multi handle and returns the easy handle's return * code. This will make everything internally use and assume multi interface. */ CURLcode curl_easy_perform(CURL *easy) { CURLM *multi; CURLMcode mcode; CURLcode code = CURLE_OK; int still_running; struct timeval timeout; int rc; CURLMsg *msg; fd_set fdread; fd_set fdwrite; fd_set fdexcep; int maxfd; if(!easy) return CURLE_BAD_FUNCTION_ARGUMENT; multi = curl_multi_init(); if(!multi) return CURLE_OUT_OF_MEMORY; mcode = curl_multi_add_handle(multi, easy); if(mcode) { curl_multi_cleanup(multi); if(mcode == CURLM_OUT_OF_MEMORY) return CURLE_OUT_OF_MEMORY; else return CURLE_FAILED_INIT; } /* we start some action by calling perform right away */ do { while(CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multi, &still_running)); if(!still_running) break; FD_ZERO(&fdread); FD_ZERO(&fdwrite); FD_ZERO(&fdexcep); /* timeout once per second */ timeout.tv_sec = 1; timeout.tv_usec = 0; /* Old deprecated style: get file descriptors from the transfers */ curl_multi_fdset(multi, &fdread, &fdwrite, &fdexcep, &maxfd); rc = Curl_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); /* The way is to extract the sockets and wait for them without using select. This whole alternative version should probably rather use the curl_multi_socket() approach. */ if(rc == -1) /* select error */ break; /* timeout or data to send/receive => loop! */ } while(still_running); msg = curl_multi_info_read(multi, &rc); if(msg) code = msg->data.result; mcode = curl_multi_remove_handle(multi, easy); /* what to do if it fails? */ mcode = curl_multi_cleanup(multi); /* what to do if it fails? */ return code; }
void LLCurl::Multi::removeEasy(Easy* easy) { curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle()); easyFree(easy); }
void rest_multi_partial_cleanup(MultiRestState *state, bool finalize, bool fast) { CURLMsg *msg; int msgs_left; while ((msg = curl_multi_info_read(state->multi_handle, &msgs_left))) { if (msg->msg == CURLMSG_DONE) { /** this handle is finished, so lets clean it */ CURL *handle = msg->easy_handle; bool found = false; int i; curl_multi_remove_handle(state->multi_handle, handle); for (i = 0; i < state->nhandles; i++) { if (state->handles[i] == handle) { int64 response_code; curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); if (msg->data.result != 0 || response_code != 200 || strstr(state->responses[i]->data, "\"errors\":true")) { /* REST endpoint messed up */ elog(ERROR, "libcurl error: handle=%p, %s: %s, response_code=%ld, result=%d", handle, state->errorbuffs[i], state->responses[i]->data, response_code, msg->data.result); } if (state->errorbuffs[i] != NULL) { pfree(state->errorbuffs[i]); state->errorbuffs[i] = NULL; } if (state->postDatas[i] != NULL) { pfree(state->postDatas[i]->data); pfree(state->postDatas[i]); state->postDatas[i] = NULL; } if (state->responses[i] != NULL) { pfree(state->responses[i]->data); pfree(state->responses[i]); state->responses[i] = NULL; } curl_easy_cleanup(handle); state->handles[i] = NULL; state->available++; if (fast) return; found = true; break; } } if (!found) { elog(ERROR, "Couldn't find easy_handle for %p", handle); } } } if (finalize) { curl_multi_cleanup(state->multi_handle); state->multi_handle = NULL; state->available = state->nhandles; } }
static int testExternalGet () { struct MHD_Daemon *d; CURL *c; char buf[2048]; struct CBC cbc; CURLM *multi; CURLMcode mret; fd_set rs; fd_set ws; fd_set es; int max; int running; struct CURLMsg *msg; time_t start; struct timeval tv; multi = NULL; cbc.buf = buf; cbc.size = 2048; cbc.pos = 0; d = MHD_start_daemon (MHD_USE_DEBUG | MHD_USE_SSL, 1082, NULL, NULL, &ahc_echo, "GET", MHD_OPTION_HTTPS_MEM_KEY, srv_key_pem, MHD_OPTION_HTTPS_MEM_CERT, srv_self_signed_cert_pem, MHD_OPTION_END); if (d == NULL) return 256; char *aes256_sha = "AES256-SHA"; if (curl_uses_nss_ssl() == 0) { aes256_sha = "rsa_aes_256_sha"; } c = curl_easy_init (); curl_easy_setopt (c, CURLOPT_URL, "https://localhost:1082/hello_world"); curl_easy_setopt (c, CURLOPT_WRITEFUNCTION, ©Buffer); curl_easy_setopt (c, CURLOPT_WRITEDATA, &cbc); /* TLS options */ curl_easy_setopt (c, CURLOPT_SSLVERSION, CURL_SSLVERSION_SSLv3); curl_easy_setopt (c, CURLOPT_SSL_CIPHER_LIST, aes256_sha); curl_easy_setopt (c, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt (c, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt (c, CURLOPT_FAILONERROR, 1); if (oneone) curl_easy_setopt (c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); else curl_easy_setopt (c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); curl_easy_setopt (c, CURLOPT_TIMEOUT, 150L); curl_easy_setopt (c, CURLOPT_CONNECTTIMEOUT, 15L); /* NOTE: use of CONNECTTIMEOUT without also setting NOSIGNAL results in really weird crashes on my system! */ curl_easy_setopt (c, CURLOPT_NOSIGNAL, 1); multi = curl_multi_init (); if (multi == NULL) { curl_easy_cleanup (c); MHD_stop_daemon (d); return 512; } mret = curl_multi_add_handle (multi, c); if (mret != CURLM_OK) { curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 1024; } start = time (NULL); while ((time (NULL) - start < 5) && (multi != NULL)) { max = 0; FD_ZERO (&rs); FD_ZERO (&ws); FD_ZERO (&es); curl_multi_perform (multi, &running); mret = curl_multi_fdset (multi, &rs, &ws, &es, &max); if (mret != CURLM_OK) { curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 2048; } if (MHD_YES != MHD_get_fdset (d, &rs, &ws, &es, &max)) { curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 4096; } tv.tv_sec = 0; tv.tv_usec = 1000; select (max + 1, &rs, &ws, &es, &tv); curl_multi_perform (multi, &running); if (running == 0) { msg = curl_multi_info_read (multi, &running); if (msg == NULL) break; if (msg->msg == CURLMSG_DONE) { if (msg->data.result != CURLE_OK) printf ("%s failed at %s:%d: `%s'\n", "curl_multi_perform", __FILE__, __LINE__, curl_easy_strerror (msg->data.result)); curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); c = NULL; multi = NULL; } } MHD_run (d); } if (multi != NULL) { curl_multi_remove_handle (multi, c); curl_easy_cleanup (c); curl_multi_cleanup (multi); } MHD_stop_daemon (d); if (cbc.pos != strlen ("/hello_world")) return 8192; if (0 != strncmp ("/hello_world", cbc.buf, strlen ("/hello_world"))) return 16384; return 0; }
/* * Source code in here hugely as reported in bug report 651464 by * Christopher R. Palmer. * * Use multi interface to get document over proxy with bad port number. * This caused the interface to "hang" in libcurl 7.10.2. */ int test(char *URL) { CURL *c; int ret=0; CURLM *m; fd_set rd, wr, exc; CURLMcode res; int running; int max_fd; int rc; curl_global_init(CURL_GLOBAL_ALL); c = curl_easy_init(); /* the point here being that there must not run anything on the given proxy port */ curl_easy_setopt(c, CURLOPT_PROXY, arg2); curl_easy_setopt(c, CURLOPT_URL, URL); curl_easy_setopt(c, CURLOPT_VERBOSE, 1); m = curl_multi_init(); res = curl_multi_add_handle(m, c); if(res && (res != CURLM_CALL_MULTI_PERFORM)) return 1; /* major failure */ do { struct timeval interval; interval.tv_sec = 1; interval.tv_usec = 0; fprintf(stderr, "curl_multi_perform()\n"); do { res = curl_multi_perform(m, &running); } while (res == CURLM_CALL_MULTI_PERFORM); if(!running) { /* This is where this code is expected to reach */ int numleft; CURLMsg *msg = curl_multi_info_read(m, &numleft); fprintf(stderr, "Expected: not running\n"); if(msg && !numleft) ret = 100; /* this is where we should be */ else ret = 99; /* not correct */ break; } fprintf(stderr, "running == %d, res == %d\n", running, res); if (res != CURLM_OK) { ret = 2; break; } FD_ZERO(&rd); FD_ZERO(&wr); FD_ZERO(&exc); max_fd = 0; fprintf(stderr, "curl_multi_fdset()\n"); if (curl_multi_fdset(m, &rd, &wr, &exc, &max_fd) != CURLM_OK) { fprintf(stderr, "unexpected failured of fdset.\n"); ret = 3; break; } rc = select(max_fd+1, &rd, &wr, &exc, &interval); fprintf(stderr, "select returned %d\n", rc); } while(rc); curl_multi_remove_handle(m, c); curl_easy_cleanup(c); curl_multi_cleanup(m); return ret; }
int main(void) { CURL *curl; CURLM *mcurl; int still_running = 1; struct timeval mp_start; char mp_timedout = 0; struct WriteThis pooh; struct curl_slist* rcpt_list = NULL; pooh.counter = 0; curl_global_init(CURL_GLOBAL_DEFAULT); curl = curl_easy_init(); if(!curl) return 1; mcurl = curl_multi_init(); if(!mcurl) return 2; rcpt_list = curl_slist_append(rcpt_list, RECEPIENT); /* more addresses can be added here rcpt_list = curl_slist_append(rcpt_list, "*****@*****.**"); */ curl_easy_setopt(curl, CURLOPT_URL, "smtp://" SMTPSERVER SMTPPORT); curl_easy_setopt(curl, CURLOPT_USERNAME, USERNAME); curl_easy_setopt(curl, CURLOPT_PASSWORD, PASSWORD); curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); curl_easy_setopt(curl, CURLOPT_MAIL_FROM, MAILFROM); curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, rcpt_list); curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER,0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_READDATA, &pooh); curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); curl_easy_setopt(curl, CURLOPT_SSLVERSION, 0); curl_easy_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE, 0); curl_multi_add_handle(mcurl, curl); mp_timedout = 0; mp_start = tvnow(); /* we start some action by calling perform right away */ curl_multi_perform(mcurl, &still_running); while(still_running) { struct timeval timeout; int rc; /* select() return code */ fd_set fdread; fd_set fdwrite; fd_set fdexcep; int maxfd = -1; long curl_timeo = -1; FD_ZERO(&fdread); FD_ZERO(&fdwrite); FD_ZERO(&fdexcep); /* set a suitable timeout to play around with */ timeout.tv_sec = 1; timeout.tv_usec = 0; curl_multi_timeout(mcurl, &curl_timeo); if(curl_timeo >= 0) { timeout.tv_sec = curl_timeo / 1000; if(timeout.tv_sec > 1) timeout.tv_sec = 1; else timeout.tv_usec = (curl_timeo % 1000) * 1000; } /* get file descriptors from the transfers */ curl_multi_fdset(mcurl, &fdread, &fdwrite, &fdexcep, &maxfd); /* In a real-world program you OF COURSE check the return code of the function calls. On success, the value of maxfd is guaranteed to be greater or equal than -1. We call select(maxfd + 1, ...), specially in case of (maxfd == -1), we call select(0, ...), which is basically equal to sleep. */ rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); if (tvdiff(tvnow(), mp_start) > MULTI_PERFORM_HANG_TIMEOUT) { fprintf(stderr, "ABORTING TEST, since it seems " "that it would have run forever.\n"); break; } switch(rc) { case -1: /* select error */ break; case 0: /* timeout */ default: /* action */ curl_multi_perform(mcurl, &still_running); break; } } curl_slist_free_all(rcpt_list); curl_multi_remove_handle(mcurl, curl); curl_multi_cleanup(mcurl); curl_easy_cleanup(curl); curl_global_cleanup(); return 0; }
/* * Simply download a HTTP file. */ int main(void) { CURL *http_handle; CURLM *multi_handle; int still_running; /* keep number of running handles */ curl_global_init(CURL_GLOBAL_DEFAULT); http_handle = curl_easy_init(); /* set the options (I left out a few, you'll get the point anyway) */ curl_easy_setopt(http_handle, CURLOPT_URL, "http://www.example.com/"); /* init a multi stack */ multi_handle = curl_multi_init(); /* add the individual transfers */ curl_multi_add_handle(multi_handle, http_handle); /* we start some action by calling perform right away */ curl_multi_perform(multi_handle, &still_running); do { struct timeval timeout; int rc; /* select() return code */ fd_set fdread; fd_set fdwrite; fd_set fdexcep; int maxfd = -1; long curl_timeo = -1; FD_ZERO(&fdread); FD_ZERO(&fdwrite); FD_ZERO(&fdexcep); /* set a suitable timeout to play around with */ timeout.tv_sec = 1; timeout.tv_usec = 0; curl_multi_timeout(multi_handle, &curl_timeo); if(curl_timeo >= 0) { timeout.tv_sec = curl_timeo / 1000; if(timeout.tv_sec > 1) timeout.tv_sec = 1; else timeout.tv_usec = (curl_timeo % 1000) * 1000; } /* get file descriptors from the transfers */ curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); /* In a real-world program you OF COURSE check the return code of the function calls. On success, the value of maxfd is guaranteed to be greater or equal than -1. We call select(maxfd + 1, ...), specially in case of (maxfd == -1), we call select(0, ...), which is basically equal to sleep. */ rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); switch(rc) { case -1: /* select error */ still_running = 0; printf("select() returns error, this is badness\n"); break; case 0: default: /* timeout or readable/writable sockets */ curl_multi_perform(multi_handle, &still_running); break; } } while(still_running); curl_multi_remove_handle(multi_handle, http_handle); curl_easy_cleanup(http_handle); curl_multi_cleanup(multi_handle); curl_global_cleanup(); return 0; }
int main(void) { CURL *curl; CURLM *mcurl; int still_running = 1; struct timeval mp_start; curl_global_init(CURL_GLOBAL_DEFAULT); curl = curl_easy_init(); if(!curl) return 1; mcurl = curl_multi_init(); if(!mcurl) return 2; /* Set username and password */ curl_easy_setopt(curl, CURLOPT_USERNAME, "user"); curl_easy_setopt(curl, CURLOPT_PASSWORD, "secret"); /* This will fetch message 1 from the user's inbox */ curl_easy_setopt(curl, CURLOPT_URL, "imap://imap.example.com/INBOX/;UID=1"); /* Tell the multi stack about our easy handle */ curl_multi_add_handle(mcurl, curl); /* Record the start time which we can use later */ mp_start = tvnow(); /* We start some action by calling perform right away */ curl_multi_perform(mcurl, &still_running); while(still_running) { struct timeval timeout; fd_set fdread; fd_set fdwrite; fd_set fdexcep; int maxfd = -1; int rc; CURLMcode mc; /* curl_multi_fdset() return code */ long curl_timeo = -1; /* Initialise the file descriptors */ FD_ZERO(&fdread); FD_ZERO(&fdwrite); FD_ZERO(&fdexcep); /* Set a suitable timeout to play around with */ timeout.tv_sec = 1; timeout.tv_usec = 0; curl_multi_timeout(mcurl, &curl_timeo); if(curl_timeo >= 0) { timeout.tv_sec = curl_timeo / 1000; if(timeout.tv_sec > 1) timeout.tv_sec = 1; else timeout.tv_usec = (curl_timeo % 1000) * 1000; } /* get file descriptors from the transfers */ mc = curl_multi_fdset(mcurl, &fdread, &fdwrite, &fdexcep, &maxfd); if(mc != CURLM_OK) { fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc); break; } /* On success the value of maxfd is guaranteed to be >= -1. We call select(maxfd + 1, ...); specially in case of (maxfd == -1) there are no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- to sleep 100ms, which is the minimum suggested value in the curl_multi_fdset() doc. */ if(maxfd == -1) { #ifdef _WIN32 Sleep(100); rc = 0; #else /* Portable sleep for platforms other than Windows. */ struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ rc = select(0, NULL, NULL, NULL, &wait); #endif } else { /* Note that on some platforms 'timeout' may be modified by select(). If you need access to the original value save a copy beforehand. */ rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); } if(tvdiff(tvnow(), mp_start) > MULTI_PERFORM_HANG_TIMEOUT) { fprintf(stderr, "ABORTING: Since it seems that we would have run forever.\n"); break; } switch(rc) { case -1: /* select error */ break; case 0: /* timeout */ default: /* action */ curl_multi_perform(mcurl, &still_running); break; } } /* Always cleanup */ curl_multi_remove_handle(mcurl, curl); curl_multi_cleanup(mcurl); curl_easy_cleanup(curl); curl_global_cleanup(); return 0; }
static int testExternalPut () { struct MHD_Daemon *d; CURL *c; char buf[2048]; struct CBC cbc; CURLM *multi; CURLMcode mret; fd_set rs; fd_set ws; fd_set es; MHD_socket max; int running; struct CURLMsg *msg; time_t start; struct timeval tv; unsigned int pos = 0; int done_flag = 0; multi = NULL; cbc.buf = buf; cbc.size = 2048; cbc.pos = 0; d = MHD_start_daemon (MHD_USE_DEBUG, 1082, NULL, NULL, &ahc_echo, &done_flag, MHD_OPTION_END); if (d == NULL) return 256; c = curl_easy_init (); curl_easy_setopt (c, CURLOPT_URL, "http://127.0.0.1:1082/hello_world"); curl_easy_setopt (c, CURLOPT_WRITEFUNCTION, ©Buffer); curl_easy_setopt (c, CURLOPT_WRITEDATA, &cbc); curl_easy_setopt (c, CURLOPT_READFUNCTION, &putBuffer); curl_easy_setopt (c, CURLOPT_READDATA, &pos); curl_easy_setopt (c, CURLOPT_UPLOAD, 1L); curl_easy_setopt (c, CURLOPT_INFILESIZE_LARGE, (curl_off_t) 8L); curl_easy_setopt (c, CURLOPT_FAILONERROR, 1); curl_easy_setopt (c, CURLOPT_TIMEOUT, 150L); if (oneone) curl_easy_setopt (c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); else curl_easy_setopt (c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); curl_easy_setopt (c, CURLOPT_CONNECTTIMEOUT, 150L); // NOTE: use of CONNECTTIMEOUT without also // setting NOSIGNAL results in really weird // crashes on my system! curl_easy_setopt (c, CURLOPT_NOSIGNAL, 1); multi = curl_multi_init (); if (multi == NULL) { curl_easy_cleanup (c); MHD_stop_daemon (d); return 512; } mret = curl_multi_add_handle (multi, c); if (mret != CURLM_OK) { curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 1024; } start = time (NULL); while ((time (NULL) - start < 5) && (multi != NULL)) { max = 0; FD_ZERO (&rs); FD_ZERO (&ws); FD_ZERO (&es); curl_multi_perform (multi, &running); mret = curl_multi_fdset (multi, &rs, &ws, &es, &max); if (mret != CURLM_OK) { curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 2048; } if (MHD_YES != MHD_get_fdset (d, &rs, &ws, &es, &max)) { curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 4096; } tv.tv_sec = 0; tv.tv_usec = 1000; select (max + 1, &rs, &ws, &es, &tv); curl_multi_perform (multi, &running); if (running == 0) { msg = curl_multi_info_read (multi, &running); if (msg == NULL) break; if (msg->msg == CURLMSG_DONE) { if (msg->data.result != CURLE_OK) printf ("%s failed at %s:%d: `%s'\n", "curl_multi_perform", __FILE__, __LINE__, curl_easy_strerror (msg->data.result)); curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); c = NULL; multi = NULL; } } MHD_run (d); } if (multi != NULL) { curl_multi_remove_handle (multi, c); curl_easy_cleanup (c); curl_multi_cleanup (multi); } MHD_stop_daemon (d); if (cbc.pos != strlen ("/hello_world")) return 8192; if (0 != strncmp ("/hello_world", cbc.buf, strlen ("/hello_world"))) return 16384; return 0; }
static int testExternalPost () { struct MHD_Daemon *d; CURL *c; char buf[2048]; struct CBC cbc; CURLM *multi; CURLMcode mret; fd_set rs; fd_set ws; fd_set es; int max; int running; time_t start; struct timeval tv; struct curl_httppost *pd; int i; multi = NULL; cbc.buf = buf; cbc.size = 2048; cbc.pos = 0; d = MHD_start_daemon (MHD_NO_FLAG /* | MHD_USE_ERROR_LOG */ , 1082, NULL, NULL, &ahc_echo, NULL, MHD_OPTION_NOTIFY_COMPLETED, &completed_cb, NULL, MHD_OPTION_END); if (d == NULL) return 256; multi = curl_multi_init (); if (multi == NULL) { MHD_stop_daemon (d); return 512; } zzuf_socat_start (); for (i = 0; i < LOOP_COUNT; i++) { fprintf (stderr, "."); c = curl_easy_init (); curl_easy_setopt (c, CURLOPT_URL, "http://localhost:1082/hello_world"); curl_easy_setopt (c, CURLOPT_WRITEFUNCTION, ©Buffer); curl_easy_setopt (c, CURLOPT_WRITEDATA, &cbc); pd = make_form (); curl_easy_setopt (c, CURLOPT_HTTPPOST, pd); curl_easy_setopt (c, CURLOPT_FAILONERROR, 1); curl_easy_setopt (c, CURLOPT_TIMEOUT, 150L); if (oneone) curl_easy_setopt (c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); else curl_easy_setopt (c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); curl_easy_setopt (c, CURLOPT_CONNECTTIMEOUT, 15L); // NOTE: use of CONNECTTIMEOUT without also // setting NOSIGNAL results in really weird // crashes on my system! curl_easy_setopt (c, CURLOPT_NOSIGNAL, 1); mret = curl_multi_add_handle (multi, c); if (mret != CURLM_OK) { curl_multi_cleanup (multi); curl_formfree (pd); curl_easy_cleanup (c); zzuf_socat_stop (); MHD_stop_daemon (d); return 1024; } start = time (NULL); while ((time (NULL) - start < 5) && (c != NULL)) { max = 0; FD_ZERO (&rs); FD_ZERO (&ws); FD_ZERO (&es); curl_multi_perform (multi, &running); mret = curl_multi_fdset (multi, &rs, &ws, &es, &max); if (mret != CURLM_OK) { curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); zzuf_socat_stop (); MHD_stop_daemon (d); curl_formfree (pd); return 2048; } if (MHD_YES != MHD_get_fdset (d, &rs, &ws, &es, &max)) { curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); curl_formfree (pd); zzuf_socat_stop (); MHD_stop_daemon (d); return 4096; } tv.tv_sec = 0; tv.tv_usec = 1000; select (max + 1, &rs, &ws, &es, &tv); curl_multi_perform (multi, &running); if (running == 0) { curl_multi_info_read (multi, &running); curl_multi_remove_handle (multi, c); curl_easy_cleanup (c); c = NULL; } MHD_run (d); } if (c != NULL) { curl_multi_remove_handle (multi, c); curl_easy_cleanup (c); } curl_formfree (pd); } fprintf (stderr, "\n"); zzuf_socat_stop (); MHD_stop_daemon (d); return 0; }
int main(void) { signal(SIGINT, sighandler); LIBXML_TEST_VERSION; curl_global_init(CURL_GLOBAL_DEFAULT); CURLM *multi_handle = curl_multi_init(); curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_con); curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 6L); /* enables http/2 if available */ #ifdef CURLPIPE_MULTIPLEX curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); #endif /* sets html start page */ curl_multi_add_handle(multi_handle, make_handle(start_page)); int msgs_left; int pending = 0; int complete = 0; int still_running = 1; while(still_running && !pending_interrupt) { int numfds; curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds); curl_multi_perform(multi_handle, &still_running); /* See how the transfers went */ CURLMsg *m = NULL; while((m = curl_multi_info_read(multi_handle, &msgs_left))) { if(m->msg == CURLMSG_DONE) { CURL *handle = m->easy_handle; char *url; memory *mem; curl_easy_getinfo(handle, CURLINFO_PRIVATE, &mem); curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &url); if(m->data.result == CURLE_OK) { long res_status; curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &res_status); if(res_status == 200) { char *ctype; curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &ctype); printf("[%d] HTTP 200 (%s): %s\n", complete, ctype, url); if(is_html(ctype) && mem->size > 100) { if(pending < max_requests && (complete + pending) < max_total) { pending += follow_links(multi_handle, mem, url); still_running = 1; } } } else { printf("[%d] HTTP %d: %s\n", complete, (int) res_status, url); } } else { printf("[%d] Connection failure: %s\n", complete, url); } curl_multi_remove_handle(multi_handle, handle); curl_easy_cleanup(handle); free(mem->buf); free(mem); complete++; pending--; } } } curl_multi_cleanup(multi_handle); curl_global_cleanup(); return 0; }
static int testExternalGet () { struct MHD_Daemon *d; CURL *c; char buf[2048]; struct CBC cbc; CURLM *multi; CURLMcode mret; fd_set rs; fd_set ws; fd_set es; MHD_socket max; int running; struct CURLMsg *msg; time_t start; struct timeval tv; multi = NULL; cbc.buf = buf; cbc.size = 2048; cbc.pos = 0; d = MHD_start_daemon (MHD_USE_DEBUG, 21080, NULL, NULL, &ahc_echo, "GET", MHD_OPTION_END); if (d == NULL) return 256; c = curl_easy_init (); curl_easy_setopt (c, CURLOPT_URL, "http://127.0.0.1:21080/hello_world"); curl_easy_setopt (c, CURLOPT_WRITEFUNCTION, ©Buffer); curl_easy_setopt (c, CURLOPT_WRITEDATA, &cbc); curl_easy_setopt (c, CURLOPT_FAILONERROR, 1); /* note that the string below intentionally uses the various ways cookies can be specified to exercise the parser! Do not change! */ curl_easy_setopt (c, CURLOPT_COOKIE, "name1=var1; name2=var2,name3 ;name4=\"var4 with spaces\";"); if (oneone) curl_easy_setopt (c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); else curl_easy_setopt (c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); curl_easy_setopt (c, CURLOPT_TIMEOUT, 150L); curl_easy_setopt (c, CURLOPT_CONNECTTIMEOUT, 150L); /* NOTE: use of CONNECTTIMEOUT without also setting NOSIGNAL results in really weird crashes on my system! */ curl_easy_setopt (c, CURLOPT_NOSIGNAL, 1); multi = curl_multi_init (); if (multi == NULL) { curl_easy_cleanup (c); MHD_stop_daemon (d); return 512; } mret = curl_multi_add_handle (multi, c); if (mret != CURLM_OK) { curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 1024; } start = time (NULL); while ((time (NULL) - start < 5) && (multi != NULL)) { max = 0; FD_ZERO (&rs); FD_ZERO (&ws); FD_ZERO (&es); curl_multi_perform (multi, &running); mret = curl_multi_fdset (multi, &rs, &ws, &es, &max); if (mret != CURLM_OK) { curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 2048; } if (MHD_YES != MHD_get_fdset (d, &rs, &ws, &es, &max)) { curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); MHD_stop_daemon (d); return 4096; } tv.tv_sec = 0; tv.tv_usec = 1000; select (max + 1, &rs, &ws, &es, &tv); curl_multi_perform (multi, &running); if (running == 0) { msg = curl_multi_info_read (multi, &running); if (msg == NULL) break; if (msg->msg == CURLMSG_DONE) { if (msg->data.result != CURLE_OK) printf ("%s failed at %s:%d: `%s'\n", "curl_multi_perform", __FILE__, __LINE__, curl_easy_strerror (msg->data.result)); curl_multi_remove_handle (multi, c); curl_multi_cleanup (multi); curl_easy_cleanup (c); c = NULL; multi = NULL; } } MHD_run (d); } if (multi != NULL) { curl_multi_remove_handle (multi, c); curl_easy_cleanup (c); curl_multi_cleanup (multi); } MHD_stop_daemon (d); if (cbc.pos != strlen ("/hello_world")) return 8192; if (0 != strncmp ("/hello_world", cbc.buf, strlen ("/hello_world"))) return 16384; return 0; }
void HttpMgr::run() { if (m_httpCmdList.empty()) { return; } int runningHandles = 0; CURLMcode code = curl_multi_perform( m_currentCurl, &runningHandles); if( CURLM_OK != code ) { LOG_ERROR << "curl_multi_perform failed!"; return; } // 用于缓存当前已执行完成的http请求 HttpCmdList doneHttpCmds; int MsgNumLeft = 0; CURLMsg* pMsg = NULL; // 获取已返回结果的http请求 while((pMsg = curl_multi_info_read( m_currentCurl, &MsgNumLeft)) != NULL) { if( pMsg->msg != CURLMSG_DONE ) { continue; } HttpCmdList::iterator itr = m_httpCmdList.begin(); HttpCmdList::iterator end = m_httpCmdList.end(); for(; itr != end; ++itr) { if( (*itr)->m_curl != pMsg->easy_handle) { continue; } break; } if( itr == m_httpCmdList.end() ) { LOG_ERROR << "find easy handle failed!"; break; } HttpCmd *cmd = *itr; cmd->m_ok = (CURLE_OK == pMsg->data.result) ? true : false; doneHttpCmds.push_back(cmd); m_httpCmdList.erase(itr); //释放 curl_multi_remove_handle(m_currentCurl, cmd->m_curl); } // 检查超时 HttpCmdList::iterator itr = m_httpCmdList.begin(); HttpCmdList::iterator end = m_httpCmdList.end(); Timestamp now = timetool::getTimeOfDay(); for(; itr != end;) { HttpCmd *cmd = *itr; if(now - cmd->m_lastAddTime > 30 * 1000) { cmd->m_ok = false; doneHttpCmds.push_back(cmd); m_httpCmdList.erase(itr++); //释放 curl_multi_remove_handle(m_currentCurl, cmd->m_curl); } else { itr++; } } // 处理已返回结果的http请求 for(HttpCmdList::iterator itr = doneHttpCmds.begin(); itr != doneHttpCmds.end(); ++itr) { HttpCmd *httpcmd = *itr; httpcmd->onExecuted(); httpcmd->release(); } }
static void tr_webThreadFunc( void * vsession ) { int unused; CURLM * multi; struct tr_web * web; int taskCount = 0; tr_session * session = vsession; /* try to enable ssl for https support; but if that fails, * try a plain vanilla init */ if( curl_global_init( CURL_GLOBAL_SSL ) ) curl_global_init( 0 ); web = tr_new0( struct tr_web, 1 ); web->close_mode = ~0; web->taskLock = tr_lockNew( ); web->tasks = NULL; multi = curl_multi_init( ); session->web = web; for( ;; ) { long msec; CURLMsg * msg; CURLMcode mcode; struct tr_web_task * task; if( web->close_mode == TR_WEB_CLOSE_NOW ) break; if( ( web->close_mode == TR_WEB_CLOSE_WHEN_IDLE ) && !taskCount ) break; /* add tasks from the queue */ tr_lockLock( web->taskLock ); while(( task = tr_list_pop_front( &web->tasks ))) { dbgmsg( "adding task to curl: [%s]", task->url ); curl_multi_add_handle( multi, createEasy( session, task )); /*fprintf( stderr, "adding a task.. taskCount is now %d\n", taskCount );*/ ++taskCount; } tr_lockUnlock( web->taskLock ); /* maybe wait a little while before calling curl_multi_perform() */ msec = 0; curl_multi_timeout( multi, &msec ); if( msec < 0 ) msec = THREADFUNC_MAX_SLEEP_MSEC; if( msec > 0 ) { int usec; int max_fd; struct timeval t; fd_set r_fd_set, w_fd_set, c_fd_set; max_fd = 0; FD_ZERO( &r_fd_set ); FD_ZERO( &w_fd_set ); FD_ZERO( &c_fd_set ); curl_multi_fdset( multi, &r_fd_set, &w_fd_set, &c_fd_set, &max_fd ); if( msec > THREADFUNC_MAX_SLEEP_MSEC ) msec = THREADFUNC_MAX_SLEEP_MSEC; usec = msec * 1000; t.tv_sec = usec / 1000000; t.tv_usec = usec % 1000000; tr_select( max_fd+1, &r_fd_set, &w_fd_set, &c_fd_set, &t ); } /* call curl_multi_perform() */ do { mcode = curl_multi_perform( multi, &unused ); } while( mcode == CURLM_CALL_MULTI_PERFORM ); /* pump completed tasks from the multi */ while(( msg = curl_multi_info_read( multi, &unused ))) { if(( msg->msg == CURLMSG_DONE ) && ( msg->easy_handle != NULL )) { struct tr_web_task * task; CURL * e = msg->easy_handle; curl_easy_getinfo( e, CURLINFO_PRIVATE, (void*)&task ); curl_easy_getinfo( e, CURLINFO_RESPONSE_CODE, &task->code ); curl_multi_remove_handle( multi, e ); curl_easy_cleanup( e ); /*fprintf( stderr, "removing a completed task.. taskCount is now %d (response code: %d, response len: %d)\n", taskCount, (int)task->code, (int)evbuffer_get_length(task->response) );*/ tr_runInEventThread( task->session, task_finish_func, task ); --taskCount; } } } /* cleanup */ curl_multi_cleanup( multi ); tr_lockFree( web->taskLock ); tr_free( web ); session->web = NULL; }
int run() { if (NULL == m_handler) { LOGE("m_handler is null."); return -1; } int msgs_in_queue = -1; int running = -1; int max_fds = -1; fd_set read_fd; fd_set write_fd; fd_set exec_fd; CURLMsg *msg; if (add_multiple_handlers() != 0) return -2; while (running) { curl_multi_perform(m_handler, &running); if (running) { FD_ZERO(&read_fd); FD_ZERO(&write_fd); FD_ZERO(&exec_fd); if (curl_multi_fdset(m_handler, &read_fd, &write_fd, &exec_fd, &max_fds)) { LOGE("curl_multi_fdset"); return -3; } if (set_timeout()) return -4; if (max_fds == -1) { sleep(MULTI_SELECT_TIMEOUT / 1000); } else { #if 0 if (0 > select(max_fds + 1, &read_fd, &write_fd, &exec_fd, &m_timeout)) { LOGE("select(%i,,,,%d)", max_fds + 1, (int)m_timeout.tv_sec); return -5; } #endif if (0 > select(max_fds + 1, &read_fd, &write_fd, &exec_fd, NULL)) { LOGE("select(%i)", max_fds + 1); return -5; } } } while ((msg = curl_multi_info_read(m_handler, &msgs_in_queue))) { if (CURLMSG_DONE == msg->msg) { Conn* conn; CURL* eh = msg->easy_handle; curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &conn); fclose(conn->output); conn->output = NULL; m_done_callback(conn); LOGI("%d - %s", msg->data.result, curl_easy_strerror(msg->data.result)); remove_done_url(conn->url); free(conn); curl_multi_remove_handle(m_handler, eh); curl_easy_cleanup(eh); m_eh_array.erase(eh); m_crawling_urls.erase(conn->url); } else { LOGE("exec_fd: CURLMsg (%d)", msg->msg); } if (add_only_one_handler() == 0) { running++; break; } } } return 0; }
static void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server) { char *eff_url; CURLMsg *msg; int msgs_left; MolochHttpRequest_t *request; CURL *easy; while ((msg = curl_multi_info_read(server->multi, &msgs_left))) { if (msg->msg == CURLMSG_DONE) { easy = msg->easy_handle; curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request); curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url); long responseCode; curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode); if (config.logESRequests) { double totalTime; double connectTime; double uploadSize; double downloadSize; curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime); curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime); curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize); curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize); LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms", request->server->outstanding, request->server->connections, responseCode, request->url, uploadSize, downloadSize, connectTime*1000, totalTime*1000); } if (request->func) { if (request->dataIn) request->dataIn[request->used] = 0; request->func(responseCode, request->dataIn, request->used, request->uw); } if (request->dataIn) { free(request->dataIn); request->dataIn = 0; } if (request->dataOut) { MOLOCH_SIZE_FREE(buffer, request->dataOut); } if (request->headerList) { curl_slist_free_all(request->headerList); } MOLOCH_TYPE_FREE(MolochHttpRequest_t, request); curl_multi_remove_handle(server->multi, easy); curl_easy_cleanup(easy); MOLOCH_LOCK(requests); server->outstanding--; MOLOCH_UNLOCK(requests); } } }
static void tr_webThreadFunc( void * vsession ) { CURLM * multi; struct tr_web * web; int taskCount = 0; struct tr_web_task * task; tr_session * session = vsession; /* try to enable ssl for https support; but if that fails, * try a plain vanilla init */ if( curl_global_init( CURL_GLOBAL_SSL ) ) curl_global_init( 0 ); web = tr_new0( struct tr_web, 1 ); web->close_mode = ~0; web->taskLock = tr_lockNew( ); web->tasks = NULL; web->curl_verbose = getenv( "TR_CURL_VERBOSE" ) != NULL; web->cookie_filename = tr_buildPath( session->configDir, "cookies.txt", NULL ); multi = curl_multi_init( ); session->web = web; for( ;; ) { long msec; int unused; CURLMsg * msg; CURLMcode mcode; if( web->close_mode == TR_WEB_CLOSE_NOW ) break; if( ( web->close_mode == TR_WEB_CLOSE_WHEN_IDLE ) && ( web->tasks == NULL ) ) break; /* add tasks from the queue */ tr_lockLock( web->taskLock ); while( web->tasks != NULL ) { /* pop the task */ task = web->tasks; web->tasks = task->next; task->next = NULL; dbgmsg( "adding task to curl: [%s]", task->url ); curl_multi_add_handle( multi, createEasy( session, web, task )); /*fprintf( stderr, "adding a task.. taskCount is now %d\n", taskCount );*/ ++taskCount; } tr_lockUnlock( web->taskLock ); /* maybe wait a little while before calling curl_multi_perform() */ msec = 0; curl_multi_timeout( multi, &msec ); if( msec < 0 ) msec = THREADFUNC_MAX_SLEEP_MSEC; if( session->isClosed ) msec = 100; /* on shutdown, call perform() more frequently */ if( msec > 0 ) { int usec; int max_fd; struct timeval t; fd_set r_fd_set, w_fd_set, c_fd_set; max_fd = 0; FD_ZERO( &r_fd_set ); FD_ZERO( &w_fd_set ); FD_ZERO( &c_fd_set ); curl_multi_fdset( multi, &r_fd_set, &w_fd_set, &c_fd_set, &max_fd ); if( msec > THREADFUNC_MAX_SLEEP_MSEC ) msec = THREADFUNC_MAX_SLEEP_MSEC; usec = msec * 1000; t.tv_sec = usec / 1000000; t.tv_usec = usec % 1000000; tr_select( max_fd+1, &r_fd_set, &w_fd_set, &c_fd_set, &t ); } /* call curl_multi_perform() */ do { mcode = curl_multi_perform( multi, &unused ); } while( mcode == CURLM_CALL_MULTI_PERFORM ); /* pump completed tasks from the multi */ while(( msg = curl_multi_info_read( multi, &unused ))) { if(( msg->msg == CURLMSG_DONE ) && ( msg->easy_handle != NULL )) { double total_time; struct tr_web_task * task; long req_bytes_sent; CURL * e = msg->easy_handle; curl_easy_getinfo( e, CURLINFO_PRIVATE, (void*)&task ); curl_easy_getinfo( e, CURLINFO_RESPONSE_CODE, &task->code ); curl_easy_getinfo( e, CURLINFO_REQUEST_SIZE, &req_bytes_sent ); curl_easy_getinfo( e, CURLINFO_TOTAL_TIME, &total_time ); task->did_connect = task->code>0 || req_bytes_sent>0; task->did_timeout = !task->code && ( total_time >= task->timeout_secs ); curl_multi_remove_handle( multi, e ); curl_easy_cleanup( e ); /*fprintf( stderr, "removing a completed task.. taskCount is now %d (response code: %d, response len: %d)\n", taskCount, (int)task->code, (int)evbuffer_get_length(task->response) );*/ tr_runInEventThread( task->session, task_finish_func, task ); --taskCount; } } } /* Discard any remaining tasks. * This is rare, but can happen on shutdown with unresponsive trackers. */ while( web->tasks != NULL ) { task = web->tasks; web->tasks = task->next; dbgmsg( "Discarding task \"%s\"", task->url ); task_free( task ); } /* cleanup */ curl_multi_cleanup( multi ); tr_lockFree( web->taskLock ); tr_free( web->cookie_filename ); tr_free( web ); session->web = NULL; }
URL_FILE * CoolUrlFopen(const char *url,const char *operation) { /* this code could check for URLs or types in the 'url' and basicly use the real fopen() for standard files */ URL_FILE *file; int ret; (void)operation; char head_buffer[BUFFER_SIZE]; file = (URL_FILE *)malloc(sizeof(URL_FILE)); if(!file) return NULL; memset(file, 0, sizeof(URL_FILE)); if(url) file->url=strdup(url); else file->url=NULL; curl_global_init(CURL_GLOBAL_ALL); //get http head file->curl = curl_easy_init(); curl_easy_setopt(file->curl, CURLOPT_URL, file->url); curl_easy_setopt(file->curl, CURLOPT_WRITEDATA, head_buffer); curl_easy_setopt(file->curl, CURLOPT_NOBODY, 1); curl_easy_setopt(file->curl, CURLOPT_HEADER, 1); curl_easy_setopt(file->curl, CURLOPT_VERBOSE, 0); curl_easy_setopt(file->curl, CURLOPT_WRITEFUNCTION, write_head_callback); ret = curl_easy_perform(file->curl); curl_easy_cleanup(file->curl); if(ret != CURLE_OK) { printf("get file size error url=%s",file->url); return NULL; } file->size = get_file_size_from_head(head_buffer); if(!file->size) return NULL; file->curl = curl_easy_init(); curl_easy_setopt(file->curl, CURLOPT_URL, file->url); curl_easy_setopt(file->curl, CURLOPT_WRITEDATA, file); curl_easy_setopt(file->curl, CURLOPT_VERBOSE, 0); curl_easy_setopt(file->curl, CURLOPT_WRITEFUNCTION, write_callback); multi_handle = curl_multi_init(); curl_multi_add_handle(multi_handle, file->curl); // lets start the fetch while(curl_multi_perform(multi_handle, &file->still_running) == CURLM_CALL_MULTI_PERFORM ); if((file->buffer_pos == 0) && (!file->still_running)) { // if still_running is 0 now, we should return NULL // make sure the easy handle is not in the multi handle anymore curl_multi_remove_handle(multi_handle, file->curl); // cleanup curl_easy_cleanup(file->curl); free(file); file = NULL; } return file; }
std::string ZLCurlNetworkManager::perform(const ZLExecutionData::Vector &dataList) const { const ZLResource &errorResource = ZLResource::resource("dialog")["networkError"]; if (dataList.empty()) { return errorResource["emptyLibrariesList"].value(); } std::set<std::string> errors; const std::string proxy = proxyHost() + ':' + proxyPort(); CURLM *handle = curl_multi_init(); std::map<CURL*,shared_ptr<ZLExecutionData> > handleToRequest; for (ZLExecutionData::Vector::const_iterator it = dataList.begin(); it != dataList.end(); ++it) { if (it->isNull() || !(*it)->isInstanceOf(ZLNetworkRequest::TYPE_ID)) { continue; } ZLNetworkRequest &request = (ZLNetworkRequest&)**it; const std::string err = doBeforeRequest(request); if (!err.empty()) { errors.insert(err); continue; } CURL *easyHandle = curl_easy_init(); if (easyHandle != 0) { handleToRequest[easyHandle] = *it; setStandardOptions(easyHandle, proxy); setRequestOptions(easyHandle, request); curl_multi_add_handle(handle, easyHandle); } } int counter; CURLMcode res; do { res = curl_multi_perform(handle, &counter); } while ((res == CURLM_CALL_MULTI_PERFORM) || (counter > 0)); CURLMsg *message; do { int queueSize; message = curl_multi_info_read(handle, &queueSize); if ((message != 0) && (message->msg == CURLMSG_DONE)) { ZLNetworkRequest &request = (ZLNetworkRequest&)*handleToRequest[message->easy_handle]; const std::string &url = request.url(); CURLcode result = message->data.result; bool doAfterResult = request.doAfter(result == CURLE_OK); if (result == CURLE_OK && !doAfterResult) { result = CURLE_WRITE_ERROR; } switch (result) { case CURLE_OK: break; case CURLE_WRITE_ERROR: if (!request.errorMessage().empty()) { errors.insert(request.errorMessage()); } else { errors.insert(ZLStringUtil::printf(errorResource["somethingWrongMessage"].value(), ZLNetworkUtil::hostFromUrl(url))); } break; default: errors.insert(ZLStringUtil::printf(errorResource["somethingWrongMessage"].value(), ZLNetworkUtil::hostFromUrl(url))); break; case CURLE_COULDNT_RESOLVE_PROXY: errors.insert(ZLStringUtil::printf(errorResource["couldntResolveProxyMessage"].value(), proxyHost())); break; case CURLE_COULDNT_RESOLVE_HOST: errors.insert(ZLStringUtil::printf(errorResource["couldntResolveHostMessage"].value(), ZLNetworkUtil::hostFromUrl(url))); break; case CURLE_COULDNT_CONNECT: errors.insert(ZLStringUtil::printf(errorResource["couldntConnectMessage"].value(), ZLNetworkUtil::hostFromUrl(url))); break; case CURLE_OPERATION_TIMEDOUT: errors.insert(errorResource["operationTimedOutMessage"].value()); break; case CURLE_SSL_CONNECT_ERROR: errors.insert(ZLStringUtil::printf(errorResource["sslConnectErrorMessage"].value(), curl_easy_strerror(CURLE_SSL_CONNECT_ERROR))); break; #if LIBCURL_VERSION_NUM > 0x071100 case CURLE_PEER_FAILED_VERIFICATION: #else case CURLE_SSL_PEER_CERTIFICATE: #endif errors.insert(ZLStringUtil::printf(errorResource["peerFailedVerificationMessage"].value(), ZLNetworkUtil::hostFromUrl(url))); break; case CURLE_SSL_CACERT: errors.insert(ZLStringUtil::printf(errorResource["sslCertificateAuthorityMessage"].value(), ZLNetworkUtil::hostFromUrl(url))); break; case CURLE_SSL_CACERT_BADFILE: errors.insert(ZLStringUtil::printf(errorResource["sslBadCertificateFileMessage"].value(), request.sslCertificate().Path)); break; case CURLE_SSL_SHUTDOWN_FAILED: errors.insert(ZLStringUtil::printf(errorResource["sslShutdownFailedMessage"].value(), ZLNetworkUtil::hostFromUrl(url))); break; } } } while ((message != 0) && (errors.size() < 3)); for (std::map<CURL*,shared_ptr<ZLExecutionData> >::const_iterator jt = handleToRequest.begin(); jt != handleToRequest.end(); ++jt) { CURL *easyHandle = jt->first; curl_multi_remove_handle(handle, easyHandle); curl_easy_cleanup(easyHandle); ZLNetworkRequest &request = (ZLNetworkRequest&)*jt->second; clearRequestOptions(request); } handleToRequest.clear(); curl_multi_cleanup(handle); std::string result; for (std::set<std::string>::const_iterator et = errors.begin(); et != errors.end(); ++et) { if (!result.empty()) { result += '\n'; } result += *et; } return result; }
/* full_download() attempts to enqueue a file for later asynchronous download - NOTE: See swupd_curl_get_file() for single file synchronous downloads. */ void full_download(struct file *file) { char *url = NULL; CURL *curl = NULL; int ret = -EFULLDOWNLOAD; char *filename = NULL; CURLMcode curlm_ret = CURLM_OK; CURLcode curl_ret = CURLE_OK; ret = swupd_curl_hashmap_insert(file); if (ret > 0) { /* no download needed */ /* File already exists - report success */ ret = 0; goto out_good; } else if (ret < 0) { /* error */ goto out_bad; } /* else (ret == 0) download needed */ curl = curl_easy_init(); if (curl == NULL) { goto out_bad; } file->curl = curl; ret = poll_fewer_than(MAX_XFER, MAX_XFER_BOTTOM); if (ret) { clean_curl_multi_queue(); goto out_bad; } string_or_die(&url, "%s/%i/files/%s.tar", content_url, file->last_change, file->hash); string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash); file->staging = filename; curl_ret = curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); if (curl_ret != CURLE_OK) { goto out_bad; } curl_ret = curl_easy_setopt(curl, CURLOPT_PRIVATE, (void *)file); if (curl_ret != CURLE_OK) { goto out_bad; } curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, swupd_download_file); if (curl_ret != CURLE_OK) { goto out_bad; } curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file); if (curl_ret != CURLE_OK) { goto out_bad; } curl_ret = swupd_curl_set_basic_options(curl, url); if (curl_ret != CURLE_OK) { goto out_bad; } curlm_ret = curl_multi_add_handle(mcurl, curl); if (curlm_ret != CURLM_OK) { goto out_bad; } if (poll_fewer_than(MAX_XFER + 10, MAX_XFER) != 0) { clean_curl_multi_queue(); } ret = 0; goto out_good; out_bad: failed = list_prepend_data(failed, file); if (curl != NULL) { /* Must remove handle out of multi queue first!*/ curl_multi_remove_handle(mcurl, curl); curl_easy_cleanup(curl); } free(filename); out_good: free(url); }
int test(char *URL) { CURL *c = NULL; CURLM *m = NULL; int res = 0; int running; start_test_timing(); global_init(CURL_GLOBAL_ALL); easy_init(c); easy_setopt(c, CURLOPT_PROXY, libtest_arg2); /* set in first.c */ easy_setopt(c, CURLOPT_URL, URL); easy_setopt(c, CURLOPT_USERPWD, "test:ing"); easy_setopt(c, CURLOPT_PROXYUSERPWD, "test:ing"); easy_setopt(c, CURLOPT_HTTPPROXYTUNNEL, 1L); easy_setopt(c, CURLOPT_HEADER, 1L); easy_setopt(c, CURLOPT_VERBOSE, 1L); multi_init(m); multi_add_handle(m, c); for(;;) { struct timeval interval; fd_set rd, wr, exc; int maxfd = -99; interval.tv_sec = 1; interval.tv_usec = 0; multi_perform(m, &running); abort_on_test_timeout(); if(!running) break; /* done */ FD_ZERO(&rd); FD_ZERO(&wr); FD_ZERO(&exc); multi_fdset(m, &rd, &wr, &exc, &maxfd); /* At this point, maxfd is guaranteed to be greater or equal than -1. */ select_test(maxfd + 1, &rd, &wr, &exc, &interval); abort_on_test_timeout(); } test_cleanup: /* proper cleanup sequence - type PA */ curl_multi_remove_handle(m, c); curl_multi_cleanup(m); curl_easy_cleanup(c); curl_global_cleanup(); return res; }
/* =============== CL_FinishHTTPDownload A download finished, find out what it was, whether there were any errors and if so, how severe. If none, rename file and other such stuff. =============== */ static void CL_FinishHTTPDownload (void) { size_t i; int msgs_in_queue; CURLMsg *msg; CURLcode result; dlhandle_t *dl; CURL *curl; long responseCode; double timeTaken; double fileSize; char tempName[MAX_OSPATH]; qboolean isFile; do { msg = curl_multi_info_read (multi, &msgs_in_queue); if (!msg) { Com_Printf ("CL_FinishHTTPDownload: Odd, no message for us...\n"); return; } if (msg->msg != CURLMSG_DONE) { Com_Printf ("CL_FinishHTTPDownload: Got some weird message...\n"); continue; } curl = msg->easy_handle; // curl doesn't provide reverse-lookup of the void * ptr, so search for it for (i = 0; i < 4; i++) { if (cls.HTTPHandles[i].curl == curl) { dl = &cls.HTTPHandles[i]; break; } } if (i == 4) Com_Error (ERR_DROP, "CL_FinishHTTPDownload: Handle not found"); // we mark everything as done even if it errored to prevent multiple // attempts. dl->queueEntry->state = DLQ_STATE_DONE; //filelist processing is done on read if (dl->file) isFile = true; else isFile = false; if (isFile) { fclose (dl->file); dl->file = NULL; } //might be aborted if (pendingCount) pendingCount--; handleCount--; //Com_Printf ("finished dl: hc = %d\n", LOG_GENERAL, handleCount); cls.downloadname[0] = 0; cls.downloadposition = 0; result = msg->data.result; switch (result) { // for some reason curl returns CURLE_OK for a 404... case CURLE_HTTP_RETURNED_ERROR: case CURLE_OK: curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &responseCode); if (responseCode == 404) { i = strlen (dl->queueEntry->quakePath); if ( !strcmp (dl->queueEntry->quakePath + i - 4, ".pak") || !strcmp (dl->queueEntry->quakePath + i - 4, ".pk3") ) downloading_pak = false; if (isFile) remove (dl->filePath); Com_Printf ("HTTP(%s): 404 File Not Found [%d remaining files]\n", dl->queueEntry->quakePath, pendingCount); curl_easy_getinfo (curl, CURLINFO_SIZE_DOWNLOAD, &fileSize); if (fileSize > 512) { // ick isFile = false; result = CURLE_FILESIZE_EXCEEDED; Com_Printf ("Oversized 404 body received (%d bytes), aborting HTTP downloading.\n", (int)fileSize); } else { curl_multi_remove_handle (multi, dl->curl); // Knightmare- fall back to UDP download for this map if failure on .bsp if ( !strncmp(dl->queueEntry->quakePath, "maps/", 5) && !strcmp(dl->queueEntry->quakePath + i - 4, ".bsp") ) { Com_Printf ("HTTP: failed to download %s, falling back to UDP until next map.\n", dl->queueEntry->quakePath); thisMapAbort = true; CL_CancelHTTPDownloads (false); CL_ResetPrecacheCheck (); } // end Knightmare continue; } } else if (responseCode == 200) { if (!isFile && !abortDownloads) CL_ParseFileList (dl); break; } //every other code is treated as fatal, fallthrough here Com_Printf ("Bad HTTP response code %d for %s, aborting HTTP downloading.\n", responseCode, dl->queueEntry->quakePath); //fatal error, disable http case CURLE_COULDNT_RESOLVE_HOST: case CURLE_COULDNT_CONNECT: case CURLE_COULDNT_RESOLVE_PROXY: if (isFile) remove (dl->filePath); // Com_Printf ("Fatal HTTP error: %s\n", curl_easy_strerror (result)); Com_Printf ("Fatal HTTP error: %s\n", CURL_ERROR(result)); curl_multi_remove_handle (multi, dl->curl); if (abortDownloads) continue; CL_CancelHTTPDownloads (true); continue; default: i = strlen (dl->queueEntry->quakePath); if ( !strcmp (dl->queueEntry->quakePath + i - 4, ".pak") || !strcmp (dl->queueEntry->quakePath + i - 4, ".pk3") ) downloading_pak = false; if (isFile) remove (dl->filePath); // Com_Printf ("HTTP download failed: %s\n", curl_easy_strerror (result)); Com_Printf ("HTTP download failed: %s\n", CURL_ERROR(result)); curl_multi_remove_handle (multi, dl->curl); continue; } if (isFile) { //rename the temp file Com_sprintf (tempName, sizeof(tempName), "%s/%s", FS_Gamedir(), dl->queueEntry->quakePath); if (rename (dl->filePath, tempName)) Com_Printf ("Failed to rename %s for some odd reason...", dl->filePath); //a pak file is very special... i = strlen (tempName); if ( !strcmp (tempName + i - 4, ".pak") || !strcmp (tempName + i - 4, ".pk3") ) { // FS_FlushCache (); // FS_ReloadPAKs (); // Knightmare- just add the pk3/ pak file if (!strcmp (tempName + i - 4, ".pk3")) FS_AddPK3File (tempName); else FS_AddPAKFile (tempName); CL_ReVerifyHTTPQueue (); downloading_pak = false; } } //show some stats curl_easy_getinfo (curl, CURLINFO_TOTAL_TIME, &timeTaken); curl_easy_getinfo (curl, CURLINFO_SIZE_DOWNLOAD, &fileSize); //FIXME: //technically i shouldn't need to do this as curl will auto reuse the //existing handle when you change the URL. however, the handleCount goes //all weird when reusing a download slot in this way. if you can figure //out why, please let me know. curl_multi_remove_handle (multi, dl->curl); Com_Printf ("HTTP(%s): %.f bytes, %.2fkB/sec [%d remaining files]\n", dl->queueEntry->quakePath, fileSize, (fileSize / 1024.0) / timeTaken, pendingCount); } while (msgs_in_queue > 0); // FS_FlushCache (); if (handleCount == 0) { if (abortDownloads == HTTPDL_ABORT_SOFT) abortDownloads = HTTPDL_ABORT_NONE; else if (abortDownloads == HTTPDL_ABORT_HARD) { // FS: Added because Whale's Weapons HTTP server rejects you after a lot of 404s. Then you lose HTTP until a hard reconnect. Q_strncpyz(cls.downloadServerRetry, cls.downloadServer, sizeof(cls.downloadServerRetry)); cls.downloadServer[0] = 0; } } // done current batch, see if we have more to dl - maybe a .bsp needs downloaded if (cls.state == ca_connected && !CL_PendingHTTPDownloads()) CL_RequestNextDownload (); }
static void curl_clean_state(CURLState *s) { if (s->s->multi) curl_multi_remove_handle(s->s->multi, s->curl); s->in_use = 0; }