/* test function */ int test(char *URL) { int res; CURLSHcode scode; char *url; struct Tdata tdata; CURL *curl; CURLSH *share; struct curl_slist *headers; int i; struct userdata user; user.text = (char *)"Pigs in space"; user.counter = 0; printf( "GLOBAL_INIT\n" ); curl_global_init( CURL_GLOBAL_ALL ); /* prepare share */ printf( "SHARE_INIT\n" ); share = curl_share_init(); curl_share_setopt( share, CURLSHOPT_LOCKFUNC, lock); curl_share_setopt( share, CURLSHOPT_UNLOCKFUNC, unlock); curl_share_setopt( share, CURLSHOPT_USERDATA, &user); printf( "CURL_LOCK_DATA_COOKIE\n" ); curl_share_setopt( share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); printf( "CURL_LOCK_DATA_DNS\n" ); curl_share_setopt( share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); res = 0; /* start treads */ for (i=1; i<=THREADS; i++ ) { /* set thread data */ tdata.url = suburl( URL, i ); /* must be freed */ tdata.share = share; /* simulate thread, direct call of "thread" function */ printf( "*** run %d\n",i ); fire( &tdata ); free( tdata.url ); } /* fetch a another one and save cookies */ printf( "*** run %d\n", i ); curl = curl_easy_init(); url = suburl( URL, i ); headers = sethost( NULL ); curl_easy_setopt( curl, CURLOPT_HTTPHEADER, (void*)headers ); curl_easy_setopt( curl, CURLOPT_URL, url ); printf( "CURLOPT_SHARE\n" ); curl_easy_setopt( curl, CURLOPT_SHARE, share ); printf( "CURLOPT_COOKIEJAR\n" ); curl_easy_setopt( curl, CURLOPT_COOKIEJAR, JAR ); printf( "PERFORM\n" ); curl_easy_perform( curl ); /* try to free share, expect to fail because share is in use*/ printf( "try SHARE_CLEANUP...\n" ); scode = curl_share_cleanup( share ); if ( scode==CURLSHE_OK ) { fprintf(stderr, "curl_share_cleanup succeed but error expected\n"); share = NULL; } else { printf( "SHARE_CLEANUP failed, correct\n" ); } /* clean up last handle */ printf( "CLEANUP\n" ); curl_easy_cleanup( curl ); curl_slist_free_all( headers ); free(url); /* free share */ printf( "SHARE_CLEANUP\n" ); scode = curl_share_cleanup( share ); if ( scode!=CURLSHE_OK ) { fprintf(stderr, "curl_share_cleanup failed, code errno %d\n", scode); } printf( "GLOBAL_CLEANUP\n" ); curl_global_cleanup(); return res; }
static VALUE ov_http_client_initialize(int argc, VALUE* argv, VALUE self) { VALUE opt; VALUE opts; long connections; long pipeline; ov_http_client_object* ptr; /* Get the pointer to the native object: */ ov_http_client_ptr(self, ptr); /* Check the number of arguments: */ if (argc > 1) { rb_raise(ov_error_class, "Expected at most one argument, 'opts', but received %d", argc); } opts = argc > 0? argv[0]: Qnil; if (NIL_P(opts)) { opts = rb_hash_new(); } else { Check_Type(opts, T_HASH); } /* Get the value of the 'ca_file' parameter: */ opt = rb_hash_aref(opts, CA_FILE_SYMBOL); if (NIL_P(opt)) { ptr->ca_file = NULL; } else { Check_Type(opt, T_STRING); ptr->ca_file = ov_string_dup(opt); } /* Get the value of the 'insecure' parameter: */ opt = rb_hash_aref(opts, INSECURE_SYMBOL); ptr->insecure = NIL_P(opt)? false: RTEST(opt); /* Get the value of the 'debug' parameter: */ opt = rb_hash_aref(opts, DEBUG_SYMBOL); ptr->debug = NIL_P(opt)? false: RTEST(opt); /* Get the value of the 'compress' parameter: */ opt = rb_hash_aref(opts, COMPRESS_SYMBOL); ptr->compress = NIL_P(opt)? true: RTEST(opt); /* Get the value of the 'timeout' parameter: */ opt = rb_hash_aref(opts, TIMEOUT_SYMBOL); if (NIL_P(opt)) { ptr->timeout = 0; } else { Check_Type(opt, T_FIXNUM); ptr->timeout = NUM2INT(opt); } /* Get the value of the 'connect_timeout' parameter: */ opt = rb_hash_aref(opts, CONNECT_TIMEOUT_SYMBOL); if (NIL_P(opt)) { ptr->connect_timeout = 0; } else { Check_Type(opt, T_FIXNUM); ptr->connect_timeout = NUM2INT(opt); } /* Get the value of the 'proxy_url' parameter: */ opt = rb_hash_aref(opts, PROXY_URL_SYMBOL); if (NIL_P(opt)) { ptr->proxy_url = NULL; } else { Check_Type(opt, T_STRING); ptr->proxy_url = ov_string_dup(opt); } /* Get the value of the 'proxy_username' parameter: */ opt = rb_hash_aref(opts, PROXY_USERNAME_SYMBOL); if (NIL_P(opt)) { ptr->proxy_username = NULL; } else { Check_Type(opt, T_STRING); ptr->proxy_username = ov_string_dup(opt); } /* Get the value of the 'proxy_password' parameter: */ opt = rb_hash_aref(opts, PROXY_PASSWORD_SYMBOL); if (NIL_P(opt)) { ptr->proxy_password = NULL; } else { Check_Type(opt, T_STRING); ptr->proxy_password = ov_string_dup(opt); } /* Get the value of the 'log' parameter: */ opt = rb_hash_aref(opts, LOG_SYMBOL); ptr->log = opt; /* Get the value of the 'pipeline' parameter: */ opt = rb_hash_aref(opts, PIPELINE_SYMBOL); if (NIL_P(opt)) { pipeline = 0; } else { Check_Type(opt, T_FIXNUM); pipeline = NUM2LONG(opt); } if (pipeline < 0) { rb_raise(rb_eArgError, "The maximum pipeline length can't be %ld, minimum is 0.", pipeline); } /* Get the value of the 'connections' parameter: */ opt = rb_hash_aref(opts, CONNECTIONS_SYMBOL); if (NIL_P(opt)) { connections = 1; } else { Check_Type(opt, T_FIXNUM); connections = NUM2LONG(opt); } if (connections < 1) { rb_raise(rb_eArgError, "The maximum number of connections can't be %ld, minimum is 1.", connections); } /* Get the value of the 'cookies' parameter. If it is a string it will be used as the path of the file where the cookies will be stored. If it is any other thing it will be treated as a boolean flag indicating if cookies should be enabled but not loaded/saved from/to any file. */ opt = rb_hash_aref(opts, COOKIES_SYMBOL); if (TYPE(opt) == T_STRING) { ptr->cookies = ov_string_dup(opt); } else if (RTEST(opt)) { ptr->cookies = ov_string_dup(rb_str_new2("")); } else { ptr->cookies = NULL; } /* Create the queue that contains requests that haven't been sent to libcurl yet: */ ptr->queue = rb_ary_new(); /* Create the hash that contains the transfers are pending an completed. Both use the identity of the request as key. */ ptr->completed = rb_funcall(rb_hash_new(), COMPARE_BY_IDENTITY_ID, 0); ptr->pending = rb_funcall(rb_hash_new(), COMPARE_BY_IDENTITY_ID, 0); /* Calculate the max number of requests that can be handled by libcurl simultaneously. For versions of libcurl newer than 7.30.0 the limit can be increased when using pipelining. For older versions it can't be increased because libcurl would create additional connections for the requests that can't be pipelined. */ ptr->limit = connections; if (pipeline > 0 && libcurl_version->version_num >= 0x071e00 /* 7.30.0 */) { ptr->limit *= pipeline; } /* Create the libcurl multi handle: */ ptr->handle = curl_multi_init(); if (ptr->handle == NULL) { rb_raise(ov_error_class, "Can't create libcurl multi object"); } /* Create the libcurl share handle in order to share cookie data: */ ptr->share = curl_share_init(); if (ptr->share == NULL) { rb_raise(ov_error_class, "Can't create libcurl share object"); } if (ptr->cookies != NULL) { curl_share_setopt(ptr->share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); } /* Enable pipelining: */ if (pipeline > 0) { curl_multi_setopt(ptr->handle, CURLMOPT_PIPELINING, CURLPIPE_HTTP1); if (libcurl_version->version_num >= 0x071e00 /* 7.30.0 */) { curl_multi_setopt(ptr->handle, CURLMOPT_MAX_PIPELINE_LENGTH, pipeline); } else { ov_http_client_log_warn( ptr->log, "Can't set maximum pipeline length to %d, it isn't supported by libcurl %s. Upgrade to 7.30.0 or " "newer to avoid this issue.", pipeline, libcurl_version->version ); } } /* Set the max number of connections: */ if (connections > 0) { if (libcurl_version->version_num >= 0x071e00 /* 7.30.0 */) { curl_multi_setopt(ptr->handle, CURLMOPT_MAX_HOST_CONNECTIONS, connections); curl_multi_setopt(ptr->handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, connections); } else { ov_http_client_log_warn( ptr->log, "Can't set maximum number of connections to %d, it isn't supported by libcurl %s. Upgrade to 7.30.0 " "or newer to avoid this issue.", connections, libcurl_version->version ); } if (libcurl_version->version_num >= 0x070f03 /* 7.16.3 */) { curl_multi_setopt(ptr->handle, CURLMOPT_MAXCONNECTS, connections); } else { ov_http_client_log_warn( ptr->log, "Can't set total maximum connection cache size to %d, it isn't supported by libcurl %s. Upgrade to " "7.16.3 or newer to avoid this issue.", connections, libcurl_version->version ); } } return self; }
static void zhttp_client_actor (zsock_t *pipe, void *args) { curl_global_init (CURL_GLOBAL_ALL); CURLM *multi = curl_multi_init (); CURLSH *share = curl_share_init (); curl_share_setopt (share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); curl_share_setopt (share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); curl_share_setopt (share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT); long verbose = (*(bool *) args) ? 1L : 0L; CURLMcode code; SOCKET pipefd = zsock_fd (pipe); struct curl_waitfd waitfd = {pipefd, CURL_WAIT_POLLIN}; // List to hold pending curl handles, in case we are destroy the client // while request are inprogress zlistx_t *pending_handles = zlistx_new (); zlistx_set_destructor (pending_handles, (zlistx_destructor_fn *) curl_destructor); zsock_signal (pipe, 0); bool terminated = false; while (!terminated) { if (!zsock_has_in (pipe)) { code = curl_multi_wait (multi, &waitfd, 1, 1000, NULL); assert (code == CURLM_OK); } if (zsock_has_in (pipe)) { char *command = zstr_recv (pipe); if (!command) break; // Interrupted // All actors must handle $TERM in this way if (streq (command, "$TERM")) terminated = true; else if (streq (command, "GET")) { char *url; zlistx_t *headers; int timeout; void *handler; void *arg; int rc = zsock_recv (pipe, "slipp", &url, &headers, &timeout, &handler, &arg); assert (rc == 0); zchunk_t *response = zchunk_new (NULL, 100); assert (response); struct curl_slist *curl_headers = zlistx_to_slist (headers); CURL *curl = curl_easy_init (); zlistx_add_end (pending_handles, curl); http_request *request = (http_request *) zmalloc (sizeof (http_request)); assert (request); request->handler = handler; request->arg = arg; request->curl = curl; request->response = response; request->headers = curl_headers; request->body = NULL; curl_easy_setopt (curl, CURLOPT_SHARE, share); curl_easy_setopt (curl, CURLOPT_TIMEOUT_MS, (long)timeout); curl_easy_setopt (curl, CURLOPT_VERBOSE, verbose); curl_easy_setopt (curl, CURLOPT_HTTPHEADER, curl_headers); curl_easy_setopt (curl, CURLOPT_URL, url); curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_data); curl_easy_setopt (curl, CURLOPT_WRITEDATA, response); curl_easy_setopt (curl, CURLOPT_PRIVATE, request); code = curl_multi_add_handle (multi, curl); assert (code == CURLM_OK); zlistx_destroy (&headers); zstr_free (&url); } else if (streq (command, "POST")) { char *url; zlistx_t *headers; zchunk_t *body; int timeout; void *handler; void *arg; int rc = zsock_recv (pipe, "slcipp", &url, &headers, &body, &timeout, &handler, &arg); assert (rc == 0); zchunk_t *response = zchunk_new (NULL, 100); assert (response); struct curl_slist *curl_headers = zlistx_to_slist (headers); CURL *curl = curl_easy_init (); zlistx_add_end (pending_handles, curl); http_request *request = (http_request *) zmalloc (sizeof (http_request)); assert (request); request->handler = handler; request->arg = arg; request->curl = curl; request->response = response; request->headers = curl_headers; request->body = body; curl_easy_setopt (curl, CURLOPT_SHARE, share); curl_easy_setopt (curl, CURLOPT_TIMEOUT_MS, (long)timeout); curl_easy_setopt (curl, CURLOPT_VERBOSE, verbose); curl_easy_setopt (curl, CURLOPT_POSTFIELDS, zchunk_data (body)); curl_easy_setopt (curl, CURLOPT_POSTFIELDSIZE, zchunk_size (body)); curl_easy_setopt (curl, CURLOPT_HTTPHEADER, curl_headers); curl_easy_setopt (curl, CURLOPT_URL, url); curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_data); curl_easy_setopt (curl, CURLOPT_WRITEDATA, response); curl_easy_setopt (curl, CURLOPT_PRIVATE, request); code = curl_multi_add_handle (multi, curl); assert (code == CURLM_OK); zlistx_destroy (&headers); zstr_free (&url); } else { puts ("E: invalid message to actor"); assert (false); } zstr_free (&command); } int still_running; code = curl_multi_perform (multi, &still_running); assert (code == CURLM_OK); int msgq = 0; struct CURLMsg *msg = curl_multi_info_read (multi, &msgq); while (msg) { if (msg->msg == CURLMSG_DONE) { CURL *curl = msg->easy_handle; http_request *request; curl_easy_getinfo (curl, CURLINFO_PRIVATE, &request); long response_code_long; curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &response_code_long); int response_code = (int) response_code_long; if (response_code == 0 && msg->data.result == CURLE_OPERATION_TIMEDOUT) response_code = -1; int rc = zsock_send (pipe, "icpp", response_code, request->response, request->handler, request->arg); assert (rc == 0); curl_multi_remove_handle (multi, curl); // Remove curl from the pending handles and delete it void *handle = zlistx_find (pending_handles, curl); assert (handle); rc = zlistx_delete (pending_handles, handle); assert (rc == 0); } msg = curl_multi_info_read (multi, &msgq); } } zlistx_destroy (&pending_handles); curl_share_cleanup (share); curl_multi_cleanup (multi); curl_global_cleanup (); }
/* test function */ int test(char *URL) { int res; CURLSHcode scode = CURLSHE_OK; CURLcode code = CURLE_OK; char *url = NULL; struct Tdata tdata; CURL *curl; CURLSH *share; struct curl_slist *headers = NULL; struct curl_slist *cookies = NULL; struct curl_slist *next_cookie = NULL; int i; struct userdata user; user.text = (char *)"Pigs in space"; user.counter = 0; printf("GLOBAL_INIT\n"); if(curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) { fprintf(stderr, "curl_global_init() failed\n"); return TEST_ERR_MAJOR_BAD; } /* prepare share */ printf("SHARE_INIT\n"); if((share = curl_share_init()) == NULL) { fprintf(stderr, "curl_share_init() failed\n"); curl_global_cleanup(); return TEST_ERR_MAJOR_BAD; } if(CURLSHE_OK == scode) { printf("CURLSHOPT_LOCKFUNC\n"); scode = curl_share_setopt(share, CURLSHOPT_LOCKFUNC, my_lock); } if(CURLSHE_OK == scode) { printf("CURLSHOPT_UNLOCKFUNC\n"); scode = curl_share_setopt(share, CURLSHOPT_UNLOCKFUNC, my_unlock); } if(CURLSHE_OK == scode) { printf("CURLSHOPT_USERDATA\n"); scode = curl_share_setopt(share, CURLSHOPT_USERDATA, &user); } if(CURLSHE_OK == scode) { printf("CURL_LOCK_DATA_COOKIE\n"); scode = curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); } if(CURLSHE_OK == scode) { printf("CURL_LOCK_DATA_DNS\n"); scode = curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); } if(CURLSHE_OK != scode) { fprintf(stderr, "curl_share_setopt() failed\n"); curl_share_cleanup(share); curl_global_cleanup(); return TEST_ERR_MAJOR_BAD; } /* initial cookie manipulation */ if((curl = curl_easy_init()) == NULL) { fprintf(stderr, "curl_easy_init() failed\n"); curl_share_cleanup(share); curl_global_cleanup(); return TEST_ERR_MAJOR_BAD; } printf("CURLOPT_SHARE\n"); test_setopt(curl, CURLOPT_SHARE, share); printf("CURLOPT_COOKIELIST injected_and_clobbered\n"); test_setopt(curl, CURLOPT_COOKIELIST, "Set-Cookie: injected_and_clobbered=yes; " "domain=host.foo.com; expires=Sat Feb 2 11:56:27 GMT 2030"); printf("CURLOPT_COOKIELIST ALL\n"); test_setopt(curl, CURLOPT_COOKIELIST, "ALL"); printf("CURLOPT_COOKIELIST session\n"); test_setopt(curl, CURLOPT_COOKIELIST, "Set-Cookie: session=elephants"); printf("CURLOPT_COOKIELIST injected\n"); test_setopt(curl, CURLOPT_COOKIELIST, "Set-Cookie: injected=yes; domain=host.foo.com; " "expires=Sat Feb 2 11:56:27 GMT 2030"); printf("CURLOPT_COOKIELIST SESS\n"); test_setopt(curl, CURLOPT_COOKIELIST, "SESS"); printf("CLEANUP\n"); curl_easy_cleanup(curl); res = 0; /* start treads */ for(i=1; i<=THREADS; i++) { /* set thread data */ tdata.url = suburl(URL, i); /* must be curl_free()d */ tdata.share = share; /* simulate thread, direct call of "thread" function */ printf("*** run %d\n",i); fire(&tdata); curl_free(tdata.url); } /* fetch a another one and save cookies */ printf("*** run %d\n", i); if((curl = curl_easy_init()) == NULL) { fprintf(stderr, "curl_easy_init() failed\n"); curl_share_cleanup(share); curl_global_cleanup(); return TEST_ERR_MAJOR_BAD; } url = suburl(URL, i); headers = sethost(NULL); test_setopt(curl, CURLOPT_HTTPHEADER, headers); test_setopt(curl, CURLOPT_URL, url); printf("CURLOPT_SHARE\n"); test_setopt(curl, CURLOPT_SHARE, share); printf("CURLOPT_COOKIEJAR\n"); test_setopt(curl, CURLOPT_COOKIEJAR, JAR); printf("CURLOPT_COOKIELIST FLUSH\n"); test_setopt(curl, CURLOPT_COOKIELIST, "FLUSH"); printf("PERFORM\n"); curl_easy_perform(curl); printf("CLEANUP\n"); curl_easy_cleanup(curl); curl_free(url); curl_slist_free_all(headers); /* load cookies */ if((curl = curl_easy_init()) == NULL) { fprintf(stderr, "curl_easy_init() failed\n"); curl_share_cleanup(share); curl_global_cleanup(); return TEST_ERR_MAJOR_BAD; } url = suburl(URL, i); headers = sethost(NULL); test_setopt(curl, CURLOPT_HTTPHEADER, headers); test_setopt(curl, CURLOPT_URL, url); printf("CURLOPT_SHARE\n"); test_setopt(curl, CURLOPT_SHARE, share); printf("CURLOPT_COOKIELIST ALL\n"); test_setopt(curl, CURLOPT_COOKIELIST, "ALL"); printf("CURLOPT_COOKIEJAR\n"); test_setopt(curl, CURLOPT_COOKIEFILE, JAR); printf("CURLOPT_COOKIELIST RELOAD\n"); test_setopt(curl, CURLOPT_COOKIELIST, "RELOAD"); code = curl_easy_getinfo(curl, CURLINFO_COOKIELIST, &cookies); if(code != CURLE_OK) { fprintf(stderr, "curl_easy_getinfo() failed\n"); res = TEST_ERR_MAJOR_BAD; goto test_cleanup; } printf("loaded cookies:\n"); if(!cookies) { fprintf(stderr, " reloading cookies from '%s' failed\n", JAR); res = TEST_ERR_MAJOR_BAD; goto test_cleanup; } printf("-----------------\n"); next_cookie = cookies; while(next_cookie) { printf(" %s\n", next_cookie->data); next_cookie = next_cookie->next; } printf("-----------------\n"); curl_slist_free_all(cookies); /* try to free share, expect to fail because share is in use*/ printf("try SHARE_CLEANUP...\n"); scode = curl_share_cleanup(share); if(scode==CURLSHE_OK) { fprintf(stderr, "curl_share_cleanup succeed but error expected\n"); share = NULL; } else { printf("SHARE_CLEANUP failed, correct\n"); } test_cleanup: /* clean up last handle */ printf("CLEANUP\n"); curl_easy_cleanup(curl); curl_slist_free_all(headers); curl_free(url); /* free share */ printf("SHARE_CLEANUP\n"); scode = curl_share_cleanup(share); if(scode!=CURLSHE_OK) fprintf(stderr, "curl_share_cleanup failed, code errno %d\n", (int)scode); printf("GLOBAL_CLEANUP\n"); curl_global_cleanup(); return res; }
int test(char *URL) { CURLM *cm = NULL; CURLSH *sh = NULL; CURL *ch = NULL; int unfinished; cm = curl_multi_init(); if(!cm) return 1; sh = curl_share_init(); if(!sh) goto cleanup; curl_share_setopt(sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); curl_share_setopt(sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); ch = curl_easy_init(); if(!ch) goto cleanup; curl_easy_setopt(ch, CURLOPT_SHARE, sh); curl_easy_setopt(ch, CURLOPT_URL, URL); curl_easy_setopt(ch, CURLOPT_COOKIEFILE, "log/cookies1905"); curl_easy_setopt(ch, CURLOPT_COOKIEJAR, "log/cookies1905"); curl_multi_add_handle(cm, ch); unfinished = 1; while(unfinished) { int MAX = 0; long max_tout; fd_set R, W, E; struct timeval timeout; FD_ZERO(&R); FD_ZERO(&W); FD_ZERO(&E); curl_multi_perform(cm, &unfinished); curl_multi_fdset(cm, &R, &W, &E, &MAX); curl_multi_timeout(cm, &max_tout); if(max_tout > 0) { timeout.tv_sec = max_tout / 1000; timeout.tv_usec = (max_tout % 1000) * 1000; } else { timeout.tv_sec = 0; timeout.tv_usec = 1000; } select(MAX + 1, &R, &W, &E, &timeout); } curl_easy_setopt(ch, CURLOPT_COOKIELIST, "FLUSH"); curl_easy_setopt(ch, CURLOPT_SHARE, NULL); curl_multi_remove_handle(cm, ch); cleanup: curl_easy_cleanup(ch); curl_share_cleanup(sh); curl_multi_cleanup(cm); curl_global_cleanup(); return 0; }