gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, char *headers, gboolean dropable, MolochResponse_cb func, gpointer uw) { MolochRequest_t *request; MolochHttp_t *server = serverV; if (!config.exiting && dropable && server->requestQ.r_count > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->requestQ.r_count); if (data) { MOLOCH_SIZE_FREE(buffer, data); } return 1; } request = MOLOCH_TYPE_ALLOC(MolochRequest_t); request->key_len = MIN(key_len, sizeof(request->key)-1); memcpy(request->key, key, request->key_len); request->key[request->key_len] = 0; strncpy(request->method, method, sizeof(request->method)); request->compress = 0; if (server->compress && data && data_len > 1000) { char *buf = moloch_http_get_buffer(data_len); int ret; z_strm.avail_in = data_len; z_strm.next_in = (unsigned char *)data; z_strm.avail_out = data_len; z_strm.next_out = (unsigned char *)buf; ret = deflate(&z_strm, Z_FINISH); if (ret == Z_STREAM_END) { request->compress = 1; MOLOCH_SIZE_FREE(buffer, data); data_len = data_len - z_strm.avail_out; data = buf; } else { MOLOCH_SIZE_FREE(buffer, buf); } deflateReset(&z_strm); } request->data_len = data_len; request->data = data; request->func = func; request->uw = uw; if (headers) strcpy(request->headers, headers); else request->headers[0] = 0; DLL_PUSH_TAIL(r_, &server->requestQ, request); moloch_http_do_requests(server); return 0; }
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw) { MolochHttpServer_t *server = serverV; // Are we overloaded if (!config.exiting && dropable && server->outstanding > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding); if (data) { MOLOCH_SIZE_FREE(buffer, data); } return 1; } MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t); if (headers) { int i; for (i = 0; headers[i]; i++) { request->headerList = curl_slist_append(request->headerList, headers[i]); } } // Do we need to compress item if (server->compress && data && data_len > 1000) { char *buf = moloch_http_get_buffer(data_len); int ret; z_strm.avail_in = data_len; z_strm.next_in = (unsigned char *)data; z_strm.avail_out = data_len; z_strm.next_out = (unsigned char *)buf; ret = deflate(&z_strm, Z_FINISH); if (ret == Z_STREAM_END) { request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate"); MOLOCH_SIZE_FREE(buffer, data); data_len = data_len - z_strm.avail_out; data = buf; } else { MOLOCH_SIZE_FREE(buffer, buf); } deflateReset(&z_strm); } request->server = server; request->func = func; request->uw = uw; request->dataOut = data; request->dataOutLen = data_len; request->easy = curl_easy_init(); if (config.debug >= 2) { curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1); } curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback); curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request); curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request); curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback); curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETDATA, server); curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback); curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETDATA, server); if (request->headerList) { curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList); } if (method[0] != 'G') { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data); } else { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL); curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L); } if (server->headerCb) { curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function); curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request); } curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L); char *host = server->names[server->namesPos]; server->namesPos = (server->namesPos + 1) % server->namesCnt; if (strchr(host, ':') == 0) { snprintf(request->url, sizeof(request->url), "%s://%s:%d%.*s", (server->https?"https":"http"), host, server->defaultPort, key_len, key); } else { snprintf(request->url, sizeof(request->url), "%s://%s%.*s", (server->https?"https":"http"), host, key_len, key); } curl_easy_setopt(request->easy, CURLOPT_URL, request->url); MOLOCH_LOCK(requests); server->outstanding++; DLL_PUSH_TAIL(rqt_, &requests, request); if (!requestsTimer) requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL); MOLOCH_UNLOCK(requests); return 0; }
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw) { MolochHttpServer_t *server = serverV; if (key_len > 1000) { LOGEXIT("Url too long %.*s", key_len, key); } // Are we overloaded if (dropable && !config.quitting && server->outstanding > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding); MOLOCH_THREAD_INCR(server->dropped); if (data) { MOLOCH_SIZE_FREE(buffer, data); } return 1; } MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t); if (headers) { int i; for (i = 0; headers[i]; i++) { request->headerList = curl_slist_append(request->headerList, headers[i]); } } if (server->defaultHeaders) { int i; for (i = 0; server->defaultHeaders[i]; i++) { request->headerList = curl_slist_append(request->headerList, server->defaultHeaders[i]); } } // Do we need to compress item if (server->compress && data && data_len > 1000) { char *buf = moloch_http_get_buffer(data_len); int ret; MOLOCH_LOCK(z_strm); z_strm.avail_in = data_len; z_strm.next_in = (unsigned char *)data; z_strm.avail_out = data_len; z_strm.next_out = (unsigned char *)buf; ret = deflate(&z_strm, Z_FINISH); if (ret == Z_STREAM_END) { request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate"); MOLOCH_SIZE_FREE(buffer, data); data_len = data_len - z_strm.avail_out; data = buf; } else { MOLOCH_SIZE_FREE(buffer, buf); } deflateReset(&z_strm); MOLOCH_UNLOCK(z_strm); } request->server = server; request->func = func; request->uw = uw; request->dataOut = data; request->dataOutLen = data_len; request->easy = curl_easy_init(); if (config.debug >= 2) { curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1); } if (config.insecure) { curl_easy_setopt(request->easy, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(request->easy, CURLOPT_SSL_VERIFYHOST, 0L); } curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback); curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request); curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request); curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback); curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback); curl_easy_setopt(request->easy, CURLOPT_ACCEPT_ENCODING, ""); // https://curl.haxx.se/libcurl/c/CURLOPT_ACCEPT_ENCODING.html if (request->headerList) { curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList); } if (method[0] != 'G') { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data); } else { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL); curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L); } if (server->headerCb) { curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function); curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request); } curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L); curl_easy_setopt(request->easy, CURLOPT_TIMEOUT, 60L); memcpy(request->key, key, key_len); request->key[key_len] = 0; MOLOCH_LOCK(requests); moloch_http_add_request(server, request, TRUE); MOLOCH_UNLOCK(requests); return 0; }
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw) { MolochRequest_t *request; MolochConn_t *conn; MolochHttp_t *server = serverV; request = MOLOCH_TYPE_ALLOC(MolochRequest_t); memcpy(request->key, key, MIN(key_len, sizeof(request->key))); strncpy(request->method, method, sizeof(request->method)); request->key_len = key_len; request->compress = 0; if (server->compress && data && data_len > 1000) { char *buf = moloch_http_get_buffer(data_len); int ret; z_strm.avail_in = data_len; z_strm.next_in = (unsigned char *)data; z_strm.avail_out = data_len; z_strm.next_out = (unsigned char *)buf; ret = deflate(&z_strm, Z_FINISH); if (ret == Z_STREAM_END) { request->compress = 1; MOLOCH_SIZE_FREE(buffer, data); data_len = data_len - z_strm.avail_out; data = buf; } else { MOLOCH_SIZE_FREE(buffer, buf); } deflateReset(&z_strm); } request->data_len = data_len; request->data = data; request->func = func; request->uw = uw; int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0; // Already have outstanding requests, see if we can process them if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) { while (DLL_POP_HEAD(e_, &server->connQ, conn)) { DLL_POP_HEAD(r_, &server->requestQ[q], conn->request); if (conn->request) { if (!moloch_http_process_send(conn, 0)) { LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key); DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request); conn->request = 0; DLL_PUSH_TAIL(e_, &server->connQ, conn); break; } } else { DLL_PUSH_TAIL(e_, &server->connQ, conn); break; } } } // Now see if we can send something new if (DLL_POP_HEAD(e_, &server->connQ, conn)) { conn->request = request; if (!moloch_http_process_send(conn, FALSE)) { conn->request = 0; DLL_PUSH_TAIL(r_, &server->requestQ[q], request); DLL_PUSH_TAIL(e_, &server->connQ, conn); } } else { request->data = data; if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count); if (data) { MOLOCH_SIZE_FREE(buffer, data); } MOLOCH_TYPE_FREE(MolochRequest_t, request); return 1; } else { DLL_PUSH_TAIL(r_, &server->requestQ[q], request); } } return 0; }