gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, char *headers, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochHttp_t *server = serverV;
if (!config.exiting && dropable && server->requestQ.r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->requestQ.r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
request->key_len = MIN(key_len, sizeof(request->key)-1);
memcpy(request->key, key, request->key_len);
request->key[request->key_len] = 0;
strncpy(request->method, method, sizeof(request->method));
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
if (headers)
strcpy(request->headers, headers);
else
request->headers[0] = 0;
DLL_PUSH_TAIL(r_, &server->requestQ, request);
moloch_http_do_requests(server);
return 0;
}
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw)
{
MolochHttpServer_t *server = serverV;
// Are we overloaded
if (!config.exiting && dropable && server->outstanding > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t);
if (headers) {
int i;
for (i = 0; headers[i]; i++) {
request->headerList = curl_slist_append(request->headerList, headers[i]);
}
}
// Do we need to compress item
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate");
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->server = server;
request->func = func;
request->uw = uw;
request->dataOut = data;
request->dataOutLen = data_len;
request->easy = curl_easy_init();
if (config.debug >= 2) {
curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1);
}
curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback);
curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETDATA, server);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETDATA, server);
if (request->headerList) {
curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList);
}
if (method[0] != 'G') {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method);
curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data);
} else {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL);
curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L);
}
if (server->headerCb) {
curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function);
curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request);
}
curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L);
char *host = server->names[server->namesPos];
server->namesPos = (server->namesPos + 1) % server->namesCnt;
if (strchr(host, ':') == 0) {
snprintf(request->url, sizeof(request->url), "%s://%s:%d%.*s", (server->https?"https":"http"), host, server->defaultPort, key_len, key);
} else {
snprintf(request->url, sizeof(request->url), "%s://%s%.*s", (server->https?"https":"http"), host, key_len, key);
}
curl_easy_setopt(request->easy, CURLOPT_URL, request->url);
MOLOCH_LOCK(requests);
server->outstanding++;
DLL_PUSH_TAIL(rqt_, &requests, request);
if (!requestsTimer)
requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL);
MOLOCH_UNLOCK(requests);
return 0;
}
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw)
{
MolochHttpServer_t *server = serverV;
if (key_len > 1000) {
LOGEXIT("Url too long %.*s", key_len, key);
}
// Are we overloaded
if (dropable && !config.quitting && server->outstanding > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding);
MOLOCH_THREAD_INCR(server->dropped);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t);
if (headers) {
int i;
for (i = 0; headers[i]; i++) {
request->headerList = curl_slist_append(request->headerList, headers[i]);
}
}
if (server->defaultHeaders) {
int i;
for (i = 0; server->defaultHeaders[i]; i++) {
request->headerList = curl_slist_append(request->headerList, server->defaultHeaders[i]);
}
}
// Do we need to compress item
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
MOLOCH_LOCK(z_strm);
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate");
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
MOLOCH_UNLOCK(z_strm);
}
request->server = server;
request->func = func;
request->uw = uw;
request->dataOut = data;
request->dataOutLen = data_len;
request->easy = curl_easy_init();
if (config.debug >= 2) {
curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1);
}
if (config.insecure) {
curl_easy_setopt(request->easy, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(request->easy, CURLOPT_SSL_VERIFYHOST, 0L);
}
curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback);
curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback);
curl_easy_setopt(request->easy, CURLOPT_ACCEPT_ENCODING, ""); // https://curl.haxx.se/libcurl/c/CURLOPT_ACCEPT_ENCODING.html
if (request->headerList) {
curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList);
}
if (method[0] != 'G') {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method);
curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data);
} else {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL);
curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L);
}
if (server->headerCb) {
curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function);
curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request);
}
curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L);
curl_easy_setopt(request->easy, CURLOPT_TIMEOUT, 60L);
memcpy(request->key, key, key_len);
request->key[key_len] = 0;
MOLOCH_LOCK(requests);
moloch_http_add_request(server, request, TRUE);
MOLOCH_UNLOCK(requests);
return 0;
}
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochConn_t *conn;
MolochHttp_t *server = serverV;
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
memcpy(request->key, key, MIN(key_len, sizeof(request->key)));
strncpy(request->method, method, sizeof(request->method));
request->key_len = key_len;
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0;
// Already have outstanding requests, see if we can process them
if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) {
while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
DLL_POP_HEAD(r_, &server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
else {
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
}
// Now see if we can send something new
if (DLL_POP_HEAD(e_, &server->connQ, conn)) {
conn->request = request;
if (!moloch_http_process_send(conn, FALSE)) {
conn->request = 0;
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
} else {
request->data = data;
if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
MOLOCH_TYPE_FREE(MolochRequest_t, request);
return 1;
} else {
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
}
}
return 0;
}