gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, char *headers, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochHttp_t *server = serverV;
if (!config.exiting && dropable && server->requestQ.r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->requestQ.r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
request->key_len = MIN(key_len, sizeof(request->key)-1);
memcpy(request->key, key, request->key_len);
request->key[request->key_len] = 0;
strncpy(request->method, method, sizeof(request->method));
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
if (headers)
strcpy(request->headers, headers);
else
request->headers[0] = 0;
DLL_PUSH_TAIL(r_, &server->requestQ, request);
moloch_http_do_requests(server);
return 0;
}
void moloch_http_finish( MolochConn_t *conn, gboolean sync)
{
char line[1000];
strcpy(line, conn->line);
conn->hp_complete = 0;
http_parser_init(&conn->parser, HTTP_RESPONSE);
if (!sync && conn->request->data) {
MOLOCH_SIZE_FREE(buffer, conn->request->data);
conn->request->data = 0;
}
while (sync) {
moloch_http_read_cb(0, 0, conn);
if (conn->hp_complete)
break;
}
gettimeofday(&conn->endTime, NULL);
if (config.logESRequests)
LOG("%s %ldms %ldms",
line,
(conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec/1000 - conn->startTime.tv_usec/1000),
(conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec/1000 - conn->startTime.tv_usec/1000)
);
}
LOCAL void moloch_session_free (MolochSession_t *session)
{
if (session->tcp_next) {
DLL_REMOVE(tcp_, &tcpWriteQ[session->thread], session);
}
g_array_free(session->filePosArray, TRUE);
g_array_free(session->fileLenArray, TRUE);
g_array_free(session->fileNumArray, TRUE);
if (session->rootId && session->rootId[0] != 'R')
g_free(session->rootId);
if (session->parserInfo) {
int i;
for (i = 0; i < session->parserNum; i++) {
if (session->parserInfo[i].parserFreeFunc)
session->parserInfo[i].parserFreeFunc(session, session->parserInfo[i].uw);
}
free(session->parserInfo);
}
if (session->pluginData)
MOLOCH_SIZE_FREE(pluginData, session->pluginData);
moloch_field_free(session);
moloch_packet_tcp_free(session);
MOLOCH_TYPE_FREE(MolochSession_t, session);
}
void moloch_http_finish( MolochConn_t *conn, gboolean sync)
{
char line[1000];
strcpy(line, conn->line);
conn->hp_complete = 0;
http_parser_init(&conn->parser, HTTP_RESPONSE);
if (!sync && conn->request->data) {
MOLOCH_SIZE_FREE(buffer, conn->request->data);
conn->request->data = 0;
}
while (sync) {
moloch_http_read_cb(0, 0, conn);
if (conn->hp_complete)
break;
}
gettimeofday(&conn->sentTime, NULL);
}
gboolean moloch_http_finish( MolochConn_t *conn, gboolean sync)
{
conn->hp_complete = 0;
http_parser_init(&conn->parser, HTTP_RESPONSE);
if (!sync && conn->request->data) {
MOLOCH_SIZE_FREE(buffer, conn->request->data);
conn->request->data = 0;
}
while (sync) {
if (!moloch_http_read_cb(0, 0, conn)) {
gettimeofday(&conn->sentTime, NULL);
return FALSE;
}
if (conn->hp_complete)
break;
}
gettimeofday(&conn->sentTime, NULL);
return TRUE;
}
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw)
{
MolochHttpServer_t *server = serverV;
// Are we overloaded
if (!config.exiting && dropable && server->outstanding > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t);
if (headers) {
int i;
for (i = 0; headers[i]; i++) {
request->headerList = curl_slist_append(request->headerList, headers[i]);
}
}
// Do we need to compress item
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate");
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->server = server;
request->func = func;
request->uw = uw;
request->dataOut = data;
request->dataOutLen = data_len;
request->easy = curl_easy_init();
if (config.debug >= 2) {
curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1);
}
curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback);
curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETDATA, server);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETDATA, server);
if (request->headerList) {
curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList);
}
if (method[0] != 'G') {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method);
curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data);
} else {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL);
curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L);
}
if (server->headerCb) {
curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function);
curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request);
}
curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L);
char *host = server->names[server->namesPos];
server->namesPos = (server->namesPos + 1) % server->namesCnt;
if (strchr(host, ':') == 0) {
snprintf(request->url, sizeof(request->url), "%s://%s:%d%.*s", (server->https?"https":"http"), host, server->defaultPort, key_len, key);
} else {
snprintf(request->url, sizeof(request->url), "%s://%s%.*s", (server->https?"https":"http"), host, key_len, key);
}
curl_easy_setopt(request->easy, CURLOPT_URL, request->url);
MOLOCH_LOCK(requests);
server->outstanding++;
DLL_PUSH_TAIL(rqt_, &requests, request);
if (!requestsTimer)
requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL);
MOLOCH_UNLOCK(requests);
return 0;
}
static void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server)
{
char *eff_url;
CURLMsg *msg;
int msgs_left;
MolochHttpRequest_t *request;
CURL *easy;
while ((msg = curl_multi_info_read(server->multi, &msgs_left))) {
if (msg->msg == CURLMSG_DONE) {
easy = msg->easy_handle;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request);
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);
long responseCode;
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode);
if (config.logESRequests) {
double totalTime;
double connectTime;
double uploadSize;
double downloadSize;
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime);
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime);
curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize);
curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize);
LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms",
request->server->outstanding,
request->server->connections,
responseCode,
request->url,
uploadSize,
downloadSize,
connectTime*1000,
totalTime*1000);
}
if (request->func) {
if (request->dataIn)
request->dataIn[request->used] = 0;
request->func(responseCode, request->dataIn, request->used, request->uw);
}
if (request->dataIn) {
free(request->dataIn);
request->dataIn = 0;
}
if (request->dataOut) {
MOLOCH_SIZE_FREE(buffer, request->dataOut);
}
if (request->headerList) {
curl_slist_free_all(request->headerList);
}
MOLOCH_TYPE_FREE(MolochHttpRequest_t, request);
curl_multi_remove_handle(server->multi, easy);
curl_easy_cleanup(easy);
MOLOCH_LOCK(requests);
server->outstanding--;
MOLOCH_UNLOCK(requests);
}
}
}
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw)
{
MolochHttpServer_t *server = serverV;
if (key_len > 1000) {
LOGEXIT("Url too long %.*s", key_len, key);
}
// Are we overloaded
if (dropable && !config.quitting && server->outstanding > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding);
MOLOCH_THREAD_INCR(server->dropped);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t);
if (headers) {
int i;
for (i = 0; headers[i]; i++) {
request->headerList = curl_slist_append(request->headerList, headers[i]);
}
}
if (server->defaultHeaders) {
int i;
for (i = 0; server->defaultHeaders[i]; i++) {
request->headerList = curl_slist_append(request->headerList, server->defaultHeaders[i]);
}
}
// Do we need to compress item
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
MOLOCH_LOCK(z_strm);
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate");
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
MOLOCH_UNLOCK(z_strm);
}
request->server = server;
request->func = func;
request->uw = uw;
request->dataOut = data;
request->dataOutLen = data_len;
request->easy = curl_easy_init();
if (config.debug >= 2) {
curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1);
}
if (config.insecure) {
curl_easy_setopt(request->easy, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(request->easy, CURLOPT_SSL_VERIFYHOST, 0L);
}
curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback);
curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback);
curl_easy_setopt(request->easy, CURLOPT_ACCEPT_ENCODING, ""); // https://curl.haxx.se/libcurl/c/CURLOPT_ACCEPT_ENCODING.html
if (request->headerList) {
curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList);
}
if (method[0] != 'G') {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method);
curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data);
} else {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL);
curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L);
}
if (server->headerCb) {
curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function);
curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request);
}
curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L);
curl_easy_setopt(request->easy, CURLOPT_TIMEOUT, 60L);
memcpy(request->key, key, key_len);
request->key[key_len] = 0;
MOLOCH_LOCK(requests);
moloch_http_add_request(server, request, TRUE);
MOLOCH_UNLOCK(requests);
return 0;
}
LOCAL void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server)
{
char *eff_url;
CURLMsg *msg;
int msgs_left;
MolochHttpRequest_t *request;
CURL *easy;
while ((msg = curl_multi_info_read(server->multi, &msgs_left))) {
if (msg->msg == CURLMSG_DONE) {
easy = msg->easy_handle;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request);
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);
long responseCode;
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode);
if (config.logESRequests || (server->printErrors && responseCode/100 != 2)) {
double totalTime;
double connectTime;
double uploadSize;
double downloadSize;
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime);
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime);
curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize);
curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize);
LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms",
request->server->outstanding,
request->server->connections,
responseCode,
request->url,
uploadSize,
downloadSize,
connectTime*1000,
totalTime*1000);
}
#ifdef MOLOCH_HTTP_DEBUG
LOG("HTTPDEBUG DECR %p %d %s", request, server->outstanding, request->url);
#endif
if (responseCode == 0 && request->retries < server->maxRetries) {
curl_multi_remove_handle(server->multi, easy);
request->retries++;
struct timeval now;
gettimeofday(&now, NULL);
MOLOCH_LOCK(requests);
server->snames[request->namePos].allowedAtSeconds = now.tv_sec + 30;
server->outstanding--;
moloch_http_add_request(server, request, TRUE);
MOLOCH_UNLOCK(requests);
} else {
if (server->printErrors && responseCode/100 != 2) {
LOG("Response length=%d :>\n%.*s", request->used, MIN(request->used, 4000), request->dataIn);
}
if (request->func) {
if (request->dataIn)
request->dataIn[request->used] = 0;
request->func(responseCode, request->dataIn, request->used, request->uw);
}
if (request->dataIn) {
free(request->dataIn);
request->dataIn = 0;
}
if (request->dataOut) {
MOLOCH_SIZE_FREE(buffer, request->dataOut);
}
if (request->headerList) {
curl_slist_free_all(request->headerList);
}
MOLOCH_TYPE_FREE(MolochHttpRequest_t, request);
curl_multi_remove_handle(server->multi, easy);
curl_easy_cleanup(easy);
MOLOCH_LOCK(requests);
server->outstanding--;
MOLOCH_UNLOCK(requests);
}
}
}
}
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochConn_t *conn;
MolochHttp_t *server = serverV;
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
memcpy(request->key, key, MIN(key_len, sizeof(request->key)));
strncpy(request->method, method, sizeof(request->method));
request->key_len = key_len;
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0;
// Already have outstanding requests, see if we can process them
if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) {
while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
DLL_POP_HEAD(r_, &server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
else {
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
}
// Now see if we can send something new
if (DLL_POP_HEAD(e_, &server->connQ, conn)) {
conn->request = request;
if (!moloch_http_process_send(conn, FALSE)) {
conn->request = 0;
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
} else {
request->data = data;
if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
MOLOCH_TYPE_FREE(MolochRequest_t, request);
return 1;
} else {
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
}
}
return 0;
}
