void moloch_http_do_requests(MolochHttp_t *server)
{
// Something waiting
while (DLL_COUNT(r_, &server->requestQ)) {
// No free connections
if (DLL_COUNT(e_, &server->connQ) == 0) {
// Not at our max, create one a second
if (server->numConns < server->maxConns && time(0) - server->lastFailedConnect > 0) {
moloch_http_create(server, FALSE);
}
return;
}
// Have a free conn and a request
MolochConn_t *conn;
DLL_POP_HEAD(e_, &server->connQ, conn);
DLL_POP_HEAD(r_, &server->requestQ, conn->request);
server->inProgress++;
if (!moloch_http_process_send(conn, FALSE)) {
server->inProgress--;
LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
DLL_PUSH_HEAD(r_, &server->requestQ, conn->request);
moloch_http_free_conn(conn, FALSE);
}
}
}
void wise_cb(unsigned char *data, int data_len, gpointer uw)
{
BSB bsb;
WiseRequest_t *request = uw;
int i;
inflight -= request->numItems;
BSB_INIT(bsb, data, data_len);
uint32_t fts = 0, ver = 0;
BSB_IMPORT_u32(bsb, fts);
BSB_IMPORT_u32(bsb, ver);
if (BSB_IS_ERROR(bsb) || ver != 0) {
for (i = 0; i < request->numItems; i++) {
wise_free_item(request->items[i]);
}
MOLOCH_TYPE_FREE(WiseRequest_t, request);
return;
}
if (fts != fieldsTS)
wise_load_fields();
struct timeval currentTime;
gettimeofday(¤tTime, NULL);
for (i = 0; i < request->numItems; i++) {
WiseItem_t *wi = request->items[i];
BSB_IMPORT_u08(bsb, wi->numOps);
if (wi->numOps > 0) {
wi->ops = malloc(wi->numOps * sizeof(WiseOp_t));
int i;
for (i = 0; i < wi->numOps; i++) {
WiseOp_t *op = &(wi->ops[i]);
int rfield = 0;
BSB_IMPORT_u08(bsb, rfield);
op->fieldPos = fieldsMap[rfield];
int len = 0;
BSB_IMPORT_u08(bsb, len);
char *str = (char*)BSB_WORK_PTR(bsb);
BSB_IMPORT_skip(bsb, len);
switch (config.fields[op->fieldPos]->type) {
case MOLOCH_FIELD_TYPE_INT_HASH:
if (op->fieldPos == tagsField) {
moloch_db_get_tag(NULL, tagsField, str, NULL); // Preload the tagname -> tag mapping
op->str = g_strdup(str);
op->strLenOrInt = len - 1;
continue;
}
// Fall thru
case MOLOCH_FIELD_TYPE_INT:
case MOLOCH_FIELD_TYPE_INT_ARRAY:
op->str = 0;
op->strLenOrInt = atoi(str);
break;
case MOLOCH_FIELD_TYPE_STR:
case MOLOCH_FIELD_TYPE_STR_ARRAY:
case MOLOCH_FIELD_TYPE_STR_HASH:
op->str = g_strdup(str);
op->strLenOrInt = len - 1;
break;
case MOLOCH_FIELD_TYPE_IP:
case MOLOCH_FIELD_TYPE_IP_HASH:
op->str = 0;
op->strLenOrInt = inet_addr(str);
break;
default:
LOG("WARNING - Unsupported expression type for %s", str);
continue;
}
}
}
wi->loadTime = currentTime.tv_sec;
int s;
for (s = 0; s < wi->numSessions; s++) {
wise_process_ops(wi->sessions[s], wi);
moloch_nids_decr_outstanding(wi->sessions[s]);
}
g_free(wi->sessions);
wi->sessions = 0;
wi->numSessions = 0;
DLL_PUSH_HEAD(wil_, &itemList[(int)wi->type], wi);
// Cache needs to be reduced
if (itemList[(int)wi->type].wil_count > maxCache) {
DLL_POP_TAIL(wil_, &itemList[(int)wi->type], wi);
wise_free_item(wi);
}
}
MOLOCH_TYPE_FREE(WiseRequest_t, request);
}
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochConn_t *conn;
MolochHttp_t *server = serverV;
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
memcpy(request->key, key, MIN(key_len, sizeof(request->key)));
strncpy(request->method, method, sizeof(request->method));
request->key_len = key_len;
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0;
// Already have outstanding requests, see if we can process them
if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) {
while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
DLL_POP_HEAD(r_, &server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
else {
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
}
// Now see if we can send something new
if (DLL_POP_HEAD(e_, &server->connQ, conn)) {
conn->request = request;
if (!moloch_http_process_send(conn, FALSE)) {
conn->request = 0;
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
} else {
request->data = data;
if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
MOLOCH_TYPE_FREE(MolochRequest_t, request);
return 1;
} else {
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
}
}
return 0;
}
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
MolochConn_t *conn = data;
char buffer[0xffff];
int len;
GError *gerror = 0;
len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);
if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
if (gerror) {
LOG("ERROR: %p:%p Receive Error: %s", (void*)conn, conn->request, gerror->message);
g_error_free(gerror);
} else if (cond & (G_IO_HUP | G_IO_ERR))
LOG("ERROR: %p:%p Lost connection to %s", (void*)conn, conn->request, conn->name);
else if (len <= 0)
LOG("ERROR: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
else
LOG("ERROR HMM: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
if (conn->request) {
// Must save, free, then call function because of recursive sync GETs
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
if (func) {
func(0, 0, uw);
}
}
g_object_unref (conn->conn);
conn->conn = 0;
if (conn != conn->server->syncConn && conn->request) {
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
conn->request = 0;
return FALSE;
}
http_parser_execute(&conn->parser, &parserSettings, buffer, len);
if (conn->hp_complete) {
gettimeofday(&conn->endTime, NULL);
if (config.logESRequests)
LOG("%s %ldms %ldms %ldms",
conn->line,
(conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
);
conn->hp_data[conn->hp_len] = 0;
/* Must save, free, then call function because of recursive sync GETs */
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
conn->request = 0;
if (func) {
func((unsigned char*)conn->hp_data, conn->hp_len, uw);
}
if (conn == conn->server->syncConn)
return TRUE;
int q;
for (q = 0; q < 2; q++) {
DLL_POP_HEAD(r_, &conn->server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
DLL_PUSH_HEAD(r_, &conn->server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
return TRUE;
}
}
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
return TRUE;
}
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
MolochConn_t *conn = data;
char buffer[0xffff];
int len;
GError *gerror = 0;
len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);
if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
if (gerror) {
LOG("ERROR: %s %p:%p Receive Error: %s", conn->name, (void*)conn, conn->request, gerror->message);
g_error_free(gerror);
} else if (cond & (G_IO_HUP | G_IO_ERR))
LOG("ERROR: %s %p:%p Lost connection to %s", conn->name, (void*)conn, conn->request, conn->name);
else if (len <= 0)
LOG("ERROR: %s %p:%p len: %d cond: %x", conn->name, (void*)conn, conn->request, len, cond);
else
LOG("ERROR HMM: %s %p:%p len: %d cond: %x", conn->name, (void*)conn, conn->request, len, cond);
if (conn == conn->server->syncConn) {
if (!conn->request) {
DEBUGCONN("AAA zerosync %s %p", conn->server->names[0], conn);
conn->server->syncConn = 0;
moloch_http_free_conn(conn, FALSE);
} else {
DEBUGCONN("AAA complete %s %p", conn->server->names[0], conn);
conn->hp_complete = 1;
}
return FALSE;
}
if (conn->request && conn->request->data) {
conn->server->inProgress--;
DLL_PUSH_HEAD(r_, &conn->server->requestQ, conn->request);
} else if (conn->request) {
conn->server->inProgress--;
// Must save, free, then call function because of recursive sync GETs
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
if (func) {
func(0, 0, uw);
}
}
moloch_http_free_conn(conn, TRUE);
return FALSE;
}
http_parser_execute(&conn->parser, &conn->server->parserSettings, buffer, len);
if (conn->hp_complete) {
gettimeofday(&conn->endTime, NULL);
if (config.logESRequests)
LOG("%s %ldms %ldms %ldms",
conn->line,
(conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
);
conn->hp_data[conn->hp_len] = 0;
/* Must save, free, then call function because of recursive sync GETs */
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
conn->request = 0;
if (func) {
func((unsigned char*)conn->hp_data, conn->hp_len, uw);
}
if (conn == conn->server->syncConn) {
if (conn->doClose) {
DEBUGCONN("AAA zerosync %s %p", conn->server->names[0], conn);
conn->server->syncConn = 0;
moloch_http_free_conn(conn, FALSE);
return FALSE;
}
return TRUE;
}
// Not syncConn below this
if (conn->doClose) {
moloch_http_free_conn(conn, TRUE);
return FALSE;
}
conn->server->inProgress--;
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
moloch_http_do_requests(conn->server);
}
return TRUE;
}