void moloch_http_do_requests(MolochHttp_t *server)
{
// Something waiting
while (DLL_COUNT(r_, &server->requestQ)) {
// No free connections
if (DLL_COUNT(e_, &server->connQ) == 0) {
// Not at our max, create one a second
if (server->numConns < server->maxConns && time(0) - server->lastFailedConnect > 0) {
moloch_http_create(server, FALSE);
}
return;
}
// Have a free conn and a request
MolochConn_t *conn;
DLL_POP_HEAD(e_, &server->connQ, conn);
DLL_POP_HEAD(r_, &server->requestQ, conn->request);
server->inProgress++;
if (!moloch_http_process_send(conn, FALSE)) {
server->inProgress--;
LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
DLL_PUSH_HEAD(r_, &server->requestQ, conn->request);
moloch_http_free_conn(conn, FALSE);
}
}
}
MolochSimple_t *writer_simple_alloc(int thread, MolochSimple_t *previous)
{
MolochSimple_t *info;
MOLOCH_LOCK(freeList[thread].lock);
DLL_POP_HEAD(simple_, &freeList[thread], info);
MOLOCH_UNLOCK(freeList[thread].lock);
if (!info) {
info = MOLOCH_TYPE_ALLOC0(MolochSimple_t);
info->buf = mmap (0, config.pcapWriteSize + MOLOCH_PACKET_MAX_LEN, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0);
info->thread = thread;
} else {
info->bufpos = 0;
info->closing = 0;
}
if (previous) {
info->file = previous->file;
} else {
info->file = MOLOCH_TYPE_ALLOC0(MolochSimpleFile_t);
switch(simpleMode) {
case MOLOCH_SIMPLE_NORMAL:
break;
case MOLOCH_SIMPLE_XOR2048:
break;
case MOLOCH_SIMPLE_AES256CTR:
info->file->cipher_ctx = EVP_CIPHER_CTX_new();
break;
}
}
return info;
}
void *writer_simple_thread(void *UNUSED(arg))
{
MolochSimple_t *info;
LOG("THREAD %p", (gpointer)pthread_self());
while (1) {
MOLOCH_LOCK(simpleQ);
while (DLL_COUNT(simple_, &simpleQ) == 0) {
MOLOCH_COND_WAIT(simpleQ);
}
DLL_POP_HEAD(simple_, &simpleQ, info);
MOLOCH_UNLOCK(simpleQ);
uint32_t pos = 0;
uint32_t total;
if (info->closing) {
total = info->bufpos;
if (total % pageSize != 0) {
total = (total - (total % pageSize) + pageSize);
}
} else {
total = config.pcapWriteSize;
}
switch(simpleMode) {
case MOLOCH_SIMPLE_NORMAL:
break;
case MOLOCH_SIMPLE_XOR2048: {
uint32_t i;
for (i = 0; i < total; i++)
info->buf[i] ^= info->file->dek[i % 256];
break;
}
case MOLOCH_SIMPLE_AES256CTR: {
int outl;
if (!EVP_EncryptUpdate(info->file->cipher_ctx, (uint8_t *)info->buf, &outl, (uint8_t *)info->buf, total))
LOGEXIT("Encrypting data failed");
if ((int)total != outl)
LOGEXIT("Encryption in (%d) and out (%d) didn't match", total, outl);
break;
}
}
while (pos < total) {
int len = write(info->file->fd, info->buf + pos, total - pos);
if (len >= 0) {
pos += len;
} else {
LOGEXIT("ERROR writing - %d %s", len, strerror(errno));
}
}
if (info->closing) {
ftruncate(info->file->fd, info->file->pos);
close(info->file->fd);
}
writer_simple_free(info);
}
return NULL;
}
void moloch_packet_tcp_free(MolochSession_t *session)
{
MolochTcpData_t *td;
while (DLL_POP_HEAD(td_, &session->tcpData, td)) {
moloch_packet_free(td->packet);
MOLOCH_TYPE_FREE(MolochTcpData_t, td);
}
}
static gboolean moloch_http_send_timer_callback(gpointer UNUSED(unused))
{
MolochHttpRequest_t *request;
while (1) {
MOLOCH_LOCK(requests);
DLL_POP_HEAD(rqt_, &requests, request);
if (!request) {
requestsTimer = 0;
MOLOCH_UNLOCK(requests);
return G_SOURCE_REMOVE;
}
MOLOCH_UNLOCK(requests);
curl_multi_add_handle(request->server->multi, request->easy);
}
return G_SOURCE_REMOVE;
}
static gboolean moloch_http_send_timer_callback(gpointer UNUSED(unused))
{
MolochHttpRequest_t *request;
while (1) {
MOLOCH_LOCK(requests);
DLL_POP_HEAD(rqt_, &requests, request);
if (!request) {
requestsTimer = 0;
MOLOCH_UNLOCK(requests);
return G_SOURCE_REMOVE;
}
MOLOCH_UNLOCK(requests);
#ifdef MOLOCH_HTTP_DEBUG
LOG("HTTPDEBUG DO %s %p %d %s", request->server->names[0], request, request->server->outstanding, request->url);
#endif
curl_multi_add_handle(request->server->multi, request->easy);
}
return G_SOURCE_REMOVE;
}
void moloch_http_free_server(void *serverV)
{
MolochHttp_t *server = serverV;
int q;
for (q = 0; q < 2; q++) {
while (server->requestQ[q].r_count > 0 || server->connQ.e_count != server->maxConns) {
g_main_context_iteration (g_main_context_default(), FALSE);
}
}
MolochConn_t *es = 0;
while (DLL_POP_HEAD(e_, &server->connQ, es)) {
MOLOCH_TYPE_FREE(MolochConn_t, es);
}
MOLOCH_TYPE_FREE(MolochConn_t, server->syncConn);
server->syncConn = 0;
g_strfreev(server->names);
MOLOCH_TYPE_FREE(MolochHttp_t, server);
}
void moloch_http_free_server(void *serverV)
{
MolochHttp_t *server = serverV;
int n=0;
while (server->requestQ.r_count > 0 || server->inProgress > 0) {
n++;
if (n%10000 == 0)
LOG("%s %d %d", server->names[0], server->requestQ.r_count, server->inProgress);
g_main_context_iteration (g_main_context_default(), FALSE);
}
MolochConn_t *conn = 0;
while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
moloch_http_free_conn(conn, FALSE);
}
moloch_http_free_conn(server->syncConn, FALSE);
server->syncConn = 0;
g_strfreev(server->names);
MOLOCH_TYPE_FREE(MolochHttp_t, server);
}
int reader_libpcapfile_next()
{
char errbuf[1024];
gchar *fullfilename;
pcap = 0;
if (config.pcapReadFiles) {
static int pcapFilePos = 0;
fullfilename = config.pcapReadFiles[pcapFilePos];
errbuf[0] = 0;
if (!fullfilename) {
goto filesDone;
}
pcapFilePos++;
LOG ("Processing %s", fullfilename);
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
return reader_libpcapfile_next();
}
if (!realpath(fullfilename, offlinePcapFilename)) {
LOG("ERROR - pcap open failed - Couldn't realpath file: '%s' with %d", fullfilename, errno);
exit(1);
}
reader_libpcapfile_opened();
return 1;
}
filesDone:
if (config.pcapReadDirs) {
static int pcapDirPos = 0;
static GDir *pcapGDir[21];
static char *pcapBase[21];
static int pcapGDirLevel = -1;
GError *error = 0;
if (pcapGDirLevel == -2) {
goto dirsDone;
}
if (pcapGDirLevel == -1) {
pcapGDirLevel = 0;
pcapBase[0] = config.pcapReadDirs[pcapDirPos];
if (!pcapBase[0]) {
pcapGDirLevel = -2;
goto dirsDone;
}
}
if (!pcapGDir[pcapGDirLevel]) {
pcapGDir[pcapGDirLevel] = g_dir_open(pcapBase[pcapGDirLevel], 0, &error);
if (error) {
LOG("ERROR: Couldn't open pcap directory: Receive Error: %s", error->message);
exit(0);
}
}
const gchar *filename;
while (1) {
filename = g_dir_read_name(pcapGDir[pcapGDirLevel]);
// No more files, stop processing this directory
if (!filename) {
break;
}
// Skip hidden files/directories
if (filename[0] == '.')
continue;
fullfilename = g_build_filename (pcapBase[pcapGDirLevel], filename, NULL);
// If recursive option and a directory then process all the files in that dir
if (config.pcapRecursive && g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
if (pcapGDirLevel >= 20)
continue;
pcapBase[pcapGDirLevel+1] = fullfilename;
pcapGDirLevel++;
return reader_libpcapfile_next();
}
if (!g_regex_match(config.offlineRegex, filename, 0, NULL)) {
g_free(fullfilename);
continue;
}
if (!realpath(fullfilename, offlinePcapFilename)) {
g_free(fullfilename);
continue;
}
if (config.pcapSkip && moloch_db_file_exists(offlinePcapFilename)) {
if (config.debug)
LOG("Skipping %s", fullfilename);
g_free(fullfilename);
continue;
}
LOG ("Processing %s", fullfilename);
errbuf[0] = 0;
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
g_free(fullfilename);
continue;
}
reader_libpcapfile_opened();
g_free(fullfilename);
return 1;
}
g_dir_close(pcapGDir[pcapGDirLevel]);
pcapGDir[pcapGDirLevel] = 0;
if (pcapGDirLevel > 0) {
g_free(pcapBase[pcapGDirLevel]);
pcapGDirLevel--;
return reader_libpcapfile_next();
} else {
pcapDirPos++;
pcapGDirLevel = -1;
return reader_libpcapfile_next();
}
}
dirsDone:
while (DLL_COUNT(s_, &monitorQ) > 0) {
MolochString_t *string;
DLL_POP_HEAD(s_, &monitorQ, string);
fullfilename = string->str;
MOLOCH_TYPE_FREE(MolochString_t, string);
if (!realpath(fullfilename, offlinePcapFilename)) {
g_free(fullfilename);
continue;
}
if (config.pcapSkip && moloch_db_file_exists(offlinePcapFilename)) {
if (config.debug)
LOG("Skipping %s", fullfilename);
g_free(fullfilename);
continue;
}
LOG ("Processing %s", fullfilename);
errbuf[0] = 0;
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
g_free(fullfilename);
continue;
}
reader_libpcapfile_opened();
g_free(fullfilename);
return 1;
}
return 0;
}
LOCAL int reader_libpcapfile_next()
{
gchar *fullfilename;
pcap = 0;
if (config.pcapReadFiles) {
static int pcapFilePos = 0;
fullfilename = config.pcapReadFiles[pcapFilePos];
if (!fullfilename) {
goto filesDone;
}
pcapFilePos++;
if (reader_libpcapfile_process(fullfilename)) {
return reader_libpcapfile_next();
}
return 1;
}
filesDone:
if (config.pcapFileLists) {
static int pcapFileListsPos;
static FILE *file;
char line[PATH_MAX];
if (!file && !config.pcapFileLists[pcapFileListsPos]) {
goto fileListsDone;
}
if (!file) {
if (strcmp(config.pcapFileLists[pcapFileListsPos], "-") == 0)
file = stdin;
else
file = fopen(config.pcapFileLists[pcapFileListsPos], "r");
pcapFileListsPos++;
if (!file) {
LOG("ERROR - Couldn't open %s", config.pcapFileLists[pcapFileListsPos - 1]);
return reader_libpcapfile_next();
}
}
if (feof(file)) {
fclose(file);
file = NULL;
return reader_libpcapfile_next();
}
if (!fgets(line, sizeof(line), file)) {
fclose(file);
file = NULL;
return reader_libpcapfile_next();
}
int lineLen = strlen(line);
if (line[lineLen-1] == '\n') {
line[lineLen-1] = 0;
}
g_strstrip(line);
if (!line[0] || line[0] == '#')
return reader_libpcapfile_next();
if (reader_libpcapfile_process(line)) {
return reader_libpcapfile_next();
}
return 1;
}
fileListsDone:
if (config.pcapReadDirs) {
static int pcapDirPos = 0;
static GDir *pcapGDir[21];
static char *pcapBase[21];
static int pcapGDirLevel = -1;
GError *error = 0;
if (pcapGDirLevel == -2) {
goto dirsDone;
}
if (pcapGDirLevel == -1) {
pcapGDirLevel = 0;
pcapBase[0] = config.pcapReadDirs[pcapDirPos];
if (!pcapBase[0]) {
pcapGDirLevel = -2;
goto dirsDone;
}
}
if (!pcapGDir[pcapGDirLevel]) {
pcapGDir[pcapGDirLevel] = g_dir_open(pcapBase[pcapGDirLevel], 0, &error);
if (error) {
LOGEXIT("ERROR: Couldn't open pcap directory: Receive Error: %s", error->message);
}
}
while (1) {
const gchar *filename = g_dir_read_name(pcapGDir[pcapGDirLevel]);
// No more files, stop processing this directory
if (!filename) {
break;
}
// Skip hidden files/directories
if (filename[0] == '.')
continue;
fullfilename = g_build_filename (pcapBase[pcapGDirLevel], filename, NULL);
// If recursive option and a directory then process all the files in that dir
if (config.pcapRecursive && g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
if (pcapGDirLevel >= 20)
continue;
pcapBase[pcapGDirLevel+1] = fullfilename;
pcapGDirLevel++;
return reader_libpcapfile_next();
}
if (!g_regex_match(config.offlineRegex, filename, 0, NULL)) {
g_free(fullfilename);
continue;
}
if (reader_libpcapfile_process(fullfilename)) {
g_free(fullfilename);
continue;
}
g_free(fullfilename);
return 1;
}
g_dir_close(pcapGDir[pcapGDirLevel]);
pcapGDir[pcapGDirLevel] = 0;
if (pcapGDirLevel > 0) {
g_free(pcapBase[pcapGDirLevel]);
pcapGDirLevel--;
return reader_libpcapfile_next();
} else {
pcapDirPos++;
pcapGDirLevel = -1;
return reader_libpcapfile_next();
}
}
dirsDone:
while (DLL_COUNT(s_, &monitorQ) > 0) {
MolochString_t *string;
DLL_POP_HEAD(s_, &monitorQ, string);
fullfilename = string->str;
MOLOCH_TYPE_FREE(MolochString_t, string);
if (reader_libpcapfile_process(fullfilename)) {
g_free(fullfilename);
continue;
}
g_free(fullfilename);
return 1;
}
return 0;
}
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochConn_t *conn;
MolochHttp_t *server = serverV;
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
memcpy(request->key, key, MIN(key_len, sizeof(request->key)));
strncpy(request->method, method, sizeof(request->method));
request->key_len = key_len;
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0;
// Already have outstanding requests, see if we can process them
if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) {
while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
DLL_POP_HEAD(r_, &server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
else {
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
}
// Now see if we can send something new
if (DLL_POP_HEAD(e_, &server->connQ, conn)) {
conn->request = request;
if (!moloch_http_process_send(conn, FALSE)) {
conn->request = 0;
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
} else {
request->data = data;
if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
MOLOCH_TYPE_FREE(MolochRequest_t, request);
return 1;
} else {
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
}
}
return 0;
}
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
MolochConn_t *conn = data;
char buffer[0xffff];
int len;
GError *gerror = 0;
len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);
if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
if (gerror) {
LOG("ERROR: %p:%p Receive Error: %s", (void*)conn, conn->request, gerror->message);
g_error_free(gerror);
} else if (cond & (G_IO_HUP | G_IO_ERR))
LOG("ERROR: %p:%p Lost connection to %s", (void*)conn, conn->request, conn->name);
else if (len <= 0)
LOG("ERROR: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
else
LOG("ERROR HMM: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
if (conn->request) {
// Must save, free, then call function because of recursive sync GETs
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
if (func) {
func(0, 0, uw);
}
}
g_object_unref (conn->conn);
conn->conn = 0;
if (conn != conn->server->syncConn && conn->request) {
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
conn->request = 0;
return FALSE;
}
http_parser_execute(&conn->parser, &parserSettings, buffer, len);
if (conn->hp_complete) {
gettimeofday(&conn->endTime, NULL);
if (config.logESRequests)
LOG("%s %ldms %ldms %ldms",
conn->line,
(conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
);
conn->hp_data[conn->hp_len] = 0;
/* Must save, free, then call function because of recursive sync GETs */
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
conn->request = 0;
if (func) {
func((unsigned char*)conn->hp_data, conn->hp_len, uw);
}
if (conn == conn->server->syncConn)
return TRUE;
int q;
for (q = 0; q < 2; q++) {
DLL_POP_HEAD(r_, &conn->server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
DLL_PUSH_HEAD(r_, &conn->server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
return TRUE;
}
}
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
return TRUE;
}
void moloch_session_process_commands(int thread)
{
// Commands
MolochSesCmd_t *cmd = 0;
int count;
for (count = 0; count < 50; count++) {
MOLOCH_LOCK(sessionCmds[thread].lock);
DLL_POP_HEAD(cmd_, &sessionCmds[thread], cmd);
MOLOCH_UNLOCK(sessionCmds[thread].lock);
if (!cmd)
break;
switch (cmd->cmd) {
case MOLOCH_SES_CMD_ADD_TAG:
moloch_field_int_add((long)cmd->uw1, cmd->session, (long)cmd->uw2);
moloch_session_decr_outstanding(cmd->session);
break;
case MOLOCH_SES_CMD_FUNC:
cmd->func(cmd->session, cmd->uw1, cmd->uw2);
break;
default:
LOG ("Unknown cmd %d", cmd->cmd);
}
MOLOCH_TYPE_FREE(MolochSesCmd_t, cmd);
}
// Closing Q
for (count = 0; count < 10; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(q_, &closingQ[thread]);
if (session && session->saveTime < (uint64_t)lastPacketSecs[thread]) {
moloch_session_save(session);
} else {
break;
}
}
// Sessions Idle Long Time
int ses;
for (ses = 0; ses < SESSION_MAX; ses++) {
for (count = 0; count < 10; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(q_, &sessionsQ[thread][ses]);
if (session && (DLL_COUNT(q_, &sessionsQ[thread][ses]) > (int)config.maxStreams ||
((uint64_t)session->lastPacket.tv_sec + config.timeouts[ses] < (uint64_t)lastPacketSecs[thread]))) {
moloch_session_save(session);
} else {
break;
}
}
}
// TCP Sessions Open Long Time
for (count = 0; count < 50; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(tcp_, &tcpWriteQ[thread]);
if (session && (uint64_t)session->saveTime < (uint64_t)lastPacketSecs[thread]) {
moloch_session_mid_save(session, lastPacketSecs[thread]);
} else {
break;
}
}
}
