void wise_print_stats()
{
int i;
for (i = 0; i < 4; i++) {
LOG("%8s lookups:%7d cache:%7d requests:%7d inprogress:%7d fail:%7d hash:%7d list:%7d",
wiseStrings[i],
stats[i][0],
stats[i][1],
stats[i][2],
stats[i][3],
stats[i][4],
HASH_COUNT(wih_, itemHash[i]),
DLL_COUNT(wil_, &itemList[i]));
}
}
DLL_NODE_PTR dll_init(void)
{
/* Allocate the header node */
#ifdef DLL_CHK
DLL_NODE_PTR head = (DLL_NODE_PTR)dll_node_create(NULL);
#else
DLL_NODE_PTR head = (DLL_NODE_PTR)dll_node_create();
#endif
if (head)
{ /* Successful Allocation */
#ifdef DLL_CHK
DLL_COUNT(head) = 0;
DLL_SET_HEAD_NODE(head);
#endif
head->data.type = DLL_HEAD_NODE;
head->next = head->previous = head;
}
return(head);
}
static void dll_disconnect_node(DLL_NODE_PTR node)
{
FF_VALIDATE(node);
#ifdef DLL_CHK
assert(!DLL_IS_HEAD_NODE(node));
assert(!DLL_IS_FREED(node));
DLL_MARK_FREED(node);
DLL_COUNT(find_head_node(node))--;
#endif
node->previous->next = dll_next(node); /* set next_node of prev_node */
node->next->previous = dll_previous(node); /* Set prev_node of next_node */
node->previous = node->next = NULL;
#ifdef FF_CHK_ADDR
node->check_address = NULL;
#endif
}
int reader_libpcapfile_next()
{
char errbuf[1024];
gchar *fullfilename;
pcap = 0;
if (config.pcapReadFiles) {
static int pcapFilePos = 0;
fullfilename = config.pcapReadFiles[pcapFilePos];
errbuf[0] = 0;
if (!fullfilename) {
goto filesDone;
}
pcapFilePos++;
LOG ("Processing %s", fullfilename);
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
return reader_libpcapfile_next();
}
if (!realpath(fullfilename, offlinePcapFilename)) {
LOG("ERROR - pcap open failed - Couldn't realpath file: '%s' with %d", fullfilename, errno);
exit(1);
}
reader_libpcapfile_opened();
return 1;
}
filesDone:
if (config.pcapReadDirs) {
static int pcapDirPos = 0;
static GDir *pcapGDir[21];
static char *pcapBase[21];
static int pcapGDirLevel = -1;
GError *error = 0;
if (pcapGDirLevel == -2) {
goto dirsDone;
}
if (pcapGDirLevel == -1) {
pcapGDirLevel = 0;
pcapBase[0] = config.pcapReadDirs[pcapDirPos];
if (!pcapBase[0]) {
pcapGDirLevel = -2;
goto dirsDone;
}
}
if (!pcapGDir[pcapGDirLevel]) {
pcapGDir[pcapGDirLevel] = g_dir_open(pcapBase[pcapGDirLevel], 0, &error);
if (error) {
LOG("ERROR: Couldn't open pcap directory: Receive Error: %s", error->message);
exit(0);
}
}
const gchar *filename;
while (1) {
filename = g_dir_read_name(pcapGDir[pcapGDirLevel]);
// No more files, stop processing this directory
if (!filename) {
break;
}
// Skip hidden files/directories
if (filename[0] == '.')
continue;
fullfilename = g_build_filename (pcapBase[pcapGDirLevel], filename, NULL);
// If recursive option and a directory then process all the files in that dir
if (config.pcapRecursive && g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
if (pcapGDirLevel >= 20)
continue;
pcapBase[pcapGDirLevel+1] = fullfilename;
pcapGDirLevel++;
return reader_libpcapfile_next();
}
if (!g_regex_match(config.offlineRegex, filename, 0, NULL)) {
g_free(fullfilename);
continue;
}
if (!realpath(fullfilename, offlinePcapFilename)) {
g_free(fullfilename);
continue;
}
if (config.pcapSkip && moloch_db_file_exists(offlinePcapFilename)) {
if (config.debug)
LOG("Skipping %s", fullfilename);
g_free(fullfilename);
continue;
}
LOG ("Processing %s", fullfilename);
errbuf[0] = 0;
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
g_free(fullfilename);
continue;
}
reader_libpcapfile_opened();
g_free(fullfilename);
return 1;
}
g_dir_close(pcapGDir[pcapGDirLevel]);
pcapGDir[pcapGDirLevel] = 0;
if (pcapGDirLevel > 0) {
g_free(pcapBase[pcapGDirLevel]);
pcapGDirLevel--;
return reader_libpcapfile_next();
} else {
pcapDirPos++;
pcapGDirLevel = -1;
return reader_libpcapfile_next();
}
}
dirsDone:
while (DLL_COUNT(s_, &monitorQ) > 0) {
MolochString_t *string;
DLL_POP_HEAD(s_, &monitorQ, string);
fullfilename = string->str;
MOLOCH_TYPE_FREE(MolochString_t, string);
if (!realpath(fullfilename, offlinePcapFilename)) {
g_free(fullfilename);
continue;
}
if (config.pcapSkip && moloch_db_file_exists(offlinePcapFilename)) {
if (config.debug)
LOG("Skipping %s", fullfilename);
g_free(fullfilename);
continue;
}
LOG ("Processing %s", fullfilename);
errbuf[0] = 0;
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
g_free(fullfilename);
continue;
}
reader_libpcapfile_opened();
g_free(fullfilename);
return 1;
}
return 0;
}
LOCAL int reader_libpcapfile_next()
{
gchar *fullfilename;
pcap = 0;
if (config.pcapReadFiles) {
static int pcapFilePos = 0;
fullfilename = config.pcapReadFiles[pcapFilePos];
if (!fullfilename) {
goto filesDone;
}
pcapFilePos++;
if (reader_libpcapfile_process(fullfilename)) {
return reader_libpcapfile_next();
}
return 1;
}
filesDone:
if (config.pcapFileLists) {
static int pcapFileListsPos;
static FILE *file;
char line[PATH_MAX];
if (!file && !config.pcapFileLists[pcapFileListsPos]) {
goto fileListsDone;
}
if (!file) {
if (strcmp(config.pcapFileLists[pcapFileListsPos], "-") == 0)
file = stdin;
else
file = fopen(config.pcapFileLists[pcapFileListsPos], "r");
pcapFileListsPos++;
if (!file) {
LOG("ERROR - Couldn't open %s", config.pcapFileLists[pcapFileListsPos - 1]);
return reader_libpcapfile_next();
}
}
if (feof(file)) {
fclose(file);
file = NULL;
return reader_libpcapfile_next();
}
if (!fgets(line, sizeof(line), file)) {
fclose(file);
file = NULL;
return reader_libpcapfile_next();
}
int lineLen = strlen(line);
if (line[lineLen-1] == '\n') {
line[lineLen-1] = 0;
}
g_strstrip(line);
if (!line[0] || line[0] == '#')
return reader_libpcapfile_next();
if (reader_libpcapfile_process(line)) {
return reader_libpcapfile_next();
}
return 1;
}
fileListsDone:
if (config.pcapReadDirs) {
static int pcapDirPos = 0;
static GDir *pcapGDir[21];
static char *pcapBase[21];
static int pcapGDirLevel = -1;
GError *error = 0;
if (pcapGDirLevel == -2) {
goto dirsDone;
}
if (pcapGDirLevel == -1) {
pcapGDirLevel = 0;
pcapBase[0] = config.pcapReadDirs[pcapDirPos];
if (!pcapBase[0]) {
pcapGDirLevel = -2;
goto dirsDone;
}
}
if (!pcapGDir[pcapGDirLevel]) {
pcapGDir[pcapGDirLevel] = g_dir_open(pcapBase[pcapGDirLevel], 0, &error);
if (error) {
LOGEXIT("ERROR: Couldn't open pcap directory: Receive Error: %s", error->message);
}
}
while (1) {
const gchar *filename = g_dir_read_name(pcapGDir[pcapGDirLevel]);
// No more files, stop processing this directory
if (!filename) {
break;
}
// Skip hidden files/directories
if (filename[0] == '.')
continue;
fullfilename = g_build_filename (pcapBase[pcapGDirLevel], filename, NULL);
// If recursive option and a directory then process all the files in that dir
if (config.pcapRecursive && g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
if (pcapGDirLevel >= 20)
continue;
pcapBase[pcapGDirLevel+1] = fullfilename;
pcapGDirLevel++;
return reader_libpcapfile_next();
}
if (!g_regex_match(config.offlineRegex, filename, 0, NULL)) {
g_free(fullfilename);
continue;
}
if (reader_libpcapfile_process(fullfilename)) {
g_free(fullfilename);
continue;
}
g_free(fullfilename);
return 1;
}
g_dir_close(pcapGDir[pcapGDirLevel]);
pcapGDir[pcapGDirLevel] = 0;
if (pcapGDirLevel > 0) {
g_free(pcapBase[pcapGDirLevel]);
pcapGDirLevel--;
return reader_libpcapfile_next();
} else {
pcapDirPos++;
pcapGDirLevel = -1;
return reader_libpcapfile_next();
}
}
dirsDone:
while (DLL_COUNT(s_, &monitorQ) > 0) {
MolochString_t *string;
DLL_POP_HEAD(s_, &monitorQ, string);
fullfilename = string->str;
MOLOCH_TYPE_FREE(MolochString_t, string);
if (reader_libpcapfile_process(fullfilename)) {
g_free(fullfilename);
continue;
}
g_free(fullfilename);
return 1;
}
return 0;
}
uint32_t writer_simple_queue_length()
{
return DLL_COUNT(simple_, &simpleQ);
}
static DLL_NODE_PTR dll_node_create(void) #endif /***************************************************************************** * NAME: dll_node_create() * * PURPOSE: Allocate a memory block for a new DLL_NODE, plus dsize for data * * USAGE: node = dll_node_create(); * * RETURNS: NULL if operation fails, otherwise a pointer to an allocated * node/data block * * DESCRIPTION: Allocates space for DLL_NODE. * * AUTHOR: Mark Ohrenschall, NGDC, (303) 497-6124, [email protected] * * SYSTEM DEPENDENT FUNCTIONS: * * GLOBALS: * * COMMENTS: * * KEYWORDS: * * ERRORS: ****************************************************************************/ { DLL_NODE_PTR node; /* Get memory for data element and set dll_data(node) to point to it */ node = (DLL_NODE_PTR)memMalloc(sizeof(DLL_NODE), "List Node"); if (node == NULL) { err_push(ERR_MEM_LACK, "Allocating list node"); return(NULL); } #ifdef FF_CHK_ADDR node->check_address = node; #endif #ifdef DLL_CHK DLL_SET_NEW(node); #endif node->next = node->previous = NULL; #ifdef DLL_CHK node->count = node->data.type = 0; #endif node->data.u.var = NULL; /* Find header */ #ifdef DLL_CHK if (link_node) DLL_COUNT(find_head_node(link_node))++; #endif return(node); }
void moloch_session_process_commands(int thread)
{
// Commands
MolochSesCmd_t *cmd = 0;
int count;
for (count = 0; count < 50; count++) {
MOLOCH_LOCK(sessionCmds[thread].lock);
DLL_POP_HEAD(cmd_, &sessionCmds[thread], cmd);
MOLOCH_UNLOCK(sessionCmds[thread].lock);
if (!cmd)
break;
switch (cmd->cmd) {
case MOLOCH_SES_CMD_ADD_TAG:
moloch_field_int_add((long)cmd->uw1, cmd->session, (long)cmd->uw2);
moloch_session_decr_outstanding(cmd->session);
break;
case MOLOCH_SES_CMD_FUNC:
cmd->func(cmd->session, cmd->uw1, cmd->uw2);
break;
default:
LOG ("Unknown cmd %d", cmd->cmd);
}
MOLOCH_TYPE_FREE(MolochSesCmd_t, cmd);
}
// Closing Q
for (count = 0; count < 10; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(q_, &closingQ[thread]);
if (session && session->saveTime < (uint64_t)lastPacketSecs[thread]) {
moloch_session_save(session);
} else {
break;
}
}
// Sessions Idle Long Time
int ses;
for (ses = 0; ses < SESSION_MAX; ses++) {
for (count = 0; count < 10; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(q_, &sessionsQ[thread][ses]);
if (session && (DLL_COUNT(q_, &sessionsQ[thread][ses]) > (int)config.maxStreams ||
((uint64_t)session->lastPacket.tv_sec + config.timeouts[ses] < (uint64_t)lastPacketSecs[thread]))) {
moloch_session_save(session);
} else {
break;
}
}
}
// TCP Sessions Open Long Time
for (count = 0; count < 50; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(tcp_, &tcpWriteQ[thread]);
if (session && (uint64_t)session->saveTime < (uint64_t)lastPacketSecs[thread]) {
moloch_session_mid_save(session, lastPacketSecs[thread]);
} else {
break;
}
}
}
int moloch_session_thread_outstanding(int thread)
{
return DLL_COUNT(q_, &closingQ[thread]) + DLL_COUNT(cmd_, &sessionCmds[thread]);
}
