void reader_libpcapfile_start() {
reader_libpcapfile_next();
if (!pcap) {
if (config.pcapMonitor) {
g_timeout_add(100, reader_libpcapfile_monitor_gfunc, 0);
} else {
moloch_quit();
}
}
}
gboolean reader_libpcapfile_monitor_gfunc (gpointer UNUSED(user_data))
{
if (DLL_COUNT(s_, &monitorQ) == 0)
return TRUE;
if (reader_libpcapfile_next()) {
return FALSE;
}
return TRUE;
}
LOCAL gboolean reader_libpcapfile_read()
{
// pause reading if too many waiting disk operations
if (moloch_writer_queue_length() > 10) {
return TRUE;
}
// pause reading if too many waiting ES operations
if (moloch_http_queue_length(esServer) > 50) {
return TRUE;
}
// pause reading if too many packets are waiting to be processed
if (moloch_packet_outstanding() > (int32_t)(config.maxPacketsInQueue/3)) {
return TRUE;
}
int r;
if (pktsToRead > 0) {
r = pcap_dispatch(pcap, MIN(pktsToRead, 5000), reader_libpcapfile_pcap_cb, NULL);
if (r > 0)
pktsToRead -= r;
if (pktsToRead == 0)
r = 0;
} else {
r = pcap_dispatch(pcap, 5000, reader_libpcapfile_pcap_cb, NULL);
}
moloch_packet_batch_flush(&batch);
// Some kind of failure, move to the next file or quit
if (r <= 0) {
if (config.pcapDelete && r == 0) {
if (config.debug)
LOG("Deleting %s", offlinePcapFilename);
int rc = unlink(offlinePcapFilename);
if (rc != 0)
LOG("Failed to delete file %s %s (%d)", offlinePcapFilename, strerror(errno), errno);
}
pcap_close(pcap);
if (reader_libpcapfile_next()) {
return FALSE;
}
if (config.pcapMonitor)
g_timeout_add(100, reader_libpcapfile_monitor_gfunc, 0);
else
moloch_quit();
return FALSE;
}
return TRUE;
}
int reader_libpcapfile_next()
{
char errbuf[1024];
gchar *fullfilename;
pcap = 0;
if (config.pcapReadFiles) {
static int pcapFilePos = 0;
fullfilename = config.pcapReadFiles[pcapFilePos];
errbuf[0] = 0;
if (!fullfilename) {
goto filesDone;
}
pcapFilePos++;
LOG ("Processing %s", fullfilename);
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
return reader_libpcapfile_next();
}
if (!realpath(fullfilename, offlinePcapFilename)) {
LOG("ERROR - pcap open failed - Couldn't realpath file: '%s' with %d", fullfilename, errno);
exit(1);
}
reader_libpcapfile_opened();
return 1;
}
filesDone:
if (config.pcapReadDirs) {
static int pcapDirPos = 0;
static GDir *pcapGDir[21];
static char *pcapBase[21];
static int pcapGDirLevel = -1;
GError *error = 0;
if (pcapGDirLevel == -2) {
goto dirsDone;
}
if (pcapGDirLevel == -1) {
pcapGDirLevel = 0;
pcapBase[0] = config.pcapReadDirs[pcapDirPos];
if (!pcapBase[0]) {
pcapGDirLevel = -2;
goto dirsDone;
}
}
if (!pcapGDir[pcapGDirLevel]) {
pcapGDir[pcapGDirLevel] = g_dir_open(pcapBase[pcapGDirLevel], 0, &error);
if (error) {
LOG("ERROR: Couldn't open pcap directory: Receive Error: %s", error->message);
exit(0);
}
}
const gchar *filename;
while (1) {
filename = g_dir_read_name(pcapGDir[pcapGDirLevel]);
// No more files, stop processing this directory
if (!filename) {
break;
}
// Skip hidden files/directories
if (filename[0] == '.')
continue;
fullfilename = g_build_filename (pcapBase[pcapGDirLevel], filename, NULL);
// If recursive option and a directory then process all the files in that dir
if (config.pcapRecursive && g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
if (pcapGDirLevel >= 20)
continue;
pcapBase[pcapGDirLevel+1] = fullfilename;
pcapGDirLevel++;
return reader_libpcapfile_next();
}
if (!g_regex_match(config.offlineRegex, filename, 0, NULL)) {
g_free(fullfilename);
continue;
}
if (!realpath(fullfilename, offlinePcapFilename)) {
g_free(fullfilename);
continue;
}
if (config.pcapSkip && moloch_db_file_exists(offlinePcapFilename)) {
if (config.debug)
LOG("Skipping %s", fullfilename);
g_free(fullfilename);
continue;
}
LOG ("Processing %s", fullfilename);
errbuf[0] = 0;
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
g_free(fullfilename);
continue;
}
reader_libpcapfile_opened();
g_free(fullfilename);
return 1;
}
g_dir_close(pcapGDir[pcapGDirLevel]);
pcapGDir[pcapGDirLevel] = 0;
if (pcapGDirLevel > 0) {
g_free(pcapBase[pcapGDirLevel]);
pcapGDirLevel--;
return reader_libpcapfile_next();
} else {
pcapDirPos++;
pcapGDirLevel = -1;
return reader_libpcapfile_next();
}
}
dirsDone:
while (DLL_COUNT(s_, &monitorQ) > 0) {
MolochString_t *string;
DLL_POP_HEAD(s_, &monitorQ, string);
fullfilename = string->str;
MOLOCH_TYPE_FREE(MolochString_t, string);
if (!realpath(fullfilename, offlinePcapFilename)) {
g_free(fullfilename);
continue;
}
if (config.pcapSkip && moloch_db_file_exists(offlinePcapFilename)) {
if (config.debug)
LOG("Skipping %s", fullfilename);
g_free(fullfilename);
continue;
}
LOG ("Processing %s", fullfilename);
errbuf[0] = 0;
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
g_free(fullfilename);
continue;
}
reader_libpcapfile_opened();
g_free(fullfilename);
return 1;
}
return 0;
}
LOCAL void reader_libpcapfile_start() {
// Compile all the filename ops. The formation is fieldexpr=value%value
// value is expanded using the g_regex_replace rules (\1 being the first capture group)
// https://developer.gnome.org/glib/stable/glib-Perl-compatible-regular-expressions.html#g-regex-replace
char **filenameOpsStr;
filenameOpsStr = moloch_config_str_list(NULL, "filenameOps", "");
int i;
for (i = 0; filenameOpsStr && filenameOpsStr[i] && i < 100; i++) {
if (!filenameOpsStr[i][0])
continue;
char *equal = strchr(filenameOpsStr[i], '=');
if (!equal) {
LOGEXIT("Must be FieldExpr=regex%%value, missing equal '%s'", filenameOpsStr[i]);
}
char *percent = strchr(equal+1, '%');
if (!percent) {
LOGEXIT("Must be FieldExpr=regex%%value, missing percent '%s'", filenameOpsStr[i]);
}
*equal = 0;
*percent = 0;
int elen = strlen(equal+1);
if (!elen) {
LOGEXIT("Must be FieldExpr=regex%%value, empty regex for '%s'", filenameOpsStr[i]);
}
int vlen = strlen(percent+1);
if (!vlen) {
LOGEXIT("Must be FieldExpr=regex%%value, empty value for '%s'", filenameOpsStr[i]);
}
int fieldPos = moloch_field_by_exp(filenameOpsStr[i]);
if (fieldPos == -1) {
LOGEXIT("Must be FieldExpr=regex?value, Unknown field expression '%s'", filenameOpsStr[i]);
}
filenameOps[filenameOpsNum].regex = g_regex_new(equal+1, 0, 0, 0);
filenameOps[filenameOpsNum].expand = g_strdup(percent+1);
if (!filenameOps[filenameOpsNum].regex)
LOGEXIT("Couldn't compile regex '%s'", equal+1);
filenameOps[filenameOpsNum].field = fieldPos;
filenameOpsNum++;
}
g_strfreev(filenameOpsStr);
// Now actually start
reader_libpcapfile_next();
if (!pcap) {
if (config.pcapMonitor) {
g_timeout_add(100, reader_libpcapfile_monitor_gfunc, 0);
} else {
moloch_quit();
}
}
}
LOCAL int reader_libpcapfile_next()
{
gchar *fullfilename;
pcap = 0;
if (config.pcapReadFiles) {
static int pcapFilePos = 0;
fullfilename = config.pcapReadFiles[pcapFilePos];
if (!fullfilename) {
goto filesDone;
}
pcapFilePos++;
if (reader_libpcapfile_process(fullfilename)) {
return reader_libpcapfile_next();
}
return 1;
}
filesDone:
if (config.pcapFileLists) {
static int pcapFileListsPos;
static FILE *file;
char line[PATH_MAX];
if (!file && !config.pcapFileLists[pcapFileListsPos]) {
goto fileListsDone;
}
if (!file) {
if (strcmp(config.pcapFileLists[pcapFileListsPos], "-") == 0)
file = stdin;
else
file = fopen(config.pcapFileLists[pcapFileListsPos], "r");
pcapFileListsPos++;
if (!file) {
LOG("ERROR - Couldn't open %s", config.pcapFileLists[pcapFileListsPos - 1]);
return reader_libpcapfile_next();
}
}
if (feof(file)) {
fclose(file);
file = NULL;
return reader_libpcapfile_next();
}
if (!fgets(line, sizeof(line), file)) {
fclose(file);
file = NULL;
return reader_libpcapfile_next();
}
int lineLen = strlen(line);
if (line[lineLen-1] == '\n') {
line[lineLen-1] = 0;
}
g_strstrip(line);
if (!line[0] || line[0] == '#')
return reader_libpcapfile_next();
if (reader_libpcapfile_process(line)) {
return reader_libpcapfile_next();
}
return 1;
}
fileListsDone:
if (config.pcapReadDirs) {
static int pcapDirPos = 0;
static GDir *pcapGDir[21];
static char *pcapBase[21];
static int pcapGDirLevel = -1;
GError *error = 0;
if (pcapGDirLevel == -2) {
goto dirsDone;
}
if (pcapGDirLevel == -1) {
pcapGDirLevel = 0;
pcapBase[0] = config.pcapReadDirs[pcapDirPos];
if (!pcapBase[0]) {
pcapGDirLevel = -2;
goto dirsDone;
}
}
if (!pcapGDir[pcapGDirLevel]) {
pcapGDir[pcapGDirLevel] = g_dir_open(pcapBase[pcapGDirLevel], 0, &error);
if (error) {
LOGEXIT("ERROR: Couldn't open pcap directory: Receive Error: %s", error->message);
}
}
while (1) {
const gchar *filename = g_dir_read_name(pcapGDir[pcapGDirLevel]);
// No more files, stop processing this directory
if (!filename) {
break;
}
// Skip hidden files/directories
if (filename[0] == '.')
continue;
fullfilename = g_build_filename (pcapBase[pcapGDirLevel], filename, NULL);
// If recursive option and a directory then process all the files in that dir
if (config.pcapRecursive && g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
if (pcapGDirLevel >= 20)
continue;
pcapBase[pcapGDirLevel+1] = fullfilename;
pcapGDirLevel++;
return reader_libpcapfile_next();
}
if (!g_regex_match(config.offlineRegex, filename, 0, NULL)) {
g_free(fullfilename);
continue;
}
if (reader_libpcapfile_process(fullfilename)) {
g_free(fullfilename);
continue;
}
g_free(fullfilename);
return 1;
}
g_dir_close(pcapGDir[pcapGDirLevel]);
pcapGDir[pcapGDirLevel] = 0;
if (pcapGDirLevel > 0) {
g_free(pcapBase[pcapGDirLevel]);
pcapGDirLevel--;
return reader_libpcapfile_next();
} else {
pcapDirPos++;
pcapGDirLevel = -1;
return reader_libpcapfile_next();
}
}
dirsDone:
while (DLL_COUNT(s_, &monitorQ) > 0) {
MolochString_t *string;
DLL_POP_HEAD(s_, &monitorQ, string);
fullfilename = string->str;
MOLOCH_TYPE_FREE(MolochString_t, string);
if (reader_libpcapfile_process(fullfilename)) {
g_free(fullfilename);
continue;
}
g_free(fullfilename);
return 1;
}
return 0;
}
