void moloch_config_load()
{
gboolean status;
GError *error = 0;
GKeyFile *keyfile;
int i;
keyfile = molochKeyFile = g_key_file_new();
status = g_key_file_load_from_file(keyfile, config.configFile, G_KEY_FILE_NONE, &error);
if (!status || error) {
printf("Couldn't load config file (%s) %s\n", config.configFile, (error?error->message:""));
exit(1);
}
char **includes = moloch_config_str_list(keyfile, "includes", NULL);
if (includes) {
moloch_config_load_includes(includes);
g_strfreev(includes);
//LOG("KEYFILE:\n%s", g_key_file_to_data(molochKeyFile, NULL, NULL));
}
char *rotateIndex = moloch_config_str(keyfile, "rotateIndex", "daily");
if (strcmp(rotateIndex, "hourly") == 0)
config.rotate = MOLOCH_ROTATE_HOURLY;
else if (strcmp(rotateIndex, "daily") == 0)
config.rotate = MOLOCH_ROTATE_DAILY;
else if (strcmp(rotateIndex, "weekly") == 0)
config.rotate = MOLOCH_ROTATE_WEEKLY;
else if (strcmp(rotateIndex, "monthly") == 0)
config.rotate = MOLOCH_ROTATE_MONTHLY;
else {
printf("Unknown rotateIndex '%s'\n", rotateIndex);
exit(1);
}
g_free(rotateIndex);
config.nodeClass = moloch_config_str(keyfile, "nodeClass", NULL);
gchar **tags = moloch_config_str_list(keyfile, "dontSaveTags", NULL);
if (tags) {
for (i = 0; tags[i]; i++) {
if (!(*tags[i]))
continue;
int num = 1;
char *colon = strchr(tags[i], ':');
if (colon) {
*colon = 0;
num = atoi(colon+1);
if (num < 1)
num = 1;
if (num > 0xffff)
num = 0xffff;
}
moloch_string_add((MolochStringHash_t *)(char*)&config.dontSaveTags, tags[i], (gpointer)(long)num, TRUE);
}
g_strfreev(tags);
}
config.dontSaveBPFs = moloch_config_str_list(keyfile, "dontSaveBPFs", NULL);
if (config.dontSaveBPFs) {
for (i = 0; config.dontSaveBPFs[i]; i++);
config.dontSaveBPFsNum = i;
config.dontSaveBPFsStop = malloc(config.dontSaveBPFsNum*sizeof(int));
GRegex *regex = g_regex_new(":\\s*(\\d+)\\s*$", 0, 0, 0);
GMatchInfo *match_info;
for (i = 0; config.dontSaveBPFs[i]; i++) {
g_regex_match(regex, config.dontSaveBPFs[i], 0, &match_info);
if (g_match_info_matches(match_info)) {
config.dontSaveBPFsStop[i] = atoi(g_match_info_fetch(match_info, 1));
gint pos;
g_match_info_fetch_pos(match_info, 0, &pos, NULL);
config.dontSaveBPFs[i][pos] = 0;
} else {
config.dontSaveBPFsStop[i] = 1;
}
g_match_info_free(match_info);
}
g_regex_unref(regex);
}
config.plugins = moloch_config_str_list(keyfile, "plugins", NULL);
config.smtpIpHeaders = moloch_config_str_list(keyfile, "smtpIpHeaders", NULL);
if (config.smtpIpHeaders) {
for (i = 0; config.smtpIpHeaders[i]; i++) {
int len = strlen(config.smtpIpHeaders[i]);
char *lower = g_ascii_strdown(config.smtpIpHeaders[i], len);
g_free(config.smtpIpHeaders[i]);
config.smtpIpHeaders[i] = lower;
if (lower[len-1] == ':')
lower[len-1] = 0;
}
}
config.prefix = moloch_config_str(keyfile, "prefix", "");
int len = strlen(config.prefix);
if (len > 0 && config.prefix[len - 1] != '_') {
char *tmp = malloc(len + 2);
memcpy(tmp, config.prefix, len);
tmp[len] = '_';
tmp[len+1] = 0;
g_free(config.prefix);
config.prefix = tmp;
}
config.elasticsearch = moloch_config_str(keyfile, "elasticsearch", "localhost:9200");
config.interface = moloch_config_str(keyfile, "interface", NULL);
config.pcapDir = moloch_config_str_list(keyfile, "pcapDir", NULL);
config.bpf = moloch_config_str(keyfile, "bpf", NULL);
config.yara = moloch_config_str(keyfile, "yara", NULL);
config.emailYara = moloch_config_str(keyfile, "emailYara", NULL);
config.geoipFile = moloch_config_str(keyfile, "geoipFile", NULL);
config.rirFile = moloch_config_str(keyfile, "rirFile", NULL);
config.geoipASNFile = moloch_config_str(keyfile, "geoipASNFile", NULL);
config.dropUser = moloch_config_str(keyfile, "dropUser", NULL);
config.dropGroup = moloch_config_str(keyfile, "dropGroup", NULL);
config.pluginsDir = moloch_config_str_list(keyfile, "pluginsDir", NULL);
config.parsersDir = moloch_config_str_list(keyfile, "parsersDir", " /data/moloch/parsers ; ./parsers ");
char *offlineRegex = moloch_config_str(keyfile, "offlineFilenameRegex", "(?i)\\.(pcap|cap)$");
config.offlineRegex = g_regex_new(offlineRegex, 0, 0, &error);
if (!config.offlineRegex || error) {
printf("Couldn't parse offlineRegex (%s) %s\n", offlineRegex, (error?error->message:""));
exit(1);
}
config.maxFileSizeG = moloch_config_double(keyfile, "maxFileSizeG", 4, 0.01, 1024);
config.maxFileSizeB = config.maxFileSizeG*1024LL*1024LL*1024LL;
config.maxFileTimeM = moloch_config_int(keyfile, "maxFileTimeM", 0, 0, 0xffff);
config.icmpTimeout = moloch_config_int(keyfile, "icmpTimeout", 10, 1, 0xffff);
config.udpTimeout = moloch_config_int(keyfile, "udpTimeout", 60, 1, 0xffff);
config.tcpTimeout = moloch_config_int(keyfile, "tcpTimeout", 60*8, 10, 0xffff);
config.tcpSaveTimeout = moloch_config_int(keyfile, "tcpSaveTimeout", 60*8, 10, 60*120);
config.maxStreams = moloch_config_int(keyfile, "maxStreams", 1500000, 1, 16777215);
config.maxPackets = moloch_config_int(keyfile, "maxPackets", 10000, 1, 1000000);
config.minFreeSpaceG = moloch_config_int(keyfile, "freeSpaceG", 100, 1, 100000);
config.dbBulkSize = moloch_config_int(keyfile, "dbBulkSize", 200000, MOLOCH_HTTP_BUFFER_SIZE*2, 1000000);
config.dbFlushTimeout = moloch_config_int(keyfile, "dbFlushTimeout", 5, 1, 60*30);
config.maxESConns = moloch_config_int(keyfile, "maxESConns", 20, 5, 1000);
config.maxESRequests = moloch_config_int(keyfile, "maxESRequests", 500, 10, 5000);
config.logEveryXPackets = moloch_config_int(keyfile, "logEveryXPackets", 50000, 1000, 1000000);
config.packetsPerPoll = moloch_config_int(keyfile, "packetsPerPoll", 50000, 1000, 1000000);
config.pcapBufferSize = moloch_config_int(keyfile, "pcapBufferSize", 300000000, 100000, 0xffffffff);
config.pcapWriteSize = moloch_config_int(keyfile, "pcapWriteSize", 0x40000, 0x40000, 0x800000);
config.maxFreeOutputBuffers = moloch_config_int(keyfile, "maxFreeOutputBuffers", 50, 0, 0xffff);
config.logUnknownProtocols = moloch_config_boolean(keyfile, "logUnknownProtocols", config.debug);
config.logESRequests = moloch_config_boolean(keyfile, "logESRequests", config.debug);
config.logFileCreation = moloch_config_boolean(keyfile, "logFileCreation", config.debug);
config.parseSMTP = moloch_config_boolean(keyfile, "parseSMTP", TRUE);
config.parseSMB = moloch_config_boolean(keyfile, "parseSMB", TRUE);
config.parseQSValue = moloch_config_boolean(keyfile, "parseQSValue", FALSE);
config.parseCookieValue = moloch_config_boolean(keyfile, "parseCookieValue", FALSE);
config.compressES = moloch_config_boolean(keyfile, "compressES", FALSE);
config.antiSynDrop = moloch_config_boolean(keyfile, "antiSynDrop", TRUE);
}
void moloch_config_load()
{
gboolean status;
GError *error = 0;
GKeyFile *keyfile;
int i;
keyfile = molochKeyFile = g_key_file_new();
status = g_key_file_load_from_file(keyfile, config.configFile, G_KEY_FILE_NONE, &error);
if (!status || error) {
printf("Couldn't load config file (%s) %s\n", config.configFile, (error?error->message:""));
exit(1);
}
char **includes = moloch_config_str_list(keyfile, "includes", NULL);
if (includes) {
moloch_config_load_includes(includes);
g_strfreev(includes);
//LOG("KEYFILE:\n%s", g_key_file_to_data(molochKeyFile, NULL, NULL));
}
char *rotateIndex = moloch_config_str(keyfile, "rotateIndex", "daily");
if (strcmp(rotateIndex, "hourly") == 0)
config.rotate = MOLOCH_ROTATE_HOURLY;
else if (strcmp(rotateIndex, "hourly6") == 0)
config.rotate = MOLOCH_ROTATE_HOURLY6;
else if (strcmp(rotateIndex, "daily") == 0)
config.rotate = MOLOCH_ROTATE_DAILY;
else if (strcmp(rotateIndex, "weekly") == 0)
config.rotate = MOLOCH_ROTATE_WEEKLY;
else if (strcmp(rotateIndex, "monthly") == 0)
config.rotate = MOLOCH_ROTATE_MONTHLY;
else {
printf("Unknown rotateIndex '%s'\n", rotateIndex);
exit(1);
}
g_free(rotateIndex);
config.nodeClass = moloch_config_str(keyfile, "nodeClass", NULL);
gchar **tags = moloch_config_str_list(keyfile, "dontSaveTags", NULL);
if (tags) {
for (i = 0; tags[i]; i++) {
if (!(*tags[i]))
continue;
int num = 1;
char *colon = strchr(tags[i], ':');
if (colon) {
*colon = 0;
num = atoi(colon+1);
if (num < 1)
num = 1;
if (num > 0xffff)
num = 0xffff;
}
moloch_string_add((MolochStringHash_t *)(char*)&config.dontSaveTags, tags[i], (gpointer)(long)num, TRUE);
}
g_strfreev(tags);
}
config.plugins = moloch_config_str_list(keyfile, "plugins", NULL);
config.rootPlugins = moloch_config_str_list(keyfile, "rootPlugins", NULL);
config.smtpIpHeaders = moloch_config_str_list(keyfile, "smtpIpHeaders", NULL);
if (config.smtpIpHeaders) {
for (i = 0; config.smtpIpHeaders[i]; i++) {
int len = strlen(config.smtpIpHeaders[i]);
char *lower = g_ascii_strdown(config.smtpIpHeaders[i], len);
g_free(config.smtpIpHeaders[i]);
config.smtpIpHeaders[i] = lower;
if (lower[len-1] == ':')
lower[len-1] = 0;
}
}
config.prefix = moloch_config_str(keyfile, "prefix", "");
int len = strlen(config.prefix);
if (len > 0 && config.prefix[len - 1] != '_') {
char *tmp = malloc(len + 2);
memcpy(tmp, config.prefix, len);
tmp[len] = '_';
tmp[len+1] = 0;
g_free(config.prefix);
config.prefix = tmp;
}
config.elasticsearch = moloch_config_str(keyfile, "elasticsearch", "localhost:9200");
config.interface = moloch_config_str_list(keyfile, "interface", NULL);
config.pcapDir = moloch_config_str_list(keyfile, "pcapDir", NULL);
config.bpf = moloch_config_str(keyfile, "bpf", NULL);
config.yara = moloch_config_str(keyfile, "yara", NULL);
config.emailYara = moloch_config_str(keyfile, "emailYara", NULL);
config.rirFile = moloch_config_str(keyfile, "rirFile", NULL);
config.ouiFile = moloch_config_str(keyfile, "ouiFile", NULL);
config.geoLite2ASN = moloch_config_str(keyfile, "geoLite2ASN", "/data/moloch/etc/GeoLite2-ASN.mmdb");
config.geoLite2Country = moloch_config_str(keyfile, "geoLite2Country", "/data/moloch/etc/GeoLite2-Country.mmdb");
config.dropUser = moloch_config_str(keyfile, "dropUser", NULL);
config.dropGroup = moloch_config_str(keyfile, "dropGroup", NULL);
config.pluginsDir = moloch_config_str_list(keyfile, "pluginsDir", NULL);
config.parsersDir = moloch_config_str_list(keyfile, "parsersDir", " /data/moloch/parsers ; ./parsers ");
char *offlineRegex = moloch_config_str(keyfile, "offlineFilenameRegex", "(?i)\\.(pcap|cap)$");
config.offlineRegex = g_regex_new(offlineRegex, 0, 0, &error);
if (!config.offlineRegex || error) {
printf("Couldn't parse offlineRegex (%s) %s\n", offlineRegex, (error?error->message:""));
exit(1);
}
g_free(offlineRegex);
config.pcapDirTemplate = moloch_config_str(keyfile, "pcapDirTemplate", NULL);
if (config.pcapDirTemplate && config.pcapDirTemplate[0] != '/') {
printf("pcapDirTemplate MUST start with a / '%s'\n", config.pcapDirTemplate);
exit(1);
}
config.pcapDirAlgorithm = moloch_config_str(keyfile, "pcapDirAlgorithm", "round-robin");
if (strcmp(config.pcapDirAlgorithm, "round-robin") != 0
&& strcmp(config.pcapDirAlgorithm, "max-free-percent") != 0
&& strcmp(config.pcapDirAlgorithm, "max-free-bytes") != 0) {
printf("'%s' is not a valid value for pcapDirAlgorithm. Supported algorithms are round-robin, max-free-percent, and max-free-bytes.\n", config.pcapDirAlgorithm);
exit(1);
}
config.maxFileSizeG = moloch_config_double(keyfile, "maxFileSizeG", 4, 0.01, 1024);
config.maxFileSizeB = config.maxFileSizeG*1024LL*1024LL*1024LL;
config.maxFileTimeM = moloch_config_int(keyfile, "maxFileTimeM", 0, 0, 0xffff);
config.timeouts[SESSION_ICMP]= moloch_config_int(keyfile, "icmpTimeout", 10, 1, 0xffff);
config.timeouts[SESSION_UDP] = moloch_config_int(keyfile, "udpTimeout", 60, 1, 0xffff);
config.timeouts[SESSION_TCP] = moloch_config_int(keyfile, "tcpTimeout", 60*8, 10, 0xffff);
config.timeouts[SESSION_SCTP]= moloch_config_int(keyfile, "sctpTimeout", 60, 10, 0xffff);
config.timeouts[SESSION_ESP] = moloch_config_int(keyfile, "espTimeout", 60*10, 10, 0xffff);
config.tcpSaveTimeout = moloch_config_int(keyfile, "tcpSaveTimeout", 60*8, 10, 60*120);
int maxStreams = moloch_config_int(keyfile, "maxStreams", 1500000, 1, 16777215);
config.maxPackets = moloch_config_int(keyfile, "maxPackets", 10000, 1, 0xffff);
config.maxPacketsInQueue = moloch_config_int(keyfile, "maxPacketsInQueue", 200000, 10000, 5000000);
config.dbBulkSize = moloch_config_int(keyfile, "dbBulkSize", 200000, MOLOCH_HTTP_BUFFER_SIZE*2, 1000000);
config.dbFlushTimeout = moloch_config_int(keyfile, "dbFlushTimeout", 5, 1, 60*30);
config.maxESConns = moloch_config_int(keyfile, "maxESConns", 20, 5, 1000);
config.maxESRequests = moloch_config_int(keyfile, "maxESRequests", 500, 10, 5000);
config.logEveryXPackets = moloch_config_int(keyfile, "logEveryXPackets", 50000, 1000, 0xffffffff);
config.pcapBufferSize = moloch_config_int(keyfile, "pcapBufferSize", 300000000, 100000, 0xffffffff);
config.pcapWriteSize = moloch_config_int(keyfile, "pcapWriteSize", 0x40000, 0x10000, 0x800000);
config.maxFreeOutputBuffers = moloch_config_int(keyfile, "maxFreeOutputBuffers", 50, 0, 0xffff);
config.fragsTimeout = moloch_config_int(keyfile, "fragsTimeout", 60*8, 60, 0xffff);
config.maxFrags = moloch_config_int(keyfile, "maxFrags", 10000, 100, 0xffffff);
config.snapLen = moloch_config_int(keyfile, "snapLen", 16384, 1, MOLOCH_PACKET_MAX_LEN);
config.maxMemPercentage = moloch_config_int(keyfile, "maxMemPercentage", 100, 5, 100);
config.maxReqBody = moloch_config_int(keyfile, "maxReqBody", 256, 0, 0x7fff);
config.packetThreads = moloch_config_int(keyfile, "packetThreads", 1, 1, MOLOCH_MAX_PACKET_THREADS);
config.logUnknownProtocols = moloch_config_boolean(keyfile, "logUnknownProtocols", config.debug);
config.logESRequests = moloch_config_boolean(keyfile, "logESRequests", config.debug);
config.logFileCreation = moloch_config_boolean(keyfile, "logFileCreation", config.debug);
config.logHTTPConnections = moloch_config_boolean(keyfile, "logHTTPConnections", TRUE);
config.parseSMTP = moloch_config_boolean(keyfile, "parseSMTP", TRUE);
config.parseSMB = moloch_config_boolean(keyfile, "parseSMB", TRUE);
config.parseQSValue = moloch_config_boolean(keyfile, "parseQSValue", FALSE);
config.parseCookieValue = moloch_config_boolean(keyfile, "parseCookieValue", FALSE);
config.supportSha256 = moloch_config_boolean(keyfile, "supportSha256", FALSE);
config.reqBodyOnlyUtf8 = moloch_config_boolean(keyfile, "reqBodyOnlyUtf8", TRUE);
config.compressES = moloch_config_boolean(keyfile, "compressES", FALSE);
config.antiSynDrop = moloch_config_boolean(keyfile, "antiSynDrop", TRUE);
config.readTruncatedPackets = moloch_config_boolean(keyfile, "readTruncatedPackets", FALSE);
config.trackESP = moloch_config_boolean(keyfile, "trackESP", FALSE);
config.maxStreams[SESSION_TCP] = maxStreams/config.packetThreads*1.25;
config.maxStreams[SESSION_UDP] = maxStreams/config.packetThreads/20;
config.maxStreams[SESSION_SCTP] = maxStreams/config.packetThreads/20;
config.maxStreams[SESSION_ICMP] = maxStreams/config.packetThreads/200;
config.maxStreams[SESSION_ESP] = maxStreams/config.packetThreads/200;
gchar **saveUnknownPackets = moloch_config_str_list(keyfile, "saveUnknownPackets", NULL);
if (saveUnknownPackets) {
for (i = 0; saveUnknownPackets[i]; i++) {
char *s = saveUnknownPackets[i];
if (strcmp(s, "all") == 0) {
memset(&config.etherSavePcap, 0xff, 1024);
memset(&config.ipSavePcap, 0xff, 4);
} else if (strcmp(s, "ip:all") == 0) {
memset(&config.ipSavePcap, 0xff, 4);
} else if (strcmp(s, "ether:all") == 0) {
memset(&config.etherSavePcap, 0xff, 1024);
} else if (strncmp(s, "ip:", 3) == 0) {
int n = atoi(s+3);
if (n < 0 || n > 0xff)
LOGEXIT("Bad value: %s", s);
BIT_SET(n, config.ipSavePcap);
} else if (strncmp(s, "-ip:", 4) == 0) {
int n = atoi(s+4);
if (n < 0 || n > 0xff)
LOGEXIT("Bad value: %s", s);
BIT_CLR(n, config.ipSavePcap);
} else if (strncmp(s, "ether:", 6) == 0) {
int n = atoi(s+6);
if (n < 0 || n > 0xffff)
LOGEXIT("Bad value: %s", s);
BIT_SET(n, config.etherSavePcap);
} else if (strncmp(s, "-ether:", 7) == 0) {
int n = atoi(s+7);
if (n < 0 || n > 0xffff)
LOGEXIT("Bad value: %s", s);
BIT_CLR(n, config.etherSavePcap);
} else {
LOGEXIT("Not sure what %s is", s);
}
}
}
}