void moloch_rules_load_complete()
{
char **bpfs;
GRegex *regex = g_regex_new(":\\s*(\\d+)\\s*$", 0, 0, 0);
int i;
bpfs = moloch_config_str_list(NULL, "dontSaveBPFs", NULL);
int pos = moloch_field_by_exp("_maxPacketsToSave");
gint start_pos;
if (bpfs) {
for (i = 0; bpfs[i]; i++) {
int n = loading.rulesLen[MOLOCH_RULE_TYPE_SESSION_SETUP]++;
MolochRule_t *rule = loading.rules[MOLOCH_RULE_TYPE_SESSION_SETUP][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t);
rule->filename = "dontSaveBPFs";
moloch_field_ops_init(&rule->ops, 1, MOLOCH_FIELD_OPS_FLAGS_COPY);
GMatchInfo *match_info = 0;
g_regex_match(regex, bpfs[i], 0, &match_info);
if (g_match_info_matches(match_info)) {
g_match_info_fetch_pos (match_info, 1, &start_pos, NULL);
rule->bpf = g_strndup(bpfs[i], start_pos-1);
moloch_field_ops_add(&rule->ops, pos, g_match_info_fetch(match_info, 1), -1);
} else {
rule->bpf = g_strdup(bpfs[i]);
moloch_field_ops_add(&rule->ops, pos, "1", -1);
}
g_match_info_free(match_info);
}
g_strfreev(bpfs);
}
bpfs = moloch_config_str_list(NULL, "minPacketsSaveBPFs", NULL);
pos = moloch_field_by_exp("_minPacketsBeforeSavingSPI");
if (bpfs) {
for (i = 0; bpfs[i]; i++) {
int n = loading.rulesLen[MOLOCH_RULE_TYPE_SESSION_SETUP]++;
MolochRule_t *rule = loading.rules[MOLOCH_RULE_TYPE_SESSION_SETUP][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t);
rule->filename = "minPacketsSaveBPFs";
moloch_field_ops_init(&rule->ops, 1, MOLOCH_FIELD_OPS_FLAGS_COPY);
GMatchInfo *match_info = 0;
g_regex_match(regex, bpfs[i], 0, &match_info);
if (g_match_info_matches(match_info)) {
g_match_info_fetch_pos (match_info, 1, &start_pos, NULL);
rule->bpf = g_strndup(bpfs[i], start_pos-1);
moloch_field_ops_add(&rule->ops, pos, g_match_info_fetch(match_info, 1), -1);
} else {
rule->bpf = g_strdup(bpfs[i]);
moloch_field_ops_add(&rule->ops, pos, "1", -1);
}
g_match_info_free(match_info);
}
g_strfreev(bpfs);
}
g_regex_unref(regex);
memcpy(¤t, &loading, sizeof(loading));
memset(&loading, 0, sizeof(loading));
}
static int MS_add_int(lua_State *L)
{
if (config.debug > 2)
molua_stackDump(L);
if (lua_gettop(L) != 3 || !lua_isuserdata(L, 1) || !(lua_isstring(L, 2) || lua_isinteger(L, 2)) || !lua_isinteger(L, 3)) {
return luaL_error(L, "usage: <session> <field string or field num(faster)> <integer>");
}
MolochSession_t *session = checkMolochSession(L, 1);
int value = lua_tointeger(L, 3);
int pos;
if (lua_isinteger(L, 2)) {
pos = lua_tointeger(L, 2);
} else {
pos = moloch_field_by_exp(lua_tostring(L, 2));
}
gboolean result;
result = moloch_field_int_add(pos, session, value);
lua_pushboolean(L, result);
return 1;
}
LOCAL void reader_libpcapfile_start() {
// Compile all the filename ops. The formation is fieldexpr=value%value
// value is expanded using the g_regex_replace rules (\1 being the first capture group)
// https://developer.gnome.org/glib/stable/glib-Perl-compatible-regular-expressions.html#g-regex-replace
char **filenameOpsStr;
filenameOpsStr = moloch_config_str_list(NULL, "filenameOps", "");
int i;
for (i = 0; filenameOpsStr && filenameOpsStr[i] && i < 100; i++) {
if (!filenameOpsStr[i][0])
continue;
char *equal = strchr(filenameOpsStr[i], '=');
if (!equal) {
LOGEXIT("Must be FieldExpr=regex%%value, missing equal '%s'", filenameOpsStr[i]);
}
char *percent = strchr(equal+1, '%');
if (!percent) {
LOGEXIT("Must be FieldExpr=regex%%value, missing percent '%s'", filenameOpsStr[i]);
}
*equal = 0;
*percent = 0;
int elen = strlen(equal+1);
if (!elen) {
LOGEXIT("Must be FieldExpr=regex%%value, empty regex for '%s'", filenameOpsStr[i]);
}
int vlen = strlen(percent+1);
if (!vlen) {
LOGEXIT("Must be FieldExpr=regex%%value, empty value for '%s'", filenameOpsStr[i]);
}
int fieldPos = moloch_field_by_exp(filenameOpsStr[i]);
if (fieldPos == -1) {
LOGEXIT("Must be FieldExpr=regex?value, Unknown field expression '%s'", filenameOpsStr[i]);
}
filenameOps[filenameOpsNum].regex = g_regex_new(equal+1, 0, 0, 0);
filenameOps[filenameOpsNum].expand = g_strdup(percent+1);
if (!filenameOps[filenameOpsNum].regex)
LOGEXIT("Couldn't compile regex '%s'", equal+1);
filenameOps[filenameOpsNum].field = fieldPos;
filenameOpsNum++;
}
g_strfreev(filenameOpsStr);
// Now actually start
reader_libpcapfile_next();
if (!pcap) {
if (config.pcapMonitor) {
g_timeout_add(100, reader_libpcapfile_monitor_gfunc, 0);
} else {
moloch_quit();
}
}
}
int moloch_field_define_text(char *text, int *shortcut)
{
int count = 0;
char *field = 0;
char *kind = 0;
char *help = 0;
char *db = 0;
char *group = 0;
char *friendly = 0;
char *category = 0;
if (config.debug)
LOG("Parsing %s", text);
char **elements = g_strsplit(text, ";", 0);
int e;
for (e = 0; elements[e]; e++) {
char *colon = strchr(elements[e], ':');
if (!colon)
continue;
*colon = 0;
colon++;
if (strcmp(elements[e], "field") == 0)
field = colon;
else if (strcmp(elements[e], "kind") == 0)
kind = colon;
else if (strcmp(elements[e], "group") == 0)
group = colon;
else if (strcmp(elements[e], "count") == 0)
count = strcmp(colon, "true") == 0;
else if (strcmp(elements[e], "friendly") == 0)
friendly = colon;
else if (strcmp(elements[e], "db") == 0)
db = colon;
else if (strcmp(elements[e], "help") == 0)
help = colon;
else if (strcmp(elements[e], "category") == 0)
category = colon;
else if (strcmp(elements[e], "shortcut") == 0) {
if (shortcut)
*shortcut = atoi(colon);
}
}
if (!field) {
LOG("Didn't find field 'field:' in '%s'", text);
g_strfreev(elements);
return -1;
}
if (!db) {
int pos = moloch_field_by_exp(field);
g_strfreev(elements);
if (pos != -1)
return pos;
LOG("Didn't find field 'db:' in '%s'", text);
return -1;
}
if (!kind) {
LOG("Didn't find field 'kind:' in '%s'", text);
g_strfreev(elements);
return -1;
}
if (strstr(kind, "termfield") != 0 && strstr(db, "-term") == 0) {
LOG("ERROR - db field %s for %s should end with -term in '%s'", kind, db, text);
exit(1);
}
char groupbuf[100];
if (!group) {
char *dot = strchr(field, '.');
if (dot) {
memcpy(groupbuf, field, MIN(100, dot-field));
groupbuf[dot-field] = 0;
group = groupbuf;
} else {
group = "general";
}
}
if (!friendly)
friendly = field;
if (!help)
help = field;
int type, flags = 0;
if (strcmp(kind, "integer") == 0 ||
strcmp(kind, "seconds") == 0)
type = MOLOCH_FIELD_TYPE_INT_GHASH;
else if (strcmp(kind, "ip") == 0) {
type = MOLOCH_FIELD_TYPE_IP_GHASH;
if (!category)
category = "ip";
} else
type = MOLOCH_FIELD_TYPE_STR_HASH;
if (count)
flags |= MOLOCH_FIELD_FLAG_COUNT;
int pos = moloch_field_define(group, kind, field, friendly, db, help, type, flags, "category", category, NULL);
g_strfreev(elements);
return pos;
}
void moloch_rules_process_rule(char *filename, YamlNode_t *parent)
{
char *name = moloch_rules_get_value(parent, "name");
if (!name)
LOGEXIT("%s: name required for rule", filename);
char *when = moloch_rules_get_value(parent, "when");
if (!when)
LOGEXIT("%s: when required for rule '%s'", filename, name);
char *bpf = moloch_rules_get_value(parent, "bpf");
GPtrArray *fields = moloch_rules_get_values(parent, "fields");
char *expression = moloch_rules_get_value(parent, "expression");
if (!bpf && !fields && !expression)
LOGEXIT("%s: bpf, fields, or expressions required for rule '%s'", filename, name);
if ((bpf && fields) || (bpf && expression) || (fields && expression))
LOGEXIT("%s: Only one of bpf, fields, or expressions can be set for rule '%s'", filename, name);
GPtrArray *ops = moloch_rules_get_values(parent, "ops");
if (!ops)
LOGEXIT("%s: ops required for rule '%s'", filename, name);
if (expression) {
LOGEXIT("Currently don't support expression, hopefully soon!");
}
int type;
int saveFlags = 0;
if (strcmp(when, "everyPacket") == 0) {
type = MOLOCH_RULE_TYPE_EVERY_PACKET;
if (!bpf)
LOGEXIT("%s: everyPacket only supports bpf", filename);
} else if (strcmp(when, "sessionSetup") == 0) {
type = MOLOCH_RULE_TYPE_SESSION_SETUP;
} else if (strcmp(when, "afterClassify") == 0) {
type = MOLOCH_RULE_TYPE_AFTER_CLASSIFY;
if (bpf)
LOGEXIT("%s: %s doesn't support bpf", filename, when);
} else if (strcmp(when, "fieldSet") == 0) {
type = MOLOCH_RULE_TYPE_FIELD_SET;
if (bpf)
LOGEXIT("%s: %s doesn't support bpf", filename, when);
} else if (strcmp(when, "beforeMiddleSave") == 0) {
type = MOLOCH_RULE_TYPE_BEFORE_SAVE;
saveFlags = 1;
if (bpf)
LOGEXIT("%s: %s doesn't support bpf", filename, when);
} else if (strcmp(when, "beforeFinalSave") == 0) {
type = MOLOCH_RULE_TYPE_BEFORE_SAVE;
saveFlags = 2;
if (bpf)
LOGEXIT("%s: %s doesn't support bpf", filename, when);
} else if (strcmp(when, "beforeBothSave") == 0) {
type = MOLOCH_RULE_TYPE_BEFORE_SAVE;
saveFlags = 3;
if (bpf)
LOGEXIT("%s: %s doesn't support bpf", filename, when);
} else {
LOGEXIT("%s: Unknown when '%s'", filename, when);
}
if (rulesLen[type] >= MOLOCH_RULES_MAX)
LOGEXIT("Too many %s rules", when);
int n = rulesLen[type]++;
MolochRule_t *rule = rules[type][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t);
rule->filename = filename;
rule->saveFlags = saveFlags;
if (bpf)
rule->bpf = g_strdup(bpf);
if (fields) {
int i;
rule->fields = malloc((int)fields->len);
for (i = 0; i < (int)fields->len; i++) {
YamlNode_t *node = g_ptr_array_index(fields, i);
int pos = moloch_field_by_exp(node->key);
if (pos == -1)
LOGEXIT("%s Couldn't find field '%s'", filename, node->key);
rule->fields[(int)rule->fieldsLen++] = pos;
switch (config.fields[pos]->type) {
case MOLOCH_FIELD_TYPE_INT:
case MOLOCH_FIELD_TYPE_INT_ARRAY:
case MOLOCH_FIELD_TYPE_INT_HASH:
case MOLOCH_FIELD_TYPE_IP:
case MOLOCH_FIELD_TYPE_IP_GHASH:
case MOLOCH_FIELD_TYPE_INT_GHASH:
case MOLOCH_FIELD_TYPE_IP_HASH:
rule->hash[pos] = g_hash_table_new(NULL, NULL);
break;
case MOLOCH_FIELD_TYPE_STR:
case MOLOCH_FIELD_TYPE_STR_ARRAY:
case MOLOCH_FIELD_TYPE_STR_HASH:
rule->hash[pos] = g_hash_table_new(g_str_hash, g_str_equal);
break;
case MOLOCH_FIELD_TYPE_CERTSINFO:
LOGEXIT("%s: Currently don't support any certs fields", filename);
}
if (node->value)
moloch_rules_process_add_field(rule, pos, node->value);
else {
int j;
for (j = 0; j < (int)node->values->len; j++) {
YamlNode_t *fnode = g_ptr_array_index(node->values, j);
moloch_rules_process_add_field(rule, pos, fnode->key);
}
}
}
}
moloch_field_ops_init(&rule->ops, ops->len, MOLOCH_FIELD_OPS_FLAGS_COPY);
int i;
for (i = 0; i < (int)ops->len; i++) {
YamlNode_t *node = g_ptr_array_index(ops, i);
int pos = moloch_field_by_exp(node->key);
if (pos == -1)
LOGEXIT("%s Couldn't find field '%s'", filename, node->key);
moloch_field_ops_add(&rule->ops, pos, node->value, strlen(node->value));
}
}
