void moloch_parser_init()
{
ipField = moloch_field_define("socks", "ip",
"ip.socks", "IP", "socksip",
"SOCKS destination IP",
MOLOCH_FIELD_TYPE_IP, MOLOCH_FIELD_FLAG_IPPRE,
"aliases", "[\"ip.socks\"]",
"portField", "sockspo", NULL);
hostField = moloch_field_define("socks", "lotermfield",
"host.socks", "Host", "socksho",
"SOCKS destination host",
MOLOCH_FIELD_TYPE_STR, 0,
"aliases", "[\"socks.host\"]", NULL);
portField = moloch_field_define("socks", "integer",
"port.socks", "Port", "sockspo",
"SOCKS destination port",
MOLOCH_FIELD_TYPE_INT, 0,
"aliases", "[\"socks.port\"]", NULL);
userField = moloch_field_define("socks", "termfield",
"socks.user", "User", "socksuser",
"SOCKS authenticated user",
MOLOCH_FIELD_TYPE_STR, 0,
"aliases", "[\"socksuser\"]", NULL);
moloch_parsers_classifier_register_tcp("socks5", 0, (unsigned char*)"\005", 1, socks5_classify);
moloch_parsers_classifier_register_tcp("socks4", 0, (unsigned char*)"\004\000", 2, socks4_classify);
moloch_parsers_classifier_register_tcp("socks4", 0, (unsigned char*)"\004\001", 2, socks4_classify);
}
void moloch_parser_init()
{
userField = moloch_field_define("radius", "termfield",
"radius.user", "User", "radius.user-term",
"RADIUS user",
MOLOCH_FIELD_TYPE_STR_HASH, 0,
"category", "user",
NULL);
macField = moloch_field_define("radius", "lotermfield",
"radius.mac", "MAC", "radius.mac-term",
"Radius Mac",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT,
NULL);
endpointIpField = moloch_field_define("radius", "ip",
"radius.endpoint-ip", "Endpoint IP", "radius.eip",
"Radius endpoint ip addresses for session",
MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_COUNT,
NULL);
framedIpField = moloch_field_define("radius", "ip",
"radius.framed-ip", "Framed IP", "radius.fip",
"Radius framed ip addresses for session",
MOLOCH_FIELD_TYPE_IP_GHASH, MOLOCH_FIELD_FLAG_COUNT,
NULL);
moloch_parsers_classifier_register_udp("radius", NULL, 0, (const unsigned char *)"\x01", 1, radius_udp_classify);
moloch_parsers_classifier_register_udp("radius", NULL, 0, (const unsigned char *)"\x02", 1, radius_udp_classify);
moloch_parsers_classifier_register_udp("radius", NULL, 0, (const unsigned char *)"\x03", 1, radius_udp_classify);
moloch_parsers_classifier_register_udp("radius", NULL, 0, (const unsigned char *)"\x04", 1, radius_udp_classify);
moloch_parsers_classifier_register_udp("radius", NULL, 0, (const unsigned char *)"\x05", 1, radius_udp_classify);
}
void moloch_parser_init()
{
realmField = moloch_field_define("krb5", "termfield",
"krb5.realm", "Realm", "krb5.realm",
"Kerberos 5 Realm",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT,
(char *)NULL);
cnameField = moloch_field_define("krb5", "termfield",
"krb5.cname", "cname", "krb5.cname",
"Kerberos 5 cname",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT,
(char *)NULL);
snameField = moloch_field_define("krb5", "termfield",
"krb5.sname", "sname", "krb5.sname",
"Kerberos 5 sname",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT,
(char *)NULL);
moloch_parsers_classifier_register_udp("krb5", 0, 7, (unsigned char*)"\x03\x02\x01\x05", 4, krb5_udp_classify);
moloch_parsers_classifier_register_udp("krb5", 0, 9, (unsigned char*)"\x03\x02\x01\x05", 4, krb5_udp_classify);
moloch_parsers_classifier_register_tcp("krb5", 0, 11, (unsigned char*)"\x03\x02\x01\x05", 4, krb5_tcp_classify);
moloch_parsers_classifier_register_tcp("krb5", 0, 13, (unsigned char*)"\x03\x02\x01\x05", 4, krb5_tcp_classify);
}
void moloch_parser_init()
{
verField = moloch_field_define("ssh", "lotermfield",
"ssh.ver", "Version", "sshver",
"SSH Software Version",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT,
NULL);
keyField = moloch_field_define("ssh", "termfield",
"ssh.key", "Key", "sshkey",
"SSH Key",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT,
NULL);
moloch_parsers_classifier_register_tcp("ssh", NULL, 0, (unsigned char*)"SSH", 3, ssh_classify);
}
void moloch_parser_init()
{
moloch_parsers_classifier_register_tcp("mysql", 1, (unsigned char*)"\x00\x00\x00\x0a", 4, mysql_classify);
userField = moloch_field_define("mysql", "lotermfield",
"mysql.user", "User", "mysql.user-term",
"Mysql user name",
MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);
versionField = moloch_field_define("mysql", "termfield",
"mysql.ver", "Version", "mysql.ver-term",
"Mysql server version string",
MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);
}
void moloch_parser_init()
{
nickField = moloch_field_define("irc", "termfield",
"irc.nick", "Nickname", "ircnck",
"Nicknames set",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT,
NULL);
channelsField = moloch_field_define("irc", "termfield",
"irc.channel", "Channel", "ircch",
"Channels joined",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT,
NULL);
moloch_parsers_classifier_register_tcp("irc", 0, (unsigned char*)":", 1, irc_classify);
moloch_parsers_classifier_register_tcp("irc", 0, (unsigned char*)"NOTICE AUTH", 11, irc_classify);
moloch_parsers_classifier_register_tcp("irc", 0, (unsigned char*)"NICK ", 5, irc_classify);
moloch_parsers_classifier_register_tcp("irc", 0, (unsigned char*)"USER ", 5, irc_classify);
}
void moloch_parser_init()
{
moloch_parsers_classifier_register_tcp("oracle", NULL, 2, (unsigned char*)"\x00\x00\x01\x00\x00\x00", 6, oracle_classify);
userField = moloch_field_define("oracle", "lotermfield",
"oracle.user", "User", "oracle.user-term",
"Oracle User",
MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
"category", "user",
NULL);
hostField = moloch_field_define("oracle", "lotermfield",
"oracle.host", "Host", "oracle.host-term",
"Oracle Host",
MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);
serviceField = moloch_field_define("oracle", "lotermfield",
"oracle.service", "Service", "oracle.service-term",
"Oracle Service",
MOLOCH_FIELD_TYPE_STR, MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);
}
void moloch_session_init()
{
uint32_t primes[] = {10007, 49999, 99991, 199799, 400009, 500009, 732209, 1092757, 1299827, 1500007, 1987411, 2999999};
int p;
for (p = 0; p < 12; p++) {
if (primes[p] >= config.maxStreams/2)
break;
}
if (p == 12) p = 11;
protocolField = moloch_field_define("general", "termfield",
"protocols", "Protocols", "prot-term",
"Protocols set for session",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);
if (config.debug)
LOG("session hash size %d", primes[p]);
int t;
for (t = 0; t < config.packetThreads; t++) {
HASHP_INIT(h_, sessions[t][SESSION_UDP], primes[p], moloch_session_hash, moloch_session_cmp);
HASHP_INIT(h_, sessions[t][SESSION_TCP], primes[p], moloch_session_hash, moloch_session_cmp);
HASHP_INIT(h_, sessions[t][SESSION_ICMP], primes[p], moloch_session_hash, moloch_session_cmp);
DLL_INIT(q_, &sessionsQ[t][SESSION_UDP]);
DLL_INIT(q_, &sessionsQ[t][SESSION_TCP]);
DLL_INIT(q_, &sessionsQ[t][SESSION_ICMP]);
DLL_INIT(tcp_, &tcpWriteQ[t]);
DLL_INIT(q_, &closingQ[t]);
DLL_INIT(cmd_, &sessionCmds[t]);
MOLOCH_LOCK_INIT(sessionCmds[t].lock);
}
moloch_add_can_quit(moloch_session_cmd_outstanding, "session commands outstanding");
moloch_add_can_quit(moloch_session_close_outstanding, "session close outstanding");
moloch_add_can_quit(moloch_session_need_save_outstanding, "session save outstanding");
}
void moloch_config_load_header(char *section, char *group, char *helpBase, char *expBase, char *dbBase, MolochStringHashStd_t *hash, int flags)
{
GError *error = 0;
char name[100];
if (!g_key_file_has_group(molochKeyFile, section))
return;
gsize keys_len;
gchar **keys = g_key_file_get_keys (molochKeyFile, section, &keys_len, &error);
if (error) {
LOG("Error with %s: %s", section, error->message);
exit(1);
}
gsize k, v;
for (k = 0 ; k < keys_len; k++) {
gsize values_len;
gchar **values = g_key_file_get_string_list(molochKeyFile,
section,
keys[k],
&values_len,
NULL);
snprintf(name, sizeof(name), "%s", keys[k]);
int type = 0;
int t = 0;
int unique = 1;
int count = 0;
char *kind = 0;
for (v = 0; v < values_len; v++) {
if (strcmp(values[v], "type:integer") == 0 ||
strcmp(values[v], "type:seconds") == 0) {
type = 1;
kind = values[v] + 5;
} else if (strcmp(values[v], "type:ip") == 0) {
type = 2;
} else if (strcmp(values[v], "unique:false") == 0) {
unique = 0;
} else if (strcmp(values[v], "count:true") == 0) {
count = 1;
}
}
int f = flags;
if (count)
f |= MOLOCH_FIELD_FLAG_CNT;
switch (type) {
case 0:
kind = "textfield";
if (unique)
t = MOLOCH_FIELD_TYPE_STR_HASH;
else
t = MOLOCH_FIELD_TYPE_STR_ARRAY;
break;
case 1:
if (unique)
t = MOLOCH_FIELD_TYPE_INT_HASH;
else
t = MOLOCH_FIELD_TYPE_INT_ARRAY;
break;
case 2:
kind = "ip";
t = MOLOCH_FIELD_TYPE_IP_HASH;
break;
}
MolochString_t *hstring;
HASH_FIND(s_, *hash, keys[k], hstring);
if (hstring) {
LOG("WARNING - ignoring field %s for %s", keys[k], section);
g_strfreev(values);
continue;
}
char expression[100];
char field[100];
char rawfield[100];
char help[100];
if (type == 0) {
sprintf(expression, "%s%s", expBase, name);
sprintf(field, "%s%s.snow", dbBase, name);
sprintf(rawfield, "%s%s.raw", dbBase, name);
sprintf(help, "%s%s", helpBase, name);
} else {
sprintf(expression, "%s%s", expBase, name);
sprintf(field, "%s%s", dbBase, name);
rawfield[0] = 0;
sprintf(help, "%s%s", helpBase, name);
}
int pos;
if (rawfield[0]) {
pos = moloch_field_define(group, kind,
expression, expression, field,
help,
t, f,
"rawField", rawfield, NULL);
} else {
pos = moloch_field_define(group, kind,
expression, expression, field,
help,
t, f, NULL);
}
moloch_config_add_header(hash, g_strdup(keys[k]), pos);
g_strfreev(values);
}
g_strfreev(keys);
}
void moloch_parser_init()
{
ipField = moloch_field_define("dns", "ip",
"ip.dns", "IP", "dnsip",
"IP from DNS result",
MOLOCH_FIELD_TYPE_IP_HASH, MOLOCH_FIELD_FLAG_CNT | MOLOCH_FIELD_FLAG_IPPRE,
"aliases", "[\"dns.ip\"]",
"category", "ip",
NULL);
hostField = moloch_field_define("dns", "lotermfield",
"host.dns", "Host", "dnsho",
"DNS host looked up",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_CNT,
"aliases", "[\"dns.host\"]",
"category", "host",
NULL);
statusField = moloch_field_define("dns", "uptermfield",
"dns.status", "Status Code", "dns.status-term",
"DNS lookup return code",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT,
NULL);
queryTypeField = moloch_field_define("dns", "uptermfield",
"dns.query.type", "Query Type", "dns.qt-term",
"DNS lookup query type",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT,
NULL);
queryClassField = moloch_field_define("dns", "uptermfield",
"dns.query.class", "Query Class", "dns.qc-term",
"DNS lookup query class",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT,
NULL);
qclasses[1] = "IN";
qclasses[2] = "CS";
qclasses[3] = "CH";
qclasses[4] = "HS";
qclasses[255] = "ANY";
//http://en.wikipedia.org/wiki/List_of_DNS_record_types
qtypes[1] = "A";
qtypes[2] = "NS";
qtypes[3] = "MD";
qtypes[4] = "MF";
qtypes[5] = "CNAME";
qtypes[6] = "SOA";
qtypes[7] = "MB";
qtypes[8] = "MG";
qtypes[9] = "MR";
qtypes[10] = "NULL";
qtypes[11] = "WKS";
qtypes[12] = "PTR";
qtypes[13] = "HINFO";
qtypes[14] = "MINFO";
qtypes[15] = "MX";
qtypes[16] = "TXT";
qtypes[17] = "RP";
qtypes[18] = "AFSDB";
qtypes[19] = "X25";
qtypes[20] = "ISDN";
qtypes[21] = "RT";
qtypes[22] = "NSAP";
qtypes[23] = "NSAPPTR";
qtypes[24] = "SIG";
qtypes[25] = "KEY";
qtypes[26] = "PX";
qtypes[27] = "GPOS";
qtypes[28] = "AAAA";
qtypes[29] = "LOC";
qtypes[30] = "NXT";
qtypes[31] = "EID";
qtypes[32] = "NIMLOC";
qtypes[33] = "SRV";
qtypes[34] = "ATMA";
qtypes[35] = "NAPTR";
qtypes[36] = "KX";
qtypes[37] = "CERT";
qtypes[38] = "A6";
qtypes[39] = "DNAME";
qtypes[40] = "SINK";
qtypes[41] = "OPT";
qtypes[42] = "APL";
qtypes[43] = "DS";
qtypes[44] = "SSHFP";
qtypes[46] = "RRSIG";
qtypes[47] = "NSEC";
qtypes[48] = "DNSKEY";
qtypes[49] = "DHCID";
qtypes[50] = "NSEC3";
qtypes[51] = "NSEC3PARAM";
qtypes[52] = "TLSA";
qtypes[55] = "HIP";
qtypes[99] = "SPF";
qtypes[249] = "TKEY";
qtypes[250] = "TSIG";
qtypes[252] = "AXFR";
qtypes[253] = "MAILB";
qtypes[254] = "MAILA";
qtypes[255] = "ANY";
#define DNS_CLASSIFY(_str) \
moloch_parsers_classifier_register_tcp("dns", 4, (unsigned char*)_str, 2, dns_tcp_classify); \
moloch_parsers_classifier_register_udp("dns", 2, (unsigned char*)_str, 2, dns_udp_classify);
DNS_CLASSIFY("\x00\x00");
DNS_CLASSIFY("\x00\x10");
DNS_CLASSIFY("\x01\x00");
DNS_CLASSIFY("\x01\x10");
DNS_CLASSIFY("\x01\x82");
DNS_CLASSIFY("\x81\x00");
DNS_CLASSIFY("\x81\x80");
DNS_CLASSIFY("\x81\x82");
DNS_CLASSIFY("\x81\x90");
DNS_CLASSIFY("\x81\xb0");
DNS_CLASSIFY("\x84\x10");
DNS_CLASSIFY("\x85\x00");
DNS_CLASSIFY("\x85\x83");
}
int moloch_field_define_text(char *text, int *shortcut)
{
int count = 0;
char *field = 0;
char *kind = 0;
char *help = 0;
char *db = 0;
char *group = 0;
char *friendly = 0;
char *category = 0;
if (config.debug)
LOG("Parsing %s", text);
char **elements = g_strsplit(text, ";", 0);
int e;
for (e = 0; elements[e]; e++) {
char *colon = strchr(elements[e], ':');
if (!colon)
continue;
*colon = 0;
colon++;
if (strcmp(elements[e], "field") == 0)
field = colon;
else if (strcmp(elements[e], "kind") == 0)
kind = colon;
else if (strcmp(elements[e], "group") == 0)
group = colon;
else if (strcmp(elements[e], "count") == 0)
count = strcmp(colon, "true") == 0;
else if (strcmp(elements[e], "friendly") == 0)
friendly = colon;
else if (strcmp(elements[e], "db") == 0)
db = colon;
else if (strcmp(elements[e], "help") == 0)
help = colon;
else if (strcmp(elements[e], "category") == 0)
category = colon;
else if (strcmp(elements[e], "shortcut") == 0) {
if (shortcut)
*shortcut = atoi(colon);
}
}
if (!field) {
LOG("Didn't find field 'field:' in '%s'", text);
g_strfreev(elements);
return -1;
}
if (!db) {
int pos = moloch_field_by_exp(field);
g_strfreev(elements);
if (pos != -1)
return pos;
LOG("Didn't find field 'db:' in '%s'", text);
return -1;
}
if (!kind) {
LOG("Didn't find field 'kind:' in '%s'", text);
g_strfreev(elements);
return -1;
}
if (strstr(kind, "termfield") != 0 && strstr(db, "-term") == 0) {
LOG("ERROR - db field %s for %s should end with -term in '%s'", kind, db, text);
exit(1);
}
char groupbuf[100];
if (!group) {
char *dot = strchr(field, '.');
if (dot) {
memcpy(groupbuf, field, MIN(100, dot-field));
groupbuf[dot-field] = 0;
group = groupbuf;
} else {
group = "general";
}
}
if (!friendly)
friendly = field;
if (!help)
help = field;
int type, flags = 0;
if (strcmp(kind, "integer") == 0 ||
strcmp(kind, "seconds") == 0)
type = MOLOCH_FIELD_TYPE_INT_GHASH;
else if (strcmp(kind, "ip") == 0) {
type = MOLOCH_FIELD_TYPE_IP_GHASH;
if (!category)
category = "ip";
} else
type = MOLOCH_FIELD_TYPE_STR_HASH;
if (count)
flags |= MOLOCH_FIELD_FLAG_COUNT;
int pos = moloch_field_define(group, kind, field, friendly, db, help, type, flags, "category", category, NULL);
g_strfreev(elements);
return pos;
}
void moloch_config_load_header(char *section, char *base, MolochStringHashStd_t *hash, int flags)
{
GError *error = 0;
char name[100];
if (!g_key_file_has_group(molochKeyFile, section))
return;
gsize keys_len;
gchar **keys = g_key_file_get_keys (molochKeyFile, section, &keys_len, &error);
if (error) {
LOG("Error with %s: %s", section, error->message);
exit(1);
}
gsize k, v;
for (k = 0 ; k < keys_len; k++) {
gsize values_len;
gchar **values = g_key_file_get_string_list(molochKeyFile,
section,
keys[k],
&values_len,
NULL);
snprintf(name, sizeof(name), "%s%s", base, keys[k]);
int type = 0;
int unique = 1;
int count = 0;
for (v = 0; v < values_len; v++) {
if (strcmp(values[v], "type:integer") == 0) {
type = 1;
} else if (strcmp(values[v], "type:ip") == 0) {
type = 2;
} else if (strcmp(values[v], "unique:false") == 0) {
unique = 0;
} else if (strcmp(values[v], "count:true") == 0) {
count = 1;
}
}
g_strfreev(values);
flags |= MOLOCH_FIELD_FLAG_HEADERS;
if (count)
flags |= MOLOCH_FIELD_FLAG_CNT;
switch (type) {
case 0:
if (unique)
type = MOLOCH_FIELD_TYPE_STR_HASH;
else
type = MOLOCH_FIELD_TYPE_STR_ARRAY;
break;
case 1:
if (unique)
type = MOLOCH_FIELD_TYPE_INT_HASH;
else
type = MOLOCH_FIELD_TYPE_INT_ARRAY;
break;
case 2:
flags |= MOLOCH_FIELD_FLAG_IPPOST;
type = MOLOCH_FIELD_TYPE_IP_HASH;
break;
}
MolochString_t *hstring;
HASH_FIND(s_, *hash, keys[k], hstring);
if (hstring) {
LOG("WARNING - ignoring field %s for %s", keys[k], section);
continue;
}
moloch_config_add_header(hash, g_strdup(keys[k]), moloch_field_define(g_strdup(name), type, flags));
}
g_strfreev(keys);
}