void http_add_value(MolochSession_t *session, HTTPInfo_t *http) { int pos = http->pos[session->which]; char *s = http->valueString[session->which]->str; int l = http->valueString[session->which]->len; while (isspace(*s)) { s++; l--; } switch (config.fields[pos]->type) { case MOLOCH_FIELD_TYPE_INT: case MOLOCH_FIELD_TYPE_INT_ARRAY: case MOLOCH_FIELD_TYPE_INT_HASH: moloch_field_int_add(pos, session, atoi(s)); g_string_free(http->valueString[session->which], TRUE); break; case MOLOCH_FIELD_TYPE_STR: case MOLOCH_FIELD_TYPE_STR_ARRAY: case MOLOCH_FIELD_TYPE_STR_HASH: moloch_field_string_add(pos, session, s, l, TRUE); g_string_free(http->valueString[session->which], TRUE); break; case MOLOCH_FIELD_TYPE_IP_HASH: { int i; gchar **parts = g_strsplit(http->valueString[session->which]->str, ",", 0); for (i = 0; parts[i]; i++) { gchar *ip = parts[i]; while (*ip == ' ') ip++; in_addr_t ia = inet_addr(ip); if (ia == 0 || ia == 0xffffffff) { moloch_nids_add_tag(session, "http:bad-xff"); LOG("ERROR - Didn't understand ip: %s %s %d", http->valueString[session->which]->str, ip, ia); continue; } moloch_field_int_add(pos, session, ia); } g_strfreev(parts); g_string_free(http->valueString[session->which], TRUE); break; } } /* SWITCH */ http->valueString[session->which] = 0; http->pos[session->which] = 0; }
void wise_process_ops(MolochSession_t *session, WiseItem_t *wi) { int i; for (i = 0; i < wi->numOps; i++) { WiseOp_t *op = &(wi->ops[i]); switch (config.fields[op->fieldPos]->type) { case MOLOCH_FIELD_TYPE_INT_HASH: if (op->fieldPos == tagsField) { moloch_nids_add_tag(session, op->str); continue; } // Fall Thru case MOLOCH_FIELD_TYPE_INT: case MOLOCH_FIELD_TYPE_INT_ARRAY: case MOLOCH_FIELD_TYPE_IP: case MOLOCH_FIELD_TYPE_IP_HASH: moloch_field_int_add(op->fieldPos, session, op->strLenOrInt); break; case MOLOCH_FIELD_TYPE_STR: case MOLOCH_FIELD_TYPE_STR_ARRAY: case MOLOCH_FIELD_TYPE_STR_HASH: moloch_field_string_add(op->fieldPos, session, op->str, op->strLenOrInt, TRUE); break; } } }
int socks4_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining) { SocksInfo_t *socks = uw; switch(socks->state4) { case SOCKS4_STATE_REPLY: if (session->which == socks->which) return 0; if (remaining >= 8 && data[0] == 0 && data[1] >= 0x5a && data[1] <= 0x5d) { if (socks->ip) moloch_field_int_add(ipField, session, socks->ip); moloch_field_int_add(portField, session, socks->port); moloch_nids_add_tag(session, "protocol:socks"); moloch_nids_add_protocol(session, "socks"); if (socks->user) { if (!moloch_field_string_add(userField, session, socks->user, socks->userlen, FALSE)) { g_free(socks->user); } socks->user = 0; } if (socks->host) { if (!moloch_field_string_add(hostField, session, socks->host, socks->hostlen, FALSE)) { g_free(socks->host); } socks->host = 0; } moloch_parsers_classify_tcp(session, data+8, remaining-8); socks->state4 = SOCKS4_STATE_DATA; return 8; } break; case SOCKS4_STATE_DATA: if (session->which != socks->which) return 0; moloch_parsers_classify_tcp(session, data, remaining); moloch_parsers_unregister(session, uw); break; } return 0; }
void moloch_session_get_tag_cb(void *sessionV, int tagType, const char *tagName, uint32_t tag, gboolean async) { MolochSession_t *session = sessionV; if (tag == 0) { LOG("ERROR - Not adding tag %s type %d couldn't get tag num", tagName, tagType); moloch_session_decr_outstanding(session); } else if (async) { moloch_session_add_cmd(session, MOLOCH_SES_CMD_ADD_TAG, (gpointer)(long)tagType, (gpointer)(long)tag, NULL); } else { moloch_field_int_add(tagType, session, tag); moloch_session_decr_outstanding(session); } }
MolochIpInfo_t *moloch_db_get_local_ip(MolochSession_t *session, uint32_t ip) { prefix_t prefix; patricia_node_t *node; prefix.family = AF_INET; prefix.bitlen = 32; prefix.add.sin.s_addr = ip; if ((node = patricia_search_best (ipTree, &prefix)) == NULL) return 0; MolochIpInfo_t *ii = node->data; int t; for (t = 0; t < ii->numtags; t++) { moloch_field_int_add(MOLOCH_FIELD_TAGS, session, ii->tags[t]); } return ii; }
int moloch_hp_cb_on_headers_complete (http_parser *parser) { HTTPInfo_t *http = parser->data; MolochSession_t *session = http->session; char tag[200]; char version[20]; #ifdef HTTPDEBUG LOG("HTTPDEBUG: which: %d code: %d method: %d", session->which, parser->status_code, parser->method); #endif int len = snprintf(version, sizeof(version), "%d.%d", parser->http_major, parser->http_minor); if (parser->status_code == 0) { #ifndef REMOVEOLD snprintf(tag, sizeof(tag), "http:method:%s", http_method_str(parser->method)); moloch_nids_add_tag(session, tag); #endif moloch_field_string_add(methodField, session, http_method_str(parser->method), -1, TRUE); moloch_field_string_add(verReqField, session, version, len, TRUE); } else { #ifndef REMOVEOLD snprintf(tag, sizeof(tag), "http:statuscode:%d", parser->status_code); moloch_nids_add_tag(session, tag); #endif moloch_field_int_add(statuscodeField, session, parser->status_code); moloch_field_string_add(verResField, session, version, len, TRUE); } if (http->inValue & (1 << session->which) && http->pos[session->which]) { http_add_value(session, http); } if (pluginsCbs & MOLOCH_PLUGIN_HP_OHC) moloch_plugins_cb_hp_ohc(session, parser); return 0; }
static int MS_add_int(lua_State *L) { if (config.debug > 2) molua_stackDump(L); if (lua_gettop(L) != 3 || !lua_isuserdata(L, 1) || !(lua_isstring(L, 2) || lua_isinteger(L, 2)) || !lua_isinteger(L, 3)) { return luaL_error(L, "usage: <session> <field string or field num(faster)> <integer>"); } MolochSession_t *session = checkMolochSession(L, 1); int value = lua_tointeger(L, 3); int pos; if (lua_isinteger(L, 2)) { pos = lua_tointeger(L, 2); } else { pos = moloch_field_by_exp(lua_tostring(L, 2)); } gboolean result; result = moloch_field_int_add(pos, session, value); lua_pushboolean(L, result); return 1; }
int socks5_parser(MolochSession_t *session, void *uw, const unsigned char *data, int remaining, int which) { SocksInfo_t *socks = uw; int consumed; //LOG("%d %d %d", which, socks->which, socks->state5[which]); //moloch_print_hex_string(data, remaining); switch(socks->state5[which]) { case SOCKS5_STATE_VER_REQUEST: if (data[2] == 0) { socks->state5[which] = SOCKS5_STATE_CONN_REQUEST; } else { socks->state5[which] = SOCKS5_STATE_USER_REQUEST; } socks->state5[(which+1)%2] = SOCKS5_STATE_VER_REPLY; break; case SOCKS5_STATE_VER_REPLY: if (remaining != 2 || data[0] != 5 || data[1] > 2) { moloch_parsers_unregister(session, uw); return 0; } moloch_nids_add_protocol(session, "socks"); if (socks->state5[socks->which] == SOCKS5_STATE_CONN_DATA) { // Other side of connection already in data state socks->state5[which] = SOCKS5_STATE_CONN_REPLY; } else if (data[1] == 0) { socks->state5[socks->which] = SOCKS5_STATE_CONN_REQUEST; socks->state5[which] = SOCKS5_STATE_CONN_REPLY; } else if (data[1] == 2) { socks->state5[socks->which] = SOCKS5_STATE_USER_REQUEST; socks->state5[which] = SOCKS5_STATE_USER_REPLY; } else { // We don't handle other auth methods moloch_parsers_unregister(session, uw); } return 2; case SOCKS5_STATE_USER_REQUEST: if ((2 + data[1] > (int)remaining) || (2 + data[1] + 1 + data[data[1]+2] > (int)remaining)) { moloch_parsers_unregister(session, uw); return 0; } moloch_field_string_add(userField, session, (char *)data + 2, data[1], TRUE); moloch_nids_add_tag(session, "socks:password"); socks->state5[which] = SOCKS5_STATE_CONN_REQUEST; return data[1] + 1 + data[data[1]+2]; case SOCKS5_STATE_USER_REPLY: socks->state5[which] = SOCKS5_STATE_CONN_REPLY; return 2; case SOCKS5_STATE_CONN_REQUEST: if (remaining < 6 || data[0] != 5 || data[1] != 1 || data[2] != 0) { moloch_parsers_unregister(session, uw); return 0; } socks->state5[which] = SOCKS5_STATE_CONN_DATA; if (data[3] == 1) { // IPV4 socks->port = (data[8]&0xff) << 8 | (data[9]&0xff); memcpy(&socks->ip, data+4, 4); moloch_field_int_add(ipField, session, socks->ip); moloch_field_int_add(portField, session, socks->port); consumed = 4 + 4 + 2; } else if (data[3] == 3) { // Domain Name socks->port = (data[5+data[4]]&0xff) << 8 | (data[6+data[4]]&0xff); char *lower = g_ascii_strdown((char*)data+5, data[4]); if (!moloch_field_string_add(hostField, session, lower, data[4], FALSE)) { g_free(lower); } moloch_field_int_add(portField, session, socks->port); consumed = 4 + 1 + data[4] + 2; } else if (data[3] == 4) { // IPV6 consumed = 4 + 16 + 2; } else { break; } moloch_parsers_classify_tcp(session, data+consumed, remaining-consumed, which); return consumed; case SOCKS5_STATE_CONN_REPLY: { if (remaining < 6) { moloch_parsers_unregister(session, uw); return 0; } socks->state5[which] = SOCKS5_STATE_CONN_DATA; if (data[3] == 1) { // IPV4 consumed = 4 + 4 + 2; } else if (data[3] == 3) { // Domain Name consumed = 4 + 1 + data[4] + 2; } else if (data[3] == 4) { // IPV6 consumed = 4 + 16 + 2; } else { break; } moloch_parsers_classify_tcp(session, data+consumed, remaining-consumed, which); return consumed; } case SOCKS5_STATE_CONN_DATA: moloch_parsers_classify_tcp(session, data, remaining, which); moloch_parsers_unregister(session, uw); return 0; default: moloch_parsers_unregister(session, uw); } return 0; }
int moloch_hp_cb_on_headers_complete (http_parser *parser) { HTTPInfo_t *http = parser->data; MolochSession_t *session = http->session; char tag[200]; char version[20]; #ifdef HTTPDEBUG LOG("HTTPDEBUG: which: %d code: %d method: %d", http->which, parser->status_code, parser->method); #endif int len = snprintf(version, sizeof(version), "%d.%d", parser->http_major, parser->http_minor); if (parser->status_code == 0) { #ifndef REMOVEOLD snprintf(tag, sizeof(tag), "http:method:%s", http_method_str(parser->method)); moloch_nids_add_tag(session, tag); #endif moloch_field_string_add(methodField, session, http_method_str(parser->method), -1, TRUE); moloch_field_string_add(verReqField, session, version, len, TRUE); } else { #ifndef REMOVEOLD snprintf(tag, sizeof(tag), "http:statuscode:%d", parser->status_code); moloch_nids_add_tag(session, tag); #endif moloch_field_int_add(statuscodeField, session, parser->status_code); moloch_field_string_add(verResField, session, version, len, TRUE); } if (http->inValue & (1 << http->which) && http->pos[http->which]) { http_add_value(session, http); } http->header[0][0] = http->header[1][0] = 0; if (http->urlString) { char *ch = http->urlString->str; while (*ch) { if (*ch < 32) { moloch_nids_add_tag(session, "http:control-char"); break; } ch++; } } if (http->cookieString && http->cookieString->str[0]) { char *start = http->cookieString->str; while (1) { while (isspace(*start)) start++; char *equal = strchr(start, '='); if (!equal) break; moloch_field_string_add(cookieKeyField, session, start, equal-start, TRUE); start = strchr(equal+1, ';'); if(!start) break; start++; } g_string_truncate(http->cookieString, 0); } if (http->authString && http->authString->str[0]) { moloch_http_parse_authorization(session, http->authString->str); g_string_truncate(http->authString, 0); } if (http->hostString) { g_string_ascii_down(http->hostString); } if (http->urlString && http->hostString) { char *colon = strchr(http->hostString->str+2, ':'); if (colon) { moloch_field_string_add(hostField, session, http->hostString->str+2, colon - http->hostString->str-2, TRUE); } else { moloch_field_string_add(hostField, session, http->hostString->str+2, http->hostString->len-2, TRUE); } char *question = strchr(http->urlString->str, '?'); if (question) { moloch_field_string_add(pathField, session, http->urlString->str, question - http->urlString->str, TRUE); char *start = question+1; char *ch; int field = keyField; for (ch = start; *ch; ch++) { if (*ch == '&') { if (ch != start && (config.parseQSValue || field == keyField)) { char *str = g_uri_unescape_segment(start, ch, NULL); if (!str) { moloch_field_string_add(field, session, start, ch-start, TRUE); } else if (!moloch_field_string_add(field, session, str, strlen(str), FALSE)) { g_free(str); } } start = ch+1; field = keyField; continue; } else if (*ch == '=') { if (ch != start && (config.parseQSValue || field == keyField)) { char *str = g_uri_unescape_segment(start, ch, NULL); if (!str) { moloch_field_string_add(field, session, start, ch-start, TRUE); } else if (!moloch_field_string_add(field, session, str, strlen(str), FALSE)) { g_free(str); } } start = ch+1; field = valueField; } } if (config.parseQSValue && field == valueField && ch > start) { char *str = g_uri_unescape_segment(start, ch, NULL); if (!str) { moloch_field_string_add(field, session, start, ch-start, TRUE); } else if (!moloch_field_string_add(field, session, str, strlen(str), FALSE)) { g_free(str); } } } else { moloch_field_string_add(pathField, session, http->urlString->str, http->urlString->len, TRUE); } if (http->urlString->str[0] != '/') { char *result = strstr(http->urlString->str, http->hostString->str+2); /* If the host header is in the first 8 bytes of url then just use the url */ if (result && result - http->urlString->str <= 8) { moloch_field_string_add(urlsField, session, http->urlString->str, http->urlString->len, FALSE); g_string_free(http->urlString, FALSE); g_string_free(http->hostString, TRUE); } else { /* Host header doesn't match the url */ g_string_append(http->hostString, ";"); g_string_append(http->hostString, http->urlString->str); moloch_field_string_add(urlsField, session, http->hostString->str, http->hostString->len, FALSE); g_string_free(http->urlString, TRUE); g_string_free(http->hostString, FALSE); } } else { /* Normal case, url starts with /, so no extra host in url */ g_string_append(http->hostString, http->urlString->str); moloch_field_string_add(urlsField, session, http->hostString->str, http->hostString->len, FALSE); g_string_free(http->urlString, TRUE); g_string_free(http->hostString, FALSE); } http->urlString = NULL; http->hostString = NULL; } else if (http->urlString) { moloch_field_string_add(urlsField, session, http->urlString->str, http->urlString->len, FALSE); g_string_free(http->urlString, FALSE); http->urlString = NULL; } else if (http->hostString) { char *colon = strchr(http->hostString->str+2, ':'); if (colon) { moloch_field_string_add(hostField, session, http->hostString->str+2, colon - http->hostString->str-2, TRUE); } else { moloch_field_string_add(hostField, session, http->hostString->str+2, http->hostString->len-2, TRUE); } g_string_free(http->hostString, TRUE); http->hostString = NULL; } moloch_nids_add_tag(session, "protocol:http"); moloch_nids_add_protocol(session, "http"); if (pluginsCbs & MOLOCH_PLUGIN_HP_OHC) moloch_plugins_cb_hp_ohc(session, parser); return 0; }
void dns_parser(MolochSession_t *session, const unsigned char *data, int len) { if (len < 18) return; int qr = (data[2] >> 7) & 0x1; int opcode = (data[2] >> 3) & 0xf; if (opcode != 0) return; int qdcount = (data[4] << 8) | data[5]; int ancount = (data[6] << 8) | data[7]; if (qdcount > 10 || qdcount <= 0) return; BSB bsb; BSB_INIT(bsb, data + 12, len - 12); /* QD Section */ int i; for (i = 0; BSB_NOT_ERROR(bsb) && i < qdcount; i++) { int namelen; unsigned char *name = dns_name(data, len, &bsb, &namelen); if (BSB_IS_ERROR(bsb)) break; if (!namelen) { name = (unsigned char*)"<root>"; namelen = 6; } unsigned short qtype = 0 , qclass = 0 ; BSB_IMPORT_u16(bsb, qtype); BSB_IMPORT_u16(bsb, qclass); char *lower = g_ascii_strdown((char*)name, namelen); if (qclass <= 255 && qclasses[qclass]) { moloch_field_string_add(queryClassField, session, qclasses[qclass], -1, TRUE); } if (qtype <= 255 && qtypes[qtype]) { moloch_field_string_add(queryTypeField, session, qtypes[qtype], -1, TRUE); } if (lower && !moloch_field_string_add(hostField, session, lower, namelen, FALSE)) { g_free(lower); } } moloch_nids_add_protocol(session, "dns"); if (qr == 0) return; int rcode = data[3] & 0xf; moloch_field_string_add(statusField, session, statuses[rcode], -1, TRUE); for (i = 0; BSB_NOT_ERROR(bsb) && i < ancount; i++) { int namelen; dns_name(data, len, &bsb, &namelen); if (BSB_IS_ERROR(bsb)) break; uint16_t antype = 0; BSB_IMPORT_u16 (bsb, antype); uint16_t anclass = 0; BSB_IMPORT_u16 (bsb, anclass); BSB_IMPORT_skip(bsb, 4); // ttl uint16_t rdlength = 0; BSB_IMPORT_u16 (bsb, rdlength); if (BSB_REMAINING(bsb) < rdlength) { break; } if (anclass != 1) { BSB_IMPORT_skip(bsb, rdlength); continue; } switch (antype) { case 1: { if (rdlength != 4) break; struct in_addr in; unsigned char *ptr = BSB_WORK_PTR(bsb); in.s_addr = ptr[3] << 24 | ptr[2] << 16 | ptr[1] << 8 | ptr[0]; moloch_field_int_add(ipField, session, in.s_addr); break; } case 5: { BSB rdbsb; BSB_INIT(rdbsb, BSB_WORK_PTR(bsb), rdlength); int namelen; unsigned char *name = dns_name(data, len, &rdbsb, &namelen); if (!namelen || BSB_IS_ERROR(rdbsb)) continue; char *lower = g_ascii_strdown((char*)name, namelen); if (lower && !moloch_field_string_add(hostField, session, lower, namelen, FALSE)) { g_free(lower); } break; } case 15: { BSB rdbsb; BSB_INIT(rdbsb, BSB_WORK_PTR(bsb), rdlength); BSB_IMPORT_skip(rdbsb, 2); // preference int namelen; unsigned char *name = dns_name(data, len, &rdbsb, &namelen); if (!namelen || BSB_IS_ERROR(rdbsb)) continue; char *lower = g_ascii_strdown((char*)name, namelen); if (lower && !moloch_field_string_add(hostField, session, lower, namelen, FALSE)) { g_free(lower); } } } /* switch */ BSB_IMPORT_skip(bsb, rdlength); } }
int radius_udp_parser(MolochSession_t *session, void *UNUSED(uw), const unsigned char *data, int len, int UNUSED(which)) { BSB bsb; BSB_INIT(bsb, data, len); BSB_IMPORT_skip(bsb, 20); unsigned char type = 0, length = 0; unsigned char *value; char str[256]; struct in_addr in; int i; while (BSB_REMAINING(bsb) > 2) { BSB_IMPORT_u08(bsb, type); BSB_IMPORT_u08(bsb, length); length -= 2; // length includes the type/length BSB_IMPORT_ptr(bsb, value, length); if (BSB_IS_ERROR(bsb)) { break; } switch (type) { case 1: moloch_field_string_add(userField, session, (char *)value, length, TRUE); break; /* case 4: LOG("NAS-IP-Address: %d %d %u.%u.%u.%u", type, length, value[0], value[1], value[2], value[3]); break;*/ case 8: memcpy(&in.s_addr, value, 4); moloch_field_int_add(framedIpField, session, in.s_addr); break; case 31: if (length == 12) { snprintf(str, sizeof(str), "%c%c:%c%c:%c%c:%c%c:%c%c:%c%c", value[0], value[1], value[2], value[3], value[4], value[5], value[6], value[7], value[8], value[9], value[10], value[11]); for (i = 0; i < 17; i++) { if (isupper (str[i])) str[i] = tolower (str[i]); } moloch_field_string_add(macField, session, str, 17, TRUE); } break; case 66: memcpy(str, value, length); str[length] = 0; inet_aton(str, &in); moloch_field_int_add(endpointIpField, session, in.s_addr); break; /* default: LOG("%d %d %.*s", type, length, length, value);*/ } } return 0; }
void moloch_session_process_commands(int thread) { // Commands MolochSesCmd_t *cmd = 0; int count; for (count = 0; count < 50; count++) { MOLOCH_LOCK(sessionCmds[thread].lock); DLL_POP_HEAD(cmd_, &sessionCmds[thread], cmd); MOLOCH_UNLOCK(sessionCmds[thread].lock); if (!cmd) break; switch (cmd->cmd) { case MOLOCH_SES_CMD_ADD_TAG: moloch_field_int_add((long)cmd->uw1, cmd->session, (long)cmd->uw2); moloch_session_decr_outstanding(cmd->session); break; case MOLOCH_SES_CMD_FUNC: cmd->func(cmd->session, cmd->uw1, cmd->uw2); break; default: LOG ("Unknown cmd %d", cmd->cmd); } MOLOCH_TYPE_FREE(MolochSesCmd_t, cmd); } // Closing Q for (count = 0; count < 10; count++) { MolochSession_t *session = DLL_PEEK_HEAD(q_, &closingQ[thread]); if (session && session->saveTime < (uint64_t)lastPacketSecs[thread]) { moloch_session_save(session); } else { break; } } // Sessions Idle Long Time int ses; for (ses = 0; ses < SESSION_MAX; ses++) { for (count = 0; count < 10; count++) { MolochSession_t *session = DLL_PEEK_HEAD(q_, &sessionsQ[thread][ses]); if (session && (DLL_COUNT(q_, &sessionsQ[thread][ses]) > (int)config.maxStreams || ((uint64_t)session->lastPacket.tv_sec + config.timeouts[ses] < (uint64_t)lastPacketSecs[thread]))) { moloch_session_save(session); } else { break; } } } // TCP Sessions Open Long Time for (count = 0; count < 50; count++) { MolochSession_t *session = DLL_PEEK_HEAD(tcp_, &tcpWriteQ[thread]); if (session && (uint64_t)session->saveTime < (uint64_t)lastPacketSecs[thread]) { moloch_session_mid_save(session, lastPacketSecs[thread]); } else { break; } } }