MolochSimple_t *writer_simple_alloc(int thread, MolochSimple_t *previous)
{
MolochSimple_t *info;
MOLOCH_LOCK(freeList[thread].lock);
DLL_POP_HEAD(simple_, &freeList[thread], info);
MOLOCH_UNLOCK(freeList[thread].lock);
if (!info) {
info = MOLOCH_TYPE_ALLOC0(MolochSimple_t);
info->buf = mmap (0, config.pcapWriteSize + MOLOCH_PACKET_MAX_LEN, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0);
info->thread = thread;
} else {
info->bufpos = 0;
info->closing = 0;
}
if (previous) {
info->file = previous->file;
} else {
info->file = MOLOCH_TYPE_ALLOC0(MolochSimpleFile_t);
switch(simpleMode) {
case MOLOCH_SIMPLE_NORMAL:
break;
case MOLOCH_SIMPLE_XOR2048:
break;
case MOLOCH_SIMPLE_AES256CTR:
info->file->cipher_ctx = EVP_CIPHER_CTX_new();
break;
}
}
return info;
}
void moloch_rules_load_complete()
{
char **bpfs;
GRegex *regex = g_regex_new(":\\s*(\\d+)\\s*$", 0, 0, 0);
int i;
bpfs = moloch_config_str_list(NULL, "dontSaveBPFs", NULL);
int pos = moloch_field_by_exp("_maxPacketsToSave");
gint start_pos;
if (bpfs) {
for (i = 0; bpfs[i]; i++) {
int n = loading.rulesLen[MOLOCH_RULE_TYPE_SESSION_SETUP]++;
MolochRule_t *rule = loading.rules[MOLOCH_RULE_TYPE_SESSION_SETUP][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t);
rule->filename = "dontSaveBPFs";
moloch_field_ops_init(&rule->ops, 1, MOLOCH_FIELD_OPS_FLAGS_COPY);
GMatchInfo *match_info = 0;
g_regex_match(regex, bpfs[i], 0, &match_info);
if (g_match_info_matches(match_info)) {
g_match_info_fetch_pos (match_info, 1, &start_pos, NULL);
rule->bpf = g_strndup(bpfs[i], start_pos-1);
moloch_field_ops_add(&rule->ops, pos, g_match_info_fetch(match_info, 1), -1);
} else {
rule->bpf = g_strdup(bpfs[i]);
moloch_field_ops_add(&rule->ops, pos, "1", -1);
}
g_match_info_free(match_info);
}
g_strfreev(bpfs);
}
bpfs = moloch_config_str_list(NULL, "minPacketsSaveBPFs", NULL);
pos = moloch_field_by_exp("_minPacketsBeforeSavingSPI");
if (bpfs) {
for (i = 0; bpfs[i]; i++) {
int n = loading.rulesLen[MOLOCH_RULE_TYPE_SESSION_SETUP]++;
MolochRule_t *rule = loading.rules[MOLOCH_RULE_TYPE_SESSION_SETUP][n] = MOLOCH_TYPE_ALLOC0(MolochRule_t);
rule->filename = "minPacketsSaveBPFs";
moloch_field_ops_init(&rule->ops, 1, MOLOCH_FIELD_OPS_FLAGS_COPY);
GMatchInfo *match_info = 0;
g_regex_match(regex, bpfs[i], 0, &match_info);
if (g_match_info_matches(match_info)) {
g_match_info_fetch_pos (match_info, 1, &start_pos, NULL);
rule->bpf = g_strndup(bpfs[i], start_pos-1);
moloch_field_ops_add(&rule->ops, pos, g_match_info_fetch(match_info, 1), -1);
} else {
rule->bpf = g_strdup(bpfs[i]);
moloch_field_ops_add(&rule->ops, pos, "1", -1);
}
g_match_info_free(match_info);
}
g_strfreev(bpfs);
}
g_regex_unref(regex);
memcpy(¤t, &loading, sizeof(loading));
memset(&loading, 0, sizeof(loading));
}
void molua_http_on_body_cb (MolochSession_t *session, http_parser *UNUSED(hp), const char *at, size_t length)
{
MoluaPlugin_t *mp = session->pluginData[molua_pluginIndex];
lua_State *L = Ls[session->thread];
int i;
for (i = 0; i < callbackRefsCnt[MOLUA_REF_HTTP]; i++) {
if (mp && mp->callbackOff[MOLUA_REF_HTTP] & (1 << i))
continue;
lua_getglobal(L, callbackRefs[MOLUA_REF_HTTP][i]);
molua_pushMolochSession(L, session);
molua_pushMolochData(L, at, length);
if (lua_pcall(L, 2, 1, 0) != 0) {
molua_stackDump(L);
LOG("error running http callback function %s", lua_tostring(L, -1));
exit(0);
}
int num = lua_tointeger(L, -1);
if (num == -1) {
if (!mp) {
mp = session->pluginData[molua_pluginIndex] = MOLOCH_TYPE_ALLOC0(MoluaPlugin_t);
}
mp->callbackOff[MOLUA_REF_HTTP] |= (1 << i);
}
lua_pop(L, 1);
}
}
void *moloch_http_create_server(char *hostnames, int defaultPort, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttp_t *server = MOLOCH_TYPE_ALLOC0(MolochHttp_t);
DLL_INIT(r_, &server->requestQ[0]);
DLL_INIT(r_, &server->requestQ[1]);
DLL_INIT(e_, &server->connQ);
server->names = g_strsplit(hostnames, ",", 0);
uint32_t i;
for (i = 0; server->names[i]; i++) {
if (strncmp(server->names[i], "http://", 7) == 0) {
char *tmp = g_strdup(server->names[i] + 7);
g_free(server->names[i]);
server->names[i] = tmp;
} else if (strncmp(server->names[i], "https://", 8) == 0) {
LOG("https not supported yet %s", server->names[i]);
exit(0);
}
}
server->namesCnt = i;
server->port = defaultPort;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
server->syncConn = moloch_http_create(server, TRUE);
for (i = 0; i < server->maxConns; i++) {
MolochConn_t *conn = moloch_http_create(server, FALSE);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
return server;
}
int moloch_plugins_register_internal(const char * name,
gboolean storeData,
size_t sessionsize,
int apiversion)
{
MolochPlugin_t *plugin;
if (sizeof(MolochSession_t) != sessionsize) {
LOG("Plugin '%s' built with different version of moloch.h", name);
exit(-1);
}
if (MOLOCH_API_VERSION != apiversion) {
LOG("Plugin '%s' built with different version of moloch.h", name);
exit(-1);
}
HASH_FIND(p_, plugins, name, plugin);
if (plugin) {
LOG("Plugin %s is already registered", name);
exit(-1);
}
plugin = MOLOCH_TYPE_ALLOC0(MolochPlugin_t);
plugin->name = strdup(name);
if (storeData) {
plugin->num = config.numPlugins++;
} else {
plugin->num = -1;
}
HASH_ADD(p_, plugins, name, plugin);
return plugin->num;
}
LOCAL void reader_libpcapfile_monitor_do(struct inotify_event *event)
{
gchar *dirname = g_hash_table_lookup(wdHashTable, (void *)(long)event->wd);
gchar *fullfilename = g_build_filename (dirname, event->name, NULL);
if (config.pcapRecursive &&
(event->mask & IN_CREATE) &&
g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
reader_libpcapfile_monitor_dir(fullfilename);
g_free(fullfilename);
return;
}
if ((event->mask & IN_CLOSE_WRITE) == 0) {
g_free(fullfilename);
return;
}
if (!g_regex_match(config.offlineRegex, fullfilename, 0, NULL)) {
g_free(fullfilename);
return;
}
MolochString_t *string = MOLOCH_TYPE_ALLOC0(MolochString_t);
string->str = fullfilename;
if (config.debug)
LOG("Monitor enqueing %s", string->str);
DLL_PUSH_TAIL(s_, &monitorQ, string);
return;
}
void moloch_field_define_json(unsigned char *expression, int expression_len, unsigned char *data, int data_len)
{
MolochFieldInfo_t *info = MOLOCH_TYPE_ALLOC0(MolochFieldInfo_t);
int i;
uint32_t out[4*100]; // Can have up to 100 elements at any level
memset(out, 0, sizeof(out));
if (js0n(data, data_len, out) != 0) {
LOG("ERROR: Parse error for >%.*s<\n", data_len, data);
fflush(stdout);
exit(0);
}
info->expression = g_strndup((char*)expression, expression_len);
for (i = 0; out[i]; i += 4) {
if (strncmp("group", (char*)data + out[i], 5) == 0) {
info->group = g_strndup((char*)data + out[i+2], out[i+3]);
} else if (strncmp("dbField", (char*)data + out[i], 7) == 0) {
info->dbFieldMem = info->dbField = g_strndup((char*)data + out[i+2], out[i+3]);
info->dbFieldLen = out[i+3];
}
}
info->pos = -1;
HASH_ADD(f_, fields, info->dbField, info);
return;
}
void *moloch_http_create_server(char *hostnames, int defaultPort, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttp_t *server = MOLOCH_TYPE_ALLOC0(MolochHttp_t);
DLL_INIT(r_, &server->requestQ);
DLL_INIT(e_, &server->connQ);
server->parserSettings.on_message_begin = moloch_http_hp_cb_on_message_begin;
server->parserSettings.on_body = moloch_http_hp_cb_on_body;
server->parserSettings.on_message_complete = moloch_http_hp_cb_on_message_complete;
server->parserSettings.on_header_field = moloch_http_hp_cb_on_header_field;
server->parserSettings.on_header_value = moloch_http_hp_cb_on_header_value;
server->names = g_strsplit(hostnames, ",", 0);
uint32_t i;
for (i = 0; server->names[i]; i++) {
if (strncmp(server->names[i], "http://", 7) == 0) {
char *tmp = g_strdup(server->names[i] + 7);
g_free(server->names[i]);
server->names[i] = tmp;
} else if (strncmp(server->names[i], "https://", 8) == 0) {
LOG("https not supported yet %s", server->names[i]);
exit(0);
}
}
server->namesCnt = i;
server->port = defaultPort;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
return server;
}
LOCAL void *reader_snf_thread(gpointer ring)
{
struct snf_recv_req req;
while (!config.quitting) {
int err = snf_ring_recv(ring, -1, &req);
if (err) {
if (err == EBUSY || err == EAGAIN || err == EINTR)
continue;
LOG("SNF quiting %d", err);
moloch_quit();
break;
}
MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t);
packet->pkt = (u_char *)req.pkt_addr;
packet->ts.tv_sec = req.timestamp / 1000000000;
packet->ts.tv_usec = req.timestamp % 1000000000000;
packet->pktlen = req.length;
moloch_packet(packet);
}
return NULL;
}
void moloch_field_define_json(unsigned char *expression, int expression_len, unsigned char *data, int data_len)
{
MolochFieldInfo_t *info = MOLOCH_TYPE_ALLOC0(MolochFieldInfo_t);
int i;
uint32_t out[4*100]; // Can have up to 100 elements at any level
memset(out, 0, sizeof(out));
if (js0n(data, data_len, out) != 0) {
LOGEXIT("ERROR: Parse error for >%.*s<\n", data_len, data);
}
info->expression = g_strndup((char*)expression, expression_len);
for (i = 0; out[i]; i += 4) {
if (strncmp("group", (char*)data + out[i], 5) == 0) {
info->group = g_strndup((char*)data + out[i+2], out[i+3]);
} else if (strncmp("dbField", (char*)data + out[i], 7) == 0) {
info->dbFieldFull = info->dbField = g_strndup((char*)data + out[i+2], out[i+3]);
info->dbFieldLen = out[i+3];
} else if (strncmp("type", (char*)data + out[i], 4) == 0) {
info->kind = g_strndup((char*)data + out[i+2], out[i+3]);
} else if (strncmp("category", (char*)data + out[i], 8) == 0) {
info->category = g_strndup((char*)data + out[i+2], out[i+3]);
} else if (strncmp("disabled", (char*)data + out[i], 8) == 0) {
if (strncmp((char *)data + out[i+2], "true", 4) == 0) {
info->flags |= MOLOCH_FIELD_FLAG_DISABLED;
}
}
}
info->pos = -1;
HASH_ADD(d_, fieldsByDb, info->dbField, info);
HASH_ADD(e_, fieldsByExp, info->expression, info);
return;
}
void moloch_field_by_exp_add_special(char *exp, int pos)
{
MolochFieldInfo_t *info = MOLOCH_TYPE_ALLOC0(MolochFieldInfo_t);
info->expression = exp;
info->pos = pos;
HASH_ADD(e_, fieldsByExp, info->expression, info);
}
LOCAL void mysql_classify(MolochSession_t *session, const unsigned char *data, int len, int which, void *UNUSED(uw))
{
if (which != 1)
return;
if (moloch_session_has_protocol(session, "mysql"))
return;
unsigned char *ptr = (unsigned char*)data + 5;
unsigned char *end = (unsigned char*)data + len;
while (ptr < end) {
if (*ptr == 0)
break;
if (!isprint(*ptr)) {
return;
}
ptr++;
}
if (ptr == end || ptr == data + 5) {
return;
}
Info_t *info = MOLOCH_TYPE_ALLOC0(Info_t);
info->versionLen = ptr - (data + 5);
info->version = g_strndup((char*)data + 5, info->versionLen);
moloch_parsers_register(session, mysql_parser, info, mysql_free);
}
void dns_tcp_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int which, void *UNUSED(uw))
{
if (/*which == 0 &&*/ session->port2 == 53 && !moloch_session_has_protocol(session, "dns")) {
moloch_session_add_protocol(session, "dns");
DNSInfo_t *info= MOLOCH_TYPE_ALLOC0(DNSInfo_t);
moloch_parsers_register(session, dns_tcp_parser, info, dns_free);
}
}
void moloch_config_add_header(MolochStringHashStd_t *hash, char *key, int pos)
{
MolochString_t *hstring;
hstring = MOLOCH_TYPE_ALLOC0(MolochString_t);
hstring->str = key;
hstring->len = strlen(key);
hstring->uw = (gpointer)(long)pos;
HASH_ADD(s_, *hash, hstring->str, hstring);
}
void ssh_classify(MolochSession_t *session, const unsigned char *UNUSED(data), int UNUSED(len), int UNUSED(which), void *UNUSED(uw))
{
if (moloch_session_has_protocol(session, "ssh"))
return;
moloch_session_add_protocol(session, "ssh");
SSHInfo_t *ssh = MOLOCH_TYPE_ALLOC0(SSHInfo_t);
moloch_parsers_register(session, ssh_parser, ssh, ssh_free);
}
static gboolean moloch_http_curl_watch_open_callback(int fd, GIOCondition condition, gpointer serverV)
{
MolochHttpServer_t *server = serverV;
struct sockaddr_in localAddress, remoteAddress;
socklen_t addressLength = sizeof(localAddress);
int rc = getsockname(fd, (struct sockaddr*)&localAddress, &addressLength);
if (rc != 0)
return FALSE;
addressLength = sizeof(remoteAddress);
rc = getpeername(fd, (struct sockaddr*)&remoteAddress, &addressLength);
if (rc != 0)
return FALSE;
char sessionId[MOLOCH_SESSIONID_LEN];
moloch_session_id(sessionId, localAddress.sin_addr.s_addr, localAddress.sin_port,
remoteAddress.sin_addr.s_addr, remoteAddress.sin_port);
LOG("Connected %d/%d - %s %d->%s:%d - fd:%d",
server->outstanding,
server->connections,
server->names[0],
ntohs(localAddress.sin_port),
inet_ntoa(remoteAddress.sin_addr),
ntohs(remoteAddress.sin_port),
fd);
MolochHttpConn_t *conn;
MOLOCH_LOCK(connections);
BIT_SET(fd, connectionsSet);
HASH_FIND(h_, connections, sessionId, conn);
if (!conn) {
conn = MOLOCH_TYPE_ALLOC0(MolochHttpConn_t);
HASH_ADD(h_, connections, sessionId, conn);
memcpy(&conn->sessionId, sessionId, sessionId[0]);
server->connections++;
} else {
char buf[1000];
LOG("ERROR - Already added %x %s", condition, moloch_session_id_string(sessionId, buf));
}
MOLOCH_UNLOCK(connections);
moloch_http_curlm_check_multi_info(server);
return FALSE;
}
MolochConn_t *
moloch_http_create(MolochHttp_t *server) {
MolochConn_t *conn;
conn = MOLOCH_TYPE_ALLOC0(MolochConn_t);
conn->parser.data = conn;
conn->server = server;
if (moloch_http_connect(conn, server->name, server->port)) {
printf("Couldn't connect to elastic search at '%s'", server->name);
exit (1);
}
return conn;
}
void reader_pfring_packet_cb(const struct pfring_pkthdr *h, const u_char *p, const u_char *UNUSED(user_bytes))
{
if (unlikely(h->caplen != h->len)) {
LOG("ERROR - Moloch requires full packet captures caplen: %d pktlen: %d", h->caplen, h->len);
exit (0);
}
MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t);
packet->pkt = (u_char *)p;
packet->ts = h->ts;
packet->pktlen = h->len;
moloch_packet(packet);
}
static int MS_table(lua_State *L)
{
MolochSession_t *session = checkMolochSession(L, 1);
MoluaPlugin_t *mp = session->pluginData[molua_pluginIndex];
if (!mp) {
mp = session->pluginData[molua_pluginIndex] = MOLOCH_TYPE_ALLOC0(MoluaPlugin_t);
}
if (!mp->table) {
lua_newtable(L);
mp->table = luaL_ref(L, LUA_REGISTRYINDEX);
}
lua_rawgeti(L, LUA_REGISTRYINDEX, mp->table);
return 1;
}
// Should only be used by packet, lots of side effects
MolochSession_t *moloch_session_find_or_create(int ses, uint32_t hash, char *sessionId, int *isNew)
{
MolochSession_t *session;
if (hash == 0) {
hash = moloch_session_hash(sessionId);
}
int thread = hash % config.packetThreads;
HASH_FIND_HASH(h_, sessions[thread][ses], hash, sessionId, session);
if (session) {
if (!session->closingQ) {
DLL_MOVE_TAIL(q_, &sessionsQ[thread][ses], session);
}
*isNew = 0;
return session;
}
*isNew = 1;
session = MOLOCH_TYPE_ALLOC0(MolochSession_t);
session->ses = ses;
memcpy(session->sessionId, sessionId, sessionId[0]);
HASH_ADD_HASH(h_, sessions[thread][ses], hash, sessionId, session);
DLL_PUSH_TAIL(q_, &sessionsQ[thread][ses], session);
if (HASH_BUCKET_COUNT(h_, sessions[thread][ses], hash) > 10) {
char buf[100];
LOG("Large number of chains: %s %u %u %u %u", moloch_session_id_string(sessionId, buf), hash, hash % sessions[thread][ses].size, thread, HASH_BUCKET_COUNT(h_, sessions[thread][ses], hash));
}
session->filePosArray = g_array_sized_new(FALSE, FALSE, sizeof(uint64_t), 100);
session->fileLenArray = g_array_sized_new(FALSE, FALSE, sizeof(uint16_t), 100);
session->fileNumArray = g_array_new(FALSE, FALSE, 4);
session->fields = MOLOCH_SIZE_ALLOC0(fields, sizeof(MolochField_t *)*config.maxField);
session->maxFields = config.maxField;
session->thread = thread;
DLL_INIT(td_, &session->tcpData);
if (config.numPlugins > 0)
session->pluginData = MOLOCH_SIZE_ALLOC0(pluginData, sizeof(void *)*config.numPlugins);
return session;
}
void socks5_classify(MolochSession_t *session, const unsigned char *data, int len, int which)
{
#ifdef SOCKSDEBUG
LOG("SOCKSDEBUG: enter %d %d", data[0], len);
#endif
if ((len >=3 && len <= 5) && data[1] == len - 2 && data[2] <= 3) {
SocksInfo_t *socks;
socks = MOLOCH_TYPE_ALLOC0(SocksInfo_t);
socks->which = which;
socks->state5[which] = SOCKS5_STATE_VER_REQUEST;
moloch_parsers_register(session, socks5_parser, socks, socks_free);
return;
}
return;
}
void reader_libpcap_pcap_cb(u_char *UNUSED(user), const struct pcap_pkthdr *h, const u_char *bytes)
{
if (unlikely(h->caplen != h->len)) {
LOG("ERROR - Moloch requires full packet captures caplen: %d pktlen: %d\n"
"turning offloading off may fix, something like 'ethtool -K INTERFACE tx off sg off gro off gso off lro off tso off'",
h->caplen, h->len);
exit (0);
}
MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t);
packet->pkt = (u_char *)bytes;
packet->ts = h->ts;
packet->pktlen = h->len;
moloch_packet(packet);
}
MolochConn_t *
moloch_http_create(MolochHttp_t *server, int blocking) {
MolochConn_t *conn;
int tries = server->namesCnt;
conn = MOLOCH_TYPE_ALLOC0(MolochConn_t);
conn->parser.data = conn;
conn->server = server;
while (tries > 0) {
if (!moloch_http_connect(conn, moloch_http_get_name(server), server->port, blocking)) {
return conn;
}
tries--;
LOG("Couldn't connect to '%s'", conn->name);
}
exit (1);
}
void irc_classify(MolochSession_t *session, const unsigned char *data, int len, int which)
{
if (data[0] == ':' && !moloch_memstr((char *)data, len, " NOTICE ", 8))
return;
//If a USER packet must have NICK with it so we don't pickup FTP
if (data[0] == 'U' && !moloch_memstr((char *)data, len, "\nNICK ", 6)) {
return;
}
if (moloch_nids_has_protocol(session, "irc"))
return;
moloch_nids_add_protocol(session, "irc");
IRCInfo_t *irc = MOLOCH_TYPE_ALLOC0(IRCInfo_t);
moloch_parsers_register(session, irc_parser, irc, irc_free);
irc_parser(session, irc, data, len, which);
}
void moloch_config_load_local_ips()
{
GError *error = 0;
if (!g_key_file_has_group(molochKeyFile, "override-ips"))
return;
gsize keys_len;
gchar **keys = g_key_file_get_keys (molochKeyFile, "override-ips", &keys_len, &error);
if (error) {
LOGEXIT("Error with override-ips: %s", error->message);
}
gsize k, v;
for (k = 0 ; k < keys_len; k++) {
gsize values_len;
gchar **values = g_key_file_get_string_list(molochKeyFile,
"override-ips",
keys[k],
&values_len,
NULL);
MolochIpInfo_t *ii = MOLOCH_TYPE_ALLOC0(MolochIpInfo_t);
for (v = 0; v < values_len; v++) {
if (strncmp(values[v], "asn:", 4) == 0) {
ii->asn = g_strdup(values[v]+4);
} else if (strncmp(values[v], "rir:", 4) == 0) {
ii->rir = g_strdup(values[v]+4);
} else if (strncmp(values[v], "tag:", 4) == 0) {
if (ii->numtags < 10) {
ii->tagsStr[ii->numtags] = strdup(values[v]+4);
ii->numtags++;
}
} else if (strncmp(values[v], "country:", 8) == 0) {
ii->country = g_strdup(values[v]+8);
}
}
moloch_db_add_local_ip(keys[k], ii);
g_strfreev(values);
}
g_strfreev(keys);
}
LOCAL void reader_libpcapfile_pcap_cb(u_char *UNUSED(user), const struct pcap_pkthdr *h, const u_char *bytes)
{
MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t);
if (unlikely(h->caplen != h->len)) {
if (!config.readTruncatedPackets) {
LOGEXIT("ERROR - Moloch requires full packet captures caplen: %d pktlen: %d. "
"If using tcpdump use the \"-s0\" option, or set readTruncatedPackets in ini file",
h->caplen, h->len);
}
packet->pktlen = h->caplen;
} else {
packet->pktlen = h->len;
}
packet->pkt = (u_char *)bytes;
packet->ts = h->ts;
packet->readerFilePos = ftell(offlineFile) - 16 - h->len;
packet->readerPos = readerPos;
moloch_packet_batch(&batch, packet);
}
gboolean moloch_string_add(void *hashv, char *string, gpointer uw, gboolean copy)
{
MolochStringHash_t *hash = hashv;
MolochString_t *hstring;
HASH_FIND(s_, *hash, string, hstring);
if (hstring) {
hstring->uw = uw;
return FALSE;
}
hstring = MOLOCH_TYPE_ALLOC0(MolochString_t);
if (copy) {
hstring->str = g_strdup(string);
} else {
hstring->str = string;
}
hstring->len = strlen(string);
hstring->uw = uw;
HASH_ADD(s_, *hash, hstring->str, hstring);
return TRUE;
}
void *moloch_http_create_server(char *hostname, int defaultPort, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttp_t *server = MOLOCH_TYPE_ALLOC0(MolochHttp_t);
DLL_INIT(r_, &server->requestQ[0]);
DLL_INIT(r_, &server->requestQ[1]);
DLL_INIT(e_, &server->connQ);
server->name = strdup(hostname);
server->port = defaultPort;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
server->syncConn = moloch_http_create(server);
uint32_t i;
for (i = 0; i < server->maxConns; i++) {
MolochConn_t *conn = moloch_http_create(server);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
return server;
}
void socks4_classify(MolochSession_t *session, const unsigned char *data, int len, int which)
{
#ifdef SOCKSDEBUG
LOG("SOCKSDEBUG: enter %d %d", data[0], len);
#endif
if (data[len-1] == 0) {
SocksInfo_t *socks;
socks = MOLOCH_TYPE_ALLOC0(SocksInfo_t);
socks->which = which;
socks->port = (data[2]&0xff) << 8 | (data[3]&0xff);
if (data[4] == 0 && data[5] == 0 && data[6] == 0 && data[7] != 0) {
socks->ip = 0;
} else {
memcpy(&socks->ip, data+4, 4);
}
int i;
for(i = 8; i < len && data[i]; i++);
if (i > 8 && i != len ) {
socks->user = g_strndup((char *)data+8, i-8);
socks->userlen = i - 8;
}
if (socks->ip == 0) {
i++;
int start;
for(start = i; i < len && data[i]; i++);
if (i > start && i != len ) {
socks->hostlen = i-start;
socks->host = g_ascii_strdown((char*)data+start, i-start);
}
}
moloch_parsers_register(session, socks4_parser, socks, socks_free);
}
}
void *moloch_http_create_server(const char *hostnames, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttpServer_t *server = MOLOCH_TYPE_ALLOC0(MolochHttpServer_t);
server->names = g_strsplit(hostnames, ",", 0);
uint32_t i;
for (i = 0; server->names[i]; i++) {
// Count entries
}
server->namesCnt = i;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
server->snames = malloc(server->namesCnt * sizeof(MolochHttpServer_t));
server->maxRetries = 3;
for (i = 0; server->names[i]; i++) {
server->snames[i].server = server;
server->snames[i].name = server->names[i];
server->snames[i].allowedAtSeconds = 0;
}
server->multi = curl_multi_init();
curl_multi_setopt(server->multi, CURLMOPT_SOCKETFUNCTION, moloch_http_curlm_socket_callback);
curl_multi_setopt(server->multi, CURLMOPT_SOCKETDATA, server);
curl_multi_setopt(server->multi, CURLMOPT_TIMERFUNCTION, moloch_http_curlm_timeout_callback);
curl_multi_setopt(server->multi, CURLMOPT_TIMERDATA, server);
curl_multi_setopt(server->multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, server->maxConns);
curl_multi_setopt(server->multi, CURLMOPT_MAXCONNECTS, server->maxConns);
server->multiTimer = g_timeout_add(50, moloch_http_timer_callback, server);
server->fd2ev = g_hash_table_new(NULL, NULL);
MOLOCH_LOCK_INIT(server->syncRequest);
return server;
}
