void *moloch_http_create_server(char *hostnames, int defaultPort, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttp_t *server = MOLOCH_TYPE_ALLOC0(MolochHttp_t);
DLL_INIT(r_, &server->requestQ[0]);
DLL_INIT(r_, &server->requestQ[1]);
DLL_INIT(e_, &server->connQ);
server->names = g_strsplit(hostnames, ",", 0);
uint32_t i;
for (i = 0; server->names[i]; i++) {
if (strncmp(server->names[i], "http://", 7) == 0) {
char *tmp = g_strdup(server->names[i] + 7);
g_free(server->names[i]);
server->names[i] = tmp;
} else if (strncmp(server->names[i], "https://", 8) == 0) {
LOG("https not supported yet %s", server->names[i]);
exit(0);
}
}
server->namesCnt = i;
server->port = defaultPort;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
server->syncConn = moloch_http_create(server, TRUE);
for (i = 0; i < server->maxConns; i++) {
MolochConn_t *conn = moloch_http_create(server, FALSE);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
return server;
}
void *moloch_http_create_server(char *hostnames, int defaultPort, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttp_t *server = MOLOCH_TYPE_ALLOC0(MolochHttp_t);
DLL_INIT(r_, &server->requestQ);
DLL_INIT(e_, &server->connQ);
server->parserSettings.on_message_begin = moloch_http_hp_cb_on_message_begin;
server->parserSettings.on_body = moloch_http_hp_cb_on_body;
server->parserSettings.on_message_complete = moloch_http_hp_cb_on_message_complete;
server->parserSettings.on_header_field = moloch_http_hp_cb_on_header_field;
server->parserSettings.on_header_value = moloch_http_hp_cb_on_header_value;
server->names = g_strsplit(hostnames, ",", 0);
uint32_t i;
for (i = 0; server->names[i]; i++) {
if (strncmp(server->names[i], "http://", 7) == 0) {
char *tmp = g_strdup(server->names[i] + 7);
g_free(server->names[i]);
server->names[i] = tmp;
} else if (strncmp(server->names[i], "https://", 8) == 0) {
LOG("https not supported yet %s", server->names[i]);
exit(0);
}
}
server->namesCnt = i;
server->port = defaultPort;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
return server;
}
void writer_simple_init(char *UNUSED(name))
{
moloch_writer_queue_length = writer_simple_queue_length;
moloch_writer_exit = writer_simple_exit;
moloch_writer_write = writer_simple_write;
char *mode = moloch_config_str(NULL, "simpleEncoding", NULL);
simpleKEKId = moloch_config_str(NULL, "simpleKEKId", NULL);
if (simpleKEKId) {
char *key = moloch_config_section_str(NULL, "keks", simpleKEKId, NULL);
if (!key) {
LOGEXIT("No kek with id '%s' found in keks config section", simpleKEKId);
}
simpleKEKLen = EVP_BytesToKey(EVP_aes_192_cbc(), EVP_md5(), NULL, (uint8_t *)key, strlen(key), 1, simpleKEK, simpleIV);
}
if (mode == NULL) {
} else if (strcmp(mode, "aes-256-ctr") == 0) {
if (!simpleKEKId)
LOGEXIT("Must set simpleKEKId");
simpleMode = MOLOCH_SIMPLE_AES256CTR;
cipher = EVP_aes_256_ctr();
} else if (strcmp(mode, "xor-2048") == 0) {
if (!simpleKEKId)
LOGEXIT("Must set simpleKEKId");
LOG("WARNING - simpleEncoding of xor-2048 is not actually secure");
simpleMode = MOLOCH_SIMPLE_XOR2048;
} else {
LOGEXIT("Unknown simpleEncoding '%s'", mode);
}
pageSize = getpagesize();
if (config.pcapWriteSize % pageSize != 0) {
config.pcapWriteSize = ((config.pcapWriteSize + pageSize - 1) / pageSize) * pageSize;
LOG ("INFO: Reseting pcapWriteSize to %u since it must be a multiple of %u", config.pcapWriteSize, pageSize);
}
DLL_INIT(simple_, &simpleQ);
int thread;
for (thread = 0; thread < config.packetThreads; thread++) {
DLL_INIT(simple_, &freeList[thread]);
MOLOCH_LOCK_INIT(freeList[thread].lock);
}
g_thread_new("moloch-simple", &writer_simple_thread, NULL);
}
void reader_libpcapfile_init(char *UNUSED(name))
{
moloch_reader_start = reader_libpcapfile_start;
moloch_reader_stats = reader_libpcapfile_stats;
if (config.pcapMonitor)
reader_libpcapfile_init_monitor();
DLL_INIT(s_, &monitorQ);
}
void moloch_http_init()
{
z_strm.zalloc = Z_NULL;
z_strm.zfree = Z_NULL;
z_strm.opaque = Z_NULL;
deflateInit(&z_strm, Z_DEFAULT_COMPRESSION);
curl_global_init(CURL_GLOBAL_SSL);
HASH_INIT(h_, connections, moloch_session_hash, moloch_http_conn_cmp);
DLL_INIT(rqt_, &requests);
}
void *moloch_http_create_server(char *hostname, int defaultPort, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttp_t *server = MOLOCH_TYPE_ALLOC0(MolochHttp_t);
DLL_INIT(r_, &server->requestQ[0]);
DLL_INIT(r_, &server->requestQ[1]);
DLL_INIT(e_, &server->connQ);
server->name = strdup(hostname);
server->port = defaultPort;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
server->syncConn = moloch_http_create(server);
uint32_t i;
for (i = 0; i < server->maxConns; i++) {
MolochConn_t *conn = moloch_http_create(server);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
return server;
}
// Should only be used by packet, lots of side effects
MolochSession_t *moloch_session_find_or_create(int ses, uint32_t hash, char *sessionId, int *isNew)
{
MolochSession_t *session;
if (hash == 0) {
hash = moloch_session_hash(sessionId);
}
int thread = hash % config.packetThreads;
HASH_FIND_HASH(h_, sessions[thread][ses], hash, sessionId, session);
if (session) {
if (!session->closingQ) {
DLL_MOVE_TAIL(q_, &sessionsQ[thread][ses], session);
}
*isNew = 0;
return session;
}
*isNew = 1;
session = MOLOCH_TYPE_ALLOC0(MolochSession_t);
session->ses = ses;
memcpy(session->sessionId, sessionId, sessionId[0]);
HASH_ADD_HASH(h_, sessions[thread][ses], hash, sessionId, session);
DLL_PUSH_TAIL(q_, &sessionsQ[thread][ses], session);
if (HASH_BUCKET_COUNT(h_, sessions[thread][ses], hash) > 10) {
char buf[100];
LOG("Large number of chains: %s %u %u %u %u", moloch_session_id_string(sessionId, buf), hash, hash % sessions[thread][ses].size, thread, HASH_BUCKET_COUNT(h_, sessions[thread][ses], hash));
}
session->filePosArray = g_array_sized_new(FALSE, FALSE, sizeof(uint64_t), 100);
session->fileLenArray = g_array_sized_new(FALSE, FALSE, sizeof(uint16_t), 100);
session->fileNumArray = g_array_new(FALSE, FALSE, 4);
session->fields = MOLOCH_SIZE_ALLOC0(fields, sizeof(MolochField_t *)*config.maxField);
session->maxFields = config.maxField;
session->thread = thread;
DLL_INIT(td_, &session->tcpData);
if (config.numPlugins > 0)
session->pluginData = MOLOCH_SIZE_ALLOC0(pluginData, sizeof(void *)*config.numPlugins);
return session;
}
void moloch_session_init()
{
uint32_t primes[] = {10007, 49999, 99991, 199799, 400009, 500009, 732209, 1092757, 1299827, 1500007, 1987411, 2999999};
int p;
for (p = 0; p < 12; p++) {
if (primes[p] >= config.maxStreams/2)
break;
}
if (p == 12) p = 11;
protocolField = moloch_field_define("general", "termfield",
"protocols", "Protocols", "prot-term",
"Protocols set for session",
MOLOCH_FIELD_TYPE_STR_HASH, MOLOCH_FIELD_FLAG_COUNT | MOLOCH_FIELD_FLAG_LINKED_SESSIONS,
NULL);
if (config.debug)
LOG("session hash size %d", primes[p]);
int t;
for (t = 0; t < config.packetThreads; t++) {
HASHP_INIT(h_, sessions[t][SESSION_UDP], primes[p], moloch_session_hash, moloch_session_cmp);
HASHP_INIT(h_, sessions[t][SESSION_TCP], primes[p], moloch_session_hash, moloch_session_cmp);
HASHP_INIT(h_, sessions[t][SESSION_ICMP], primes[p], moloch_session_hash, moloch_session_cmp);
DLL_INIT(q_, &sessionsQ[t][SESSION_UDP]);
DLL_INIT(q_, &sessionsQ[t][SESSION_TCP]);
DLL_INIT(q_, &sessionsQ[t][SESSION_ICMP]);
DLL_INIT(tcp_, &tcpWriteQ[t]);
DLL_INIT(q_, &closingQ[t]);
DLL_INIT(cmd_, &sessionCmds[t]);
MOLOCH_LOCK_INIT(sessionCmds[t].lock);
}
moloch_add_can_quit(moloch_session_cmd_outstanding, "session commands outstanding");
moloch_add_can_quit(moloch_session_close_outstanding, "session close outstanding");
moloch_add_can_quit(moloch_session_need_save_outstanding, "session save outstanding");
}
void tls_process_server_certificate(MolochSession_t *session, const unsigned char *data, int len)
{
BSB cbsb;
BSB_INIT(cbsb, data, len);
BSB_IMPORT_skip(cbsb, 3); // Length again
while(BSB_REMAINING(cbsb) > 3) {
int badreason = 0;
unsigned char *cdata = BSB_WORK_PTR(cbsb);
int clen = MIN(BSB_REMAINING(cbsb) - 3, (cdata[0] << 16 | cdata[1] << 8 | cdata[2]));
MolochCertsInfo_t *certs = MOLOCH_TYPE_ALLOC0(MolochCertsInfo_t);
DLL_INIT(s_, &certs->alt);
DLL_INIT(s_, &certs->subject.commonName);
DLL_INIT(s_, &certs->issuer.commonName);
int atag, alen, apc;
unsigned char *value;
BSB bsb;
BSB_INIT(bsb, cdata + 3, clen);
/* Certificate */
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 1; goto bad_cert;}
BSB_INIT(bsb, value, alen);
/* signedCertificate */
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 2; goto bad_cert;}
BSB_INIT(bsb, value, alen);
/* serialNumber or version*/
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 3; goto bad_cert;}
if (apc) {
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 4; goto bad_cert;}
}
certs->serialNumberLen = alen;
certs->serialNumber = malloc(alen);
memcpy(certs->serialNumber, value, alen);
/* signature */
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 5; goto bad_cert;}
/* issuer */
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 6; goto bad_cert;}
BSB tbsb;
BSB_INIT(tbsb, value, alen);
tls_certinfo_process(&certs->issuer, &tbsb);
/* validity */
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 7; goto bad_cert;}
/* subject */
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 8; goto bad_cert;}
BSB_INIT(tbsb, value, alen);
tls_certinfo_process(&certs->subject, &tbsb);
/* subjectPublicKeyInfo */
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 9; goto bad_cert;}
/* extensions */
if (BSB_REMAINING(bsb)) {
if (!(value = moloch_parsers_asn_get_tlv(&bsb, &apc, &atag, &alen)))
{badreason = 10; goto bad_cert;}
BSB tbsb;
BSB_INIT(tbsb, value, alen);
char lastOid[100];
lastOid[0] = 0;
tls_alt_names(certs, &tbsb, lastOid);
}
if (!moloch_field_certsinfo_add(certsField, session, certs, clen*2)) {
moloch_field_certsinfo_free(certs);
}
BSB_IMPORT_skip(cbsb, clen + 3);
continue;
bad_cert:
if (config.debug)
LOG("bad cert %d - %d", badreason, clen);
moloch_field_certsinfo_free(certs);
break;
}
}
