void moloch_http_free_server(void *serverV)
{
MolochHttp_t *server = serverV;
int n=0;
while (server->requestQ.r_count > 0 || server->inProgress > 0) {
n++;
if (n%10000 == 0)
LOG("%s %d %d", server->names[0], server->requestQ.r_count, server->inProgress);
g_main_context_iteration (g_main_context_default(), FALSE);
}
MolochConn_t *conn = 0;
while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
moloch_http_free_conn(conn, FALSE);
}
moloch_http_free_conn(server->syncConn, FALSE);
server->syncConn = 0;
g_strfreev(server->names);
MOLOCH_TYPE_FREE(MolochHttp_t, server);
}
int reader_libpcapfile_next()
{
char errbuf[1024];
gchar *fullfilename;
pcap = 0;
if (config.pcapReadFiles) {
static int pcapFilePos = 0;
fullfilename = config.pcapReadFiles[pcapFilePos];
errbuf[0] = 0;
if (!fullfilename) {
goto filesDone;
}
pcapFilePos++;
LOG ("Processing %s", fullfilename);
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
return reader_libpcapfile_next();
}
if (!realpath(fullfilename, offlinePcapFilename)) {
LOG("ERROR - pcap open failed - Couldn't realpath file: '%s' with %d", fullfilename, errno);
exit(1);
}
reader_libpcapfile_opened();
return 1;
}
filesDone:
if (config.pcapReadDirs) {
static int pcapDirPos = 0;
static GDir *pcapGDir[21];
static char *pcapBase[21];
static int pcapGDirLevel = -1;
GError *error = 0;
if (pcapGDirLevel == -2) {
goto dirsDone;
}
if (pcapGDirLevel == -1) {
pcapGDirLevel = 0;
pcapBase[0] = config.pcapReadDirs[pcapDirPos];
if (!pcapBase[0]) {
pcapGDirLevel = -2;
goto dirsDone;
}
}
if (!pcapGDir[pcapGDirLevel]) {
pcapGDir[pcapGDirLevel] = g_dir_open(pcapBase[pcapGDirLevel], 0, &error);
if (error) {
LOG("ERROR: Couldn't open pcap directory: Receive Error: %s", error->message);
exit(0);
}
}
const gchar *filename;
while (1) {
filename = g_dir_read_name(pcapGDir[pcapGDirLevel]);
// No more files, stop processing this directory
if (!filename) {
break;
}
// Skip hidden files/directories
if (filename[0] == '.')
continue;
fullfilename = g_build_filename (pcapBase[pcapGDirLevel], filename, NULL);
// If recursive option and a directory then process all the files in that dir
if (config.pcapRecursive && g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
if (pcapGDirLevel >= 20)
continue;
pcapBase[pcapGDirLevel+1] = fullfilename;
pcapGDirLevel++;
return reader_libpcapfile_next();
}
if (!g_regex_match(config.offlineRegex, filename, 0, NULL)) {
g_free(fullfilename);
continue;
}
if (!realpath(fullfilename, offlinePcapFilename)) {
g_free(fullfilename);
continue;
}
if (config.pcapSkip && moloch_db_file_exists(offlinePcapFilename)) {
if (config.debug)
LOG("Skipping %s", fullfilename);
g_free(fullfilename);
continue;
}
LOG ("Processing %s", fullfilename);
errbuf[0] = 0;
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
g_free(fullfilename);
continue;
}
reader_libpcapfile_opened();
g_free(fullfilename);
return 1;
}
g_dir_close(pcapGDir[pcapGDirLevel]);
pcapGDir[pcapGDirLevel] = 0;
if (pcapGDirLevel > 0) {
g_free(pcapBase[pcapGDirLevel]);
pcapGDirLevel--;
return reader_libpcapfile_next();
} else {
pcapDirPos++;
pcapGDirLevel = -1;
return reader_libpcapfile_next();
}
}
dirsDone:
while (DLL_COUNT(s_, &monitorQ) > 0) {
MolochString_t *string;
DLL_POP_HEAD(s_, &monitorQ, string);
fullfilename = string->str;
MOLOCH_TYPE_FREE(MolochString_t, string);
if (!realpath(fullfilename, offlinePcapFilename)) {
g_free(fullfilename);
continue;
}
if (config.pcapSkip && moloch_db_file_exists(offlinePcapFilename)) {
if (config.debug)
LOG("Skipping %s", fullfilename);
g_free(fullfilename);
continue;
}
LOG ("Processing %s", fullfilename);
errbuf[0] = 0;
pcap = pcap_open_offline(fullfilename, errbuf);
if (!pcap) {
LOG("Couldn't process '%s' error '%s'", fullfilename, errbuf);
g_free(fullfilename);
continue;
}
reader_libpcapfile_opened();
g_free(fullfilename);
return 1;
}
return 0;
}
int moloch_http_curl_close_callback(void *snameV, curl_socket_t fd)
{
MolochHttpServerName_t *sname = snameV;
MolochHttpServer_t *server = sname->server;
if (! BIT_ISSET(fd, connectionsSet)) {
long ev = (long)g_hash_table_lookup(server->fd2ev, (void *)(long)fd);
LOG("Couldn't connect %s (%d, %ld) ", sname->name, fd, ev);
close(fd);
GSource *source = g_main_context_find_source_by_id (NULL, ev);
if (source)
g_source_destroy (source);
g_hash_table_remove(server->fd2ev, (void *)(long)fd);
return 0;
}
struct sockaddr_storage localAddressStorage, remoteAddressStorage;
socklen_t addressLength = sizeof(localAddressStorage);
int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength);
if (rc != 0)
return 0;
addressLength = sizeof(remoteAddressStorage);
rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength);
if (rc != 0)
return 0;
char sessionId[MOLOCH_SESSIONID_LEN];
int localPort, remotePort;
char remoteIp[INET6_ADDRSTRLEN+2];
if (localAddressStorage.ss_family == AF_INET) {
struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage;
struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage;
moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port,
remoteAddress->sin_addr.s_addr, remoteAddress->sin_port);
localPort = ntohs(localAddress->sin_port);
remotePort = ntohs(remoteAddress->sin_port);
inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp));
} else {
struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage;
struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage;
moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port,
remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port);
localPort = ntohs(localAddress->sin6_port);
remotePort = ntohs(remoteAddress->sin6_port);
inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2);
remoteIp[0] = '[';
strcat(remoteIp, "]");
}
MolochHttpConn_t *conn;
BIT_CLR(fd, connectionsSet);
MOLOCH_LOCK(connections);
HASH_FIND(h_, connections, sessionId, conn);
if (conn) {
HASH_REMOVE(h_, connections, conn);
MOLOCH_TYPE_FREE(MolochHttpConn_t, conn);
}
MOLOCH_UNLOCK(connections);
server->connections--;
if (config.logHTTPConnections) {
LOG("Close %d/%d - %s %d->%s:%d fd:%d removed: %s",
server->outstanding,
server->connections,
sname->name,
localPort,
remoteIp,
remotePort,
fd,
conn?"true":"false");
}
close (fd);
return 0;
}
LOCAL void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server)
{
char *eff_url;
CURLMsg *msg;
int msgs_left;
MolochHttpRequest_t *request;
CURL *easy;
while ((msg = curl_multi_info_read(server->multi, &msgs_left))) {
if (msg->msg == CURLMSG_DONE) {
easy = msg->easy_handle;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request);
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);
long responseCode;
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode);
if (config.logESRequests || (server->printErrors && responseCode/100 != 2)) {
double totalTime;
double connectTime;
double uploadSize;
double downloadSize;
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime);
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime);
curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize);
curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize);
LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms",
request->server->outstanding,
request->server->connections,
responseCode,
request->url,
uploadSize,
downloadSize,
connectTime*1000,
totalTime*1000);
}
#ifdef MOLOCH_HTTP_DEBUG
LOG("HTTPDEBUG DECR %p %d %s", request, server->outstanding, request->url);
#endif
if (responseCode == 0 && request->retries < server->maxRetries) {
curl_multi_remove_handle(server->multi, easy);
request->retries++;
struct timeval now;
gettimeofday(&now, NULL);
MOLOCH_LOCK(requests);
server->snames[request->namePos].allowedAtSeconds = now.tv_sec + 30;
server->outstanding--;
moloch_http_add_request(server, request, TRUE);
MOLOCH_UNLOCK(requests);
} else {
if (server->printErrors && responseCode/100 != 2) {
LOG("Response length=%d :>\n%.*s", request->used, MIN(request->used, 4000), request->dataIn);
}
if (request->func) {
if (request->dataIn)
request->dataIn[request->used] = 0;
request->func(responseCode, request->dataIn, request->used, request->uw);
}
if (request->dataIn) {
free(request->dataIn);
request->dataIn = 0;
}
if (request->dataOut) {
MOLOCH_SIZE_FREE(buffer, request->dataOut);
}
if (request->headerList) {
curl_slist_free_all(request->headerList);
}
MOLOCH_TYPE_FREE(MolochHttpRequest_t, request);
curl_multi_remove_handle(server->multi, easy);
curl_easy_cleanup(easy);
MOLOCH_LOCK(requests);
server->outstanding--;
MOLOCH_UNLOCK(requests);
}
}
}
}
LOCAL int reader_libpcapfile_next()
{
gchar *fullfilename;
pcap = 0;
if (config.pcapReadFiles) {
static int pcapFilePos = 0;
fullfilename = config.pcapReadFiles[pcapFilePos];
if (!fullfilename) {
goto filesDone;
}
pcapFilePos++;
if (reader_libpcapfile_process(fullfilename)) {
return reader_libpcapfile_next();
}
return 1;
}
filesDone:
if (config.pcapFileLists) {
static int pcapFileListsPos;
static FILE *file;
char line[PATH_MAX];
if (!file && !config.pcapFileLists[pcapFileListsPos]) {
goto fileListsDone;
}
if (!file) {
if (strcmp(config.pcapFileLists[pcapFileListsPos], "-") == 0)
file = stdin;
else
file = fopen(config.pcapFileLists[pcapFileListsPos], "r");
pcapFileListsPos++;
if (!file) {
LOG("ERROR - Couldn't open %s", config.pcapFileLists[pcapFileListsPos - 1]);
return reader_libpcapfile_next();
}
}
if (feof(file)) {
fclose(file);
file = NULL;
return reader_libpcapfile_next();
}
if (!fgets(line, sizeof(line), file)) {
fclose(file);
file = NULL;
return reader_libpcapfile_next();
}
int lineLen = strlen(line);
if (line[lineLen-1] == '\n') {
line[lineLen-1] = 0;
}
g_strstrip(line);
if (!line[0] || line[0] == '#')
return reader_libpcapfile_next();
if (reader_libpcapfile_process(line)) {
return reader_libpcapfile_next();
}
return 1;
}
fileListsDone:
if (config.pcapReadDirs) {
static int pcapDirPos = 0;
static GDir *pcapGDir[21];
static char *pcapBase[21];
static int pcapGDirLevel = -1;
GError *error = 0;
if (pcapGDirLevel == -2) {
goto dirsDone;
}
if (pcapGDirLevel == -1) {
pcapGDirLevel = 0;
pcapBase[0] = config.pcapReadDirs[pcapDirPos];
if (!pcapBase[0]) {
pcapGDirLevel = -2;
goto dirsDone;
}
}
if (!pcapGDir[pcapGDirLevel]) {
pcapGDir[pcapGDirLevel] = g_dir_open(pcapBase[pcapGDirLevel], 0, &error);
if (error) {
LOGEXIT("ERROR: Couldn't open pcap directory: Receive Error: %s", error->message);
}
}
while (1) {
const gchar *filename = g_dir_read_name(pcapGDir[pcapGDirLevel]);
// No more files, stop processing this directory
if (!filename) {
break;
}
// Skip hidden files/directories
if (filename[0] == '.')
continue;
fullfilename = g_build_filename (pcapBase[pcapGDirLevel], filename, NULL);
// If recursive option and a directory then process all the files in that dir
if (config.pcapRecursive && g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
if (pcapGDirLevel >= 20)
continue;
pcapBase[pcapGDirLevel+1] = fullfilename;
pcapGDirLevel++;
return reader_libpcapfile_next();
}
if (!g_regex_match(config.offlineRegex, filename, 0, NULL)) {
g_free(fullfilename);
continue;
}
if (reader_libpcapfile_process(fullfilename)) {
g_free(fullfilename);
continue;
}
g_free(fullfilename);
return 1;
}
g_dir_close(pcapGDir[pcapGDirLevel]);
pcapGDir[pcapGDirLevel] = 0;
if (pcapGDirLevel > 0) {
g_free(pcapBase[pcapGDirLevel]);
pcapGDirLevel--;
return reader_libpcapfile_next();
} else {
pcapDirPos++;
pcapGDirLevel = -1;
return reader_libpcapfile_next();
}
}
dirsDone:
while (DLL_COUNT(s_, &monitorQ) > 0) {
MolochString_t *string;
DLL_POP_HEAD(s_, &monitorQ, string);
fullfilename = string->str;
MOLOCH_TYPE_FREE(MolochString_t, string);
if (reader_libpcapfile_process(fullfilename)) {
g_free(fullfilename);
continue;
}
g_free(fullfilename);
return 1;
}
return 0;
}
void wise_cb(unsigned char *data, int data_len, gpointer uw)
{
BSB bsb;
WiseRequest_t *request = uw;
int i;
inflight -= request->numItems;
BSB_INIT(bsb, data, data_len);
uint32_t fts = 0, ver = 0;
BSB_IMPORT_u32(bsb, fts);
BSB_IMPORT_u32(bsb, ver);
if (BSB_IS_ERROR(bsb) || ver != 0) {
for (i = 0; i < request->numItems; i++) {
wise_free_item(request->items[i]);
}
MOLOCH_TYPE_FREE(WiseRequest_t, request);
return;
}
if (fts != fieldsTS)
wise_load_fields();
struct timeval currentTime;
gettimeofday(¤tTime, NULL);
for (i = 0; i < request->numItems; i++) {
WiseItem_t *wi = request->items[i];
BSB_IMPORT_u08(bsb, wi->numOps);
if (wi->numOps > 0) {
wi->ops = malloc(wi->numOps * sizeof(WiseOp_t));
int i;
for (i = 0; i < wi->numOps; i++) {
WiseOp_t *op = &(wi->ops[i]);
int rfield = 0;
BSB_IMPORT_u08(bsb, rfield);
op->fieldPos = fieldsMap[rfield];
int len = 0;
BSB_IMPORT_u08(bsb, len);
char *str = (char*)BSB_WORK_PTR(bsb);
BSB_IMPORT_skip(bsb, len);
switch (config.fields[op->fieldPos]->type) {
case MOLOCH_FIELD_TYPE_INT_HASH:
if (op->fieldPos == tagsField) {
moloch_db_get_tag(NULL, tagsField, str, NULL); // Preload the tagname -> tag mapping
op->str = g_strdup(str);
op->strLenOrInt = len - 1;
continue;
}
// Fall thru
case MOLOCH_FIELD_TYPE_INT:
case MOLOCH_FIELD_TYPE_INT_ARRAY:
op->str = 0;
op->strLenOrInt = atoi(str);
break;
case MOLOCH_FIELD_TYPE_STR:
case MOLOCH_FIELD_TYPE_STR_ARRAY:
case MOLOCH_FIELD_TYPE_STR_HASH:
op->str = g_strdup(str);
op->strLenOrInt = len - 1;
break;
case MOLOCH_FIELD_TYPE_IP:
case MOLOCH_FIELD_TYPE_IP_HASH:
op->str = 0;
op->strLenOrInt = inet_addr(str);
break;
default:
LOG("WARNING - Unsupported expression type for %s", str);
continue;
}
}
}
wi->loadTime = currentTime.tv_sec;
int s;
for (s = 0; s < wi->numSessions; s++) {
wise_process_ops(wi->sessions[s], wi);
moloch_nids_decr_outstanding(wi->sessions[s]);
}
g_free(wi->sessions);
wi->sessions = 0;
wi->numSessions = 0;
DLL_PUSH_HEAD(wil_, &itemList[(int)wi->type], wi);
// Cache needs to be reduced
if (itemList[(int)wi->type].wil_count > maxCache) {
DLL_POP_TAIL(wil_, &itemList[(int)wi->type], wi);
wise_free_item(wi);
}
}
MOLOCH_TYPE_FREE(WiseRequest_t, request);
}
int moloch_field_define(char *group, char *kind, char *expression, char *friendlyName, char *dbField, char *help, int type, int flags, ...)
{
char dbField2[100];
char expression2[1000];
char friendlyName2[1000];
char help2[1000];
char rawField[100];
MolochFieldInfo_t *minfo = 0;
HASH_FIND(d_, fieldsByDb, dbField, minfo);
if (!minfo) {
minfo = MOLOCH_TYPE_ALLOC0(MolochFieldInfo_t);
minfo->dbFieldFull = g_strdup(dbField);
minfo->dbField = minfo->dbFieldFull;
minfo->dbFieldLen = strlen(minfo->dbField);
minfo->pos = -1;
minfo->expression = g_strdup(expression);
minfo->group = g_strdup(group);
minfo->kind = g_strdup(kind);
HASH_ADD(d_, fieldsByDb, minfo->dbField, minfo);
HASH_ADD(e_, fieldsByExp, minfo->expression, minfo);
if ((flags & MOLOCH_FIELD_FLAG_NODB) == 0) {
va_list args;
va_start(args, flags);
moloch_db_add_field(group, kind, expression, friendlyName, dbField, help, args);
va_end(args);
}
} else {
char *category = NULL;
if (strcmp(kind, minfo->kind) != 0) {
LOG("WARNING - Field kind in db %s doesn't match field kind %s in capture for field %s", minfo->kind, kind, expression);
}
va_list args;
va_start(args, flags);
while(1) {
char *field = va_arg(args, char *);
if (!field) break;
char *value = va_arg(args, char *);
if (strcmp(field, "category") == 0 && value) {
category = value;
}
}
va_end(args);
if (category && (!minfo->category || strcmp(category, minfo->category) != 0)) {
LOG("UPDATING - Field category in db %s doesn't match field category %s in capture for field %s", minfo->category, category, expression);
moloch_db_update_field(expression, "category", category);
}
}
// Hack to remove trailing .snow on capture side
int dbLen = strlen(minfo->dbField);
if (dbLen > 5 && memcmp(".snow", minfo->dbField+dbLen-5, 5) == 0) {
minfo->dbField[dbLen-5] = 0;
minfo->dbFieldLen -= 5;
}
minfo->type = type;
minfo->flags = flags;
if ((flags & MOLOCH_FIELD_FLAG_FAKE) == 0) {
if (minfo->pos == -1) {
minfo->pos = config.maxField++;
}
config.fields[minfo->pos] = minfo;
// Change leading part to dbGroup
char *firstdot = strchr(minfo->dbField, '.');
if (firstdot) {
static char lastGroup[100] = "";
static int groupNum = 0;
if (memcmp(minfo->dbField, lastGroup, (firstdot-minfo->dbField)+1) == 0) {
minfo->dbGroupNum = groupNum;
} else {
groupNum++;
minfo->dbGroupNum = groupNum;
memcpy(lastGroup, minfo->dbField, (firstdot-minfo->dbField)+1);
}
minfo->dbGroup = minfo->dbField;
minfo->dbGroupLen = firstdot - minfo->dbField;
minfo->dbField += (firstdot - minfo->dbField) + 1;
minfo->dbFieldLen = strlen(minfo->dbField);
}
}
if (flags & MOLOCH_FIELD_FLAG_NODB)
return minfo->pos;
MolochFieldInfo_t *info = 0;
if (flags & MOLOCH_FIELD_FLAG_CNT) {
sprintf(dbField2, "%scnt", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(expression2, "%s.cnt", expression);
sprintf(friendlyName2, "%s Cnt", friendlyName);
sprintf(help2, "Unique number of %s", help);
moloch_db_add_field(group, "integer", expression2, friendlyName2, dbField2, help2, NULL);
}
}
if (flags & MOLOCH_FIELD_FLAG_SCNT) {
sprintf(dbField2, "%sscnt", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(expression2, "%s.cnt", expression);
sprintf(friendlyName2, "%s Cnt", friendlyName);
sprintf(help2, "Unique number of %s", help);
moloch_db_add_field(group, "integer", expression2, friendlyName2, dbField2, help2, NULL);
}
}
if (flags & MOLOCH_FIELD_FLAG_COUNT) {
sprintf(dbField2, "%s-cnt", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(expression2, "%s.cnt", expression);
sprintf(friendlyName2, "%s Cnt", friendlyName);
sprintf(help2, "Unique number of %s", help);
moloch_db_add_field(group, "integer", expression2, friendlyName2, dbField2, help2, NULL);
}
}
if (flags & MOLOCH_FIELD_FLAG_FAKE) {
g_free(minfo->expression);
g_free(minfo->dbField);
g_free(minfo->group);
HASH_REMOVE(d_, fieldsByDb, minfo);
HASH_REMOVE(e_, fieldsByExp, minfo);
MOLOCH_TYPE_FREE(MolochFieldInfo_t, minfo);
return -1;
}
if (flags & MOLOCH_FIELD_FLAG_IPPRE) {
int fnlen = strlen(friendlyName);
sprintf(dbField2, "g%s", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(expression2, "country.%s", expression+3);
sprintf(friendlyName2, "%.*s GEO", fnlen-2, friendlyName);
sprintf(help2, "GeoIP country string calculated from the %s", help);
moloch_db_add_field(group, "uptermfield", expression2, friendlyName2, dbField2, help2, NULL);
}
sprintf(dbField2, "as%s", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(expression2, "asn.%s", expression+3);
sprintf(friendlyName2, "%.*s ASN", fnlen-2, friendlyName);
sprintf(help2, "GeoIP ASN string calculated from the %s", help);
sprintf(rawField, "raw%s", dbField2);
moloch_nids_add_field_proxy(group, "textfield", expression2, friendlyName2, dbField2, help2, "rawField", rawField, NULL);
}
sprintf(dbField2, "rir%s", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(expression2, "rir.%s", expression+3);
sprintf(friendlyName2, "%.*s RIR", fnlen-2, friendlyName);
sprintf(help2, "Regional Internet Registry string calculated from %s", help);
moloch_db_add_field(group, "uptermfield", expression2, friendlyName2, dbField2, help2, NULL);
}
} else if (type == MOLOCH_FIELD_TYPE_IP || type == MOLOCH_FIELD_TYPE_IP_HASH || type == MOLOCH_FIELD_TYPE_IP_GHASH) {
sprintf(dbField2, "%s-geo", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(expression2, "%s.country", expression);
sprintf(friendlyName2, "%s GEO", friendlyName);
sprintf(help2, "GeoIP country string calculated from the %s", help);
moloch_db_add_field(group, "uptermfield", expression2, friendlyName2, dbField2, help2, NULL);
}
sprintf(dbField2, "%s-asn.snow", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(dbField2, "%s-asn.snow", dbField);
sprintf(expression2, "%s.asn", expression);
sprintf(friendlyName2, "%s ASN", friendlyName);
sprintf(rawField, "%s-asn.raw", dbField);
sprintf(help2, "GeoIP ASN string calculated from the %s", help);
moloch_nids_add_field_proxy(group, "textfield", expression2, friendlyName2, dbField2, help2, "rawField", rawField, NULL);
}
sprintf(dbField2, "%s-rir", dbField);
HASH_FIND(d_, fieldsByDb, dbField2, info);
if (!info) {
sprintf(expression2, "%s.rir", expression);
sprintf(friendlyName2, "%s RIR", friendlyName);
sprintf(help2, "Regional Internet Registry string calculated from %s", help);
moloch_db_add_field(group, "uptermfield", expression2, friendlyName2, dbField2, help2, NULL);
}
}
return minfo->pos;
}
void moloch_session_process_commands(int thread)
{
// Commands
MolochSesCmd_t *cmd = 0;
int count;
for (count = 0; count < 50; count++) {
MOLOCH_LOCK(sessionCmds[thread].lock);
DLL_POP_HEAD(cmd_, &sessionCmds[thread], cmd);
MOLOCH_UNLOCK(sessionCmds[thread].lock);
if (!cmd)
break;
switch (cmd->cmd) {
case MOLOCH_SES_CMD_ADD_TAG:
moloch_field_int_add((long)cmd->uw1, cmd->session, (long)cmd->uw2);
moloch_session_decr_outstanding(cmd->session);
break;
case MOLOCH_SES_CMD_FUNC:
cmd->func(cmd->session, cmd->uw1, cmd->uw2);
break;
default:
LOG ("Unknown cmd %d", cmd->cmd);
}
MOLOCH_TYPE_FREE(MolochSesCmd_t, cmd);
}
// Closing Q
for (count = 0; count < 10; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(q_, &closingQ[thread]);
if (session && session->saveTime < (uint64_t)lastPacketSecs[thread]) {
moloch_session_save(session);
} else {
break;
}
}
// Sessions Idle Long Time
int ses;
for (ses = 0; ses < SESSION_MAX; ses++) {
for (count = 0; count < 10; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(q_, &sessionsQ[thread][ses]);
if (session && (DLL_COUNT(q_, &sessionsQ[thread][ses]) > (int)config.maxStreams ||
((uint64_t)session->lastPacket.tv_sec + config.timeouts[ses] < (uint64_t)lastPacketSecs[thread]))) {
moloch_session_save(session);
} else {
break;
}
}
}
// TCP Sessions Open Long Time
for (count = 0; count < 50; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(tcp_, &tcpWriteQ[thread]);
if (session && (uint64_t)session->saveTime < (uint64_t)lastPacketSecs[thread]) {
moloch_session_mid_save(session, lastPacketSecs[thread]);
} else {
break;
}
}
}
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
MolochConn_t *conn = data;
char buffer[0xffff];
int len;
GError *gerror = 0;
len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);
if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
if (gerror) {
LOG("ERROR: %p:%p Receive Error: %s", (void*)conn, conn->request, gerror->message);
g_error_free(gerror);
} else if (cond & (G_IO_HUP | G_IO_ERR))
LOG("ERROR: %p:%p Lost connection to %s", (void*)conn, conn->request, conn->name);
else if (len <= 0)
LOG("ERROR: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
else
LOG("ERROR HMM: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
if (conn->request) {
// Must save, free, then call function because of recursive sync GETs
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
if (func) {
func(0, 0, uw);
}
}
g_object_unref (conn->conn);
conn->conn = 0;
if (conn != conn->server->syncConn && conn->request) {
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
conn->request = 0;
return FALSE;
}
http_parser_execute(&conn->parser, &parserSettings, buffer, len);
if (conn->hp_complete) {
gettimeofday(&conn->endTime, NULL);
if (config.logESRequests)
LOG("%s %ldms %ldms %ldms",
conn->line,
(conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
);
conn->hp_data[conn->hp_len] = 0;
/* Must save, free, then call function because of recursive sync GETs */
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
conn->request = 0;
if (func) {
func((unsigned char*)conn->hp_data, conn->hp_len, uw);
}
if (conn == conn->server->syncConn)
return TRUE;
int q;
for (q = 0; q < 2; q++) {
DLL_POP_HEAD(r_, &conn->server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
DLL_PUSH_HEAD(r_, &conn->server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
return TRUE;
}
}
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
return TRUE;
}
LOCAL void irc_free(MolochSession_t UNUSED(*session), void *uw)
{
IRCInfo_t *irc = uw;
MOLOCH_TYPE_FREE(IRCInfo_t, irc);
}
LOCAL void tds_free(MolochSession_t UNUSED(*session), void *uw)
{
TDSInfo_t *tds = uw;
MOLOCH_TYPE_FREE(TDSInfo_t, tds);
}
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
MolochConn_t *conn = data;
char buffer[0xffff];
int len;
GError *gerror = 0;
len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);
if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
if (gerror) {
LOG("ERROR: %s %p:%p Receive Error: %s", conn->name, (void*)conn, conn->request, gerror->message);
g_error_free(gerror);
} else if (cond & (G_IO_HUP | G_IO_ERR))
LOG("ERROR: %s %p:%p Lost connection to %s", conn->name, (void*)conn, conn->request, conn->name);
else if (len <= 0)
LOG("ERROR: %s %p:%p len: %d cond: %x", conn->name, (void*)conn, conn->request, len, cond);
else
LOG("ERROR HMM: %s %p:%p len: %d cond: %x", conn->name, (void*)conn, conn->request, len, cond);
if (conn == conn->server->syncConn) {
if (!conn->request) {
DEBUGCONN("AAA zerosync %s %p", conn->server->names[0], conn);
conn->server->syncConn = 0;
moloch_http_free_conn(conn, FALSE);
} else {
DEBUGCONN("AAA complete %s %p", conn->server->names[0], conn);
conn->hp_complete = 1;
}
return FALSE;
}
if (conn->request && conn->request->data) {
conn->server->inProgress--;
DLL_PUSH_HEAD(r_, &conn->server->requestQ, conn->request);
} else if (conn->request) {
conn->server->inProgress--;
// Must save, free, then call function because of recursive sync GETs
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
if (func) {
func(0, 0, uw);
}
}
moloch_http_free_conn(conn, TRUE);
return FALSE;
}
http_parser_execute(&conn->parser, &conn->server->parserSettings, buffer, len);
if (conn->hp_complete) {
gettimeofday(&conn->endTime, NULL);
if (config.logESRequests)
LOG("%s %ldms %ldms %ldms",
conn->line,
(conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
);
conn->hp_data[conn->hp_len] = 0;
/* Must save, free, then call function because of recursive sync GETs */
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
conn->request = 0;
if (func) {
func((unsigned char*)conn->hp_data, conn->hp_len, uw);
}
if (conn == conn->server->syncConn) {
if (conn->doClose) {
DEBUGCONN("AAA zerosync %s %p", conn->server->names[0], conn);
conn->server->syncConn = 0;
moloch_http_free_conn(conn, FALSE);
return FALSE;
}
return TRUE;
}
// Not syncConn below this
if (conn->doClose) {
moloch_http_free_conn(conn, TRUE);
return FALSE;
}
conn->server->inProgress--;
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
moloch_http_do_requests(conn->server);
}
return TRUE;
}
int moloch_http_curl_close_callback(void *serverV, curl_socket_t fd)
{
MolochHttpServer_t *server = serverV;
if (! BIT_ISSET(fd, connectionsSet)) {
LOG("Couldn't connect %s defaultPort: %d", server->names[0], server->defaultPort);
return 0;
}
struct sockaddr_storage localAddressStorage, remoteAddressStorage;
socklen_t addressLength = sizeof(localAddressStorage);
int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength);
if (rc != 0)
return 0;
addressLength = sizeof(remoteAddressStorage);
rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength);
if (rc != 0)
return 0;
char sessionId[MOLOCH_SESSIONID_LEN];
int localPort, remotePort;
char remoteIp[INET6_ADDRSTRLEN+2];
if (localAddressStorage.ss_family == AF_INET) {
struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage;
struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage;
moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port,
remoteAddress->sin_addr.s_addr, remoteAddress->sin_port);
localPort = ntohs(localAddress->sin_port);
remotePort = ntohs(remoteAddress->sin_port);
inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp));
} else {
struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage;
struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage;
moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port,
remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port);
localPort = ntohs(localAddress->sin6_port);
remotePort = ntohs(remoteAddress->sin6_port);
inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2);
remoteIp[0] = '[';
strcat(remoteIp, "]");
}
MolochHttpConn_t *conn;
BIT_CLR(fd, connectionsSet);
MOLOCH_LOCK(connections);
HASH_FIND(h_, connections, sessionId, conn);
if (conn) {
HASH_REMOVE(h_, connections, conn);
MOLOCH_TYPE_FREE(MolochHttpConn_t, conn);
}
MOLOCH_UNLOCK(connections);
server->connections--;
LOG("Close %d/%d - %s %d->%s:%d fd:%d removed: %s",
server->outstanding,
server->connections,
server->names[0],
localPort,
remoteIp,
remotePort,
fd,
conn?"true":"false");
close (fd);
return 0;
}
void ssh_free(MolochSession_t UNUSED(*session), void *uw)
{
SSHInfo_t *ssh = uw;
MOLOCH_TYPE_FREE(SSHInfo_t, ssh);
}
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochConn_t *conn;
MolochHttp_t *server = serverV;
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
memcpy(request->key, key, MIN(key_len, sizeof(request->key)));
strncpy(request->method, method, sizeof(request->method));
request->key_len = key_len;
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0;
// Already have outstanding requests, see if we can process them
if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) {
while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
DLL_POP_HEAD(r_, &server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
else {
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
}
// Now see if we can send something new
if (DLL_POP_HEAD(e_, &server->connQ, conn)) {
conn->request = request;
if (!moloch_http_process_send(conn, FALSE)) {
conn->request = 0;
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
} else {
request->data = data;
if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
MOLOCH_TYPE_FREE(MolochRequest_t, request);
return 1;
} else {
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
}
}
return 0;
}
static void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server)
{
char *eff_url;
CURLMsg *msg;
int msgs_left;
MolochHttpRequest_t *request;
CURL *easy;
while ((msg = curl_multi_info_read(server->multi, &msgs_left))) {
if (msg->msg == CURLMSG_DONE) {
easy = msg->easy_handle;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request);
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);
long responseCode;
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode);
if (config.logESRequests) {
double totalTime;
double connectTime;
double uploadSize;
double downloadSize;
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime);
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime);
curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize);
curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize);
LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms",
request->server->outstanding,
request->server->connections,
responseCode,
request->url,
uploadSize,
downloadSize,
connectTime*1000,
totalTime*1000);
}
if (request->func) {
if (request->dataIn)
request->dataIn[request->used] = 0;
request->func(responseCode, request->dataIn, request->used, request->uw);
}
if (request->dataIn) {
free(request->dataIn);
request->dataIn = 0;
}
if (request->dataOut) {
MOLOCH_SIZE_FREE(buffer, request->dataOut);
}
if (request->headerList) {
curl_slist_free_all(request->headerList);
}
MOLOCH_TYPE_FREE(MolochHttpRequest_t, request);
curl_multi_remove_handle(server->multi, easy);
curl_easy_cleanup(easy);
MOLOCH_LOCK(requests);
server->outstanding--;
MOLOCH_UNLOCK(requests);
}
}
}
LOCAL void krb5_free(MolochSession_t UNUSED(*session), void *uw)
{
KRB5Info_t *krb5 = uw;
MOLOCH_TYPE_FREE(KRB5Info_t, krb5);
}
