void *moloch_http_create_server(char *hostnames, int defaultPort, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttp_t *server = MOLOCH_TYPE_ALLOC0(MolochHttp_t);
DLL_INIT(r_, &server->requestQ[0]);
DLL_INIT(r_, &server->requestQ[1]);
DLL_INIT(e_, &server->connQ);
server->names = g_strsplit(hostnames, ",", 0);
uint32_t i;
for (i = 0; server->names[i]; i++) {
if (strncmp(server->names[i], "http://", 7) == 0) {
char *tmp = g_strdup(server->names[i] + 7);
g_free(server->names[i]);
server->names[i] = tmp;
} else if (strncmp(server->names[i], "https://", 8) == 0) {
LOG("https not supported yet %s", server->names[i]);
exit(0);
}
}
server->namesCnt = i;
server->port = defaultPort;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
server->syncConn = moloch_http_create(server, TRUE);
for (i = 0; i < server->maxConns; i++) {
MolochConn_t *conn = moloch_http_create(server, FALSE);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
return server;
}
LOCAL void reader_libpcapfile_monitor_do(struct inotify_event *event)
{
gchar *dirname = g_hash_table_lookup(wdHashTable, (void *)(long)event->wd);
gchar *fullfilename = g_build_filename (dirname, event->name, NULL);
if (config.pcapRecursive &&
(event->mask & IN_CREATE) &&
g_file_test(fullfilename, G_FILE_TEST_IS_DIR)) {
reader_libpcapfile_monitor_dir(fullfilename);
g_free(fullfilename);
return;
}
if ((event->mask & IN_CLOSE_WRITE) == 0) {
g_free(fullfilename);
return;
}
if (!g_regex_match(config.offlineRegex, fullfilename, 0, NULL)) {
g_free(fullfilename);
return;
}
MolochString_t *string = MOLOCH_TYPE_ALLOC0(MolochString_t);
string->str = fullfilename;
if (config.debug)
LOG("Monitor enqueing %s", string->str);
DLL_PUSH_TAIL(s_, &monitorQ, string);
return;
}
void writer_simple_free(MolochSimple_t *info)
{
int thread = info->thread;
if (info->closing) {
switch(simpleMode) {
case MOLOCH_SIMPLE_NORMAL:
break;
case MOLOCH_SIMPLE_XOR2048:
break;
case MOLOCH_SIMPLE_AES256CTR:
EVP_CIPHER_CTX_free(info->file->cipher_ctx);
break;
}
MOLOCH_TYPE_FREE(MolochSimpleFile_t, info->file);
}
info->file = 0;
if (DLL_COUNT(simple_, &freeList[thread]) < 16) {
MOLOCH_LOCK(freeList[thread].lock);
DLL_PUSH_TAIL(simple_, &freeList[thread], info);
MOLOCH_UNLOCK(freeList[thread].lock);
} else {
munmap(info->buf, config.pcapWriteSize + MOLOCH_PACKET_MAX_LEN);
MOLOCH_TYPE_FREE(MolochSimple_t, info);
}
}
gboolean moloch_http_conn_cb(gint UNUSED(fd), GIOCondition UNUSED(cond), gpointer data) {
MolochConn_t *conn = data;
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
DEBUGCONN("AAA conn ww clear %s %p fd:%d,%d ww:%d", conn->server->names[0], conn, fd, g_socket_get_fd(conn->conn), conn->writeWatch);
conn->writeWatch = 0;
moloch_http_do_requests(conn->server);
return FALSE;
}
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, char *headers, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochHttp_t *server = serverV;
if (!config.exiting && dropable && server->requestQ.r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->requestQ.r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
request->key_len = MIN(key_len, sizeof(request->key)-1);
memcpy(request->key, key, request->key_len);
request->key[request->key_len] = 0;
strncpy(request->method, method, sizeof(request->method));
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
if (headers)
strcpy(request->headers, headers);
else
request->headers[0] = 0;
DLL_PUSH_TAIL(r_, &server->requestQ, request);
moloch_http_do_requests(server);
return 0;
}
void moloch_session_mark_for_close (MolochSession_t *session, int ses)
{
session->closingQ = 1;
session->saveTime = session->lastPacket.tv_sec + 5;
DLL_REMOVE(q_, &sessionsQ[session->thread][ses], session);
DLL_PUSH_TAIL(q_, &closingQ[session->thread], session);
if (session->tcp_next) {
DLL_REMOVE(tcp_, &tcpWriteQ[session->thread], session);
}
}
void moloch_session_add_cmd(MolochSession_t *session, MolochSesCmd icmd, gpointer uw1, gpointer uw2, MolochCmd_func func)
{
MolochSesCmd_t *cmd = MOLOCH_TYPE_ALLOC(MolochSesCmd_t);
cmd->cmd = icmd;
cmd->session = session;
cmd->uw1 = uw1;
cmd->uw2 = uw2;
cmd->func = func;
MOLOCH_LOCK(sessionCmds[session->thread].lock);
DLL_PUSH_TAIL(cmd_, &sessionCmds[session->thread], cmd);
moloch_packet_thread_wake(session->thread);
MOLOCH_UNLOCK(sessionCmds[session->thread].lock);
}
void writer_simple_process_buf(int thread, int closing)
{
MolochSimple_t *info = currentInfo[thread];
info->closing = closing;
if (!closing) {
currentInfo[thread] = writer_simple_alloc(thread, info);
memcpy(currentInfo[thread]->buf, info->buf + config.pcapWriteSize, info->bufpos - config.pcapWriteSize);
currentInfo[thread]->bufpos = info->bufpos - config.pcapWriteSize;
} else {
currentInfo[thread] = NULL;
}
MOLOCH_LOCK(simpleQ);
DLL_PUSH_TAIL(simple_, &simpleQ, info);
MOLOCH_COND_SIGNAL(simpleQ);
MOLOCH_UNLOCK(simpleQ);
}
// Should only be used by packet, lots of side effects
MolochSession_t *moloch_session_find_or_create(int ses, uint32_t hash, char *sessionId, int *isNew)
{
MolochSession_t *session;
if (hash == 0) {
hash = moloch_session_hash(sessionId);
}
int thread = hash % config.packetThreads;
HASH_FIND_HASH(h_, sessions[thread][ses], hash, sessionId, session);
if (session) {
if (!session->closingQ) {
DLL_MOVE_TAIL(q_, &sessionsQ[thread][ses], session);
}
*isNew = 0;
return session;
}
*isNew = 1;
session = MOLOCH_TYPE_ALLOC0(MolochSession_t);
session->ses = ses;
memcpy(session->sessionId, sessionId, sessionId[0]);
HASH_ADD_HASH(h_, sessions[thread][ses], hash, sessionId, session);
DLL_PUSH_TAIL(q_, &sessionsQ[thread][ses], session);
if (HASH_BUCKET_COUNT(h_, sessions[thread][ses], hash) > 10) {
char buf[100];
LOG("Large number of chains: %s %u %u %u %u", moloch_session_id_string(sessionId, buf), hash, hash % sessions[thread][ses].size, thread, HASH_BUCKET_COUNT(h_, sessions[thread][ses], hash));
}
session->filePosArray = g_array_sized_new(FALSE, FALSE, sizeof(uint64_t), 100);
session->fileLenArray = g_array_sized_new(FALSE, FALSE, sizeof(uint16_t), 100);
session->fileNumArray = g_array_new(FALSE, FALSE, 4);
session->fields = MOLOCH_SIZE_ALLOC0(fields, sizeof(MolochField_t *)*config.maxField);
session->maxFields = config.maxField;
session->thread = thread;
DLL_INIT(td_, &session->tcpData);
if (config.numPlugins > 0)
session->pluginData = MOLOCH_SIZE_ALLOC0(pluginData, sizeof(void *)*config.numPlugins);
return session;
}
void
tls_certinfo_process(MolochCertInfo_t *ci, BSB *bsb)
{
int apc, atag, alen;
char lastOid[1000];
lastOid[0] = 0;
while (BSB_REMAINING(*bsb)) {
unsigned char *value = moloch_parsers_asn_get_tlv(bsb, &apc, &atag, &alen);
if (!value)
return;
if (apc) {
BSB tbsb;
BSB_INIT(tbsb, value, alen);
tls_certinfo_process(ci, &tbsb);
} else if (atag == 6) {
moloch_parsers_asn_decode_oid(lastOid, sizeof(lastOid), value, alen);
} else if (lastOid[0] && (atag == 20 || atag == 19 || atag == 12)) {
/* 20 == BER_UNI_TAG_TeletexString
* 19 == BER_UNI_TAG_PrintableString
* 12 == BER_UNI_TAG_UTF8String
*/
if (strcmp(lastOid, "2.5.4.3") == 0) {
MolochString_t *element = MOLOCH_TYPE_ALLOC(MolochString_t);
element->utf8 = atag == 12;
if (element->utf8)
element->str = g_utf8_strdown((char*)value, alen);
else
element->str = g_ascii_strdown((char*)value, alen);
DLL_PUSH_TAIL(s_, &ci->commonName, element);
} else if (strcmp(lastOid, "2.5.4.10") == 0) {
if (ci->orgName) {
LOG("Multiple orgName %s => %.*s", ci->orgName, alen, value);
free(ci->orgName);
}
ci->orgUtf8 = atag == 12;
ci->orgName = g_strndup((char*)value, alen);
}
}
}
}
LOCAL void moloch_http_add_request(MolochHttpServer_t *server, MolochHttpRequest_t *request, gboolean async)
{
struct timeval now;
gettimeofday(&now, NULL);
int startPos = server->namesPos;
int offset = 0;
while (server->snames[server->namesPos].allowedAtSeconds > now.tv_sec) {
server->snames[server->namesPos].allowedAtSeconds -= offset;
server->namesPos = (server->namesPos + 1) % server->namesCnt;
if (startPos == server->namesPos)
offset = 1;
}
request->namePos = server->namesPos;
server->namesPos = (server->namesPos + 1) % server->namesCnt;
char *host = server->names[request->namePos];
snprintf(request->url, sizeof(request->url), "%s%s", host, request->key);
curl_easy_setopt(request->easy, CURLOPT_URL, request->url);
if (async) {
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETDATA, &server->snames[request->namePos]);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETDATA, &server->snames[request->namePos]);
#ifdef MOLOCH_HTTP_DEBUG
LOG("HTTPDEBUG INCR %p %d %s", request, server->outstanding, request->url);
#endif
server->outstanding++;
DLL_PUSH_TAIL(rqt_, &requests, request);
if (!requestsTimer)
requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL);
} else {
}
}
void *moloch_http_create_server(char *hostname, int defaultPort, int maxConns, int maxOutstandingRequests, int compress)
{
MolochHttp_t *server = MOLOCH_TYPE_ALLOC0(MolochHttp_t);
DLL_INIT(r_, &server->requestQ[0]);
DLL_INIT(r_, &server->requestQ[1]);
DLL_INIT(e_, &server->connQ);
server->name = strdup(hostname);
server->port = defaultPort;
server->maxConns = maxConns;
server->maxOutstandingRequests = maxOutstandingRequests;
server->compress = compress;
server->syncConn = moloch_http_create(server);
uint32_t i;
for (i = 0; i < server->maxConns; i++) {
MolochConn_t *conn = moloch_http_create(server);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
return server;
}
void
tls_alt_names(MolochCertsInfo_t *certs, BSB *bsb, char *lastOid)
{
int apc, atag, alen;
while (BSB_REMAINING(*bsb) >= 2) {
unsigned char *value = moloch_parsers_asn_get_tlv(bsb, &apc, &atag, &alen);
if (!value)
return;
if (apc) {
BSB tbsb;
BSB_INIT(tbsb, value, alen);
tls_alt_names(certs, &tbsb, lastOid);
if (certs->alt.s_count > 0) {
return;
}
} else if (atag == 6) {
moloch_parsers_asn_decode_oid(lastOid, 100, value, alen);
if (strcmp(lastOid, "2.5.29.17") != 0)
lastOid[0] = 0;
} else if (lastOid[0] && atag == 4) {
BSB tbsb;
BSB_INIT(tbsb, value, alen);
tls_alt_names(certs, &tbsb, lastOid);
return;
} else if (lastOid[0] && atag == 2) {
MolochString_t *element = MOLOCH_TYPE_ALLOC0(MolochString_t);
element->str = g_ascii_strdown((char*)value, alen);
element->len = alen;
DLL_PUSH_TAIL(s_, &certs->alt, element);
}
}
lastOid[0] = 0;
return;
}
static void
reader_libpcapfile_monitor_changed (GFileMonitor *UNUSED(monitor),
GFile *file,
GFile *UNUSED(other_file),
GFileMonitorEvent event_type,
gpointer UNUSED(user_data))
{
// Monitor new directories?
if (config.pcapRecursive &&
event_type == G_FILE_MONITOR_EVENT_CREATED &&
g_file_query_file_type(file, G_FILE_QUERY_INFO_NONE, NULL) == G_FILE_TYPE_DIRECTORY) {
gchar *path = g_file_get_path(file);
reader_libpcapfile_monitor_dir(path);
g_free(path);
return;
}
if (event_type != G_FILE_MONITOR_EVENT_CHANGES_DONE_HINT)
return;
gchar *basename = g_file_get_path(file);
if (!g_regex_match(config.offlineRegex, basename, 0, NULL)) {
g_free(basename);
return;
}
g_free(basename);
gchar *path = g_file_get_path(file);
MolochString_t *string = MOLOCH_TYPE_ALLOC0(MolochString_t);
string->str = path;
if (config.debug)
LOG("Monitor enqueing %s", string->str);
DLL_PUSH_TAIL(s_, &monitorQ, string);
}
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
MolochRequest_t *request;
MolochConn_t *conn;
MolochHttp_t *server = serverV;
request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
memcpy(request->key, key, MIN(key_len, sizeof(request->key)));
strncpy(request->method, method, sizeof(request->method));
request->key_len = key_len;
request->compress = 0;
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->compress = 1;
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->data_len = data_len;
request->data = data;
request->func = func;
request->uw = uw;
int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0;
// Already have outstanding requests, see if we can process them
if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) {
while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
DLL_POP_HEAD(r_, &server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
else {
DLL_PUSH_TAIL(e_, &server->connQ, conn);
break;
}
}
}
// Now see if we can send something new
if (DLL_POP_HEAD(e_, &server->connQ, conn)) {
conn->request = request;
if (!moloch_http_process_send(conn, FALSE)) {
conn->request = 0;
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
DLL_PUSH_TAIL(e_, &server->connQ, conn);
}
} else {
request->data = data;
if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
MOLOCH_TYPE_FREE(MolochRequest_t, request);
return 1;
} else {
DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
}
}
return 0;
}
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
MolochConn_t *conn = data;
char buffer[0xffff];
int len;
GError *gerror = 0;
len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);
if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
if (gerror) {
LOG("ERROR: %s %p:%p Receive Error: %s", conn->name, (void*)conn, conn->request, gerror->message);
g_error_free(gerror);
} else if (cond & (G_IO_HUP | G_IO_ERR))
LOG("ERROR: %s %p:%p Lost connection to %s", conn->name, (void*)conn, conn->request, conn->name);
else if (len <= 0)
LOG("ERROR: %s %p:%p len: %d cond: %x", conn->name, (void*)conn, conn->request, len, cond);
else
LOG("ERROR HMM: %s %p:%p len: %d cond: %x", conn->name, (void*)conn, conn->request, len, cond);
if (conn == conn->server->syncConn) {
if (!conn->request) {
DEBUGCONN("AAA zerosync %s %p", conn->server->names[0], conn);
conn->server->syncConn = 0;
moloch_http_free_conn(conn, FALSE);
} else {
DEBUGCONN("AAA complete %s %p", conn->server->names[0], conn);
conn->hp_complete = 1;
}
return FALSE;
}
if (conn->request && conn->request->data) {
conn->server->inProgress--;
DLL_PUSH_HEAD(r_, &conn->server->requestQ, conn->request);
} else if (conn->request) {
conn->server->inProgress--;
// Must save, free, then call function because of recursive sync GETs
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
if (func) {
func(0, 0, uw);
}
}
moloch_http_free_conn(conn, TRUE);
return FALSE;
}
http_parser_execute(&conn->parser, &conn->server->parserSettings, buffer, len);
if (conn->hp_complete) {
gettimeofday(&conn->endTime, NULL);
if (config.logESRequests)
LOG("%s %ldms %ldms %ldms",
conn->line,
(conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
);
conn->hp_data[conn->hp_len] = 0;
/* Must save, free, then call function because of recursive sync GETs */
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
conn->request = 0;
if (func) {
func((unsigned char*)conn->hp_data, conn->hp_len, uw);
}
if (conn == conn->server->syncConn) {
if (conn->doClose) {
DEBUGCONN("AAA zerosync %s %p", conn->server->names[0], conn);
conn->server->syncConn = 0;
moloch_http_free_conn(conn, FALSE);
return FALSE;
}
return TRUE;
}
// Not syncConn below this
if (conn->doClose) {
moloch_http_free_conn(conn, TRUE);
return FALSE;
}
conn->server->inProgress--;
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
moloch_http_do_requests(conn->server);
}
return TRUE;
}
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw)
{
MolochHttpServer_t *server = serverV;
// Are we overloaded
if (!config.exiting && dropable && server->outstanding > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t);
if (headers) {
int i;
for (i = 0; headers[i]; i++) {
request->headerList = curl_slist_append(request->headerList, headers[i]);
}
}
// Do we need to compress item
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate");
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
}
request->server = server;
request->func = func;
request->uw = uw;
request->dataOut = data;
request->dataOutLen = data_len;
request->easy = curl_easy_init();
if (config.debug >= 2) {
curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1);
}
curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback);
curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETDATA, server);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETDATA, server);
if (request->headerList) {
curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList);
}
if (method[0] != 'G') {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method);
curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data);
} else {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL);
curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L);
}
if (server->headerCb) {
curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function);
curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request);
}
curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L);
char *host = server->names[server->namesPos];
server->namesPos = (server->namesPos + 1) % server->namesCnt;
if (strchr(host, ':') == 0) {
snprintf(request->url, sizeof(request->url), "%s://%s:%d%.*s", (server->https?"https":"http"), host, server->defaultPort, key_len, key);
} else {
snprintf(request->url, sizeof(request->url), "%s://%s%.*s", (server->https?"https":"http"), host, key_len, key);
}
curl_easy_setopt(request->easy, CURLOPT_URL, request->url);
MOLOCH_LOCK(requests);
server->outstanding++;
DLL_PUSH_TAIL(rqt_, &requests, request);
if (!requestsTimer)
requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL);
MOLOCH_UNLOCK(requests);
return 0;
}
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
MolochConn_t *conn = data;
char buffer[0xffff];
int len;
GError *gerror = 0;
len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);
if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
if (gerror) {
LOG("ERROR: %p:%p Receive Error: %s", (void*)conn, conn->request, gerror->message);
g_error_free(gerror);
} else if (cond & (G_IO_HUP | G_IO_ERR))
LOG("ERROR: %p:%p Lost connection to %s", (void*)conn, conn->request, conn->name);
else if (len <= 0)
LOG("ERROR: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
else
LOG("ERROR HMM: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
if (conn->request) {
// Must save, free, then call function because of recursive sync GETs
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
if (func) {
func(0, 0, uw);
}
}
g_object_unref (conn->conn);
conn->conn = 0;
if (conn != conn->server->syncConn && conn->request) {
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
conn->request = 0;
return FALSE;
}
http_parser_execute(&conn->parser, &parserSettings, buffer, len);
if (conn->hp_complete) {
gettimeofday(&conn->endTime, NULL);
if (config.logESRequests)
LOG("%s %ldms %ldms %ldms",
conn->line,
(conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
(conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
);
conn->hp_data[conn->hp_len] = 0;
/* Must save, free, then call function because of recursive sync GETs */
MolochResponse_cb func = conn->request->func;
gpointer uw = conn->request->uw;
MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
conn->request = 0;
if (func) {
func((unsigned char*)conn->hp_data, conn->hp_len, uw);
}
if (conn == conn->server->syncConn)
return TRUE;
int q;
for (q = 0; q < 2; q++) {
DLL_POP_HEAD(r_, &conn->server->requestQ[q], conn->request);
if (conn->request) {
if (!moloch_http_process_send(conn, 0)) {
DLL_PUSH_HEAD(r_, &conn->server->requestQ[q], conn->request);
conn->request = 0;
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
return TRUE;
}
}
DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
}
return TRUE;
}
