LOCAL gboolean moloch_http_curl_watch_open_callback(int fd, GIOCondition condition, gpointer snameV)
{
MolochHttpServerName_t *sname = snameV;
MolochHttpServer_t *server = sname->server;
struct sockaddr_storage localAddressStorage, remoteAddressStorage;
socklen_t addressLength = sizeof(localAddressStorage);
int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength);
if (rc != 0)
return CURLE_OK;
addressLength = sizeof(remoteAddressStorage);
rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength);
if (rc != 0)
return CURLE_OK;
char sessionId[MOLOCH_SESSIONID_LEN];
int localPort, remotePort;
char remoteIp[INET6_ADDRSTRLEN+2];
if (localAddressStorage.ss_family == AF_INET) {
struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage;
struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage;
moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port,
remoteAddress->sin_addr.s_addr, remoteAddress->sin_port);
localPort = ntohs(localAddress->sin_port);
remotePort = ntohs(remoteAddress->sin_port);
inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp));
} else {
struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage;
struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage;
moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port,
remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port);
localPort = ntohs(localAddress->sin6_port);
remotePort = ntohs(remoteAddress->sin6_port);
inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2);
remoteIp[0] = '[';
strcat(remoteIp, "]");
}
if (config.logHTTPConnections) {
LOG("Connected %d/%d - %s %d->%s:%d - fd:%d",
server->outstanding,
server->connections,
sname->name,
localPort,
remoteIp,
remotePort,
fd);
}
MolochHttpConn_t *conn;
MOLOCH_LOCK(connections);
BIT_SET(fd, connectionsSet);
HASH_FIND(h_, connections, sessionId, conn);
if (!conn) {
conn = MOLOCH_TYPE_ALLOC0(MolochHttpConn_t);
HASH_ADD(h_, connections, sessionId, conn);
memcpy(&conn->sessionId, sessionId, sessionId[0]);
server->connections++;
} else {
char buf[1000];
LOG("ERROR - Already added %x %s", condition, moloch_session_id_string(sessionId, buf));
}
MOLOCH_UNLOCK(connections);
moloch_http_curlm_check_multi_info(server);
return CURLE_OK;
}
int moloch_http_curl_close_callback(void *snameV, curl_socket_t fd)
{
MolochHttpServerName_t *sname = snameV;
MolochHttpServer_t *server = sname->server;
if (! BIT_ISSET(fd, connectionsSet)) {
long ev = (long)g_hash_table_lookup(server->fd2ev, (void *)(long)fd);
LOG("Couldn't connect %s (%d, %ld) ", sname->name, fd, ev);
close(fd);
GSource *source = g_main_context_find_source_by_id (NULL, ev);
if (source)
g_source_destroy (source);
g_hash_table_remove(server->fd2ev, (void *)(long)fd);
return 0;
}
struct sockaddr_storage localAddressStorage, remoteAddressStorage;
socklen_t addressLength = sizeof(localAddressStorage);
int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength);
if (rc != 0)
return 0;
addressLength = sizeof(remoteAddressStorage);
rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength);
if (rc != 0)
return 0;
char sessionId[MOLOCH_SESSIONID_LEN];
int localPort, remotePort;
char remoteIp[INET6_ADDRSTRLEN+2];
if (localAddressStorage.ss_family == AF_INET) {
struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage;
struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage;
moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port,
remoteAddress->sin_addr.s_addr, remoteAddress->sin_port);
localPort = ntohs(localAddress->sin_port);
remotePort = ntohs(remoteAddress->sin_port);
inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp));
} else {
struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage;
struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage;
moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port,
remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port);
localPort = ntohs(localAddress->sin6_port);
remotePort = ntohs(remoteAddress->sin6_port);
inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2);
remoteIp[0] = '[';
strcat(remoteIp, "]");
}
MolochHttpConn_t *conn;
BIT_CLR(fd, connectionsSet);
MOLOCH_LOCK(connections);
HASH_FIND(h_, connections, sessionId, conn);
if (conn) {
HASH_REMOVE(h_, connections, conn);
MOLOCH_TYPE_FREE(MolochHttpConn_t, conn);
}
MOLOCH_UNLOCK(connections);
server->connections--;
if (config.logHTTPConnections) {
LOG("Close %d/%d - %s %d->%s:%d fd:%d removed: %s",
server->outstanding,
server->connections,
sname->name,
localPort,
remoteIp,
remotePort,
fd,
conn?"true":"false");
}
close (fd);
return 0;
}
unsigned char *moloch_http_send_sync(void *serverV, const char *method, const char *key, int32_t key_len, char *data, uint32_t data_len, char **headers, size_t *return_len)
{
MolochHttpServer_t *server = serverV;
struct curl_slist *headerList = NULL;
if (return_len)
*return_len = 0;
CURL *easy;
if (headers) {
int i;
for (i = 0; headers[i]; i++) {
headerList = curl_slist_append(headerList, headers[i]);
}
}
if (server->defaultHeaders) {
int i;
for (i = 0; server->defaultHeaders[i]; i++) {
headerList = curl_slist_append(headerList, server->defaultHeaders[i]);
}
}
MOLOCH_LOCK(server->syncRequest);
if (!server->syncRequest.easy) {
easy = server->syncRequest.easy = curl_easy_init();
if (config.debug >= 2) {
curl_easy_setopt(easy, CURLOPT_VERBOSE, 1);
}
curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback);
curl_easy_setopt(easy, CURLOPT_WRITEDATA, (void *)&server->syncRequest);
curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT, 10L);
curl_easy_setopt(easy, CURLOPT_TIMEOUT, 60L);
} else {
easy = server->syncRequest.easy;
}
if (config.insecure) {
curl_easy_setopt(easy, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(easy, CURLOPT_SSL_VERIFYHOST, 0L);
}
if (method[0] != 'G') {
curl_easy_setopt(easy, CURLOPT_CUSTOMREQUEST, method);
curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, data_len);
curl_easy_setopt(easy, CURLOPT_POSTFIELDS, data);
} else {
curl_easy_setopt(easy, CURLOPT_CUSTOMREQUEST, NULL);
curl_easy_setopt(easy, CURLOPT_HTTPGET, 1L);
}
if (headerList) {
curl_easy_setopt(easy, CURLOPT_HTTPHEADER, headerList);
}
if (key_len == -1)
key_len = strlen(key);
memcpy(server->syncRequest.key, key, key_len);
server->syncRequest.key[key_len] = 0;
server->syncRequest.retries = 0;
while (1) {
MOLOCH_LOCK(requests);
moloch_http_add_request(server, &server->syncRequest, FALSE);
MOLOCH_UNLOCK(requests);
server->syncRequest.used = 0;
int res = curl_easy_perform(easy);
if (res != CURLE_OK) {
if (server->syncRequest.retries < server->maxRetries) {
struct timeval now;
gettimeofday(&now, NULL);
server->snames[server->syncRequest.namePos].allowedAtSeconds = now.tv_sec + 30;
LOG("Retry %s error '%s'", server->syncRequest.url, curl_easy_strerror(res));
server->syncRequest.retries++;
continue;
}
LOG("libcurl failure %s error '%s'", server->syncRequest.url, curl_easy_strerror(res));
MOLOCH_UNLOCK(server->syncRequest);
if (headerList) {
curl_slist_free_all(headerList);
}
return 0;
}
break;
}
if (headerList) {
curl_slist_free_all(headerList);
}
if (server->syncRequest.dataIn)
server->syncRequest.dataIn[server->syncRequest.used] = 0;
if (return_len)
*return_len = server->syncRequest.used;
long responseCode;
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode);
if (config.logESRequests || (server->printErrors && responseCode/100 != 2)) {
double totalTime;
double connectTime;
double uploadSize;
double downloadSize;
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime);
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime);
curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize);
curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize);
LOG("%d/%d SYNC %ld %s %.0lf/%0.lf %.0lfms %.0lfms",
1, 1,
responseCode,
server->syncRequest.url,
uploadSize,
downloadSize,
connectTime*1000,
totalTime*1000);
}
uint8_t *dataIn = server->syncRequest.dataIn;
server->syncRequest.dataIn = 0;
MOLOCH_UNLOCK(server->syncRequest);
return dataIn;
}
LOCAL void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server)
{
char *eff_url;
CURLMsg *msg;
int msgs_left;
MolochHttpRequest_t *request;
CURL *easy;
while ((msg = curl_multi_info_read(server->multi, &msgs_left))) {
if (msg->msg == CURLMSG_DONE) {
easy = msg->easy_handle;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request);
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);
long responseCode;
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode);
if (config.logESRequests || (server->printErrors && responseCode/100 != 2)) {
double totalTime;
double connectTime;
double uploadSize;
double downloadSize;
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime);
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime);
curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize);
curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize);
LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms",
request->server->outstanding,
request->server->connections,
responseCode,
request->url,
uploadSize,
downloadSize,
connectTime*1000,
totalTime*1000);
}
#ifdef MOLOCH_HTTP_DEBUG
LOG("HTTPDEBUG DECR %p %d %s", request, server->outstanding, request->url);
#endif
if (responseCode == 0 && request->retries < server->maxRetries) {
curl_multi_remove_handle(server->multi, easy);
request->retries++;
struct timeval now;
gettimeofday(&now, NULL);
MOLOCH_LOCK(requests);
server->snames[request->namePos].allowedAtSeconds = now.tv_sec + 30;
server->outstanding--;
moloch_http_add_request(server, request, TRUE);
MOLOCH_UNLOCK(requests);
} else {
if (server->printErrors && responseCode/100 != 2) {
LOG("Response length=%d :>\n%.*s", request->used, MIN(request->used, 4000), request->dataIn);
}
if (request->func) {
if (request->dataIn)
request->dataIn[request->used] = 0;
request->func(responseCode, request->dataIn, request->used, request->uw);
}
if (request->dataIn) {
free(request->dataIn);
request->dataIn = 0;
}
if (request->dataOut) {
MOLOCH_SIZE_FREE(buffer, request->dataOut);
}
if (request->headerList) {
curl_slist_free_all(request->headerList);
}
MOLOCH_TYPE_FREE(MolochHttpRequest_t, request);
curl_multi_remove_handle(server->multi, easy);
curl_easy_cleanup(easy);
MOLOCH_LOCK(requests);
server->outstanding--;
MOLOCH_UNLOCK(requests);
}
}
}
}
void wise_lookup(MolochSession_t *session, WiseRequest_t *request, char *value, int type)
{
if (*value == 0)
return;
if (request->numItems >= 256)
return;
MOLOCH_LOCK(item);
static int lookups = 0;
WiseItem_t *wi;
lookups++;
if ((lookups % 10000) == 0)
wise_print_stats();
stats[type][INTEL_STAT_LOOKUP]++;
HASH_FIND(wih_, itemHash[type], value, wi);
if (wi) {
// Already being looked up
if (wi->sessions) {
if (wi->numSessions < wi->sessionsSize) {
wi->sessions[wi->numSessions++] = session;
moloch_session_incr_outstanding(session);
}
stats[type][INTEL_STAT_INPROGRESS]++;
goto cleanup;
}
struct timeval currentTime;
gettimeofday(¤tTime, NULL);
if (wi->loadTime + cacheSecs > currentTime.tv_sec) {
wise_process_ops(session, wi);
stats[type][INTEL_STAT_CACHE]++;
goto cleanup;
}
/* Had it in cache, but it is too old */
DLL_REMOVE(wil_, &itemList[type], wi);
wise_free_ops(wi);
} else {
// Know nothing about it
wi = MOLOCH_TYPE_ALLOC0(WiseItem_t);
wi->key = g_strdup(value);
wi->type = type;
wi->sessionsSize = 20;
HASH_ADD(wih_, itemHash[type], wi->key, wi);
}
wi->sessions = malloc(sizeof(MolochSession_t *) * wi->sessionsSize);
wi->sessions[wi->numSessions++] = session;
moloch_session_incr_outstanding(session);
stats[type][INTEL_STAT_REQUEST]++;
BSB_EXPORT_u08(request->bsb, type);
int len = strlen(value);
BSB_EXPORT_u16(request->bsb, len);
BSB_EXPORT_ptr(request->bsb, value, len);
request->items[request->numItems++] = wi;
cleanup:
MOLOCH_UNLOCK(item);
}
LOCAL void *reader_tpacketv3_thread(gpointer infov)
{
long info = (long)infov;
struct pollfd pfd;
int pos = -1;
memset(&pfd, 0, sizeof(pfd));
pfd.fd = infos[info].fd;
pfd.events = POLLIN | POLLERR;
pfd.revents = 0;
MolochPacketBatch_t batch;
moloch_packet_batch_init(&batch);
while (!config.quitting) {
if (pos == -1) {
MOLOCH_LOCK(infos[info].lock);
pos = infos[info].nextPos;
infos[info].nextPos = (infos[info].nextPos + 1) % infos[info].req.tp_block_nr;
MOLOCH_UNLOCK(infos[info].lock);
}
struct tpacket_block_desc *tbd = infos[info].rd[pos].iov_base;
if (config.debug > 2) {
int i;
int cnt = 0;
int waiting = 0;
for (i = 0; i < (int)infos[info].req.tp_block_nr; i++) {
struct tpacket_block_desc *stbd = infos[info].rd[i].iov_base;
if (stbd->hdr.bh1.block_status & TP_STATUS_USER) {
cnt++;
waiting += stbd->hdr.bh1.num_pkts;
}
}
LOG("Stats pos:%d info:%ld status:%x waiting:%d total cnt:%d total waiting:%d", pos, info, tbd->hdr.bh1.block_status, tbd->hdr.bh1.num_pkts, cnt, waiting);
}
// Wait until the block is owned by moloch
if ((tbd->hdr.bh1.block_status & TP_STATUS_USER) == 0) {
poll(&pfd, 1, -1);
continue;
}
struct tpacket3_hdr *th;
th = (struct tpacket3_hdr *) ((uint8_t *) tbd + tbd->hdr.bh1.offset_to_first_pkt);
uint16_t p;
for (p = 0; p < tbd->hdr.bh1.num_pkts; p++) {
if (unlikely(th->tp_snaplen != th->tp_len)) {
LOGEXIT("ERROR - Moloch requires full packet captures caplen: %d pktlen: %d\n"
"See https://github.com/aol/moloch/wiki/FAQ#Moloch_requires_full_packet_captures_error",
th->tp_snaplen, th->tp_len);
}
MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t);
packet->pkt = (u_char *)th + th->tp_mac;
packet->pktlen = th->tp_len;
packet->ts.tv_sec = th->tp_sec;
packet->ts.tv_usec = th->tp_nsec/1000;
packet->readerPos = info;
moloch_packet_batch(&batch, packet);
th = (struct tpacket3_hdr *) ((uint8_t *) th + th->tp_next_offset);
}
moloch_packet_batch_flush(&batch);
tbd->hdr.bh1.block_status = TP_STATUS_KERNEL;
pos = -1;
}
return NULL;
}
void wise_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw)
{
BSB bsb;
WiseRequest_t *request = uw;
int i;
inflight -= request->numItems;
BSB_INIT(bsb, data, data_len);
uint32_t fts = 0, ver = 0;
BSB_IMPORT_u32(bsb, fts);
BSB_IMPORT_u32(bsb, ver);
if (BSB_IS_ERROR(bsb) || ver != 0) {
MOLOCH_LOCK(item);
for (i = 0; i < request->numItems; i++) {
wise_free_item_unlocked(request->items[i]);
}
MOLOCH_UNLOCK(item);
MOLOCH_TYPE_FREE(WiseRequest_t, request);
return;
}
if (fts != fieldsTS)
wise_load_fields();
struct timeval currentTime;
gettimeofday(¤tTime, NULL);
for (i = 0; i < request->numItems; i++) {
WiseItem_t *wi = request->items[i];
BSB_IMPORT_u08(bsb, wi->numOps);
if (wi->numOps > 0) {
wi->ops = malloc(wi->numOps * sizeof(WiseOp_t));
int i;
for (i = 0; i < wi->numOps; i++) {
WiseOp_t *op = &(wi->ops[i]);
int rfield = 0;
BSB_IMPORT_u08(bsb, rfield);
op->fieldPos = fieldsMap[rfield];
int len = 0;
BSB_IMPORT_u08(bsb, len);
char *str = (char*)BSB_WORK_PTR(bsb);
BSB_IMPORT_skip(bsb, len);
switch (config.fields[op->fieldPos]->type) {
case MOLOCH_FIELD_TYPE_INT_HASH:
case MOLOCH_FIELD_TYPE_INT_GHASH:
if (op->fieldPos == tagsField) {
moloch_db_get_tag(NULL, tagsField, str, NULL); // Preload the tagname -> tag mapping
op->str = g_strdup(str);
op->strLenOrInt = len - 1;
continue;
}
// Fall thru
case MOLOCH_FIELD_TYPE_INT:
case MOLOCH_FIELD_TYPE_INT_ARRAY:
op->str = 0;
op->strLenOrInt = atoi(str);
break;
case MOLOCH_FIELD_TYPE_STR:
case MOLOCH_FIELD_TYPE_STR_ARRAY:
case MOLOCH_FIELD_TYPE_STR_HASH:
op->str = g_strdup(str);
op->strLenOrInt = len - 1;
break;
case MOLOCH_FIELD_TYPE_IP:
case MOLOCH_FIELD_TYPE_IP_HASH:
case MOLOCH_FIELD_TYPE_IP_GHASH:
op->str = 0;
op->strLenOrInt = inet_addr(str);
break;
default:
LOG("WARNING - Unsupported expression type for %s", str);
continue;
}
}
}
wi->loadTime = currentTime.tv_sec;
int s;
for (s = 0; s < wi->numSessions; s++) {
moloch_session_add_cmd(wi->sessions[s], MOLOCH_SES_CMD_FUNC, wi, NULL, wise_session_cmd_cb);
}
g_free(wi->sessions);
wi->sessions = 0;
wi->numSessions = 0;
MOLOCH_LOCK(item);
DLL_PUSH_HEAD(wil_, &itemList[(int)wi->type], wi);
// Cache needs to be reduced
if (itemList[(int)wi->type].wil_count > maxCache) {
DLL_POP_TAIL(wil_, &itemList[(int)wi->type], wi);
wise_free_item_unlocked(wi);
}
MOLOCH_UNLOCK(item);
}
MOLOCH_TYPE_FREE(WiseRequest_t, request);
}
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw)
{
MolochHttpServer_t *server = serverV;
// Are we overloaded
if (dropable && !config.quitting && server->outstanding > server->maxOutstandingRequests) {
LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding);
if (data) {
MOLOCH_SIZE_FREE(buffer, data);
}
return 1;
}
MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t);
if (headers) {
int i;
for (i = 0; headers[i]; i++) {
request->headerList = curl_slist_append(request->headerList, headers[i]);
}
}
// Do we need to compress item
if (server->compress && data && data_len > 1000) {
char *buf = moloch_http_get_buffer(data_len);
int ret;
MOLOCH_LOCK(z_strm);
z_strm.avail_in = data_len;
z_strm.next_in = (unsigned char *)data;
z_strm.avail_out = data_len;
z_strm.next_out = (unsigned char *)buf;
ret = deflate(&z_strm, Z_FINISH);
if (ret == Z_STREAM_END) {
request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate");
MOLOCH_SIZE_FREE(buffer, data);
data_len = data_len - z_strm.avail_out;
data = buf;
} else {
MOLOCH_SIZE_FREE(buffer, buf);
}
deflateReset(&z_strm);
MOLOCH_UNLOCK(z_strm);
}
request->server = server;
request->func = func;
request->uw = uw;
request->dataOut = data;
request->dataOutLen = data_len;
request->easy = curl_easy_init();
if (config.debug >= 2) {
curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1);
}
curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback);
curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback);
curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETDATA, server);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback);
curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETDATA, server);
curl_easy_setopt(request->easy, CURLOPT_ACCEPT_ENCODING, ""); // https://curl.haxx.se/libcurl/c/CURLOPT_ACCEPT_ENCODING.html
if (request->headerList) {
curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList);
}
if (method[0] != 'G') {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method);
curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len);
curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data);
} else {
curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL);
curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L);
}
if (server->headerCb) {
curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function);
curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request);
}
curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L);
char *host = server->names[server->namesPos];
server->namesPos = (server->namesPos + 1) % server->namesCnt;
if (strchr(host, ':') == 0) {
snprintf(request->url, sizeof(request->url), "%s://%s:%d%.*s", (server->https?"https":"http"), host, server->defaultPort, key_len, key);
} else {
snprintf(request->url, sizeof(request->url), "%s://%s%.*s", (server->https?"https":"http"), host, key_len, key);
}
curl_easy_setopt(request->easy, CURLOPT_URL, request->url);
MOLOCH_LOCK(requests);
#ifdef MOLOCH_HTTP_DEBUG
LOG("HTTPDEBUG INCR %s %p %d %s", server->names[0], request, server->outstanding, request->url);
#endif
server->outstanding++;
DLL_PUSH_TAIL(rqt_, &requests, request);
if (!requestsTimer)
requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL);
MOLOCH_UNLOCK(requests);
return 0;
}
int moloch_http_curl_close_callback(void *serverV, curl_socket_t fd)
{
MolochHttpServer_t *server = serverV;
if (! BIT_ISSET(fd, connectionsSet)) {
LOG("Couldn't connect %s defaultPort: %d", server->names[0], server->defaultPort);
return 0;
}
struct sockaddr_storage localAddressStorage, remoteAddressStorage;
socklen_t addressLength = sizeof(localAddressStorage);
int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength);
if (rc != 0)
return 0;
addressLength = sizeof(remoteAddressStorage);
rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength);
if (rc != 0)
return 0;
char sessionId[MOLOCH_SESSIONID_LEN];
int localPort, remotePort;
char remoteIp[INET6_ADDRSTRLEN+2];
if (localAddressStorage.ss_family == AF_INET) {
struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage;
struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage;
moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port,
remoteAddress->sin_addr.s_addr, remoteAddress->sin_port);
localPort = ntohs(localAddress->sin_port);
remotePort = ntohs(remoteAddress->sin_port);
inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp));
} else {
struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage;
struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage;
moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port,
remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port);
localPort = ntohs(localAddress->sin6_port);
remotePort = ntohs(remoteAddress->sin6_port);
inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2);
remoteIp[0] = '[';
strcat(remoteIp, "]");
}
MolochHttpConn_t *conn;
BIT_CLR(fd, connectionsSet);
MOLOCH_LOCK(connections);
HASH_FIND(h_, connections, sessionId, conn);
if (conn) {
HASH_REMOVE(h_, connections, conn);
MOLOCH_TYPE_FREE(MolochHttpConn_t, conn);
}
MOLOCH_UNLOCK(connections);
server->connections--;
LOG("Close %d/%d - %s %d->%s:%d fd:%d removed: %s",
server->outstanding,
server->connections,
server->names[0],
localPort,
remoteIp,
remotePort,
fd,
conn?"true":"false");
close (fd);
return 0;
}
static void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server)
{
char *eff_url;
CURLMsg *msg;
int msgs_left;
MolochHttpRequest_t *request;
CURL *easy;
while ((msg = curl_multi_info_read(server->multi, &msgs_left))) {
if (msg->msg == CURLMSG_DONE) {
easy = msg->easy_handle;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request);
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);
long responseCode;
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode);
if (config.logESRequests) {
double totalTime;
double connectTime;
double uploadSize;
double downloadSize;
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime);
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime);
curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize);
curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize);
LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms",
request->server->outstanding,
request->server->connections,
responseCode,
request->url,
uploadSize,
downloadSize,
connectTime*1000,
totalTime*1000);
}
#ifdef MOLOCH_HTTP_DEBUG
LOG("HTTPDEBUG DECR %s %p %d %s", server->names[0], request, server->outstanding, request->url);
#endif
if (request->func) {
if (request->dataIn)
request->dataIn[request->used] = 0;
request->func(responseCode, request->dataIn, request->used, request->uw);
}
if (request->dataIn) {
free(request->dataIn);
request->dataIn = 0;
}
if (request->dataOut) {
MOLOCH_SIZE_FREE(buffer, request->dataOut);
}
if (request->headerList) {
curl_slist_free_all(request->headerList);
}
MOLOCH_TYPE_FREE(MolochHttpRequest_t, request);
curl_multi_remove_handle(server->multi, easy);
curl_easy_cleanup(easy);
MOLOCH_LOCK(requests);
server->outstanding--;
MOLOCH_UNLOCK(requests);
}
}
}
void moloch_session_process_commands(int thread)
{
// Commands
MolochSesCmd_t *cmd = 0;
int count;
for (count = 0; count < 50; count++) {
MOLOCH_LOCK(sessionCmds[thread].lock);
DLL_POP_HEAD(cmd_, &sessionCmds[thread], cmd);
MOLOCH_UNLOCK(sessionCmds[thread].lock);
if (!cmd)
break;
switch (cmd->cmd) {
case MOLOCH_SES_CMD_ADD_TAG:
moloch_field_int_add((long)cmd->uw1, cmd->session, (long)cmd->uw2);
moloch_session_decr_outstanding(cmd->session);
break;
case MOLOCH_SES_CMD_FUNC:
cmd->func(cmd->session, cmd->uw1, cmd->uw2);
break;
default:
LOG ("Unknown cmd %d", cmd->cmd);
}
MOLOCH_TYPE_FREE(MolochSesCmd_t, cmd);
}
// Closing Q
for (count = 0; count < 10; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(q_, &closingQ[thread]);
if (session && session->saveTime < (uint64_t)lastPacketSecs[thread]) {
moloch_session_save(session);
} else {
break;
}
}
// Sessions Idle Long Time
int ses;
for (ses = 0; ses < SESSION_MAX; ses++) {
for (count = 0; count < 10; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(q_, &sessionsQ[thread][ses]);
if (session && (DLL_COUNT(q_, &sessionsQ[thread][ses]) > (int)config.maxStreams ||
((uint64_t)session->lastPacket.tv_sec + config.timeouts[ses] < (uint64_t)lastPacketSecs[thread]))) {
moloch_session_save(session);
} else {
break;
}
}
}
// TCP Sessions Open Long Time
for (count = 0; count < 50; count++) {
MolochSession_t *session = DLL_PEEK_HEAD(tcp_, &tcpWriteQ[thread]);
if (session && (uint64_t)session->saveTime < (uint64_t)lastPacketSecs[thread]) {
moloch_session_mid_save(session, lastPacketSecs[thread]);
} else {
break;
}
}
}
