LOCAL gboolean moloch_http_curl_watch_open_callback(int fd, GIOCondition condition, gpointer snameV) { MolochHttpServerName_t *sname = snameV; MolochHttpServer_t *server = sname->server; struct sockaddr_storage localAddressStorage, remoteAddressStorage; socklen_t addressLength = sizeof(localAddressStorage); int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength); if (rc != 0) return CURLE_OK; addressLength = sizeof(remoteAddressStorage); rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength); if (rc != 0) return CURLE_OK; char sessionId[MOLOCH_SESSIONID_LEN]; int localPort, remotePort; char remoteIp[INET6_ADDRSTRLEN+2]; if (localAddressStorage.ss_family == AF_INET) { struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage; struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage; moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port, remoteAddress->sin_addr.s_addr, remoteAddress->sin_port); localPort = ntohs(localAddress->sin_port); remotePort = ntohs(remoteAddress->sin_port); inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp)); } else { struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage; struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage; moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port, remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port); localPort = ntohs(localAddress->sin6_port); remotePort = ntohs(remoteAddress->sin6_port); inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2); remoteIp[0] = '['; strcat(remoteIp, "]"); } if (config.logHTTPConnections) { LOG("Connected %d/%d - %s %d->%s:%d - fd:%d", server->outstanding, server->connections, sname->name, localPort, remoteIp, remotePort, fd); } MolochHttpConn_t *conn; MOLOCH_LOCK(connections); BIT_SET(fd, connectionsSet); HASH_FIND(h_, connections, sessionId, conn); if (!conn) { conn = MOLOCH_TYPE_ALLOC0(MolochHttpConn_t); HASH_ADD(h_, connections, sessionId, conn); memcpy(&conn->sessionId, sessionId, sessionId[0]); server->connections++; } else { char buf[1000]; LOG("ERROR - Already added %x %s", condition, moloch_session_id_string(sessionId, buf)); } MOLOCH_UNLOCK(connections); moloch_http_curlm_check_multi_info(server); return CURLE_OK; }
int moloch_http_curl_close_callback(void *snameV, curl_socket_t fd) { MolochHttpServerName_t *sname = snameV; MolochHttpServer_t *server = sname->server; if (! BIT_ISSET(fd, connectionsSet)) { long ev = (long)g_hash_table_lookup(server->fd2ev, (void *)(long)fd); LOG("Couldn't connect %s (%d, %ld) ", sname->name, fd, ev); close(fd); GSource *source = g_main_context_find_source_by_id (NULL, ev); if (source) g_source_destroy (source); g_hash_table_remove(server->fd2ev, (void *)(long)fd); return 0; } struct sockaddr_storage localAddressStorage, remoteAddressStorage; socklen_t addressLength = sizeof(localAddressStorage); int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength); if (rc != 0) return 0; addressLength = sizeof(remoteAddressStorage); rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength); if (rc != 0) return 0; char sessionId[MOLOCH_SESSIONID_LEN]; int localPort, remotePort; char remoteIp[INET6_ADDRSTRLEN+2]; if (localAddressStorage.ss_family == AF_INET) { struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage; struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage; moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port, remoteAddress->sin_addr.s_addr, remoteAddress->sin_port); localPort = ntohs(localAddress->sin_port); remotePort = ntohs(remoteAddress->sin_port); inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp)); } else { struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage; struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage; moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port, remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port); localPort = ntohs(localAddress->sin6_port); remotePort = ntohs(remoteAddress->sin6_port); inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2); remoteIp[0] = '['; strcat(remoteIp, "]"); } MolochHttpConn_t *conn; BIT_CLR(fd, connectionsSet); MOLOCH_LOCK(connections); HASH_FIND(h_, connections, sessionId, conn); if (conn) { HASH_REMOVE(h_, connections, conn); MOLOCH_TYPE_FREE(MolochHttpConn_t, conn); } MOLOCH_UNLOCK(connections); server->connections--; if (config.logHTTPConnections) { LOG("Close %d/%d - %s %d->%s:%d fd:%d removed: %s", server->outstanding, server->connections, sname->name, localPort, remoteIp, remotePort, fd, conn?"true":"false"); } close (fd); return 0; }
unsigned char *moloch_http_send_sync(void *serverV, const char *method, const char *key, int32_t key_len, char *data, uint32_t data_len, char **headers, size_t *return_len) { MolochHttpServer_t *server = serverV; struct curl_slist *headerList = NULL; if (return_len) *return_len = 0; CURL *easy; if (headers) { int i; for (i = 0; headers[i]; i++) { headerList = curl_slist_append(headerList, headers[i]); } } if (server->defaultHeaders) { int i; for (i = 0; server->defaultHeaders[i]; i++) { headerList = curl_slist_append(headerList, server->defaultHeaders[i]); } } MOLOCH_LOCK(server->syncRequest); if (!server->syncRequest.easy) { easy = server->syncRequest.easy = curl_easy_init(); if (config.debug >= 2) { curl_easy_setopt(easy, CURLOPT_VERBOSE, 1); } curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback); curl_easy_setopt(easy, CURLOPT_WRITEDATA, (void *)&server->syncRequest); curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT, 10L); curl_easy_setopt(easy, CURLOPT_TIMEOUT, 60L); } else { easy = server->syncRequest.easy; } if (config.insecure) { curl_easy_setopt(easy, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(easy, CURLOPT_SSL_VERIFYHOST, 0L); } if (method[0] != 'G') { curl_easy_setopt(easy, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, data_len); curl_easy_setopt(easy, CURLOPT_POSTFIELDS, data); } else { curl_easy_setopt(easy, CURLOPT_CUSTOMREQUEST, NULL); curl_easy_setopt(easy, CURLOPT_HTTPGET, 1L); } if (headerList) { curl_easy_setopt(easy, CURLOPT_HTTPHEADER, headerList); } if (key_len == -1) key_len = strlen(key); memcpy(server->syncRequest.key, key, key_len); server->syncRequest.key[key_len] = 0; server->syncRequest.retries = 0; while (1) { MOLOCH_LOCK(requests); moloch_http_add_request(server, &server->syncRequest, FALSE); MOLOCH_UNLOCK(requests); server->syncRequest.used = 0; int res = curl_easy_perform(easy); if (res != CURLE_OK) { if (server->syncRequest.retries < server->maxRetries) { struct timeval now; gettimeofday(&now, NULL); server->snames[server->syncRequest.namePos].allowedAtSeconds = now.tv_sec + 30; LOG("Retry %s error '%s'", server->syncRequest.url, curl_easy_strerror(res)); server->syncRequest.retries++; continue; } LOG("libcurl failure %s error '%s'", server->syncRequest.url, curl_easy_strerror(res)); MOLOCH_UNLOCK(server->syncRequest); if (headerList) { curl_slist_free_all(headerList); } return 0; } break; } if (headerList) { curl_slist_free_all(headerList); } if (server->syncRequest.dataIn) server->syncRequest.dataIn[server->syncRequest.used] = 0; if (return_len) *return_len = server->syncRequest.used; long responseCode; curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode); if (config.logESRequests || (server->printErrors && responseCode/100 != 2)) { double totalTime; double connectTime; double uploadSize; double downloadSize; curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime); curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime); curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize); curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize); LOG("%d/%d SYNC %ld %s %.0lf/%0.lf %.0lfms %.0lfms", 1, 1, responseCode, server->syncRequest.url, uploadSize, downloadSize, connectTime*1000, totalTime*1000); } uint8_t *dataIn = server->syncRequest.dataIn; server->syncRequest.dataIn = 0; MOLOCH_UNLOCK(server->syncRequest); return dataIn; }
LOCAL void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server) { char *eff_url; CURLMsg *msg; int msgs_left; MolochHttpRequest_t *request; CURL *easy; while ((msg = curl_multi_info_read(server->multi, &msgs_left))) { if (msg->msg == CURLMSG_DONE) { easy = msg->easy_handle; curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request); curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url); long responseCode; curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode); if (config.logESRequests || (server->printErrors && responseCode/100 != 2)) { double totalTime; double connectTime; double uploadSize; double downloadSize; curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime); curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime); curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize); curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize); LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms", request->server->outstanding, request->server->connections, responseCode, request->url, uploadSize, downloadSize, connectTime*1000, totalTime*1000); } #ifdef MOLOCH_HTTP_DEBUG LOG("HTTPDEBUG DECR %p %d %s", request, server->outstanding, request->url); #endif if (responseCode == 0 && request->retries < server->maxRetries) { curl_multi_remove_handle(server->multi, easy); request->retries++; struct timeval now; gettimeofday(&now, NULL); MOLOCH_LOCK(requests); server->snames[request->namePos].allowedAtSeconds = now.tv_sec + 30; server->outstanding--; moloch_http_add_request(server, request, TRUE); MOLOCH_UNLOCK(requests); } else { if (server->printErrors && responseCode/100 != 2) { LOG("Response length=%d :>\n%.*s", request->used, MIN(request->used, 4000), request->dataIn); } if (request->func) { if (request->dataIn) request->dataIn[request->used] = 0; request->func(responseCode, request->dataIn, request->used, request->uw); } if (request->dataIn) { free(request->dataIn); request->dataIn = 0; } if (request->dataOut) { MOLOCH_SIZE_FREE(buffer, request->dataOut); } if (request->headerList) { curl_slist_free_all(request->headerList); } MOLOCH_TYPE_FREE(MolochHttpRequest_t, request); curl_multi_remove_handle(server->multi, easy); curl_easy_cleanup(easy); MOLOCH_LOCK(requests); server->outstanding--; MOLOCH_UNLOCK(requests); } } } }
void wise_lookup(MolochSession_t *session, WiseRequest_t *request, char *value, int type) { if (*value == 0) return; if (request->numItems >= 256) return; MOLOCH_LOCK(item); static int lookups = 0; WiseItem_t *wi; lookups++; if ((lookups % 10000) == 0) wise_print_stats(); stats[type][INTEL_STAT_LOOKUP]++; HASH_FIND(wih_, itemHash[type], value, wi); if (wi) { // Already being looked up if (wi->sessions) { if (wi->numSessions < wi->sessionsSize) { wi->sessions[wi->numSessions++] = session; moloch_session_incr_outstanding(session); } stats[type][INTEL_STAT_INPROGRESS]++; goto cleanup; } struct timeval currentTime; gettimeofday(¤tTime, NULL); if (wi->loadTime + cacheSecs > currentTime.tv_sec) { wise_process_ops(session, wi); stats[type][INTEL_STAT_CACHE]++; goto cleanup; } /* Had it in cache, but it is too old */ DLL_REMOVE(wil_, &itemList[type], wi); wise_free_ops(wi); } else { // Know nothing about it wi = MOLOCH_TYPE_ALLOC0(WiseItem_t); wi->key = g_strdup(value); wi->type = type; wi->sessionsSize = 20; HASH_ADD(wih_, itemHash[type], wi->key, wi); } wi->sessions = malloc(sizeof(MolochSession_t *) * wi->sessionsSize); wi->sessions[wi->numSessions++] = session; moloch_session_incr_outstanding(session); stats[type][INTEL_STAT_REQUEST]++; BSB_EXPORT_u08(request->bsb, type); int len = strlen(value); BSB_EXPORT_u16(request->bsb, len); BSB_EXPORT_ptr(request->bsb, value, len); request->items[request->numItems++] = wi; cleanup: MOLOCH_UNLOCK(item); }
LOCAL void *reader_tpacketv3_thread(gpointer infov) { long info = (long)infov; struct pollfd pfd; int pos = -1; memset(&pfd, 0, sizeof(pfd)); pfd.fd = infos[info].fd; pfd.events = POLLIN | POLLERR; pfd.revents = 0; MolochPacketBatch_t batch; moloch_packet_batch_init(&batch); while (!config.quitting) { if (pos == -1) { MOLOCH_LOCK(infos[info].lock); pos = infos[info].nextPos; infos[info].nextPos = (infos[info].nextPos + 1) % infos[info].req.tp_block_nr; MOLOCH_UNLOCK(infos[info].lock); } struct tpacket_block_desc *tbd = infos[info].rd[pos].iov_base; if (config.debug > 2) { int i; int cnt = 0; int waiting = 0; for (i = 0; i < (int)infos[info].req.tp_block_nr; i++) { struct tpacket_block_desc *stbd = infos[info].rd[i].iov_base; if (stbd->hdr.bh1.block_status & TP_STATUS_USER) { cnt++; waiting += stbd->hdr.bh1.num_pkts; } } LOG("Stats pos:%d info:%ld status:%x waiting:%d total cnt:%d total waiting:%d", pos, info, tbd->hdr.bh1.block_status, tbd->hdr.bh1.num_pkts, cnt, waiting); } // Wait until the block is owned by moloch if ((tbd->hdr.bh1.block_status & TP_STATUS_USER) == 0) { poll(&pfd, 1, -1); continue; } struct tpacket3_hdr *th; th = (struct tpacket3_hdr *) ((uint8_t *) tbd + tbd->hdr.bh1.offset_to_first_pkt); uint16_t p; for (p = 0; p < tbd->hdr.bh1.num_pkts; p++) { if (unlikely(th->tp_snaplen != th->tp_len)) { LOGEXIT("ERROR - Moloch requires full packet captures caplen: %d pktlen: %d\n" "See https://github.com/aol/moloch/wiki/FAQ#Moloch_requires_full_packet_captures_error", th->tp_snaplen, th->tp_len); } MolochPacket_t *packet = MOLOCH_TYPE_ALLOC0(MolochPacket_t); packet->pkt = (u_char *)th + th->tp_mac; packet->pktlen = th->tp_len; packet->ts.tv_sec = th->tp_sec; packet->ts.tv_usec = th->tp_nsec/1000; packet->readerPos = info; moloch_packet_batch(&batch, packet); th = (struct tpacket3_hdr *) ((uint8_t *) th + th->tp_next_offset); } moloch_packet_batch_flush(&batch); tbd->hdr.bh1.block_status = TP_STATUS_KERNEL; pos = -1; } return NULL; }
void wise_cb(int UNUSED(code), unsigned char *data, int data_len, gpointer uw) { BSB bsb; WiseRequest_t *request = uw; int i; inflight -= request->numItems; BSB_INIT(bsb, data, data_len); uint32_t fts = 0, ver = 0; BSB_IMPORT_u32(bsb, fts); BSB_IMPORT_u32(bsb, ver); if (BSB_IS_ERROR(bsb) || ver != 0) { MOLOCH_LOCK(item); for (i = 0; i < request->numItems; i++) { wise_free_item_unlocked(request->items[i]); } MOLOCH_UNLOCK(item); MOLOCH_TYPE_FREE(WiseRequest_t, request); return; } if (fts != fieldsTS) wise_load_fields(); struct timeval currentTime; gettimeofday(¤tTime, NULL); for (i = 0; i < request->numItems; i++) { WiseItem_t *wi = request->items[i]; BSB_IMPORT_u08(bsb, wi->numOps); if (wi->numOps > 0) { wi->ops = malloc(wi->numOps * sizeof(WiseOp_t)); int i; for (i = 0; i < wi->numOps; i++) { WiseOp_t *op = &(wi->ops[i]); int rfield = 0; BSB_IMPORT_u08(bsb, rfield); op->fieldPos = fieldsMap[rfield]; int len = 0; BSB_IMPORT_u08(bsb, len); char *str = (char*)BSB_WORK_PTR(bsb); BSB_IMPORT_skip(bsb, len); switch (config.fields[op->fieldPos]->type) { case MOLOCH_FIELD_TYPE_INT_HASH: case MOLOCH_FIELD_TYPE_INT_GHASH: if (op->fieldPos == tagsField) { moloch_db_get_tag(NULL, tagsField, str, NULL); // Preload the tagname -> tag mapping op->str = g_strdup(str); op->strLenOrInt = len - 1; continue; } // Fall thru case MOLOCH_FIELD_TYPE_INT: case MOLOCH_FIELD_TYPE_INT_ARRAY: op->str = 0; op->strLenOrInt = atoi(str); break; case MOLOCH_FIELD_TYPE_STR: case MOLOCH_FIELD_TYPE_STR_ARRAY: case MOLOCH_FIELD_TYPE_STR_HASH: op->str = g_strdup(str); op->strLenOrInt = len - 1; break; case MOLOCH_FIELD_TYPE_IP: case MOLOCH_FIELD_TYPE_IP_HASH: case MOLOCH_FIELD_TYPE_IP_GHASH: op->str = 0; op->strLenOrInt = inet_addr(str); break; default: LOG("WARNING - Unsupported expression type for %s", str); continue; } } } wi->loadTime = currentTime.tv_sec; int s; for (s = 0; s < wi->numSessions; s++) { moloch_session_add_cmd(wi->sessions[s], MOLOCH_SES_CMD_FUNC, wi, NULL, wise_session_cmd_cb); } g_free(wi->sessions); wi->sessions = 0; wi->numSessions = 0; MOLOCH_LOCK(item); DLL_PUSH_HEAD(wil_, &itemList[(int)wi->type], wi); // Cache needs to be reduced if (itemList[(int)wi->type].wil_count > maxCache) { DLL_POP_TAIL(wil_, &itemList[(int)wi->type], wi); wise_free_item_unlocked(wi); } MOLOCH_UNLOCK(item); } MOLOCH_TYPE_FREE(WiseRequest_t, request); }
gboolean moloch_http_send(void *serverV, const char *method, const char *key, uint32_t key_len, char *data, uint32_t data_len, char **headers, gboolean dropable, MolochHttpResponse_cb func, gpointer uw) { MolochHttpServer_t *server = serverV; // Are we overloaded if (dropable && !config.quitting && server->outstanding > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue %d is too big", key_len, key, data_len, server->outstanding); if (data) { MOLOCH_SIZE_FREE(buffer, data); } return 1; } MolochHttpRequest_t *request = MOLOCH_TYPE_ALLOC0(MolochHttpRequest_t); if (headers) { int i; for (i = 0; headers[i]; i++) { request->headerList = curl_slist_append(request->headerList, headers[i]); } } // Do we need to compress item if (server->compress && data && data_len > 1000) { char *buf = moloch_http_get_buffer(data_len); int ret; MOLOCH_LOCK(z_strm); z_strm.avail_in = data_len; z_strm.next_in = (unsigned char *)data; z_strm.avail_out = data_len; z_strm.next_out = (unsigned char *)buf; ret = deflate(&z_strm, Z_FINISH); if (ret == Z_STREAM_END) { request->headerList = curl_slist_append(request->headerList, "Content-Encoding: deflate"); MOLOCH_SIZE_FREE(buffer, data); data_len = data_len - z_strm.avail_out; data = buf; } else { MOLOCH_SIZE_FREE(buffer, buf); } deflateReset(&z_strm); MOLOCH_UNLOCK(z_strm); } request->server = server; request->func = func; request->uw = uw; request->dataOut = data; request->dataOutLen = data_len; request->easy = curl_easy_init(); if (config.debug >= 2) { curl_easy_setopt(request->easy, CURLOPT_VERBOSE, 1); } curl_easy_setopt(request->easy, CURLOPT_WRITEFUNCTION, moloch_http_curl_write_callback); curl_easy_setopt(request->easy, CURLOPT_WRITEDATA, (void *)request); curl_easy_setopt(request->easy, CURLOPT_PRIVATE, (void *)request); curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETFUNCTION, moloch_http_curl_open_callback); curl_easy_setopt(request->easy, CURLOPT_OPENSOCKETDATA, server); curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETFUNCTION, moloch_http_curl_close_callback); curl_easy_setopt(request->easy, CURLOPT_CLOSESOCKETDATA, server); curl_easy_setopt(request->easy, CURLOPT_ACCEPT_ENCODING, ""); // https://curl.haxx.se/libcurl/c/CURLOPT_ACCEPT_ENCODING.html if (request->headerList) { curl_easy_setopt(request->easy, CURLOPT_HTTPHEADER, request->headerList); } if (method[0] != 'G') { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(request->easy, CURLOPT_INFILESIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDSIZE, data_len); curl_easy_setopt(request->easy, CURLOPT_POSTFIELDS, data); } else { curl_easy_setopt(request->easy, CURLOPT_CUSTOMREQUEST, NULL); curl_easy_setopt(request->easy, CURLOPT_HTTPGET, 1L); } if (server->headerCb) { curl_easy_setopt(request->easy, CURLOPT_HEADERFUNCTION, moloch_http_curlm_header_function); curl_easy_setopt(request->easy, CURLOPT_HEADERDATA, request); } curl_easy_setopt(request->easy, CURLOPT_CONNECTTIMEOUT, 10L); char *host = server->names[server->namesPos]; server->namesPos = (server->namesPos + 1) % server->namesCnt; if (strchr(host, ':') == 0) { snprintf(request->url, sizeof(request->url), "%s://%s:%d%.*s", (server->https?"https":"http"), host, server->defaultPort, key_len, key); } else { snprintf(request->url, sizeof(request->url), "%s://%s%.*s", (server->https?"https":"http"), host, key_len, key); } curl_easy_setopt(request->easy, CURLOPT_URL, request->url); MOLOCH_LOCK(requests); #ifdef MOLOCH_HTTP_DEBUG LOG("HTTPDEBUG INCR %s %p %d %s", server->names[0], request, server->outstanding, request->url); #endif server->outstanding++; DLL_PUSH_TAIL(rqt_, &requests, request); if (!requestsTimer) requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL); MOLOCH_UNLOCK(requests); return 0; }
int moloch_http_curl_close_callback(void *serverV, curl_socket_t fd) { MolochHttpServer_t *server = serverV; if (! BIT_ISSET(fd, connectionsSet)) { LOG("Couldn't connect %s defaultPort: %d", server->names[0], server->defaultPort); return 0; } struct sockaddr_storage localAddressStorage, remoteAddressStorage; socklen_t addressLength = sizeof(localAddressStorage); int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength); if (rc != 0) return 0; addressLength = sizeof(remoteAddressStorage); rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength); if (rc != 0) return 0; char sessionId[MOLOCH_SESSIONID_LEN]; int localPort, remotePort; char remoteIp[INET6_ADDRSTRLEN+2]; if (localAddressStorage.ss_family == AF_INET) { struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage; struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage; moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port, remoteAddress->sin_addr.s_addr, remoteAddress->sin_port); localPort = ntohs(localAddress->sin_port); remotePort = ntohs(remoteAddress->sin_port); inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp)); } else { struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage; struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage; moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port, remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port); localPort = ntohs(localAddress->sin6_port); remotePort = ntohs(remoteAddress->sin6_port); inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2); remoteIp[0] = '['; strcat(remoteIp, "]"); } MolochHttpConn_t *conn; BIT_CLR(fd, connectionsSet); MOLOCH_LOCK(connections); HASH_FIND(h_, connections, sessionId, conn); if (conn) { HASH_REMOVE(h_, connections, conn); MOLOCH_TYPE_FREE(MolochHttpConn_t, conn); } MOLOCH_UNLOCK(connections); server->connections--; LOG("Close %d/%d - %s %d->%s:%d fd:%d removed: %s", server->outstanding, server->connections, server->names[0], localPort, remoteIp, remotePort, fd, conn?"true":"false"); close (fd); return 0; }
static void moloch_http_curlm_check_multi_info(MolochHttpServer_t *server) { char *eff_url; CURLMsg *msg; int msgs_left; MolochHttpRequest_t *request; CURL *easy; while ((msg = curl_multi_info_read(server->multi, &msgs_left))) { if (msg->msg == CURLMSG_DONE) { easy = msg->easy_handle; curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void*)&request); curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url); long responseCode; curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &responseCode); if (config.logESRequests) { double totalTime; double connectTime; double uploadSize; double downloadSize; curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &totalTime); curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connectTime); curl_easy_getinfo(easy, CURLINFO_SIZE_UPLOAD, &uploadSize); curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD, &downloadSize); LOG("%d/%d ASYNC %ld %s %.0lf/%.0lf %.0lfms %.0lfms", request->server->outstanding, request->server->connections, responseCode, request->url, uploadSize, downloadSize, connectTime*1000, totalTime*1000); } #ifdef MOLOCH_HTTP_DEBUG LOG("HTTPDEBUG DECR %s %p %d %s", server->names[0], request, server->outstanding, request->url); #endif if (request->func) { if (request->dataIn) request->dataIn[request->used] = 0; request->func(responseCode, request->dataIn, request->used, request->uw); } if (request->dataIn) { free(request->dataIn); request->dataIn = 0; } if (request->dataOut) { MOLOCH_SIZE_FREE(buffer, request->dataOut); } if (request->headerList) { curl_slist_free_all(request->headerList); } MOLOCH_TYPE_FREE(MolochHttpRequest_t, request); curl_multi_remove_handle(server->multi, easy); curl_easy_cleanup(easy); MOLOCH_LOCK(requests); server->outstanding--; MOLOCH_UNLOCK(requests); } } }
void moloch_session_process_commands(int thread) { // Commands MolochSesCmd_t *cmd = 0; int count; for (count = 0; count < 50; count++) { MOLOCH_LOCK(sessionCmds[thread].lock); DLL_POP_HEAD(cmd_, &sessionCmds[thread], cmd); MOLOCH_UNLOCK(sessionCmds[thread].lock); if (!cmd) break; switch (cmd->cmd) { case MOLOCH_SES_CMD_ADD_TAG: moloch_field_int_add((long)cmd->uw1, cmd->session, (long)cmd->uw2); moloch_session_decr_outstanding(cmd->session); break; case MOLOCH_SES_CMD_FUNC: cmd->func(cmd->session, cmd->uw1, cmd->uw2); break; default: LOG ("Unknown cmd %d", cmd->cmd); } MOLOCH_TYPE_FREE(MolochSesCmd_t, cmd); } // Closing Q for (count = 0; count < 10; count++) { MolochSession_t *session = DLL_PEEK_HEAD(q_, &closingQ[thread]); if (session && session->saveTime < (uint64_t)lastPacketSecs[thread]) { moloch_session_save(session); } else { break; } } // Sessions Idle Long Time int ses; for (ses = 0; ses < SESSION_MAX; ses++) { for (count = 0; count < 10; count++) { MolochSession_t *session = DLL_PEEK_HEAD(q_, &sessionsQ[thread][ses]); if (session && (DLL_COUNT(q_, &sessionsQ[thread][ses]) > (int)config.maxStreams || ((uint64_t)session->lastPacket.tv_sec + config.timeouts[ses] < (uint64_t)lastPacketSecs[thread]))) { moloch_session_save(session); } else { break; } } } // TCP Sessions Open Long Time for (count = 0; count < 50; count++) { MolochSession_t *session = DLL_PEEK_HEAD(tcp_, &tcpWriteQ[thread]); if (session && (uint64_t)session->saveTime < (uint64_t)lastPacketSecs[thread]) { moloch_session_mid_save(session, lastPacketSecs[thread]); } else { break; } } }