unsigned char *moloch_http_send_sync(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, size_t *return_len) { MolochRequest_t *request; gboolean sent = FALSE; MolochHttp_t *server = serverV; request = MOLOCH_TYPE_ALLOC(MolochRequest_t); memcpy(request->key, key, MIN(key_len, sizeof(request->key))); strncpy(request->method, method, sizeof(request->method)); request->method[sizeof(request->method)-1] = 0; request->key_len = key_len; request->data_len = data_len; request->data = data; request->func = 0; request->uw = 0; request->compress = 0; if (return_len) *return_len = 0; server->syncConn->request = request; sent = moloch_http_process_send(server->syncConn, TRUE); if (sent) { if (return_len) *return_len = server->syncConn->hp_len; return (unsigned char*)server->syncConn->hp_data; } return 0; }
void moloch_http_do_requests(MolochHttp_t *server) { // Something waiting while (DLL_COUNT(r_, &server->requestQ)) { // No free connections if (DLL_COUNT(e_, &server->connQ) == 0) { // Not at our max, create one a second if (server->numConns < server->maxConns && time(0) - server->lastFailedConnect > 0) { moloch_http_create(server, FALSE); } return; } // Have a free conn and a request MolochConn_t *conn; DLL_POP_HEAD(e_, &server->connQ, conn); DLL_POP_HEAD(r_, &server->requestQ, conn->request); server->inProgress++; if (!moloch_http_process_send(conn, FALSE)) { server->inProgress--; LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key); DLL_PUSH_HEAD(r_, &server->requestQ, conn->request); moloch_http_free_conn(conn, FALSE); } } }
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw) { MolochRequest_t *request; MolochConn_t *conn; MolochHttp_t *server = serverV; request = MOLOCH_TYPE_ALLOC(MolochRequest_t); memcpy(request->key, key, MIN(key_len, sizeof(request->key))); strncpy(request->method, method, sizeof(request->method)); request->key_len = key_len; request->compress = 0; if (server->compress && data && data_len > 1000) { char *buf = moloch_http_get_buffer(data_len); int ret; z_strm.avail_in = data_len; z_strm.next_in = (unsigned char *)data; z_strm.avail_out = data_len; z_strm.next_out = (unsigned char *)buf; ret = deflate(&z_strm, Z_FINISH); if (ret == Z_STREAM_END) { request->compress = 1; MOLOCH_SIZE_FREE(buffer, data); data_len = data_len - z_strm.avail_out; data = buf; } else { MOLOCH_SIZE_FREE(buffer, buf); } deflateReset(&z_strm); } request->data_len = data_len; request->data = data; request->func = func; request->uw = uw; int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0; // Already have outstanding requests, see if we can process them if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) { while (DLL_POP_HEAD(e_, &server->connQ, conn)) { DLL_POP_HEAD(r_, &server->requestQ[q], conn->request); if (conn->request) { if (!moloch_http_process_send(conn, 0)) { LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key); DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request); conn->request = 0; DLL_PUSH_TAIL(e_, &server->connQ, conn); break; } } else { DLL_PUSH_TAIL(e_, &server->connQ, conn); break; } } } // Now see if we can send something new if (DLL_POP_HEAD(e_, &server->connQ, conn)) { conn->request = request; if (!moloch_http_process_send(conn, FALSE)) { conn->request = 0; DLL_PUSH_TAIL(r_, &server->requestQ[q], request); DLL_PUSH_TAIL(e_, &server->connQ, conn); } } else { request->data = data; if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) { LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count); if (data) { MOLOCH_SIZE_FREE(buffer, data); } MOLOCH_TYPE_FREE(MolochRequest_t, request); return 1; } else { DLL_PUSH_TAIL(r_, &server->requestQ[q], request); } } return 0; }
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) { MolochConn_t *conn = data; char buffer[0xffff]; int len; GError *gerror = 0; len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror); if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) { if (gerror) { LOG("ERROR: %p:%p Receive Error: %s", (void*)conn, conn->request, gerror->message); g_error_free(gerror); } else if (cond & (G_IO_HUP | G_IO_ERR)) LOG("ERROR: %p:%p Lost connection to %s", (void*)conn, conn->request, conn->name); else if (len <= 0) LOG("ERROR: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond); else LOG("ERROR HMM: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond); if (conn->request) { // Must save, free, then call function because of recursive sync GETs MolochResponse_cb func = conn->request->func; gpointer uw = conn->request->uw; MOLOCH_TYPE_FREE(MolochRequest_t, conn->request); if (func) { func(0, 0, uw); } } g_object_unref (conn->conn); conn->conn = 0; if (conn != conn->server->syncConn && conn->request) { DLL_PUSH_TAIL(e_, &conn->server->connQ, conn); } conn->request = 0; return FALSE; } http_parser_execute(&conn->parser, &parserSettings, buffer, len); if (conn->hp_complete) { gettimeofday(&conn->endTime, NULL); if (config.logESRequests) LOG("%s %ldms %ldms %ldms", conn->line, (conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000, (conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000, (conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000 ); conn->hp_data[conn->hp_len] = 0; /* Must save, free, then call function because of recursive sync GETs */ MolochResponse_cb func = conn->request->func; gpointer uw = conn->request->uw; MOLOCH_TYPE_FREE(MolochRequest_t, conn->request); conn->request = 0; if (func) { func((unsigned char*)conn->hp_data, conn->hp_len, uw); } if (conn == conn->server->syncConn) return TRUE; int q; for (q = 0; q < 2; q++) { DLL_POP_HEAD(r_, &conn->server->requestQ[q], conn->request); if (conn->request) { if (!moloch_http_process_send(conn, 0)) { DLL_PUSH_HEAD(r_, &conn->server->requestQ[q], conn->request); conn->request = 0; DLL_PUSH_TAIL(e_, &conn->server->connQ, conn); } return TRUE; } } DLL_PUSH_TAIL(e_, &conn->server->connQ, conn); } return TRUE; }