Esempio n. 1
0
File: http.c Progetto: Amelos/moloch
void moloch_http_do_requests(MolochHttp_t *server)
{
    // Something waiting
    while (DLL_COUNT(r_, &server->requestQ)) {

        // No free connections
        if (DLL_COUNT(e_, &server->connQ) == 0) {

            // Not at our max, create one a second
            if (server->numConns < server->maxConns && time(0) - server->lastFailedConnect > 0) {
                moloch_http_create(server, FALSE);
            }
            return;
        }

        // Have a free conn and a request
        MolochConn_t *conn;
        DLL_POP_HEAD(e_, &server->connQ, conn);
        DLL_POP_HEAD(r_, &server->requestQ, conn->request);
        server->inProgress++;
        if (!moloch_http_process_send(conn, FALSE)) {
            server->inProgress--;
            LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
            DLL_PUSH_HEAD(r_, &server->requestQ, conn->request);

            moloch_http_free_conn(conn, FALSE);
        }
    }
}
Esempio n. 2
0
File: wise.c Progetto: Amelos/moloch
void wise_cb(unsigned char *data, int data_len, gpointer uw)
{

    BSB             bsb;
    WiseRequest_t *request = uw;
    int             i;

    inflight -= request->numItems;

    BSB_INIT(bsb, data, data_len);

    uint32_t fts = 0, ver = 0;
    BSB_IMPORT_u32(bsb, fts);
    BSB_IMPORT_u32(bsb, ver);

    if (BSB_IS_ERROR(bsb) || ver != 0) {
        for (i = 0; i < request->numItems; i++) {
            wise_free_item(request->items[i]);
        }
        MOLOCH_TYPE_FREE(WiseRequest_t, request);
        return;
    }

    if (fts != fieldsTS)
        wise_load_fields();

    struct timeval currentTime;
    gettimeofday(&currentTime, NULL);

    for (i = 0; i < request->numItems; i++) {
        WiseItem_t    *wi = request->items[i];
        BSB_IMPORT_u08(bsb, wi->numOps);

        if (wi->numOps > 0) {
            wi->ops = malloc(wi->numOps * sizeof(WiseOp_t));

            int i;
            for (i = 0; i < wi->numOps; i++) {
                WiseOp_t *op = &(wi->ops[i]);

                int rfield = 0;
                BSB_IMPORT_u08(bsb, rfield);
                op->fieldPos = fieldsMap[rfield];

                int len = 0;
                BSB_IMPORT_u08(bsb, len);
                char *str = (char*)BSB_WORK_PTR(bsb);
                BSB_IMPORT_skip(bsb, len);

                switch (config.fields[op->fieldPos]->type) {
                case  MOLOCH_FIELD_TYPE_INT_HASH:
                    if (op->fieldPos == tagsField) {
                        moloch_db_get_tag(NULL, tagsField, str, NULL); // Preload the tagname -> tag mapping
                        op->str = g_strdup(str);
                        op->strLenOrInt = len - 1;
                        continue;
                    }
                    // Fall thru
                case  MOLOCH_FIELD_TYPE_INT:
                case  MOLOCH_FIELD_TYPE_INT_ARRAY:
                    op->str = 0;
                    op->strLenOrInt = atoi(str);
                    break;
                case  MOLOCH_FIELD_TYPE_STR:
                case  MOLOCH_FIELD_TYPE_STR_ARRAY:
                case  MOLOCH_FIELD_TYPE_STR_HASH:
                    op->str = g_strdup(str);
                    op->strLenOrInt = len - 1;
                    break;
                case  MOLOCH_FIELD_TYPE_IP:
                case  MOLOCH_FIELD_TYPE_IP_HASH:
                    op->str = 0;
                    op->strLenOrInt = inet_addr(str);
                    break;
                default:
                    LOG("WARNING - Unsupported expression type for %s", str);
                    continue;
                }
            }
        }

        wi->loadTime = currentTime.tv_sec;

        int s;
        for (s = 0; s < wi->numSessions; s++) {
            wise_process_ops(wi->sessions[s], wi);
            moloch_nids_decr_outstanding(wi->sessions[s]);
        }
        g_free(wi->sessions);
        wi->sessions = 0;
        wi->numSessions = 0;

        DLL_PUSH_HEAD(wil_, &itemList[(int)wi->type], wi);
        // Cache needs to be reduced
        if (itemList[(int)wi->type].wil_count > maxCache) {
            DLL_POP_TAIL(wil_, &itemList[(int)wi->type], wi);
            wise_free_item(wi);
        }
    }
    MOLOCH_TYPE_FREE(WiseRequest_t, request);
}
Esempio n. 3
0
File: http.c Progetto: ariosx/moloch
gboolean moloch_http_send(void *serverV, char *method, char *key, uint32_t key_len, char *data, uint32_t data_len, gboolean dropable, MolochResponse_cb func, gpointer uw)
{
    MolochRequest_t     *request;
    MolochConn_t        *conn;
    MolochHttp_t        *server = serverV;

    request = MOLOCH_TYPE_ALLOC(MolochRequest_t);
    memcpy(request->key, key, MIN(key_len, sizeof(request->key)));
    strncpy(request->method, method, sizeof(request->method));
    request->key_len  = key_len;
    request->compress = 0;

    if (server->compress && data && data_len > 1000) {
        char            *buf = moloch_http_get_buffer(data_len);
        int              ret;

        z_strm.avail_in   = data_len;
        z_strm.next_in    = (unsigned char *)data;
        z_strm.avail_out  = data_len;
        z_strm.next_out   = (unsigned char *)buf;
        ret = deflate(&z_strm, Z_FINISH);
        if (ret == Z_STREAM_END) {
            request->compress = 1;
            MOLOCH_SIZE_FREE(buffer, data);
            data_len = data_len - z_strm.avail_out;
            data     = buf;
        } else {
            MOLOCH_SIZE_FREE(buffer, buf);
        }

        deflateReset(&z_strm);
    }

    request->data_len = data_len;
    request->data     = data;
    request->func     = func;
    request->uw       = uw;

    int q = data_len > MOLOCH_HTTP_BUFFER_SIZE?1:0;

    // Already have outstanding requests, see if we can process them
    if (server->requestQ[q].r_count && server->connQ.e_count && time(0) - server->lastFailedConnect > 0 ) {
        while (DLL_POP_HEAD(e_, &server->connQ, conn)) {
            DLL_POP_HEAD(r_, &server->requestQ[q], conn->request);
            if (conn->request) {
                if (!moloch_http_process_send(conn, 0)) {
                    LOG("ERROR - %p: Couldn't send %.*s", (void*)conn, conn->request->key_len, conn->request->key);
                    DLL_PUSH_HEAD(r_, &server->requestQ[q], conn->request);
                    conn->request = 0;
                    DLL_PUSH_TAIL(e_, &server->connQ, conn);
                    break;
                }
            }
            else {
                DLL_PUSH_TAIL(e_, &server->connQ, conn);
                break;
            }
        }
    }

    // Now see if we can send something new
    if (DLL_POP_HEAD(e_, &server->connQ, conn)) {
        conn->request = request;
        if (!moloch_http_process_send(conn, FALSE)) {
            conn->request = 0;
            DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
            DLL_PUSH_TAIL(e_, &server->connQ, conn);
        }
    } else {
        request->data = data;
        if (!config.exiting && dropable && server->requestQ[q].r_count > server->maxOutstandingRequests) {
            LOG("ERROR - Dropping request %.*s of size %d queue[%d] %d is too big", key_len, key, data_len, q, server->requestQ[q].r_count);

            if (data) {
                MOLOCH_SIZE_FREE(buffer, data);
            }
            MOLOCH_TYPE_FREE(MolochRequest_t, request);
            return 1;
        } else {
            DLL_PUSH_TAIL(r_, &server->requestQ[q], request);
        }
    }

    return 0;
}
Esempio n. 4
0
File: http.c Progetto: ariosx/moloch
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
    MolochConn_t        *conn = data;
    char                 buffer[0xffff];
    int                  len;
    GError              *gerror = 0;

    len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);

    if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
        if (gerror) {
            LOG("ERROR: %p:%p Receive Error: %s", (void*)conn, conn->request, gerror->message);
            g_error_free(gerror);
        } else if (cond & (G_IO_HUP | G_IO_ERR))
            LOG("ERROR: %p:%p Lost connection to %s", (void*)conn, conn->request, conn->name);
        else if (len <= 0)
            LOG("ERROR: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);
        else
            LOG("ERROR HMM: %p:%p len: %d cond: %x", (void*)conn, conn->request, len, cond);

        if (conn->request) {
            // Must save, free, then call function because of recursive sync GETs
            MolochResponse_cb  func = conn->request->func;
            gpointer           uw = conn->request->uw;

            MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);

            if (func) {
                func(0, 0, uw);
            }
        }

        g_object_unref (conn->conn);
        conn->conn = 0;
        if (conn != conn->server->syncConn && conn->request) {
            DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
        }
        conn->request = 0;
        return FALSE;
    }

    http_parser_execute(&conn->parser, &parserSettings, buffer, len);

    if (conn->hp_complete) {
        gettimeofday(&conn->endTime, NULL);
        if (config.logESRequests)
            LOG("%s %ldms %ldms %ldms",
               conn->line,
               (conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
               (conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
               (conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
               );

        conn->hp_data[conn->hp_len] = 0;

        /* Must save, free, then call function because of recursive sync GETs */
        MolochResponse_cb  func = conn->request->func;
        gpointer           uw = conn->request->uw;

        MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
        conn->request = 0;

        if (func) {
            func((unsigned char*)conn->hp_data, conn->hp_len, uw);
        }


        if (conn == conn->server->syncConn)
            return TRUE;


        int q;
        for (q = 0; q < 2; q++) {
            DLL_POP_HEAD(r_, &conn->server->requestQ[q], conn->request);
            if (conn->request) {
                if (!moloch_http_process_send(conn, 0)) {
                    DLL_PUSH_HEAD(r_, &conn->server->requestQ[q], conn->request);
                    conn->request = 0;
                    DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
                }
                return TRUE;
            }
        }
        DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
    }

    return TRUE;
}
Esempio n. 5
0
File: http.c Progetto: Amelos/moloch
gboolean moloch_http_read_cb(gint UNUSED(fd), GIOCondition cond, gpointer data) {
    MolochConn_t        *conn = data;
    char                 buffer[0xffff];
    int                  len;
    GError              *gerror = 0;

    len = g_socket_receive(conn->conn, buffer, sizeof(buffer)-1, NULL, &gerror);

    if (gerror || cond & (G_IO_HUP | G_IO_ERR) || len <= 0) {
        if (gerror) {
            LOG("ERROR: %s %p:%p Receive Error: %s", conn->name, (void*)conn, conn->request, gerror->message);
            g_error_free(gerror);
        } else if (cond & (G_IO_HUP | G_IO_ERR))
            LOG("ERROR: %s %p:%p Lost connection to %s", conn->name, (void*)conn, conn->request, conn->name);
        else if (len <= 0)
            LOG("ERROR: %s %p:%p len: %d cond: %x", conn->name, (void*)conn, conn->request, len, cond);
        else
            LOG("ERROR HMM: %s %p:%p len: %d cond: %x", conn->name, (void*)conn, conn->request, len, cond);

        if (conn == conn->server->syncConn) {
            if (!conn->request) {
                DEBUGCONN("AAA zerosync %s %p", conn->server->names[0], conn);
                conn->server->syncConn = 0;
                moloch_http_free_conn(conn, FALSE);
            } else {
                DEBUGCONN("AAA complete %s %p", conn->server->names[0], conn);
                conn->hp_complete = 1;
            }
            return FALSE;
        }

        if (conn->request && conn->request->data) {
            conn->server->inProgress--;
            DLL_PUSH_HEAD(r_, &conn->server->requestQ, conn->request);
        } else if (conn->request) {
            conn->server->inProgress--;
            // Must save, free, then call function because of recursive sync GETs
            MolochResponse_cb  func = conn->request->func;
            gpointer           uw = conn->request->uw;

            MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);

            if (func) {
                func(0, 0, uw);
            }
        }

        moloch_http_free_conn(conn, TRUE);
        return FALSE;
    }

    http_parser_execute(&conn->parser, &conn->server->parserSettings, buffer, len);

    if (conn->hp_complete) {
        gettimeofday(&conn->endTime, NULL);
        if (config.logESRequests)
            LOG("%s %ldms %ldms %ldms",
               conn->line,
               (conn->sendTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sendTime.tv_usec - conn->startTime.tv_usec)/1000,
               (conn->sentTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->sentTime.tv_usec - conn->startTime.tv_usec)/1000,
               (conn->endTime.tv_sec - conn->startTime.tv_sec)*1000 + (conn->endTime.tv_usec - conn->startTime.tv_usec)/1000
               );

        conn->hp_data[conn->hp_len] = 0;

        /* Must save, free, then call function because of recursive sync GETs */
        MolochResponse_cb  func = conn->request->func;
        gpointer           uw = conn->request->uw;

        MOLOCH_TYPE_FREE(MolochRequest_t, conn->request);
        conn->request = 0;

        if (func) {
            func((unsigned char*)conn->hp_data, conn->hp_len, uw);
        }

        if (conn == conn->server->syncConn) {
            if (conn->doClose) {
                DEBUGCONN("AAA zerosync %s %p", conn->server->names[0], conn);
                conn->server->syncConn = 0;
                moloch_http_free_conn(conn, FALSE);
                return FALSE;
            }

            return TRUE;
        }

        // Not syncConn below this

        if (conn->doClose) {
            moloch_http_free_conn(conn, TRUE);
            return FALSE;
        }

        conn->server->inProgress--;
        DLL_PUSH_TAIL(e_, &conn->server->connQ, conn);
        moloch_http_do_requests(conn->server);
    }

    return TRUE;
}