Example #1
0
int collection_store(modsec_rec *msr, apr_table_t *col) {
    char *dbm_filename = NULL;
    msc_string *var_name = NULL, *var_key = NULL;
    unsigned char *blob = NULL;
    unsigned int blob_size, blob_offset;
    apr_status_t rc;
    apr_sdbm_datum_t key;
    apr_sdbm_datum_t value;
    apr_sdbm_t *dbm = NULL;
    const apr_array_header_t *arr;
    apr_table_entry_t *te;
    int i;
    const apr_table_t *stored_col = NULL;
    const apr_table_t *orig_col = NULL;

    var_name = (msc_string *)apr_table_get(col, "__name");
    if (var_name == NULL) {
        goto error;
    }

    var_key = (msc_string *)apr_table_get(col, "__key");
    if (var_key == NULL) {
        goto error;
    }

    if (msr->txcfg->data_dir == NULL) {
        msr_log(msr, 1, "Unable to store collection (name \"%s\", key \"%s\"). Use "
            "SecDataDir to define data directory first.", log_escape_ex(msr->mp, var_name->value, var_name->value_len),
            log_escape_ex(msr->mp, var_key->value, var_key->value_len));
        goto error;
    }

    // ENH: lowercase the var name in the filename
    dbm_filename = apr_pstrcat(msr->mp, msr->txcfg->data_dir, "/", var_name->value, NULL);

    if (msr->txcfg->debuglog_level >= 9) {
        msr_log(msr, 9, "collection_store: Retrieving collection (name \"%s\", filename \"%s\")",log_escape(msr->mp, var_name->value),
                log_escape(msr->mp, dbm_filename));
    }

    /* Delete IS_NEW on store. */
    apr_table_unset(col, "IS_NEW");

    /* Delete UPDATE_RATE on store to save space as it is calculated */
    apr_table_unset(col, "UPDATE_RATE");

    /* Update the timeout value. */
    {
        msc_string *var = (msc_string *)apr_table_get(col, "TIMEOUT");
        if (var != NULL) {
            int timeout = atoi(var->value);
            var = (msc_string *)apr_table_get(col, "__expire_KEY");
            if (var != NULL) {
                var->value = apr_psprintf(msr->mp, "%" APR_TIME_T_FMT, (apr_time_t)(apr_time_sec(apr_time_now()) + timeout));
                var->value_len = strlen(var->value);
            }
        }
    }

    /* LAST_UPDATE_TIME */
    {
        msc_string *var = (msc_string *)apr_table_get(col, "LAST_UPDATE_TIME");
        if (var == NULL) {
            var = (msc_string *)apr_pcalloc(msr->mp, sizeof(msc_string));
            var->name = "LAST_UPDATE_TIME";
            var->name_len = strlen(var->name);
            apr_table_setn(col, var->name, (void *)var);
        }
        var->value = apr_psprintf(msr->mp, "%" APR_TIME_T_FMT, (apr_time_t)(apr_time_sec(apr_time_now())));
        var->value_len = strlen(var->value);
    }

    /* UPDATE_COUNTER */
    {
        msc_string *var = (msc_string *)apr_table_get(col, "UPDATE_COUNTER");
        int counter = 0;
        if (var == NULL) {
            var = (msc_string *)apr_pcalloc(msr->mp, sizeof(msc_string));
            var->name = "UPDATE_COUNTER";
            var->name_len = strlen(var->name);
            apr_table_setn(col, var->name, (void *)var);
        } else {
            counter = atoi(var->value);
        }
        var->value = apr_psprintf(msr->mp, "%d", counter + 1);
        var->value_len = strlen(var->value);
    }

    /* ENH Make the expiration timestamp accessible in blob form so that
     * it is easier/faster to determine expiration without having to
     * convert back to table form
     */

    rc = apr_sdbm_open(&dbm, dbm_filename, APR_CREATE | APR_WRITE | APR_SHARELOCK,
        CREATEMODE, msr->mp);
    if (rc != APR_SUCCESS) {
        msr_log(msr, 1, "Failed to access DBM file \"%s\": %s", log_escape(msr->mp, dbm_filename),
            get_apr_error(msr->mp, rc));
        dbm = NULL;
        goto error;
    }

    /* Need to lock to pull in the stored data again and apply deltas. */
    rc = apr_sdbm_lock(dbm, APR_FLOCK_EXCLUSIVE);
    if (rc != APR_SUCCESS) {
        msr_log(msr, 1, "Failed to exclusivly lock DBM file \"%s\": %s", log_escape(msr->mp, dbm_filename),
            get_apr_error(msr->mp, rc));
        goto error;
    }

    /* If there is an original value, then create a delta and
     * apply the delta to the current value */
    orig_col = (const apr_table_t *)apr_table_get(msr->collections_original, var_name->value);
    if (orig_col != NULL) {
        if (msr->txcfg->debuglog_level >= 9) {
            msr_log(msr, 9, "Re-retrieving collection prior to store: %s",
                apr_psprintf(msr->mp, "%.*s", var_name->value_len, var_name->value));
        }

        stored_col = (const apr_table_t *)collection_retrieve_ex(dbm, msr, var_name->value, var_key->value, var_key->value_len);
    }

    /* Merge deltas and calculate the size first. */
    blob_size = 3 + 2;
    arr = apr_table_elts(col);
    te = (apr_table_entry_t *)arr->elts;
    for (i = 0; i < arr->nelts; i++) {
        msc_string *var = (msc_string *)te[i].val;
        int len;

        /* If there is an original value, then apply the delta
         * to the latest stored value */
        if (stored_col != NULL) {
            const msc_string *orig_var = (const msc_string *)apr_table_get(orig_col, var->name);
            if (orig_var != NULL) {
                const msc_string *stored_var = (const msc_string *)apr_table_get(stored_col, var->name);
                if (stored_var != NULL) {
                    int origval = atoi(orig_var->value);
                    int ourval = atoi(var->value);
                    int storedval = atoi(stored_var->value);
                    int delta = ourval - origval;
                    int newval = storedval + delta;

                    if (newval < 0) newval = 0; /* Counters never go below zero. */

                    var->value = apr_psprintf(msr->mp, "%d", newval);
                    var->value_len = strlen(var->value);
                    
                    if (msr->txcfg->debuglog_level >= 9) {
                        msr_log(msr, 9, "Delta applied for %s.%s %d->%d (%d): %d + (%d) = %d [%s,%d]",
                        log_escape_ex(msr->mp, var_name->value, var_name->value_len),
                        log_escape_ex(msr->mp, var->name, var->name_len),
                        origval, ourval, delta, storedval, delta, newval, var->value, var->value_len);
                    }
                }
            }
        }

        len = var->name_len + 1;
        if (len >= 65536) len = 65536;
        blob_size += len + 2;

        len = var->value_len + 1;
        if (len >= 65536) len = 65536;
        blob_size += len + 2;
    }

    /* Now generate the binary object. */
    blob = apr_pcalloc(msr->mp, blob_size);
    if (blob == NULL) {
        goto error;
    }

    blob[0] = 0x49;
    blob[1] = 0x52;
    blob[2] = 0x01;

    blob_offset = 3;
    arr = apr_table_elts(col);
    te = (apr_table_entry_t *)arr->elts;
    for (i = 0; i < arr->nelts; i++) {
        msc_string *var = (msc_string *)te[i].val;
        int len;

        len = var->name_len + 1;
        if (len >= 65536) len = 65536;

        blob[blob_offset + 0] = (len & 0xff00) >> 8;
        blob[blob_offset + 1] = len & 0x00ff;
        memcpy(blob + blob_offset + 2, var->name, len - 1);
        blob[blob_offset + 2 + len - 1] = '\0';
        blob_offset += 2 + len;

        len = var->value_len + 1;
        if (len >= 65536) len = 65536;

        blob[blob_offset + 0] = (len & 0xff00) >> 8;
        blob[blob_offset + 1] = len & 0x00ff;
        memcpy(blob + blob_offset + 2, var->value, len - 1);
        blob[blob_offset + 2 + len - 1] = '\0';
        blob_offset += 2 + len;

        if (msr->txcfg->debuglog_level >= 9) {
            msr_log(msr, 9, "Wrote variable: name \"%s\", value \"%s\".",
                log_escape_ex(msr->mp, var->name, var->name_len),
                log_escape_ex(msr->mp, var->value, var->value_len));
        }
    }

    blob[blob_offset] = 0;
    blob[blob_offset + 1] = 0;

    /* And, finally, store it. */
    key.dptr = var_key->value;
    key.dsize = var_key->value_len + 1;

    value.dptr = (char *)blob;
    value.dsize = blob_size;

    rc = apr_sdbm_store(dbm, key, value, APR_SDBM_REPLACE);
    if (rc != APR_SUCCESS) {
        msr_log(msr, 1, "Failed to write to DBM file \"%s\": %s", dbm_filename,
            get_apr_error(msr->mp, rc));
        goto error;
    }

    apr_sdbm_close(dbm);

    if (msr->txcfg->debuglog_level >= 4) {
        msr_log(msr, 4, "Persisted collection (name \"%s\", key \"%s\").",
            log_escape_ex(msr->mp, var_name->value, var_name->value_len),
            log_escape_ex(msr->mp, var_key->value, var_key->value_len));
    }

    return 0;

error:

    if (dbm) {
        apr_sdbm_close(dbm);
    }

    return -1;
}
Example #2
0
static int dump_database(apr_pool_t *pool, apr_sdbm_t *db, int action)
{
    apr_status_t ret;
    apr_sdbm_datum_t key;
    apr_sdbm_datum_t val;
    apr_sdbm_t *db_dest;
    double elements = 0;
    int bad_datum = 0;
    int expired_datum = 0;
    int removed = 0;
    int progress = 0;
    int fret = 0;
    
    if (action & PRINT)
        v("Dumping database...\n");
    if (action & SHRINK)
        v("Starting the shrink process...\n");
    if (action & STATUS)
        v("Showing some status about the databases...\n");

    if (action & EXTRACT)
    {
        v("Exporting valid items to: /tmp/new_db.[pag,dir]...\n");
        ret = apr_sdbm_open(&db_dest, "/tmp/new_db",
                APR_CREATE | APR_WRITE | APR_SHARELOCK, 0x0777, pool);

        if (ret != APR_SUCCESS)
        {
            v("Failed to retrieve the first key of the database.\n");
            fret = -1;
            goto end;
        }
    }

    ret = apr_sdbm_firstkey(db, &key);
    if (ret != APR_SUCCESS)
    {
        v("Failed to retrieve the first key of the database.\n");
        fret = -1;
        goto end;
    }

    do {
        ret = apr_sdbm_fetch(db, &val, key);
        if (ret != APR_SUCCESS) {
            v("Failed to fetch the value of the key: %s.\n", key.dptr);
            fret = -1;
            goto end;
        }

        elements++;

        if (action & PRINT)
        {
            if ((!(action & PRINT_ONLY_EXPIRED)) ||
                    ((action & PRINT_ONLY_EXPIRED) && is_expired(pool,
                    (const unsigned char *)val.dptr, val.dsize)))
            {
                printf("Key: \"%s\", Value len: %d\n", key.dptr, val.dsize);
                if (action & PRINT_MODSEC_VARS)
                {
                    print_modsec_variables(pool,
                            (const unsigned char *)val.dptr, val.dsize);
                }
            }
        }

        if (action & SHRINK || action & STATUS || action & EXTRACT)
        {
            int selected = 0;
            if (val.dsize == 0) {
                bad_datum++;
                selected = 1;
            }

            if (is_expired(pool, (const unsigned char *)val.dptr, val.dsize))
            {
                expired_datum++;
                selected = 1;
            }

            if ((int)elements % 10 == 0)
            {
                int p2s = (int) progress++ % 4;
                p(" [%c] %.0f records so far.\r", progress_feedback[p2s],
                        elements);
                fflush(stdout);
            }

            if (selected && action & SHRINK)
            {
                ret = remote_datum_t(pool, db, &key);
                if (ret != APR_SUCCESS)
                {
                    p("Failed to delete key: \"%s\"\n", (const unsigned char *)key.dptr);
                } else {
                    removed++;
                }
                //Remove key.
            }

            if (selected == 0 && action & EXTRACT)
            {
                ret = apr_sdbm_store(db_dest, key, val, APR_SDBM_INSERT);
                if (ret != APR_SUCCESS)
                {
                    p("Failed to insert key: \"%s\"\n", (const unsigned char *)key.dptr);
                }
 
            }

        }

        ret = apr_sdbm_nextkey(db, &key);
        if (ret != APR_SUCCESS) {
            v("Failed to retrieve the next key.\n");
            fret = -1;
            goto end;
        }
    } while (key.dptr);

end:
    if (action & EXTRACT)
    {
        p("New database generated with valied keys at: /tmp/new_db\n");
        apr_sdbm_close(db_dest);
    }
    if (action & SHRINK || action & STATUS)
    {
        printf("\n");
        printf("Total of %.0f elements processed.\n", elements);
        printf("%d elements removed.\n", removed);
        printf("Expired elements: %d, inconsistent items: %d\n", expired_datum,
            bad_datum);
        if (expired_datum+bad_datum != 0 && elements !=0)
            printf("Fragmentation rate: %2.2f%% of the database is/was dirty " \
                "data.\n", 100*(expired_datum+bad_datum)/elements);
    }

    return fret;
}