Ejemplo n.º 1
0
END_TEST

START_TEST(test_bf_shared_compatible_persist)
{
    bloom_filter_params params = {0, 0, 1e6, 1e-4};
    bf_params_for_capacity(&params);
    bloom_bitmap map;
    bloom_bloomfilter filter;
    fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, PERSISTENT, &map) == 0);
    fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
    fchmod(map.fileno, 0777);

    // Check all the keys get added
    char buf[100];
    int res;
    for (int i=0;i<1000;i++) {
        snprintf((char*)&buf, 100, "test%d", i);
        res = bf_add(&filter, (char*)&buf);
        fail_unless(res == 1);
    }
    fail_unless(bf_close(&filter) == 0);

    // Test all the keys are contained
    fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, SHARED, &map) == 0);
    fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
    for (int i=0;i<1000;i++) {
        snprintf((char*)&buf, 100, "test%d", i);
        res = bf_contains(&filter, (char*)&buf);
        fail_unless(res == 1);
    }
    unlink("/tmp/shared_compat_persist.mmap");
}
Ejemplo n.º 2
0
END_TEST

START_TEST(test_sbf_close_does_flush)
{
    bloom_sbf_params params = SBF_DEFAULT_PARAMS;
    params.initial_capacity = 1e3;
    params.fp_probability = 1e-4;

    nextfile next;
    next.format = "/tmp/mmap_close.%d.data";
    next.num = 0;

    bloom_sbf sbf;
    int res = sbf_from_filters(&params, sbf_make_callback, &next, 0, NULL, &sbf);
    fail_unless(res == 0);

    char buf[100];
    for (int i=0;i<2000;i++) {
        snprintf((char*)&buf, 100, "foobar%d", i);
        sbf_add(&sbf, (char*)&buf);
    }
    fail_unless(sbf_close(&sbf) == 0);

    bloom_bitmap maps[2];
    bitmap_from_filename("/tmp/mmap_close.0.data", get_size("/tmp/mmap_close.0.data"), 1, 1, SHARED, (bloom_bitmap*)&maps);
    bitmap_from_filename("/tmp/mmap_close.1.data", get_size("/tmp/mmap_close.1.data"), 1, 1, SHARED, ((bloom_bitmap*)&maps)+1);

    bloom_bloomfilter filters[2];
    bf_from_bitmap((bloom_bitmap*)&maps, 1, 0, (bloom_bloomfilter*)&filters);
    bf_from_bitmap(((bloom_bitmap*)&maps)+1, 1, 0, ((bloom_bloomfilter*)&filters)+1);

    bloom_bloomfilter **filter_map = calloc(2, sizeof(bloom_bloomfilter*));
    filter_map[0] = (bloom_bloomfilter*)&filters;
    filter_map[1] = ((bloom_bloomfilter*)&filters)+1;

    res = sbf_from_filters(&params, sbf_make_callback, &next, 2, filter_map, &sbf);
    fail_unless(res == 0);

    fail_unless(sbf_size(&sbf) == 2000);
    fail_unless(sbf_total_capacity(&sbf) == 5*1e3);

    for (int i=0;i<2000;i++) {
        snprintf((char*)&buf, 100, "foobar%d", i);
        res = sbf_contains(&sbf, (char*)&buf);
        fail_unless(res == 1);
    }

    unlink("/tmp/mmap_close.0.data");
    unlink("/tmp/mmap_close.1.data");
}
Ejemplo n.º 3
0
END_TEST

START_TEST(make_bf_fresh_then_restore)
{
    // Use -1 for anonymous
    bloom_bitmap map;
    bloom_bloomfilter filter;
    bitmap_from_file(-1, 4096, ANONYMOUS, &map);
    int res = bf_from_bitmap(&map, 10, 1, &filter); // Make fresh
    fail_unless(res == 0);

    bloom_bloomfilter filter2;
    res = bf_from_bitmap(&map, 10, 0, &filter2); // Restore now
    fail_unless(res == 0);
}
Ejemplo n.º 4
0
END_TEST

START_TEST(test_bf_fp_prob_extended)
{
    bloom_filter_params params = {0, 0, 1e6, 0.001};
    bf_params_for_capacity(&params);
    bloom_bitmap map;
    bloom_bloomfilter filter;
    fail_unless(bitmap_from_file(-1, params.bytes, ANONYMOUS, &map) == 0);
    fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);

    // Check all the keys get added
    char buf[100];
    int res;
    int num_wrong = 0;
    for (int i=0;i<1e6;i++) {
        snprintf((char*)&buf, 100, "test%d", i);
        res = bf_add(&filter, (char*)&buf);
        if (res == 0) num_wrong++;
    }

    // We added 1M items, with a capacity of 1M and error of 1/1000.
    // Technically we should have 1K false positives
    fail_unless(num_wrong <= 1000);
}
Ejemplo n.º 5
0
END_TEST

START_TEST(test_length)
{
    bloom_filter_params params = {0, 0, 1e6, 1e-4};
    bf_params_for_capacity(&params);
    bloom_bitmap map;
    bitmap_from_file(-1, params.bytes, ANONYMOUS, &map);
    bloom_bloomfilter filter;
    bf_from_bitmap(&map, params.k_num, 1, &filter);

    // Check the size
    fail_unless(bf_size(&filter) == 0);

    // Check all the keys get added
    char buf[100];
    int res;
    for (int i=0;i<1000;i++) {
        snprintf((char*)&buf, 100, "test%d", i);
        res = bf_add(&filter, (char*)&buf);
        fail_unless(res == 1);
    }

    // Check the size
    fail_unless(bf_size(&filter) == 1000);
}
Ejemplo n.º 6
0
END_TEST


START_TEST(make_bf_no_map)
{
    // Use -1 for anonymous
    bloom_bloomfilter filter;
    int res = bf_from_bitmap(NULL, 10, 1, &filter);
    fail_unless(res == -EINVAL);
}
Ejemplo n.º 7
0
END_TEST

START_TEST(make_bf_fresh_not_new)
{
    // Use -1 for anonymous
    bloom_bitmap map;
    bloom_bloomfilter filter;
    bitmap_from_file(-1, 4096, ANONYMOUS, &map);
    int res = bf_from_bitmap(&map, 10, 0, &filter);
    fail_unless(res == -1);
}
Ejemplo n.º 8
0
END_TEST

START_TEST(make_bf_zero_k)
{
    // Use -1 for anonymous
    bloom_bitmap map;
    bloom_bloomfilter filter;
    bitmap_from_file(-1, 4096, ANONYMOUS, &map);
    int res = bf_from_bitmap(&map, 0, 1, &filter);
    fail_unless(res == -EINVAL);
}
Ejemplo n.º 9
0
END_TEST


START_TEST(test_bf_double_close)
{
    bloom_filter_params params = {0, 0, 1e6, 1e-4};
    bf_params_for_capacity(&params);
    bloom_bitmap map;
    bloom_bloomfilter filter;
    bitmap_from_file(-1, params.bytes, ANONYMOUS, &map);
    bf_from_bitmap(&map, params.k_num, 1, &filter);

    fail_unless(bf_close(&filter) == 0);
    fail_unless(bf_close(&filter) == -1);
}
Ejemplo n.º 10
0
END_TEST

START_TEST(test_bf_value_sanity)
{
    // Use -1 for anonymous
    bloom_bitmap map;
    bloom_bloomfilter filter;
    bitmap_from_file(-1, 4096, ANONYMOUS, &map);
    int res = bf_from_bitmap(&map, 10, 1, &filter); // Make fresh
    fail_unless(res == 0);

    fail_unless(filter.bitmap_size == 28672);  // Should be the bitmap size
    fail_unless(filter.offset == 2867);        // Should be size / 10
    fail_unless(filter.header->k_num == 10);
    fail_unless(filter.header->count == 0);
}
Ejemplo n.º 11
0
END_TEST

START_TEST(test_flush_close)
{
    bloom_filter_params params = {0, 0, 1e6, 1e-4};
    bf_params_for_capacity(&params);
    bloom_bitmap map;
    bloom_bloomfilter filter;
    bitmap_from_filename("/tmp/test_flush_close.mmap", params.bytes, 1, SHARED, &map);
    bf_from_bitmap(&map, params.k_num, 1, &filter);

    fail_unless(bf_flush(&filter) == 0);
    fail_unless(bf_close(&filter) == 0);

    unlink("/tmp/test_flush_close.mmap");
}
Ejemplo n.º 12
0
/**
 * This beast mode method scans the data directory
 * belonging to this filter for any existing filters,
 * and restores the SBF
 * @return 0 on success. -1 on error.
 */
static int discover_existing_filters(bloom_filter *f) {
    // Scan through the folder looking for data files
    struct dirent **namelist;
    int num;

    // Filter only data dirs, in sorted order
    num = scandir(f->full_path, &namelist, filter_data_files, alphasort);
    if (num == -1) {
        syslog(LOG_ERR, "Failed to scan files for filter '%s'. %s",
                f->filter_name, strerror(errno));
        return -1;
    }
    syslog(LOG_INFO, "Found %d files for filter %s.", num, f->filter_name);

    // Speical case when there are no filters
    if (num == 0) {
        int res = create_sbf(f, 0, NULL);
        return res;
    }

    // Allocate space for all the filter
    bloom_bitmap **maps = malloc(num * sizeof(bloom_bitmap*));
    bloom_bloomfilter **filters = malloc(num * sizeof(bloom_bloomfilter*));

    // Initialize the bitmaps and bloom filters
    int res;
    int err = 0;
    uint64_t size;
    bitmap_mode mode = (f->config->use_mmap) ? SHARED : PERSISTENT;
    for (int i=0; i < num && !err; i++) {
        // Get the full path to the bitmap
        char *bitmap_path = join_path(f->full_path, namelist[i]->d_name);
        syslog(LOG_INFO, "Discovered bloom filter: %s.", bitmap_path);

        // Get the size
        size = get_size(bitmap_path);
        if (size == 0) {
            err = 1;
            syslog(LOG_ERR, "Failed to get the filesize for: %s. %s", bitmap_path, strerror(errno));
            free(bitmap_path);
            break;
        }

        // Create the bitmap
        bloom_bitmap *bitmap = maps[num - i - 1] = malloc(sizeof(bloom_bitmap));
        res = bitmap_from_filename(bitmap_path, size, 0, mode, bitmap);
        if (res != 0) {
            err = 1;
            syslog(LOG_ERR, "Failed to load bitmap for: %s. %s", bitmap_path, strerror(errno));
            free(bitmap);
            free(bitmap_path);
            break;
        }

        // Create the bloom filter
        bloom_bloomfilter *filter = filters[num - i - 1] = malloc(sizeof(bloom_bloomfilter));
        res = bf_from_bitmap(bitmap, 1, 0, filter);
        if (res != 0) {
            err = 1;
            syslog(LOG_ERR, "Failed to load bloom filter for: %s. [%d]", bitmap_path, res);
            free(filter);
            bitmap_close(bitmap);
            free(bitmap);
            free(bitmap_path);
            break;
        }

        // Cleanup
        free(bitmap_path);
    }

    // Free the memory associated with scandir
    for (int i=0; i < num; i++) free(namelist[i]);
    free(namelist);

    // Return if there was an error
    if (err) return -1;

    // Create the SBF
    res = create_sbf(f, num, filters);

    // Cleanup on err
    if (res != 0) {
        syslog(LOG_ERR, "Failed to make scalable bloom filter for: %s.", f->filter_name);

        // For f***s sake. We need to clean up so much shit now.
        for (int i=0; i < num; i++) {
            bf_close(filters[i]);
            bitmap_close(maps[i]);
            free(filters[i]);
            free(maps[i]);
        }
    }

    // Increase our page ins
    f->counters.page_ins += 1;

    // Remove the filters list
    free(maps);
    free(filters);
    return (err) ? -1 : 0;
}