END_TEST
START_TEST(test_bf_shared_compatible_persist)
{
bloom_filter_params params = {0, 0, 1e6, 1e-4};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bloom_bloomfilter filter;
fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, PERSISTENT, &map) == 0);
fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
fchmod(map.fileno, 0777);
// Check all the keys get added
char buf[100];
int res;
for (int i=0;i<1000;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_add(&filter, (char*)&buf);
fail_unless(res == 1);
}
fail_unless(bf_close(&filter) == 0);
// Test all the keys are contained
fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, SHARED, &map) == 0);
fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
for (int i=0;i<1000;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_contains(&filter, (char*)&buf);
fail_unless(res == 1);
}
unlink("/tmp/shared_compat_persist.mmap");
}
END_TEST
START_TEST(test_sbf_close_does_flush)
{
bloom_sbf_params params = SBF_DEFAULT_PARAMS;
params.initial_capacity = 1e3;
params.fp_probability = 1e-4;
nextfile next;
next.format = "/tmp/mmap_close.%d.data";
next.num = 0;
bloom_sbf sbf;
int res = sbf_from_filters(¶ms, sbf_make_callback, &next, 0, NULL, &sbf);
fail_unless(res == 0);
char buf[100];
for (int i=0;i<2000;i++) {
snprintf((char*)&buf, 100, "foobar%d", i);
sbf_add(&sbf, (char*)&buf);
}
fail_unless(sbf_close(&sbf) == 0);
bloom_bitmap maps[2];
bitmap_from_filename("/tmp/mmap_close.0.data", get_size("/tmp/mmap_close.0.data"), 1, 1, SHARED, (bloom_bitmap*)&maps);
bitmap_from_filename("/tmp/mmap_close.1.data", get_size("/tmp/mmap_close.1.data"), 1, 1, SHARED, ((bloom_bitmap*)&maps)+1);
bloom_bloomfilter filters[2];
bf_from_bitmap((bloom_bitmap*)&maps, 1, 0, (bloom_bloomfilter*)&filters);
bf_from_bitmap(((bloom_bitmap*)&maps)+1, 1, 0, ((bloom_bloomfilter*)&filters)+1);
bloom_bloomfilter **filter_map = calloc(2, sizeof(bloom_bloomfilter*));
filter_map[0] = (bloom_bloomfilter*)&filters;
filter_map[1] = ((bloom_bloomfilter*)&filters)+1;
res = sbf_from_filters(¶ms, sbf_make_callback, &next, 2, filter_map, &sbf);
fail_unless(res == 0);
fail_unless(sbf_size(&sbf) == 2000);
fail_unless(sbf_total_capacity(&sbf) == 5*1e3);
for (int i=0;i<2000;i++) {
snprintf((char*)&buf, 100, "foobar%d", i);
res = sbf_contains(&sbf, (char*)&buf);
fail_unless(res == 1);
}
unlink("/tmp/mmap_close.0.data");
unlink("/tmp/mmap_close.1.data");
}
END_TEST
START_TEST(make_bf_fresh_then_restore)
{
// Use -1 for anonymous
bloom_bitmap map;
bloom_bloomfilter filter;
bitmap_from_file(-1, 4096, ANONYMOUS, &map);
int res = bf_from_bitmap(&map, 10, 1, &filter); // Make fresh
fail_unless(res == 0);
bloom_bloomfilter filter2;
res = bf_from_bitmap(&map, 10, 0, &filter2); // Restore now
fail_unless(res == 0);
}
END_TEST
START_TEST(test_bf_fp_prob_extended)
{
bloom_filter_params params = {0, 0, 1e6, 0.001};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bloom_bloomfilter filter;
fail_unless(bitmap_from_file(-1, params.bytes, ANONYMOUS, &map) == 0);
fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
// Check all the keys get added
char buf[100];
int res;
int num_wrong = 0;
for (int i=0;i<1e6;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_add(&filter, (char*)&buf);
if (res == 0) num_wrong++;
}
// We added 1M items, with a capacity of 1M and error of 1/1000.
// Technically we should have 1K false positives
fail_unless(num_wrong <= 1000);
}
END_TEST
START_TEST(test_length)
{
bloom_filter_params params = {0, 0, 1e6, 1e-4};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bitmap_from_file(-1, params.bytes, ANONYMOUS, &map);
bloom_bloomfilter filter;
bf_from_bitmap(&map, params.k_num, 1, &filter);
// Check the size
fail_unless(bf_size(&filter) == 0);
// Check all the keys get added
char buf[100];
int res;
for (int i=0;i<1000;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_add(&filter, (char*)&buf);
fail_unless(res == 1);
}
// Check the size
fail_unless(bf_size(&filter) == 1000);
}
END_TEST
START_TEST(make_bf_no_map)
{
// Use -1 for anonymous
bloom_bloomfilter filter;
int res = bf_from_bitmap(NULL, 10, 1, &filter);
fail_unless(res == -EINVAL);
}
END_TEST
START_TEST(make_bf_fresh_not_new)
{
// Use -1 for anonymous
bloom_bitmap map;
bloom_bloomfilter filter;
bitmap_from_file(-1, 4096, ANONYMOUS, &map);
int res = bf_from_bitmap(&map, 10, 0, &filter);
fail_unless(res == -1);
}
END_TEST
START_TEST(make_bf_zero_k)
{
// Use -1 for anonymous
bloom_bitmap map;
bloom_bloomfilter filter;
bitmap_from_file(-1, 4096, ANONYMOUS, &map);
int res = bf_from_bitmap(&map, 0, 1, &filter);
fail_unless(res == -EINVAL);
}
END_TEST
START_TEST(test_bf_double_close)
{
bloom_filter_params params = {0, 0, 1e6, 1e-4};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bloom_bloomfilter filter;
bitmap_from_file(-1, params.bytes, ANONYMOUS, &map);
bf_from_bitmap(&map, params.k_num, 1, &filter);
fail_unless(bf_close(&filter) == 0);
fail_unless(bf_close(&filter) == -1);
}
END_TEST
START_TEST(test_bf_value_sanity)
{
// Use -1 for anonymous
bloom_bitmap map;
bloom_bloomfilter filter;
bitmap_from_file(-1, 4096, ANONYMOUS, &map);
int res = bf_from_bitmap(&map, 10, 1, &filter); // Make fresh
fail_unless(res == 0);
fail_unless(filter.bitmap_size == 28672); // Should be the bitmap size
fail_unless(filter.offset == 2867); // Should be size / 10
fail_unless(filter.header->k_num == 10);
fail_unless(filter.header->count == 0);
}
END_TEST
START_TEST(test_flush_close)
{
bloom_filter_params params = {0, 0, 1e6, 1e-4};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bloom_bloomfilter filter;
bitmap_from_filename("/tmp/test_flush_close.mmap", params.bytes, 1, SHARED, &map);
bf_from_bitmap(&map, params.k_num, 1, &filter);
fail_unless(bf_flush(&filter) == 0);
fail_unless(bf_close(&filter) == 0);
unlink("/tmp/test_flush_close.mmap");
}
/**
* This beast mode method scans the data directory
* belonging to this filter for any existing filters,
* and restores the SBF
* @return 0 on success. -1 on error.
*/
static int discover_existing_filters(bloom_filter *f) {
// Scan through the folder looking for data files
struct dirent **namelist;
int num;
// Filter only data dirs, in sorted order
num = scandir(f->full_path, &namelist, filter_data_files, alphasort);
if (num == -1) {
syslog(LOG_ERR, "Failed to scan files for filter '%s'. %s",
f->filter_name, strerror(errno));
return -1;
}
syslog(LOG_INFO, "Found %d files for filter %s.", num, f->filter_name);
// Speical case when there are no filters
if (num == 0) {
int res = create_sbf(f, 0, NULL);
return res;
}
// Allocate space for all the filter
bloom_bitmap **maps = malloc(num * sizeof(bloom_bitmap*));
bloom_bloomfilter **filters = malloc(num * sizeof(bloom_bloomfilter*));
// Initialize the bitmaps and bloom filters
int res;
int err = 0;
uint64_t size;
bitmap_mode mode = (f->config->use_mmap) ? SHARED : PERSISTENT;
for (int i=0; i < num && !err; i++) {
// Get the full path to the bitmap
char *bitmap_path = join_path(f->full_path, namelist[i]->d_name);
syslog(LOG_INFO, "Discovered bloom filter: %s.", bitmap_path);
// Get the size
size = get_size(bitmap_path);
if (size == 0) {
err = 1;
syslog(LOG_ERR, "Failed to get the filesize for: %s. %s", bitmap_path, strerror(errno));
free(bitmap_path);
break;
}
// Create the bitmap
bloom_bitmap *bitmap = maps[num - i - 1] = malloc(sizeof(bloom_bitmap));
res = bitmap_from_filename(bitmap_path, size, 0, mode, bitmap);
if (res != 0) {
err = 1;
syslog(LOG_ERR, "Failed to load bitmap for: %s. %s", bitmap_path, strerror(errno));
free(bitmap);
free(bitmap_path);
break;
}
// Create the bloom filter
bloom_bloomfilter *filter = filters[num - i - 1] = malloc(sizeof(bloom_bloomfilter));
res = bf_from_bitmap(bitmap, 1, 0, filter);
if (res != 0) {
err = 1;
syslog(LOG_ERR, "Failed to load bloom filter for: %s. [%d]", bitmap_path, res);
free(filter);
bitmap_close(bitmap);
free(bitmap);
free(bitmap_path);
break;
}
// Cleanup
free(bitmap_path);
}
// Free the memory associated with scandir
for (int i=0; i < num; i++) free(namelist[i]);
free(namelist);
// Return if there was an error
if (err) return -1;
// Create the SBF
res = create_sbf(f, num, filters);
// Cleanup on err
if (res != 0) {
syslog(LOG_ERR, "Failed to make scalable bloom filter for: %s.", f->filter_name);
// For f***s sake. We need to clean up so much shit now.
for (int i=0; i < num; i++) {
bf_close(filters[i]);
bitmap_close(maps[i]);
free(filters[i]);
free(maps[i]);
}
}
// Increase our page ins
f->counters.page_ins += 1;
// Remove the filters list
free(maps);
free(filters);
return (err) ? -1 : 0;
}
