END_TEST
START_TEST(test_bf_shared_compatible_persist)
{
bloom_filter_params params = {0, 0, 1e6, 1e-4};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bloom_bloomfilter filter;
fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, PERSISTENT, &map) == 0);
fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
fchmod(map.fileno, 0777);
// Check all the keys get added
char buf[100];
int res;
for (int i=0;i<1000;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_add(&filter, (char*)&buf);
fail_unless(res == 1);
}
fail_unless(bf_close(&filter) == 0);
// Test all the keys are contained
fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, SHARED, &map) == 0);
fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
for (int i=0;i<1000;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_contains(&filter, (char*)&buf);
fail_unless(res == 1);
}
unlink("/tmp/shared_compat_persist.mmap");
}
END_TEST
START_TEST(test_sbf_close_does_flush)
{
bloom_sbf_params params = SBF_DEFAULT_PARAMS;
params.initial_capacity = 1e3;
params.fp_probability = 1e-4;
nextfile next;
next.format = "/tmp/mmap_close.%d.data";
next.num = 0;
bloom_sbf sbf;
int res = sbf_from_filters(¶ms, sbf_make_callback, &next, 0, NULL, &sbf);
fail_unless(res == 0);
char buf[100];
for (int i=0;i<2000;i++) {
snprintf((char*)&buf, 100, "foobar%d", i);
sbf_add(&sbf, (char*)&buf);
}
fail_unless(sbf_close(&sbf) == 0);
bloom_bitmap maps[2];
bitmap_from_filename("/tmp/mmap_close.0.data", get_size("/tmp/mmap_close.0.data"), 1, 1, SHARED, (bloom_bitmap*)&maps);
bitmap_from_filename("/tmp/mmap_close.1.data", get_size("/tmp/mmap_close.1.data"), 1, 1, SHARED, ((bloom_bitmap*)&maps)+1);
bloom_bloomfilter filters[2];
bf_from_bitmap((bloom_bitmap*)&maps, 1, 0, (bloom_bloomfilter*)&filters);
bf_from_bitmap(((bloom_bitmap*)&maps)+1, 1, 0, ((bloom_bloomfilter*)&filters)+1);
bloom_bloomfilter **filter_map = calloc(2, sizeof(bloom_bloomfilter*));
filter_map[0] = (bloom_bloomfilter*)&filters;
filter_map[1] = ((bloom_bloomfilter*)&filters)+1;
res = sbf_from_filters(¶ms, sbf_make_callback, &next, 2, filter_map, &sbf);
fail_unless(res == 0);
fail_unless(sbf_size(&sbf) == 2000);
fail_unless(sbf_total_capacity(&sbf) == 5*1e3);
for (int i=0;i<2000;i++) {
snprintf((char*)&buf, 100, "foobar%d", i);
res = sbf_contains(&sbf, (char*)&buf);
fail_unless(res == 1);
}
unlink("/tmp/mmap_close.0.data");
unlink("/tmp/mmap_close.1.data");
}
END_TEST
START_TEST(make_bitmap_nofile_persistent)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/asdf123", 4096, 0, 0, PERSISTENT, &map);
fail_unless(res == -ENOENT);
}
END_TEST
START_TEST(make_bitmap_nofile_create)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_nofile_create", 4096, 1, 1, SHARED, &map);
unlink("/tmp/mmap_nofile_create");
fail_unless(res == 0);
}
END_TEST
START_TEST(make_bitmap_nofile_create_persistent)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_nofile_create_persist", 4096, 1, 1, PERSISTENT, &map);
unlink("/tmp/mmap_nofile_create_persist");
fail_unless(res == 0);
}
static int sbf_make_callback(void *in, uint64_t bytes, bloom_bitmap *map) {
char buf[1000];
nextfile *n = in;
snprintf(buf, 1000, n->format, n->num);
n->num++;
int res = bitmap_from_filename((char*)&buf, bytes, 1, 1, SHARED, map);
fchmod(map->fileno, 0777);
return res;
}
END_TEST
START_TEST(close_does_flush) {
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_close_flush", 4096, 1, 1, SHARED, &map);
fchmod(map.fileno, 0777);
fail_unless(res == 0);
for (int idx = 0; idx < 4096*8 ; idx++) {
bitmap_setbit((&map), idx);
}
bitmap_close(&map);
res = bitmap_from_filename("/tmp/mmap_close_flush", 4096, 0, 0, SHARED, &map);
fail_unless(res == 0);
for (int idx = 0; idx < 4096; idx++) {
fail_unless(map.mmap[idx] == 255);
}
unlink("/tmp/mmap_close_flush");
}
END_TEST
START_TEST(flush_bitmap_file)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_flush_bitmap", 8196, 1, 1, SHARED, &map);
fail_unless(res == 0);
fail_unless(bitmap_flush(&map) == 0);
unlink("/tmp/mmap_flush_bitmap");
}
END_TEST
START_TEST(close_bitmap_file)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_close_bitmap", 8196, 1, 1, SHARED, &map);
fail_unless(res == 0);
fail_unless(bitmap_close(&map) == 0);
fail_unless(map.mmap == NULL);
unlink("/tmp/mmap_close_bitmap");
}
END_TEST
START_TEST(flush_bitmap_file_persistent)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_flush_bitmap_persist", 8196,
1, 1, PERSISTENT, &map);
fail_unless(res == 0);
fail_unless(bitmap_flush(&map) == 0);
unlink("/tmp/mmap_flush_bitmap_persist");
}
/**
* Callback used with SBF to generate file names.
*/
static int bloomf_sbf_callback(void* in, uint64_t bytes, bloom_bitmap *out) {
// Cast the input pointer
bloom_filter *filt = in;
// Check if we are in-memory
if (filt->filter_config.in_memory) {
syslog(LOG_INFO, "Creating new in-memory bitmap for filter %s. Size: %llu",
filt->filter_name, (unsigned long long)bytes);
return bitmap_from_file(-1, bytes, ANONYMOUS, out);
}
// Scan through the folder looking for data files
struct dirent **namelist = NULL;
int num_files;
// Filter only data dirs, in sorted order
num_files = scandir(filt->full_path, &namelist, filter_data_files, NULL);
syslog(LOG_INFO, "Found %d files for filter %s.", num_files, filt->filter_name);
if (num_files < 0) {
syslog(LOG_ERR, "Error discovering files for filter '%s'. %s",
filt->filter_name, strerror(errno));
return -1;
}
// Free the memory associated with scandir
for (int i=0; i < num_files; i++) {
free(namelist[i]);
}
if (namelist) free(namelist);
// Generate the new file name
char *filename = NULL;
int file_name_len;
file_name_len = asprintf(&filename, DATA_FILE_NAME, num_files);
assert(file_name_len != -1);
// Get the full path
char *full_path = join_path(filt->full_path, filename);
free(filename);
syslog(LOG_INFO, "Creating new file: %s for filter %s. Size: %llu",
full_path, filt->filter_name, (unsigned long long)bytes);
// Create the bitmap
bitmap_mode mode = (filt->config->use_mmap) ? SHARED : PERSISTENT;
int res = bitmap_from_filename(full_path, bytes, 1, mode, out);
if (res) {
syslog(LOG_CRIT, "Failed to create new file: %s for filter %s. Err: %s",
full_path, filt->filter_name, strerror(errno));
}
free(full_path);
return res;
}
END_TEST
START_TEST(close_bitmap_file_persistent)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_close_bitmap_persist", 8196, 1, 1,
PERSISTENT, &map);
fail_unless(res == 0);
fail_unless(bitmap_close(&map) == 0);
fail_unless(map.mmap == NULL);
unlink("/tmp/mmap_close_bitmap_persist");
}
END_TEST
START_TEST(flush_does_write_persist) {
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/persist_flush_write", 4096, 1, 1, PERSISTENT, &map);
fchmod(map.fileno, 0777);
fail_unless(res == 0);
for (int idx = 0; idx < 4096*8 ; idx++) {
bitmap_setbit((&map), idx);
}
bitmap_flush(&map);
bloom_bitmap map2;
res = bitmap_from_filename("/tmp/persist_flush_write", 4096, 0, 0,
PERSISTENT, &map2);
fail_unless(res == 0);
for (int idx = 0; idx < 4096; idx++) {
fail_unless(map2.mmap[idx] == 255);
}
unlink("/tmp/persist_flush_write");
}
END_TEST
START_TEST(make_bitmap_resize)
{
int fh = open("/tmp/mmap_resize", O_RDWR|O_CREAT);
fchmod(fh, 0777);
fail_unless(fh > 0);
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_resize", 4096, 0, 1, SHARED, &map);
unlink("/tmp/mmap_resize");
fail_unless(res == 0);
}
END_TEST
START_TEST(getbit_bitmap_file_persist_zero)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/persist_getbit_zero", 4096, 1, 1, PERSISTENT, &map);
fail_unless(res == 0);
for (int idx = 0; idx < 4096*8 ; idx++) {
fail_unless(bitmap_getbit((&map), idx) == 0);
}
unlink("/tmp/persist_getbit_zero");
}
END_TEST
START_TEST(getbit_bitmap_file_one)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_getbit_one", 4096, 1, 1, SHARED, &map);
fail_unless(res == 0);
memset(map.mmap, 255, 4096);
for (int idx = 0; idx < 4096*8 ; idx++) {
fail_unless(bitmap_getbit((&map), idx) == 1);
}
unlink("/tmp/mmap_getbit_one");
}
END_TEST
START_TEST(make_bitmap_resize_persistent)
{
int fh = open("/tmp/mmap_resize_persist", O_RDWR|O_CREAT);
fchmod(fh, 0777);
fail_unless(fh > 0);
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_resize_persist",
4096, 0, 1, PERSISTENT, &map);
unlink("/tmp/mmap_resize_persist");
fail_unless(res == 0);
}
END_TEST
/*
bloom_bitmap *bitmap_from_filename(char* filename, size_t len, int create, int resize) {
*/
START_TEST(make_bitmap_nofile)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/asdf123", 4096, 0, 0, SHARED, &map);
fail_unless(res == -ENOENT);
}
END_TEST
START_TEST(setbit_bitmap_file_one)
{
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_setbit_one", 4096, 1, 1, SHARED, &map);
fail_unless(res == 0);
for (int idx = 0; idx < 4096*8 ; idx++) {
bitmap_setbit((&map), idx);
}
for (int idx = 0; idx < 4096; idx++) {
fail_unless(map.mmap[idx] == 255);
}
unlink("/tmp/mmap_setbit_one");
}
END_TEST
/**
* Test that flush does indeed write to disk
*/
START_TEST(flush_does_write) {
bloom_bitmap map;
int res = bitmap_from_filename("/tmp/mmap_flush_write", 4096, 1, 1, SHARED, &map);
fchmod(map.fileno, 0777);
fail_unless(res == 0);
for (int idx = 0; idx < 4096*8 ; idx++) {
bitmap_setbit((&map), idx);
}
bitmap_flush(&map);
bloom_bitmap map2;
res = bitmap_from_filename("/tmp/mmap_flush_write", 4096, 0, 0, SHARED, &map2);
fail_unless(res == 0);
for (int idx = 0; idx < 4096; idx++) {
fail_unless(map2.mmap[idx] == 255);
}
unlink("/tmp/mmap_flush_write");
}
END_TEST
START_TEST(test_flush_close)
{
bloom_filter_params params = {0, 0, 1e6, 1e-4};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bloom_bloomfilter filter;
bitmap_from_filename("/tmp/test_flush_close.mmap", params.bytes, 1, SHARED, &map);
bf_from_bitmap(&map, params.k_num, 1, &filter);
fail_unless(bf_flush(&filter) == 0);
fail_unless(bf_close(&filter) == 0);
unlink("/tmp/test_flush_close.mmap");
}
/**
* This beast mode method scans the data directory
* belonging to this filter for any existing filters,
* and restores the SBF
* @return 0 on success. -1 on error.
*/
static int discover_existing_filters(bloom_filter *f) {
// Scan through the folder looking for data files
struct dirent **namelist;
int num;
// Filter only data dirs, in sorted order
num = scandir(f->full_path, &namelist, filter_data_files, alphasort);
if (num == -1) {
syslog(LOG_ERR, "Failed to scan files for filter '%s'. %s",
f->filter_name, strerror(errno));
return -1;
}
syslog(LOG_INFO, "Found %d files for filter %s.", num, f->filter_name);
// Speical case when there are no filters
if (num == 0) {
int res = create_sbf(f, 0, NULL);
return res;
}
// Allocate space for all the filter
bloom_bitmap **maps = malloc(num * sizeof(bloom_bitmap*));
bloom_bloomfilter **filters = malloc(num * sizeof(bloom_bloomfilter*));
// Initialize the bitmaps and bloom filters
int res;
int err = 0;
uint64_t size;
bitmap_mode mode = (f->config->use_mmap) ? SHARED : PERSISTENT;
for (int i=0; i < num && !err; i++) {
// Get the full path to the bitmap
char *bitmap_path = join_path(f->full_path, namelist[i]->d_name);
syslog(LOG_INFO, "Discovered bloom filter: %s.", bitmap_path);
// Get the size
size = get_size(bitmap_path);
if (size == 0) {
err = 1;
syslog(LOG_ERR, "Failed to get the filesize for: %s. %s", bitmap_path, strerror(errno));
free(bitmap_path);
break;
}
// Create the bitmap
bloom_bitmap *bitmap = maps[num - i - 1] = malloc(sizeof(bloom_bitmap));
res = bitmap_from_filename(bitmap_path, size, 0, mode, bitmap);
if (res != 0) {
err = 1;
syslog(LOG_ERR, "Failed to load bitmap for: %s. %s", bitmap_path, strerror(errno));
free(bitmap);
free(bitmap_path);
break;
}
// Create the bloom filter
bloom_bloomfilter *filter = filters[num - i - 1] = malloc(sizeof(bloom_bloomfilter));
res = bf_from_bitmap(bitmap, 1, 0, filter);
if (res != 0) {
err = 1;
syslog(LOG_ERR, "Failed to load bloom filter for: %s. [%d]", bitmap_path, res);
free(filter);
bitmap_close(bitmap);
free(bitmap);
free(bitmap_path);
break;
}
// Cleanup
free(bitmap_path);
}
// Free the memory associated with scandir
for (int i=0; i < num; i++) free(namelist[i]);
free(namelist);
// Return if there was an error
if (err) return -1;
// Create the SBF
res = create_sbf(f, num, filters);
// Cleanup on err
if (res != 0) {
syslog(LOG_ERR, "Failed to make scalable bloom filter for: %s.", f->filter_name);
// For f***s sake. We need to clean up so much shit now.
for (int i=0; i < num; i++) {
bf_close(filters[i]);
bitmap_close(maps[i]);
free(filters[i]);
free(maps[i]);
}
}
// Increase our page ins
f->counters.page_ins += 1;
// Remove the filters list
free(maps);
free(filters);
return (err) ? -1 : 0;
}
