/** ***************************************************************************
* A few simple tests to check if it works at all.
*
*/
static void basic()
{
(void)printf("----- basic -----\n");
struct bloom bloom;
assert(bloom_init(&bloom, 0, 1.0) == 1);
assert(bloom_init(&bloom, 10, 0) == 1);
assert(bloom.ready == 0);
assert(bloom_add(&bloom, "hello world", 11) == -1);
assert(bloom_check(&bloom, "hello world", 11) == -1);
bloom_free(&bloom);
assert(bloom_init(&bloom, 102, 0.1) == 0);
assert(bloom.ready == 1);
bloom_print(&bloom);
assert(bloom_check(&bloom, "hello world", 11) == 0);
assert(bloom_add(&bloom, "hello world", 11) == 0);
assert(bloom_check(&bloom, "hello world", 11) == 1);
assert(bloom_add(&bloom, "hello world", 11) > 0);
assert(bloom_add(&bloom, "hello", 5) == 0);
assert(bloom_add(&bloom, "hello", 5) > 0);
assert(bloom_check(&bloom, "hello", 5) == 1);
bloom_free(&bloom);
}
/** ***************************************************************************
* Simple loop to compare performance.
*
*/
static void perf_loop(int entries, int count)
{
(void)printf("----- perf_loop -----\n");
struct bloom bloom;
assert(bloom_init(&bloom, entries, 0.001) == 0);
bloom_print(&bloom);
int i;
int collisions = 0;
struct timeval tp;
(void)gettimeofday(&tp, NULL);
long before = (tp.tv_sec * 1000L) + (tp.tv_usec / 1000L);
for (i = 0; i < count; i++) {
if (bloom_add(&bloom, (void *)&i, sizeof(int))) { collisions++; }
}
(void)gettimeofday(&tp, NULL);
long after = (tp.tv_sec * 1000L) + (tp.tv_usec / 1000L);
(void)printf("Added %d elements of size %d, took %d ms (collisions=%d)\n",
count, (int)sizeof(int), (int)(after - before), collisions);
(void)printf("%d,%d,%ld\n", entries, bloom.bytes, after - before);
bloom_free(&bloom);
}
/** ***************************************************************************
* Create a bloom filter with given parameters and add 'count' random elements
* into it to see if collission rates are within expectations.
*
*/
static void add_random(int entries, double error, int count)
{
(void)printf("----- add_random(%d, %f, %d) -----\n", entries, error, count);
struct bloom bloom;
assert(bloom_init(&bloom, entries, error) == 0);
bloom_print(&bloom);
char block[32];
int collisions = 0;
int fd = open("/dev/urandom", O_RDONLY);
int n;
for (n = 0; n < count; n++) {
assert(read(fd, block, 32) == 32);
if (bloom_add(&bloom, (void *)block, 32)) { collisions++; }
}
(void)close(fd);
bloom_free(&bloom);
(void)printf("added %d elements, got %d collisions\n", count, collisions);
if (count <= entries) {
assert(collisions <= (entries * error));
} else if (count <= entries * 2) {
assert(collisions < (2 * entries * error));
}
}
static void
create_and_test_bloom(int power, int64 nelements, int callerseed)
{
int bloom_work_mem;
uint64 seed;
int64 nfalsepos;
bloom_filter *filter;
bloom_work_mem = (1L << power) / 8L / 1024L;
elog(DEBUG1, "bloom_work_mem (KB): %d", bloom_work_mem);
/*
* Generate random seed, or use caller's. Seed should always be a
* positive value less than or equal to PG_INT32_MAX, to ensure that any
* random seed can be recreated through callerseed if the need arises.
* (Don't assume that RAND_MAX cannot exceed PG_INT32_MAX.)
*/
seed = callerseed < 0 ? random() % PG_INT32_MAX : callerseed;
/* Create Bloom filter, populate it, and report on false positive rate */
filter = bloom_create(nelements, bloom_work_mem, seed);
populate_with_dummy_strings(filter, nelements);
nfalsepos = nfalsepos_for_missing_strings(filter, nelements);
ereport((nfalsepos > nelements * FPOSITIVE_THRESHOLD) ? WARNING : DEBUG1,
(errmsg_internal("seed: " UINT64_FORMAT " false positives: " INT64_FORMAT " (%.6f%%) bitset %.2f%% set",
seed, nfalsepos, (double) nfalsepos / nelements,
100.0 * bloom_prop_bits_set(filter))));
bloom_free(filter);
}
static void runtest (void)
{
unsigned char md1[SHA256_DIGEST_LENGTH];
unsigned char md2[SHA256_DIGEST_LENGTH];
sha256_Raw((unsigned char *)data1, strlen(data1), md1);
sha256_Raw((unsigned char *)data2, strlen(data2), md2);
struct bloom bloom;
assert(bloom_init(&bloom, 1000, 0.001) == true);
bloom_insert(&bloom, md1, sizeof(md1));
assert(bloom_contains(&bloom, md1, sizeof(md1)) == true);
assert(bloom_contains(&bloom, md2, sizeof(md2)) == false);
cstring *ser = cstr_new_sz(1024);
ser_bloom(ser, &bloom);
struct bloom bloom2;
__bloom_init(&bloom2);
struct const_buffer buf = { ser->str, ser->len };
assert(deser_bloom(&bloom2, &buf) == true);
assert(bloom.nHashFuncs == bloom2.nHashFuncs);
assert(bloom.vData->len == bloom2.vData->len);
assert(memcmp(bloom.vData->str, bloom2.vData->str, bloom2.vData->len) == 0);
assert(bloom_contains(&bloom2, md1, sizeof(md1)) == true);
assert(bloom_contains(&bloom2, md2, sizeof(md2)) == false);
bloom_free(&bloom2);
bloom_free(&bloom);
cstr_free(ser, true);
}
void
wallet_close(struct wallet *wallet)
{
if (wallet == NULL) {
return;
}
bloom_free(wallet->filter);
wallet->filter = NULL;
txdb_close(wallet->txdb);
hashtable_clear_with_callback(wallet->hash_keys, wallet_free_key_cb);
hashtable_destroy(wallet->hash_keys);
free(wallet->filename);
secure_free(wallet->ckey_store);
memset(wallet, 0, sizeof *wallet);
free(wallet);
}
int
ppbloom_add(const void *buffer, int len)
{
int err;
err = bloom_add(ppbloom + current, buffer, len);
if (err == -1)
return err;
bloom_count[current]++;
if (bloom_count[current] >= entries) {
bloom_count[current] = 0;
current = current == PING ? PONG : PING;
bloom_free(ppbloom + current);
bloom_init(ppbloom + current, entries, error);
}
return 0;
}
void file_hash_exit(void)
{
unsigned int i, has_entries = 0;
fio_mutex_down(hash_lock);
for (i = 0; i < HASH_BUCKETS; i++)
has_entries += !flist_empty(&file_hash[i]);
fio_mutex_up(hash_lock);
if (has_entries)
log_err("fio: file hash not empty on exit\n");
sfree(file_hash);
file_hash = NULL;
fio_mutex_remove(hash_lock);
hash_lock = NULL;
bloom_free(file_bloom);
file_bloom = NULL;
}
void
ppbloom_free()
{
bloom_free(ppbloom + PING);
bloom_free(ppbloom + PONG);
}
int dbfile_populate_hashes(struct rb_root *d_tree)
{
int ret;
sqlite3 *db;
sqlite3_stmt *stmt = NULL;
char *filename;
uint64_t ino, subvolid;
uint64_t num_hashes;
struct filerec *file;
struct bloom_cb_priv priv;
db = dbfile_get_handle();
if (!db)
return ENOENT;
ret = dbfile_count_rows(db, &num_hashes, NULL);
if (ret)
return ret;
priv.d_tree = d_tree;
ret = bloom_init(&priv.bloom, num_hashes, 0.01);
if (ret)
return ret;
ret = sqlite3_prepare_v2(db,
"SELECT ino, subvol, filename from files;", -1,
&stmt, NULL);
if (ret) {
perror_sqlite(ret, "preparing statement");
goto out_bloom;
}
while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
ino = sqlite3_column_int64(stmt, 0);
subvolid = sqlite3_column_int64(stmt, 1);
filename = (char *)sqlite3_column_text(stmt, 2);
file = filerec_new(filename, ino, subvolid);
if (!file) {
ret = ENOMEM;
goto out_finalize;
}
ret = dbfile_walk_file_hashes(db, file, load_into_bloom_cb,
&priv);
if (ret)
goto out_finalize;
}
if (ret != SQLITE_DONE) {
perror_sqlite(ret, "retrieving file info from table");
goto out_finalize;
}
ret = 0;
out_finalize:
sqlite3_finalize(stmt);
out_bloom:
bloom_free(&priv.bloom);
return ret;
}
int main(int argc, char* argv[])
{
size_t n = 1000000;
size_t m = 8;
int index = 0;
int opt;
while (true) {
opt = getopt(argc, argv, "n:m:i:");
if (opt == -1) break;
switch (opt) {
case 'n':
n = (size_t) strtoul(optarg, NULL, 10);
break;
case 'm':
m = (size_t) strtoul(optarg, NULL, 10);
break;
case 'i':
index = (size_t) strtoul(optarg, NULL, 10);
break;
case '?':
return EXIT_FAILURE;
default:
abort();
}
}
bloom_t* B = bloom_alloc(n, m);
char line[512];
kmer_t x;
size_t count = 0, unique_count = 0;
FILE *fd = NULL;
char *file = NULL;
char *files[] = {"random.txt","random1.txt","random2.txt","random3.txt","random4.txt"};
/*
* char files[16] = "random.txt";
* if(index > 0)
* snprintf(files, sizeof(files), "random%d.txt", index);
*/
if(index)
file = files[index];
else
file = files[0];
fd = fopen(file, "r");
if(fd == NULL)
fprintf(stderr, "fopen error\n");
while (fgets(line, sizeof(line), fd)) {
x = (kmer_t) strtoul(line, NULL, 10);
if (bloom_inc(B, x) == 1)
++unique_count;
else
printf("%s", line);
++count;
}
printf("\n");
printf("%zu\t%zu\n", unique_count, count);
rewind(fd);
while (fgets(line, sizeof(line), fd)) {
x = (kmer_t) strtoul(line, NULL, 10);
//bloom_del(B, x);
if(bloom_ldec(B, x) == 1)
--unique_count;
--count;
}
fclose(fd);
fd = NULL;
bloom_free(B);
printf("%zu\t%zu\n", unique_count, count);
return 0;
}
