/** *************************************************************************** * A few simple tests to check if it works at all. * */ static void basic() { (void)printf("----- basic -----\n"); struct bloom bloom; assert(bloom_init(&bloom, 0, 1.0) == 1); assert(bloom_init(&bloom, 10, 0) == 1); assert(bloom.ready == 0); assert(bloom_add(&bloom, "hello world", 11) == -1); assert(bloom_check(&bloom, "hello world", 11) == -1); bloom_free(&bloom); assert(bloom_init(&bloom, 102, 0.1) == 0); assert(bloom.ready == 1); bloom_print(&bloom); assert(bloom_check(&bloom, "hello world", 11) == 0); assert(bloom_add(&bloom, "hello world", 11) == 0); assert(bloom_check(&bloom, "hello world", 11) == 1); assert(bloom_add(&bloom, "hello world", 11) > 0); assert(bloom_add(&bloom, "hello", 5) == 0); assert(bloom_add(&bloom, "hello", 5) > 0); assert(bloom_check(&bloom, "hello", 5) == 1); bloom_free(&bloom); }
/** *************************************************************************** * Simple loop to compare performance. * */ static void perf_loop(int entries, int count) { (void)printf("----- perf_loop -----\n"); struct bloom bloom; assert(bloom_init(&bloom, entries, 0.001) == 0); bloom_print(&bloom); int i; int collisions = 0; struct timeval tp; (void)gettimeofday(&tp, NULL); long before = (tp.tv_sec * 1000L) + (tp.tv_usec / 1000L); for (i = 0; i < count; i++) { if (bloom_add(&bloom, (void *)&i, sizeof(int))) { collisions++; } } (void)gettimeofday(&tp, NULL); long after = (tp.tv_sec * 1000L) + (tp.tv_usec / 1000L); (void)printf("Added %d elements of size %d, took %d ms (collisions=%d)\n", count, (int)sizeof(int), (int)(after - before), collisions); (void)printf("%d,%d,%ld\n", entries, bloom.bytes, after - before); bloom_free(&bloom); }
/** *************************************************************************** * Create a bloom filter with given parameters and add 'count' random elements * into it to see if collission rates are within expectations. * */ static void add_random(int entries, double error, int count) { (void)printf("----- add_random(%d, %f, %d) -----\n", entries, error, count); struct bloom bloom; assert(bloom_init(&bloom, entries, error) == 0); bloom_print(&bloom); char block[32]; int collisions = 0; int fd = open("/dev/urandom", O_RDONLY); int n; for (n = 0; n < count; n++) { assert(read(fd, block, 32) == 32); if (bloom_add(&bloom, (void *)block, 32)) { collisions++; } } (void)close(fd); bloom_free(&bloom); (void)printf("added %d elements, got %d collisions\n", count, collisions); if (count <= entries) { assert(collisions <= (entries * error)); } else if (count <= entries * 2) { assert(collisions < (2 * entries * error)); } }
static void create_and_test_bloom(int power, int64 nelements, int callerseed) { int bloom_work_mem; uint64 seed; int64 nfalsepos; bloom_filter *filter; bloom_work_mem = (1L << power) / 8L / 1024L; elog(DEBUG1, "bloom_work_mem (KB): %d", bloom_work_mem); /* * Generate random seed, or use caller's. Seed should always be a * positive value less than or equal to PG_INT32_MAX, to ensure that any * random seed can be recreated through callerseed if the need arises. * (Don't assume that RAND_MAX cannot exceed PG_INT32_MAX.) */ seed = callerseed < 0 ? random() % PG_INT32_MAX : callerseed; /* Create Bloom filter, populate it, and report on false positive rate */ filter = bloom_create(nelements, bloom_work_mem, seed); populate_with_dummy_strings(filter, nelements); nfalsepos = nfalsepos_for_missing_strings(filter, nelements); ereport((nfalsepos > nelements * FPOSITIVE_THRESHOLD) ? WARNING : DEBUG1, (errmsg_internal("seed: " UINT64_FORMAT " false positives: " INT64_FORMAT " (%.6f%%) bitset %.2f%% set", seed, nfalsepos, (double) nfalsepos / nelements, 100.0 * bloom_prop_bits_set(filter)))); bloom_free(filter); }
static void runtest (void) { unsigned char md1[SHA256_DIGEST_LENGTH]; unsigned char md2[SHA256_DIGEST_LENGTH]; sha256_Raw((unsigned char *)data1, strlen(data1), md1); sha256_Raw((unsigned char *)data2, strlen(data2), md2); struct bloom bloom; assert(bloom_init(&bloom, 1000, 0.001) == true); bloom_insert(&bloom, md1, sizeof(md1)); assert(bloom_contains(&bloom, md1, sizeof(md1)) == true); assert(bloom_contains(&bloom, md2, sizeof(md2)) == false); cstring *ser = cstr_new_sz(1024); ser_bloom(ser, &bloom); struct bloom bloom2; __bloom_init(&bloom2); struct const_buffer buf = { ser->str, ser->len }; assert(deser_bloom(&bloom2, &buf) == true); assert(bloom.nHashFuncs == bloom2.nHashFuncs); assert(bloom.vData->len == bloom2.vData->len); assert(memcmp(bloom.vData->str, bloom2.vData->str, bloom2.vData->len) == 0); assert(bloom_contains(&bloom2, md1, sizeof(md1)) == true); assert(bloom_contains(&bloom2, md2, sizeof(md2)) == false); bloom_free(&bloom2); bloom_free(&bloom); cstr_free(ser, true); }
void wallet_close(struct wallet *wallet) { if (wallet == NULL) { return; } bloom_free(wallet->filter); wallet->filter = NULL; txdb_close(wallet->txdb); hashtable_clear_with_callback(wallet->hash_keys, wallet_free_key_cb); hashtable_destroy(wallet->hash_keys); free(wallet->filename); secure_free(wallet->ckey_store); memset(wallet, 0, sizeof *wallet); free(wallet); }
int ppbloom_add(const void *buffer, int len) { int err; err = bloom_add(ppbloom + current, buffer, len); if (err == -1) return err; bloom_count[current]++; if (bloom_count[current] >= entries) { bloom_count[current] = 0; current = current == PING ? PONG : PING; bloom_free(ppbloom + current); bloom_init(ppbloom + current, entries, error); } return 0; }
void file_hash_exit(void) { unsigned int i, has_entries = 0; fio_mutex_down(hash_lock); for (i = 0; i < HASH_BUCKETS; i++) has_entries += !flist_empty(&file_hash[i]); fio_mutex_up(hash_lock); if (has_entries) log_err("fio: file hash not empty on exit\n"); sfree(file_hash); file_hash = NULL; fio_mutex_remove(hash_lock); hash_lock = NULL; bloom_free(file_bloom); file_bloom = NULL; }
void ppbloom_free() { bloom_free(ppbloom + PING); bloom_free(ppbloom + PONG); }
int dbfile_populate_hashes(struct rb_root *d_tree) { int ret; sqlite3 *db; sqlite3_stmt *stmt = NULL; char *filename; uint64_t ino, subvolid; uint64_t num_hashes; struct filerec *file; struct bloom_cb_priv priv; db = dbfile_get_handle(); if (!db) return ENOENT; ret = dbfile_count_rows(db, &num_hashes, NULL); if (ret) return ret; priv.d_tree = d_tree; ret = bloom_init(&priv.bloom, num_hashes, 0.01); if (ret) return ret; ret = sqlite3_prepare_v2(db, "SELECT ino, subvol, filename from files;", -1, &stmt, NULL); if (ret) { perror_sqlite(ret, "preparing statement"); goto out_bloom; } while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) { ino = sqlite3_column_int64(stmt, 0); subvolid = sqlite3_column_int64(stmt, 1); filename = (char *)sqlite3_column_text(stmt, 2); file = filerec_new(filename, ino, subvolid); if (!file) { ret = ENOMEM; goto out_finalize; } ret = dbfile_walk_file_hashes(db, file, load_into_bloom_cb, &priv); if (ret) goto out_finalize; } if (ret != SQLITE_DONE) { perror_sqlite(ret, "retrieving file info from table"); goto out_finalize; } ret = 0; out_finalize: sqlite3_finalize(stmt); out_bloom: bloom_free(&priv.bloom); return ret; }
int main(int argc, char* argv[]) { size_t n = 1000000; size_t m = 8; int index = 0; int opt; while (true) { opt = getopt(argc, argv, "n:m:i:"); if (opt == -1) break; switch (opt) { case 'n': n = (size_t) strtoul(optarg, NULL, 10); break; case 'm': m = (size_t) strtoul(optarg, NULL, 10); break; case 'i': index = (size_t) strtoul(optarg, NULL, 10); break; case '?': return EXIT_FAILURE; default: abort(); } } bloom_t* B = bloom_alloc(n, m); char line[512]; kmer_t x; size_t count = 0, unique_count = 0; FILE *fd = NULL; char *file = NULL; char *files[] = {"random.txt","random1.txt","random2.txt","random3.txt","random4.txt"}; /* * char files[16] = "random.txt"; * if(index > 0) * snprintf(files, sizeof(files), "random%d.txt", index); */ if(index) file = files[index]; else file = files[0]; fd = fopen(file, "r"); if(fd == NULL) fprintf(stderr, "fopen error\n"); while (fgets(line, sizeof(line), fd)) { x = (kmer_t) strtoul(line, NULL, 10); if (bloom_inc(B, x) == 1) ++unique_count; else printf("%s", line); ++count; } printf("\n"); printf("%zu\t%zu\n", unique_count, count); rewind(fd); while (fgets(line, sizeof(line), fd)) { x = (kmer_t) strtoul(line, NULL, 10); //bloom_del(B, x); if(bloom_ldec(B, x) == 1) --unique_count; --count; } fclose(fd); fd = NULL; bloom_free(B); printf("%zu\t%zu\n", unique_count, count); return 0; }