const BOOL fread_bloom_032(FILE* const f, BLOOM** b)
{
assert(f != NULL);
assert(b != NULL);
*b = NULL;
uint8_t nfuncs;
hashfunc_t* hashfuncs;
if (!fread_hashspec(f, &hashfuncs, &nfuncs)) return FALSE;
size_t asize;
if (fread(&asize, sizeof(size_t), 1, f) != 1)
{
free(hashfuncs);
return FALSE;
}
*b = bloom_create(asize);
bloom_set_hashfuncs_ex(*b, hashfuncs, nfuncs);
free(hashfuncs);
if (*b == NULL) return FALSE;
const size_t numRead = fread((*b)->a, sizeof(char), (*b)->size, f);
if (numRead != (*b)->size)
{
bloom_destroy(*b);
return FALSE;
}
return TRUE;
}
struct dht_result_rating * rating_create(const struct sockaddr_storage *ss,
const struct sockaddr *from, int fromlen) {
unsigned char md[SHA_DIGEST_LENGTH];
struct dht_result_rating * result;
result = (struct dht_result_rating*) malloc(
sizeof(struct dht_result_rating));
result->frombloom = bloom_create(96, 7, hash1, hash2, hash3, hash4, hash5,
hash6, hash7);
SHA1((const unsigned char*) from, fromlen, md);
bloom_add(result->frombloom, (const char*) md);
result->next = NULL;
result->rating = 0;
result->ss = malloc(sizeof(struct sockaddr_storage));
result->ss->ss_family = ss->ss_family;
if (ss->ss_family == AF_INET) {
struct sockaddr_in* dest = (struct sockaddr_in*) result->ss;
struct sockaddr_in* src = (struct sockaddr_in*) ss;
dest->sin_port = src->sin_port;
memcpy(&dest->sin_addr, &src->sin_addr, 4);
} else if (ss->ss_family == AF_INET6) {
struct sockaddr_in6* dest = (struct sockaddr_in6*) result->ss;
struct sockaddr_in6* src = (struct sockaddr_in6*) ss;
dest->sin6_port = src->sin6_port;
memcpy(&dest->sin6_addr, &src->sin6_addr, 16);
}
return result;
}
static void
create_and_test_bloom(int power, int64 nelements, int callerseed)
{
int bloom_work_mem;
uint64 seed;
int64 nfalsepos;
bloom_filter *filter;
bloom_work_mem = (1L << power) / 8L / 1024L;
elog(DEBUG1, "bloom_work_mem (KB): %d", bloom_work_mem);
/*
* Generate random seed, or use caller's. Seed should always be a
* positive value less than or equal to PG_INT32_MAX, to ensure that any
* random seed can be recreated through callerseed if the need arises.
* (Don't assume that RAND_MAX cannot exceed PG_INT32_MAX.)
*/
seed = callerseed < 0 ? random() % PG_INT32_MAX : callerseed;
/* Create Bloom filter, populate it, and report on false positive rate */
filter = bloom_create(nelements, bloom_work_mem, seed);
populate_with_dummy_strings(filter, nelements);
nfalsepos = nfalsepos_for_missing_strings(filter, nelements);
ereport((nfalsepos > nelements * FPOSITIVE_THRESHOLD) ? WARNING : DEBUG1,
(errmsg_internal("seed: " UINT64_FORMAT " false positives: " INT64_FORMAT " (%.6f%%) bitset %.2f%% set",
seed, nfalsepos, (double) nfalsepos / nelements,
100.0 * bloom_prop_bits_set(filter))));
bloom_free(filter);
}
Datum
bloom_emptyp(PG_FUNCTION_ARGS)
{
float8 p = PG_GETARG_FLOAT8(0);
uint64_t n = PG_GETARG_INT64(1);
BloomFilter *bloom = bloom_create(p, n);
PG_RETURN_POINTER(bloom);
}
static void
wallet_filter_init(struct wallet *wallet)
{
ASSERT(wallet->filter == NULL);
wallet->filter = bloom_create(10, 0.001);
wallet_update_filter(wallet, wallet->filter);
}
int
main(void)
{
plan_tests(6);
setvbuf(stdout, 0, _IOLBF, 0);
BLOOM *bp = bloom_create(12, 16, 2);
ok(bp, "created");
int rc = bloom_chk(bp, "AB");
ok(rc == 0, "Check 'AB' not in empty table: %d", rc);
printf("# Add AB\n");
bloom_add(bp, "AB");
rc = bloom_chk(bp, "AB");
ok(rc != 0, "Check 'AB' after add: %d", rc);
printf("# Add AC\n");
bloom_add(bp, "AC");
rc = bloom_chk(bp, "!@");
ok(rc != 0, "Check '!@' without add: %d (false positive)", rc);
printf("# Before adding CA...DZ\n");
bloom_dump(bp, stdout);
int antestat = bloom_stat(bp), anteover = bloom_over(bp);
char hash[] = "__";
for (hash[1] = 'A'; hash[1] <= 'D'; ++hash[1])
for (hash[0] = 'C'; hash[0] <= 'Z'; ++hash[0])
bloom_add(bp, hash);
printf("# After adding CA...DZ:\n");
bloom_dump(bp, stdout);
for (hash[1] = 'A'; hash[1] <= 'D'; ++hash[1])
for (hash[0] = 'C'; hash[0] <= 'Z'; ++hash[0])
bloom_del(bp, hash);
printf("# After deleting CA...DZ:\n");
bloom_dump(bp, stdout);
int poststat = bloom_stat(bp), postover = bloom_over(bp);
ok(poststat == antestat, "stat %d -> %d", antestat, poststat);
ok(postover == anteover, "over %d -> %d", anteover, postover);
bloom_destroy(bp);
return exit_status();
}
int checkLastFromFiles(request_rec * r, const char * filter, const char * packet) {
BLOOM * bloom;
FILE * fp = fopen(filter, "r");
size_t read;
/* read the filter size */
size_t size;
read = fread(&size, sizeof(size_t), 1, fp);
/* Re-create the filter */
if(!(bloom=bloom_create(size, 2, sax_hash, sdbm_hash)))
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(05011) "check last: bloom filter");
/* Read the filter from file */
read = fread(bloom->a, sizeof(char), (bloom->asize+CHAR_BIT-1)/CHAR_BIT, fp);
fclose(fp);
fp = fopen(packet, "r");
/* Read len of the packet */
int buflen;
read = fread(&buflen, sizeof(int), 1, fp);
/* Alocate memory for the buffer */
char * buffer;
if((buffer = (char *) malloc(buflen * sizeof(char))) == NULL)
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(05011) "check last: buffer");
/* Read the buffer */
read = fread(buffer, sizeof(char), buflen, fp);
fclose(fp);
/* Check and return if the request was received */
if(bloom_check(bloom, buffer)){
free(buffer);
bloom_destroy(bloom);
return 1;
}
else{
free(buffer);
bloom_destroy(bloom);
return 0;
}
}
int main(int argc, char *argv[])
{
FILE *fp;
char line[1024];
char *p;
BLOOM *bloom;
if(argc<2) {
fprintf(stderr, "ERROR: No word file specified\n");
return EXIT_FAILURE;
}
if(!(bloom=bloom_create(2500000, 2, sax_hash, sdbm_hash))) {
fprintf(stderr, "ERROR: Could not create bloom filter\n");
return EXIT_FAILURE;
}
if(!(fp=fopen(argv[1], "r"))) {
fprintf(stderr, "ERROR: Could not open file %s\n", argv[1]);
return EXIT_FAILURE;
}
while(fgets(line, 1024, fp)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
bloom_add(bloom, line);
}
fclose(fp);
while(fgets(line, 1024, stdin)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
p=strtok(line, " \t,.;:\r\n?!-/()");
while(p) {
if(!bloom_check(bloom, p)) {
printf("No match for ford \"%s\"\n", p);
}
p=strtok(NULL, " \t,.;:\r\n?!-/()");
}
}
bloom_destroy(bloom);
return EXIT_SUCCESS;
}
void bf_check(FILE* fp1,FILE* fp2,FILE *fp3) {
BLOOM *bloom ;
char line[1024];
int pos = -1;
bloom = bloom_create(239620000);
printf("Create done\n");
while(fgets(line,1024,fp1))
bloom_add(bloom,line);
printf("add done\n");
fprintf(fp3,"-------------------------------------Bloom Filter Match-------------------------------------------\n");
while(fgets(line,1024,fp2)) {
pos++;
if(!bloom_check(bloom,line))
fprintf(fp3,"%d no\n",pos);
}
}
static BloomFilter *
bloom_startup(FunctionCallInfo fcinfo, float8 p, uint64_t n)
{
BloomFilter *bloom;
Oid type = AggGetInitialArgType(fcinfo);
fcinfo->flinfo->fn_extra = lookup_type_cache(type, 0);
if (p && n)
{
bloom = bloom_create(p, n);
}
else
bloom = BloomFilterCreate();
return bloom;
}
int CS_summary(struct bloom ** filter_ptr)
{
if (!filter_ptr) return -1;
*filter_ptr = bloom_create(_cs.summary_size, BLOOM_ARGS);
int i;
pthread_mutex_lock(&_cs.lock);
for (i = 0; i < _cs.table->size; i++) {
struct hash_entry * entry = _cs.table->entries[i];
if (entry && entry->valid) {
char * name = entry->key;
bloom_add(*filter_ptr, name);
}
}
pthread_mutex_unlock(&_cs.lock);
return 0;
}
int main(int argc, char *argv[])
{
FILE *fp;
/* No domain more than 1024 characs */
char line[1024];
char *p;
BLOOM *bloom;
if(argc<2) {
fprintf(stderr, "ERROR: No word file specified\n");
return EXIT_FAILURE;
}
if(!(bloom=bloom_create(2500000, 2, sax_hash, sdbm_hash))) {
fprintf(stderr, "ERROR: Could not create bloom filter\n");
return EXIT_FAILURE;
}
if(!(fp=fopen(argv[1], "r"))) {
fprintf(stderr, "ERROR: Could not open file %s\n", argv[1]);
return EXIT_FAILURE;
}
/* set block - read names from file argv[1] and add to filter */
while(fgets(line, 1024, fp)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
bloom_add(bloom, line);
}
fclose(fp);
/* TODO Dump twice - filter.bin and filter_version.bin */
const char *filepath;
filepath = "bloomfilter/filter.bin";
if (!bloom_dump(bloom, filepath)) {
fprintf(stderr, "ERROR: Could not dump bloom filter %s\n", filepath);
}
char filepath2[30];
filepath = "";
sprintf(filepath2, "%s%d.bin", "bloomfilter/filter_", bloom->timestamp);
if (!bloom_dump(bloom, filepath2)) {
fprintf(stderr, "ERROR: Could not dump bloom filter %s\n", filepath2);
}
/* check block - enter name to check */
while(fgets(line, 1024, stdin)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
/* Domain name can have ".", "/", ":" and "-"
* p=strtok(line, " \t,.;:\r\n?!-/()");
* Divide line on symbols to check each word*/
p=strtok(line, " \t,;\r\n?!()");
while(p) {
if(!bloom_check(bloom, p)) {
printf("No match for word \"%s\"\n", p);
}
p=strtok(NULL, " \t,.;:\r\n?!-/()");
}
}
bloom_destroy(bloom);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
clock_t start=clock();
//程序运行时需从cmd启动，后面两个参数依次为源字符串和待查询字符串的文件名（文件地址）
char line[1024];
if (argc < 2) {
fprintf(stderr, "ERROR: No word file specified\n");
return -1;
}
//获取时间
int big_count = 0;
int count = 0;
//bloom filter algorithm
BF *bloomfilter = bloom_create(BIG_PRIME);
freopen(argv[1], "r", stdin);
while (scanf("%s", line) != EOF) {
upper_string(line);
if (check_string(line) == 1) {
bloom_add(bloomfilter, line);
big_count++;
}
}
fclose(stdin);
freopen(argv[2], "r", stdin);
freopen("result_bf.dat", "w", stdout);
while (scanf("%s", line) != EOF) {
upper_string(line);
if (check_string(line) == 1 && bloom_check(bloomfilter, line) == 1) {
printf("yes\n");
count++;
} else {
printf("no\n");
}
}
fclose(stdin);
fclose(stdout);
freopen("/dev/stdin", "r", stdin);
freopen("computer.time", "a", stdout);
printf("%f\n",(double)(clock()-start)/CLOCKS_PER_SEC);
bloom_destroy(bloomfilter);
/*****************************************************************************/
/*
//压缩trie树
trie_Node* root;
root = trie_create();
//读文件，建立压缩trie树
freopen(argv[1], "r", stdin);
while (scanf("%s", line) != EOF) {
upper_string(line);
if (check_string(line) == 1) {
trie_add(line, root);
}
}
fclose(stdin);
//查询是否在树中
count = 0;
freopen(argv[2], "r", stdin);
freopen("result_trie.dat", "w", stdout);
while (scanf("%s", line) != EOF) {
upper_string(line);
if (check_string(line) == 1 && trie_check(line, root) == 1) {
printf("yes\n");
count++;
} else {
printf("no\n");
}
}
fclose(stdin);
fclose(stdout);
freopen("/dev/pts/3", "r", stdin);
freopen("computer.time", "a", stdout);
//printf("存在%d个\n", count);
*/
// trie_destroy(root);//运行完直接退出，自动释放内存，是否destroy不重要（取消注释可以运行destroy）
return 0;
}
int main(int argc, char *argv[])
{
FILE *fp1;
FILE *fp2;
FILE *fp3;
int i = 0;
char line[1024];
char *p;
BF *bloom;
if(argc<2) {
fprintf(stderr, "ERROR: No word file specified\n");
return EXIT_FAILURE;
}
if(!(bloom=bloom_create(200000000, 11, RSHash, JSHash, PJWHash, ELFHash, BKDRHash, SDBMHash, DJBHash, DEKHash, BPHash, FNVHash, APHash))) {
fprintf(stderr, "ERROR: Could not create bloom filter\n");
return EXIT_FAILURE;
}
if(!(fp1=fopen(argv[1], "r"))) {
fprintf(stderr, "ERROR: Could not open file %s\n", argv[1]);
return EXIT_FAILURE;
}
while(fgets(line, 1024, fp1)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
bloom_add(bloom, line);
}
fclose(fp1);
if(!(fp2=fopen(argv[2], "r"))) {
fprintf(stderr, "ERROR: Could not open file %s\n", argv[2]);
return EXIT_FAILURE;
}
if(!(fp3=fopen("checkedemailresult.dat","w"))){
fprintf(stderr, "ERROR:Could not open file");
return EXIT_FAILURE;
}
while(fgets(line, 1024, fp2)) {
i++;
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
p=strtok(line, "\r\n");
while(p) {
if(bloom_check(bloom, line)==1) {
fputs("yes\n",fp3);
}
else{fputs("no\n",fp3);}
p=strtok(NULL, "\r\n");
}
}
fclose(fp3);
fclose(fp2);
bloom_destroy(bloom);
return EXIT_SUCCESS;
}
const BOOL fread_bloomconfig(FILE* const f, const char* const key, const char* const value, void* const usr)
{
assert(usr != NULL);
container_iodata_t* const x = (container_iodata_t*) usr;
container_t* const container = (container_t*) x->data;
if (container->data == NULL)
{
BLOOM* const b = bloom_create(0);
container_set_bloomfilter(container, b);
}
char* tail;
switch (cmp(key, "hashes", "data", NULL))
{
case 0:
{
char* buf;
STRDUP(value, buf);
size_t n = 1;
char* needle = buf;
for (; *needle != '\0'; needle++) if (*needle == ',') n++;
ASSERT(n <= UINT8_MAX);
hashfunc_t* funcs = (hashfunc_t*) calloc(n, sizeof(hashfunc_t));
size_t i = 0;
char* prev = buf;
while ((needle = strchr(prev, ',')) != NULL)
{
*needle = '\0';
funcs[i++] = to_hashfunc(prev);
prev = needle +1;
}
funcs[i] = to_hashfunc(prev);
bloom_set_hashfuncs_ex(container->data, funcs, (uint8_t) n);
free(funcs); free(buf);
break;
}
case 1:
{
size_t size = strtoul(value, &tail, 10);
int has_validsize = (value != tail && (*tail == '\0' || strcmp(tail, "raw") == 0));
if (has_validsize)
{
// We assume it is an inline bloom specification
// with the specified size.
BLOOM* const b = (BLOOM*) container->data;
FN_READBYTE read = (*tail == 'r' ? read_byte : read_hexbyte);
if (!bloom_set_ex(b, read, size, f)) return FALSE;
break;
}
const char* const filename = tail;
if (x->request_file == NULL || x->host == NULL)
{
// Illegal configuration or programming error
return FALSE;
}
char* const slash = strrchr(tail, '/');
if (slash != NULL)
{
size = strtoul(slash +1, &tail, 10);
has_validsize = (tail != slash +1 && *tail == '\0');
if (has_validsize)
{
// all read
*slash = '\0';
}
}
FILE* const f = x->request_file(filename, x->host);
if (f == NULL) return FALSE;
if (!has_validsize)
{
// No size specification let's use the file size and
// assume the bit size equals the number of bytes *8
fseek_s(f, 0, SEEK_END);
size = ftell_s(f) * 8;
fseek_s(f, 0, SEEK_SET);
has_validsize = TRUE;
}
BLOOM* const b = (BLOOM*) container->data;
if (!bloom_set_ex(b, read_byte, size, f)) return FALSE;
fclose(f);
break;
}
default:
// Unknown identifier
return FALSE;
}
return TRUE;
}
