int main()
{
CS_init(OLDEST, 0.02);
//make sure its emtpy
struct bloom * filter = NULL;
CS_summary(&filter);
bloom_print(filter);
bloom_destroy(filter);
struct content_obj obj;
obj.name = content_name_create("/tom/test");
obj.timestamp = 123;
obj.size = 0;
obj.data = NULL;
CS_put(&obj);
CS_summary(&filter);
bloom_print(filter);
bloom_destroy(filter);
struct content_name * name = content_name_create("/tom");
struct content_obj * co = CS_get(name);
if (&obj != co) {
printf("failed!\n");
exit(EXIT_FAILURE);
}
exit(EXIT_SUCCESS);
}
int checkLastFromFiles(request_rec * r, const char * filter, const char * packet) {
BLOOM * bloom;
FILE * fp = fopen(filter, "r");
size_t read;
/* read the filter size */
size_t size;
read = fread(&size, sizeof(size_t), 1, fp);
/* Re-create the filter */
if(!(bloom=bloom_create(size, 2, sax_hash, sdbm_hash)))
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(05011) "check last: bloom filter");
/* Read the filter from file */
read = fread(bloom->a, sizeof(char), (bloom->asize+CHAR_BIT-1)/CHAR_BIT, fp);
fclose(fp);
fp = fopen(packet, "r");
/* Read len of the packet */
int buflen;
read = fread(&buflen, sizeof(int), 1, fp);
/* Alocate memory for the buffer */
char * buffer;
if((buffer = (char *) malloc(buflen * sizeof(char))) == NULL)
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(05011) "check last: buffer");
/* Read the buffer */
read = fread(buffer, sizeof(char), buflen, fp);
fclose(fp);
/* Check and return if the request was received */
if(bloom_check(bloom, buffer)){
free(buffer);
bloom_destroy(bloom);
return 1;
}
else{
free(buffer);
bloom_destroy(bloom);
return 0;
}
}
int main()
{
//Part 1. Evaluating Hash Functions
int bloomsize = 100;
int x;
bloom_filter_t bloomfilter;
bloom_init(&bloomfilter, bloomsize);
printf ("Hash1: %i %i %i %i %i %i\n",hash1(&bloomfilter, 0),hash1(&bloomfilter, 1),
hash1(&bloomfilter, 2),hash1(&bloomfilter, 3),hash1(&bloomfilter, 13),
hash1(&bloomfilter, 97));
printf ("Hash2: %i %i %i %i %i %i\n",hash2(&bloomfilter, 0),hash2(&bloomfilter, 1),
hash2(&bloomfilter, 2),hash2(&bloomfilter, 3),hash2(&bloomfilter, 13),
hash2(&bloomfilter, 97));
bloom_destroy(&bloomfilter);
//Part 2:
printf("\nDoing Smoke Test.\n");
bloomsize = 1000;
bloom_init(&bloomfilter, bloomsize);
for (x= 0; x< 70; x++)
{
bloom_add(&bloomfilter, x);
}
int totalbits = 0;
for (x = 0; x< bloomsize; x++)
{
totalbits += get_bit(&bloomfilter, x);
}
printf("Total bits set: %i\n",totalbits);
bloom_destroy(&bloomfilter);
//Part 3
printf("\nDoing N_HASHES Test.\n");
int array1[100];
int array2[100];
gen_rand(array1, 100, 1000000);
gen_rand(array2, 100, 1000000);
run_test3(array1, array2, 100);
}
void run_test3(int* array1, int* array2, int arraysize)
{
int bloomsize = 1000;
bloom_filter_t bloomfilter;
bloom_init(&bloomfilter, bloomsize);
int x = 0;
//set the bits in bloomfilter based on array1
for (x= 0; x< arraysize; x++)
{
bloom_add(&bloomfilter, array1[x]);
}
//First, count all the bits that are set
int totalbits = 0;
for (x = 0; x< bloomsize; x++)
{
totalbits += get_bit(&bloomfilter, x);
}
printf("Total bits set: %i\n",totalbits);
int array2bits = 0;
//Next, count all the bits in the second array that are set in bloomfiter
for (x = 0; x< arraysize; x++)
{
array2bits += bloom_check(&bloomfilter, array2[x]);
}
printf("Array2 bits set: %i\n",array2bits);
bloom_destroy(&bloomfilter);
}
const BOOL fread_bloom_032(FILE* const f, BLOOM** b)
{
assert(f != NULL);
assert(b != NULL);
*b = NULL;
uint8_t nfuncs;
hashfunc_t* hashfuncs;
if (!fread_hashspec(f, &hashfuncs, &nfuncs)) return FALSE;
size_t asize;
if (fread(&asize, sizeof(size_t), 1, f) != 1)
{
free(hashfuncs);
return FALSE;
}
*b = bloom_create(asize);
bloom_set_hashfuncs_ex(*b, hashfuncs, nfuncs);
free(hashfuncs);
if (*b == NULL) return FALSE;
const size_t numRead = fread((*b)->a, sizeof(char), (*b)->size, f);
if (numRead != (*b)->size)
{
bloom_destroy(*b);
return FALSE;
}
return TRUE;
}
void free_rating(struct dht_result_rating *head) {
struct dht_result_rating * next;
while (head) {
next = head->next;
bloom_destroy(head->frombloom);
free(head->ss);
free(head);
head = next;
}
}
int
main(void)
{
plan_tests(6);
setvbuf(stdout, 0, _IOLBF, 0);
BLOOM *bp = bloom_create(12, 16, 2);
ok(bp, "created");
int rc = bloom_chk(bp, "AB");
ok(rc == 0, "Check 'AB' not in empty table: %d", rc);
printf("# Add AB\n");
bloom_add(bp, "AB");
rc = bloom_chk(bp, "AB");
ok(rc != 0, "Check 'AB' after add: %d", rc);
printf("# Add AC\n");
bloom_add(bp, "AC");
rc = bloom_chk(bp, "!@");
ok(rc != 0, "Check '!@' without add: %d (false positive)", rc);
printf("# Before adding CA...DZ\n");
bloom_dump(bp, stdout);
int antestat = bloom_stat(bp), anteover = bloom_over(bp);
char hash[] = "__";
for (hash[1] = 'A'; hash[1] <= 'D'; ++hash[1])
for (hash[0] = 'C'; hash[0] <= 'Z'; ++hash[0])
bloom_add(bp, hash);
printf("# After adding CA...DZ:\n");
bloom_dump(bp, stdout);
for (hash[1] = 'A'; hash[1] <= 'D'; ++hash[1])
for (hash[0] = 'C'; hash[0] <= 'Z'; ++hash[0])
bloom_del(bp, hash);
printf("# After deleting CA...DZ:\n");
bloom_dump(bp, stdout);
int poststat = bloom_stat(bp), postover = bloom_over(bp);
ok(poststat == antestat, "stat %d -> %d", antestat, poststat);
ok(postover == anteover, "over %d -> %d", anteover, postover);
bloom_destroy(bp);
return exit_status();
}
static void * create_summary_response(void * arg)
{
prctl(PR_SET_NAME, "ccnfd_sum", 0, 0, 0);
log_print(g_log, "create_summary_response: handling CS summary request");
int sock;
memcpy(&sock, (int * )arg, sizeof(int));
free(arg);
struct bloom * filter;
int rv = CS_summary(&filter);
if (!filter || (rv != 0)) {
goto END_CREATE_SUMMARY_RESP;
}
/* finish rcving the summary request packet */
int payload_size;
if (recv(sock, &payload_size, sizeof(uint32_t), 0) == -1) {
log_error(g_log, "recv: %s.", strerror(errno));
goto END_CREATE_SUMMARY_RESP;
}
int ignore;
if (recv(sock, &ignore, sizeof(int), 0) == -1) {
log_error(g_log, "recv: %s.", strerror(errno));
goto END_CREATE_SUMMARY_RESP;
}
/* response format:
* vector length : int
* vector : byte[]
*/
int n;
int size = filter->vector->num_words;
if ((n = send(sock, &size, sizeof(uint32_t), 0)) <= 0) {
log_error(g_log, "create_summary_response: send - %s", strerror(errno));
}
if ((n = send(sock, filter->vector->map, size * sizeof(uint32_t), 0)) <= 0) {
log_error(g_log, "create_summary_response: send - %s", strerror(errno));
}
END_CREATE_SUMMARY_RESP:
bloom_destroy(filter);
close(sock);
return NULL;
}
void hfile_destroy(hfile_t *fp)
{
meta_t *o;
if (fp == NULL)
return;
hinfo_destroy(fp->fileinfo);
htail_destroy(fp->trailer);
for (o = index_head(fp->metas); o != NULL; o = meta_next(o))
block_destroy(fp->blocks[o->id]);
index_destroy(fp->metas);
bloom_destroy(fp->bloom);
free(fp->blocks);
hfile_free(fp);
}
int main(int argc, char *argv[])
{
FILE *fp;
char line[1024];
char *p;
BLOOM *bloom;
if(argc<2) {
fprintf(stderr, "ERROR: No word file specified\n");
return EXIT_FAILURE;
}
if(!(bloom=bloom_create(2500000, 2, sax_hash, sdbm_hash))) {
fprintf(stderr, "ERROR: Could not create bloom filter\n");
return EXIT_FAILURE;
}
if(!(fp=fopen(argv[1], "r"))) {
fprintf(stderr, "ERROR: Could not open file %s\n", argv[1]);
return EXIT_FAILURE;
}
while(fgets(line, 1024, fp)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
bloom_add(bloom, line);
}
fclose(fp);
while(fgets(line, 1024, stdin)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
p=strtok(line, " \t,.;:\r\n?!-/()");
while(p) {
if(!bloom_check(bloom, p)) {
printf("No match for ford \"%s\"\n", p);
}
p=strtok(NULL, " \t,.;:\r\n?!-/()");
}
}
bloom_destroy(bloom);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
FILE *fp;
/* No domain more than 1024 characs */
char line[1024];
char *p;
BLOOM *bloom;
if(argc<2) {
fprintf(stderr, "ERROR: No word file specified\n");
return EXIT_FAILURE;
}
if(!(bloom=bloom_create(2500000, 2, sax_hash, sdbm_hash))) {
fprintf(stderr, "ERROR: Could not create bloom filter\n");
return EXIT_FAILURE;
}
if(!(fp=fopen(argv[1], "r"))) {
fprintf(stderr, "ERROR: Could not open file %s\n", argv[1]);
return EXIT_FAILURE;
}
/* set block - read names from file argv[1] and add to filter */
while(fgets(line, 1024, fp)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
bloom_add(bloom, line);
}
fclose(fp);
/* TODO Dump twice - filter.bin and filter_version.bin */
const char *filepath;
filepath = "bloomfilter/filter.bin";
if (!bloom_dump(bloom, filepath)) {
fprintf(stderr, "ERROR: Could not dump bloom filter %s\n", filepath);
}
char filepath2[30];
filepath = "";
sprintf(filepath2, "%s%d.bin", "bloomfilter/filter_", bloom->timestamp);
if (!bloom_dump(bloom, filepath2)) {
fprintf(stderr, "ERROR: Could not dump bloom filter %s\n", filepath2);
}
/* check block - enter name to check */
while(fgets(line, 1024, stdin)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
/* Domain name can have ".", "/", ":" and "-"
* p=strtok(line, " \t,.;:\r\n?!-/()");
* Divide line on symbols to check each word*/
p=strtok(line, " \t,;\r\n?!()");
while(p) {
if(!bloom_check(bloom, p)) {
printf("No match for word \"%s\"\n", p);
}
p=strtok(NULL, " \t,.;:\r\n?!-/()");
}
}
bloom_destroy(bloom);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
clock_t start=clock();
//程序运行时需从cmd启动，后面两个参数依次为源字符串和待查询字符串的文件名（文件地址）
char line[1024];
if (argc < 2) {
fprintf(stderr, "ERROR: No word file specified\n");
return -1;
}
//获取时间
int big_count = 0;
int count = 0;
//bloom filter algorithm
BF *bloomfilter = bloom_create(BIG_PRIME);
freopen(argv[1], "r", stdin);
while (scanf("%s", line) != EOF) {
upper_string(line);
if (check_string(line) == 1) {
bloom_add(bloomfilter, line);
big_count++;
}
}
fclose(stdin);
freopen(argv[2], "r", stdin);
freopen("result_bf.dat", "w", stdout);
while (scanf("%s", line) != EOF) {
upper_string(line);
if (check_string(line) == 1 && bloom_check(bloomfilter, line) == 1) {
printf("yes\n");
count++;
} else {
printf("no\n");
}
}
fclose(stdin);
fclose(stdout);
freopen("/dev/stdin", "r", stdin);
freopen("computer.time", "a", stdout);
printf("%f\n",(double)(clock()-start)/CLOCKS_PER_SEC);
bloom_destroy(bloomfilter);
/*****************************************************************************/
/*
//压缩trie树
trie_Node* root;
root = trie_create();
//读文件，建立压缩trie树
freopen(argv[1], "r", stdin);
while (scanf("%s", line) != EOF) {
upper_string(line);
if (check_string(line) == 1) {
trie_add(line, root);
}
}
fclose(stdin);
//查询是否在树中
count = 0;
freopen(argv[2], "r", stdin);
freopen("result_trie.dat", "w", stdout);
while (scanf("%s", line) != EOF) {
upper_string(line);
if (check_string(line) == 1 && trie_check(line, root) == 1) {
printf("yes\n");
count++;
} else {
printf("no\n");
}
}
fclose(stdin);
fclose(stdout);
freopen("/dev/pts/3", "r", stdin);
freopen("computer.time", "a", stdout);
//printf("存在%d个\n", count);
*/
// trie_destroy(root);//运行完直接退出，自动释放内存，是否destroy不重要（取消注释可以运行destroy）
return 0;
}
static void stop(ErlDrvData handle) {
bloom_drv_t *driver = (bloom_drv_t*)handle;
bloom_destroy(driver->bloom);
driver_free(driver);
}
int main(int argc, char *argv[])
{
FILE *fp1;
FILE *fp2;
FILE *fp3;
int i = 0;
char line[1024];
char *p;
BF *bloom;
if(argc<2) {
fprintf(stderr, "ERROR: No word file specified\n");
return EXIT_FAILURE;
}
if(!(bloom=bloom_create(200000000, 11, RSHash, JSHash, PJWHash, ELFHash, BKDRHash, SDBMHash, DJBHash, DEKHash, BPHash, FNVHash, APHash))) {
fprintf(stderr, "ERROR: Could not create bloom filter\n");
return EXIT_FAILURE;
}
if(!(fp1=fopen(argv[1], "r"))) {
fprintf(stderr, "ERROR: Could not open file %s\n", argv[1]);
return EXIT_FAILURE;
}
while(fgets(line, 1024, fp1)) {
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
bloom_add(bloom, line);
}
fclose(fp1);
if(!(fp2=fopen(argv[2], "r"))) {
fprintf(stderr, "ERROR: Could not open file %s\n", argv[2]);
return EXIT_FAILURE;
}
if(!(fp3=fopen("checkedemailresult.dat","w"))){
fprintf(stderr, "ERROR:Could not open file");
return EXIT_FAILURE;
}
while(fgets(line, 1024, fp2)) {
i++;
if((p=strchr(line, '\r'))) *p='\0';
if((p=strchr(line, '\n'))) *p='\0';
p=strtok(line, "\r\n");
while(p) {
if(bloom_check(bloom, line)==1) {
fputs("yes\n",fp3);
}
else{fputs("no\n",fp3);}
p=strtok(NULL, "\r\n");
}
}
fclose(fp3);
fclose(fp2);
bloom_destroy(bloom);
return EXIT_SUCCESS;
}
int build_main (int argc, char **argv)
{
if (argc < 2)
build_usage ();
char *position;
BIGNUM capacity;
/*-------defaults for bloom filter building-------*/
int opt;
int k_mer = 0;
float error_rate = 0.0005;
char *list = NULL;
char *target_path = NULL;
char *source = NULL;
//XXX make -l and -r mutually exclusive
while ((opt = getopt (argc, argv, "e:k:o:r:l:h")) != -1)
{
switch (opt)
{
case 'e':
(optarg) && ((error_rate = atof (optarg)), 1);
break;
case 'k':
(optarg) && ((k_mer = atoi (optarg)), 1);
break;
case 'o':
(optarg) && ((target_path = optarg), 1);
break;
case 'r':
(optarg) && (source = optarg, 1);
break;
case 'l':
(optarg) && (list = optarg, 1);
break;
case 'h':
return build_usage ();
default:
printf ("Unknown option: -%c\n", (char) optopt);
return build_usage ();
}
}
if (!list && !source)
{
fprintf (stderr, "\nPlease, at least specify a reference file (-r) and an output bloom filter (-o)\n");
exit (-1);
}
if (!list)
{
#ifdef DEBUG
printf ("[bloom build]: source is %s\n", source);
printf ("[bloom build]: target is %s\n", target_path);
#endif
build (source, target_path, k_mer, error_rate, argv[0]);
}
else
{
bloom *bl_2 = NEW (bloom);
Queue *head = NEW (Queue);
Queue *tail = NEW (Queue);
head->next = tail;
F_set *File_head = NEW (F_set);
File_head = make_list (source, list);
while (File_head)
{
//map query- into memory--------------
position = mmaping (File_head->filename);
if (*position == '>')
capacity = strlen (position);
else
capacity = strlen (position) / 2;
init_bloom (bl_2, capacity, error_rate, k_mer, File_head->filename);
ref_add (bl_2, position);
save_bloom (File_head->filename, bl_2, argv[0], target_path);
bloom_destroy (bl_2);
munmap (position, strlen (position));
File_head = File_head->next;
}
}
return 0;
}
char *query (char *query, char *bloom_filter, double tole_rate, double sampling_rate, char *list, char *target_path, char *report_fmt, char mode)
{
gzFile zip = NULL;
char type = '@';
int normal = 0;
int threads = 0;
BIGCAST offset = 0;
char *position = NULL;
static char timestamp[40] = {0};
// Get current timestamp, for benchmarking purposes (begin of run timestamp)
isodate(timestamp);
bloom *bl_2 = NEW (bloom);
F_set *File_head = make_list (bloom_filter, list);
/*initialization for python interface*/
File_head->hits = 0;
File_head->all_k = 0;
File_head->reads_num = 0;
File_head->reads_contam = 0;
File_head->filename = bloom_filter; //extra initialization for python interface
if (load_bloom (File_head->filename, bl_2)<=0) //load a bloom filter
exit(-1);
if (tole_rate == 0)
{
tole_rate = mco_suggestion (bl_2->k_mer); // suggest an optimal match cut-off
}
if (mode == 'r')
{
init_string(ONEG); // initialize strings for containing reads
}
/*
if ((get_size (query) < 2 * ONEG) && !strstr (query, ".gz") && !strstr (query, ".tar"))
normal = 0;
else
{
if ((zip = gzopen (query, "rb")) < 0)
{
perror ("query open error...\n");
exit (-1);
}
normal = 0;
}
*/
if ((zip = gzopen (query, "rb")) <= 0)
{
fprintf(stderr, "%s\n", strerror(errno));
exit(EXIT_FAILURE);
}
if (strstr (query, ".fastq") != NULL || strstr (query, ".fq") != NULL)
type = '@';
else
type = '>';
if (normal == 0)
position = (char *) calloc (1,(ONEG+1)*sizeof (char));
while (offset != -1)
{
if (normal == 1)
{
position = mmaping (query);
offset = -1;
}
else
{
offset = CHUNKer (zip, offset, ONEG, position, type);
}
Queue *head = NEW (Queue);
head->location = NULL;
Queue *tail = NEW (Queue);
head->next = tail;
Queue *head2 = head;
get_parainfo (position, head, type);
#pragma omp parallel
{
// XXX: Awesome will be the day when OpenMP is in OSX
#ifndef __APPLE__
threads = omp_get_num_threads();
#endif
#pragma omp single nowait
{
while (head != tail)
{
#pragma omp task firstprivate(head)
{
if (head->location != NULL)
{
read_process (bl_2, head, tail, File_head, sampling_rate, tole_rate, mode, type);
}
}
head = head->next;
} // End of firstprivate
} // End of single - no implied barrier (nowait)
} // End of parallel region - implied barrier
if (position != NULL && normal == 0)
{
memset (position, 0, strlen (position));
}
else if (normal == 1)
{
munmap (position, strlen (position));
}
else
{
perror ("Cannot memset, wrong position on fastq file\n");
exit (-1);
}
clean_list (head2, tail);
if (mode == 'r')
{
if (target_path!=NULL)
{
save_result (query, File_head->filename, type, target_path, re_clean(), re_contam()); //save results into file if facs remove is called
}
else
{
write_default(re_clean(), re_contam(), offset);
}
reset_string();
}
} //end while
if (normal == 0)
{
bloom_destroy(bl_2);
gzclose(zip);
free (position); //dont like file mapping, strings need to be freed in a normal way
}
/*
mode c and r refer to contamination checking and removal function respectively.
The following 9 lines make sure that json/tsv output is printed after the checking
process, but willnot be written in stdout when running removal process.
*/
if (target_path!=NULL || mode == 'c')
{
return report(File_head, query, report_fmt, target_path, timestamp, prob_suggestion(bl_2->k_mer), threads);
}
else
{
char *s = "";
return s;
}
}
