Beispiel #1
0
int mode_filter(int argc, char** argv)
{
	uint argo = 0;

	uint negative = FALSE;
	if (strcmp(argv[2 + argo], "-n") == 0) {
		negative = TRUE;
		argo++;
	}

	uint update = FALSE;
	if (strcmp(argv[2 + argo], "-u") == 0) {
		update = TRUE;
		argo++;
	}

	char* bloom_file = argv[2 + argo];
	bf_t* filter = bf_load(bloom_file, BLOOM_HASH_FUNC);
	if (!filter) {
		printf("File %s has wrong format or doesn't exist.\n", bloom_file);

		return 2;
	}


	char input_buffer[BF_KEY_BUFFER_SIZE] = { 0 };
	uint actual_length = 0;
	while (gets_skip(input_buffer, BF_KEY_BUFFER_SIZE, &actual_length)) {
		uint has = bf_has(filter, (const char*)input_buffer, actual_length);
		if (negative) {
			if (!has) {
				printf("%s\n", input_buffer);
				fflush(stdout);
				if (update) {
					bf_add(filter, input_buffer, actual_length);
				}
			}
		} else {
			if (has) {
				printf("%s\n", input_buffer);
				fflush(stdout);
			} else {
				if (update) {
					bf_add(filter, input_buffer, actual_length);
				}
			}
		}
	}

	if (update) {
		bf_save(filter, bloom_file);
	}

	bf_destroy(filter);

	return 0;
}
Beispiel #2
0
/*  add two complex numbers.
	c = a + b 
*/
void bf_add_cmplx( COMPLEX *a, COMPLEX *b, COMPLEX *c)
{
	COMPLEX mya, myb;

	bf_copy_cmplx( a, &mya);
	bf_copy_cmplx( b, &myb);
	
	bf_add( &mya.real, &myb.real, &c->real);
	bf_add( &mya.imag, &myb.imag, &c->imag);
}
Beispiel #3
0
END_TEST

START_TEST(test_bf_shared_compatible_persist)
{
    bloom_filter_params params = {0, 0, 1e6, 1e-4};
    bf_params_for_capacity(&params);
    bloom_bitmap map;
    bloom_bloomfilter filter;
    fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, PERSISTENT, &map) == 0);
    fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
    fchmod(map.fileno, 0777);

    // Check all the keys get added
    char buf[100];
    int res;
    for (int i=0;i<1000;i++) {
        snprintf((char*)&buf, 100, "test%d", i);
        res = bf_add(&filter, (char*)&buf);
        fail_unless(res == 1);
    }
    fail_unless(bf_close(&filter) == 0);

    // Test all the keys are contained
    fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, SHARED, &map) == 0);
    fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
    for (int i=0;i<1000;i++) {
        snprintf((char*)&buf, 100, "test%d", i);
        res = bf_contains(&filter, (char*)&buf);
        fail_unless(res == 1);
    }
    unlink("/tmp/shared_compat_persist.mmap");
}
Beispiel #4
0
END_TEST

START_TEST(test_bf_fp_prob_extended)
{
    bloom_filter_params params = {0, 0, 1e6, 0.001};
    bf_params_for_capacity(&params);
    bloom_bitmap map;
    bloom_bloomfilter filter;
    fail_unless(bitmap_from_file(-1, params.bytes, ANONYMOUS, &map) == 0);
    fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);

    // Check all the keys get added
    char buf[100];
    int res;
    int num_wrong = 0;
    for (int i=0;i<1e6;i++) {
        snprintf((char*)&buf, 100, "test%d", i);
        res = bf_add(&filter, (char*)&buf);
        if (res == 0) num_wrong++;
    }

    // We added 1M items, with a capacity of 1M and error of 1/1000.
    // Technically we should have 1K false positives
    fail_unless(num_wrong <= 1000);
}
Beispiel #5
0
END_TEST

START_TEST(test_length)
{
    bloom_filter_params params = {0, 0, 1e6, 1e-4};
    bf_params_for_capacity(&params);
    bloom_bitmap map;
    bitmap_from_file(-1, params.bytes, ANONYMOUS, &map);
    bloom_bloomfilter filter;
    bf_from_bitmap(&map, params.k_num, 1, &filter);

    // Check the size
    fail_unless(bf_size(&filter) == 0);

    // Check all the keys get added
    char buf[100];
    int res;
    for (int i=0;i<1000;i++) {
        snprintf((char*)&buf, 100, "test%d", i);
        res = bf_add(&filter, (char*)&buf);
        fail_unless(res == 1);
    }

    // Check the size
    fail_unless(bf_size(&filter) == 1000);
}
Beispiel #6
0
void bf_magnitude_cmplx( COMPLEX *x, FLOAT *m)
{
	FLOAT x2, y2;
	
	bf_multiply( &x->real, &x->real, &x2);
	bf_multiply( &x->imag, &x->imag, &y2);
	bf_add( &x2, &y2, m);
	bf_square_root( m, m);
}
Beispiel #7
0
/*  multiply two complex numbers.

	c = (a.real * b.real - a.imag * b.imag) + i(a.imag * b.real + a.real * b.imag)
*/
void bf_multiply_cmplx(  COMPLEX *a, COMPLEX *b, COMPLEX *c)
{
	COMPLEX mya, myb;
	FLOAT	temp1, temp2;

	bf_copy_cmplx( a, &mya);
	bf_copy_cmplx( b, &myb);
	bf_multiply( &mya.real, &myb.real, &temp1);
	bf_multiply( &mya.imag, &myb.imag, &temp2);
	bf_subtract( &temp1, &temp2, &c->real);
	bf_multiply( &mya.real, &myb.imag, &temp1);
	bf_multiply( &mya.imag, &myb.real, &temp2);
	bf_add( &temp1, &temp2, &c->imag);
}
Beispiel #8
0
/*  divide two complex numbers.
	To keep things in Cartesian form multiply top by
	conjugate of bottom.  Scale result by magnitude of
	bottom.
	
		output is c = ( a * b^*) / |b|
	
	returns 1 if b != 0, 0 if |b| = 0
*/
int bf_divide_cmplx( COMPLEX *a, COMPLEX *b, COMPLEX *c)
{
	FLOAT	mag1, mag2;
	COMPLEX	myb;
	
	bf_copy_cmplx( b, &myb);
	bf_multiply( &myb.real, &myb.real, &mag1);
	bf_multiply( &myb.imag, &myb.imag, &mag2);
	bf_add( &mag1, &mag2, &mag1);
	bf_negate( &myb.imag);
	bf_multiply_cmplx( a, &myb, c);
	if( ! bf_divide( &c->real, &mag1, &c->real)) return 0;
	bf_divide( &c->imag, &mag1, &c->imag);
	return 1;
}
Beispiel #9
0
int mode_create(int argc, char** argv)
{
	uint argo = 0;
	uint progress = FALSE;
	if (strcmp(argv[2], "-p") == 0) {
		argo++;
		progress = TRUE;
	}

	if (argc - argo < 5) {
		show_help(argv[0]);

		return 1;
	}

	char* bloom_file = argv[2 + argo];
	double error_rate = atof(argv[3 + argo]);
	bf_index_t key_count = atol(argv[4 + argo]);

	bf_t* filter = bf_create(error_rate, key_count, BLOOM_HASH_FUNC);

	char input_buffer[BF_KEY_BUFFER_SIZE] = { 0 };
	bf_index_t key = 0;
	uint actual_length;
	while (gets_skip(input_buffer, BF_KEY_BUFFER_SIZE, &actual_length)) {
		bf_add(filter, (const char*)input_buffer, actual_length);
		key = key + 1;
		if (progress && key % 1000000 == 0) {
			printf("%s: %s <- %ld\n", argv[0], bloom_file, key);
		}
	}

	bf_save(filter, bloom_file);

	bf_destroy(filter);

	return 0;
}
Beispiel #10
0
const char *CmdAddHandler(bloom_filter_t *bloom, const char element[]) {
    bf_add(bloom, element);
    return added_response;
}
Beispiel #11
0
main()
{
	FLOAT o1, dcubed, n, *offset;
	INDEX 	i, j, k, limit;
	MULTIPOLY 	sigma3;
	MULTIPOLY	q24, tau1, tau2;
	MULTIPOLY	joftop, jofbot, joftau;
	FLOAT	*coef, *tsubj, *tnew, *prevc;
	FLOAT	bctop, bcbottom;
	int		shift, maxstore;
	MULTIPOLY	cheb[100];
	
	struct
	{
		ELEMENT x, y;
		COMPLEX start, jt;
	} datablock;
	FILE *svplot;
	COMPLEX tau, jtau, arc[512], q, qn;
	FLOAT theta, dtheta;
	COMPLEX	temp;
	
	limit = 50;
	maxstore = limit+5;
		
	bf_init_ram_space();
	bf_init_float();

/*  create table of sigma_3(n) (sum of cube of all factors
	of n).
*/			
	sigma3.degree = limit;
	if( !bf_get_space( &sigma3)) 
	{
		printf("no space for that much data\n");
		exit(0);
	}
	bf_one( &o1);
	bf_null( &n);
	for( i=1; i<limit; i++)
	{
		bf_add( &o1,  &n, &n);
		bf_multiply( &n, &n, &dcubed);
		bf_multiply( &n, &dcubed, &dcubed);
		for( j=i; j<limit; j+=i)
		{
			offset = Address( sigma3) + j;
			bf_add( &dcubed, offset, offset);
		}
	}
	/*  save to disk here if necessary */
/*	for( i=0; i<limit; i++)
	{
		tsubj = Address( sigma3) + i;
		printf("i=%d\n", i);
		printfloat("sigma3(i) = ", tsubj);
	}

/*  compute Ramanujan's tau function to some ridiculous degree.
	First step is compute coefficients of (1-x)^24 using
	binomial expansion.
*/

	q24.degree = 24;
	if( !bf_get_space( &q24))
	{
		printf("no room for binomial coefficients?\n");
		exit(0);
	}
	coef = Address(q24);
	bf_int_to_float( 1, coef);
	bf_int_to_float( 24, &bctop);
	bf_int_to_float( 1, &bcbottom);
	for( i=1; i<=24; i++)
	{
		prevc = coef; 
		coef = Address(q24) + i;
		bf_multiply( prevc, &bctop, coef);
		bf_divide( coef, &bcbottom, coef);
		bf_negate( coef);
		bf_round( coef, coef);
/*		printf("i= %d\n", i);
		printfloat("  coef =", coef);

/*  decrement top and increment bottom for next term
	in binomial expansion */

		bf_subtract( &bctop, &o1, &bctop);
		bf_add( &o1, &bcbottom, &bcbottom);
	}

/*  now compute Ramanujan's tau.
	Keep two versions, a source and a destination.
	work back and forth multiplying source by
	binomial coefficients and sum to power shifted
	location.  Seeking coefficients for each power 
	of q:  q*product( 1- q^n)^24  n = 1 to infinity.
*/

	tau1.degree = maxstore;
	tau2.degree = maxstore;
	if( !bf_get_space( &tau1))
	{
		printf("can't allocate first tau block.\n");
		exit(0);
	}
	if( !bf_get_space( &tau2))
	{
		printf("can't allocate second tau block.\n");
		exit(0);
	}
	coef = Address( q24);
	tnew = Address( tau1);
	bf_multi_copy( 25, coef, tnew);
	k = 2;
	while( k<maxstore )  /*  loop over products  */
	{
/*  multiply by 1, first coefficient of each product term  */

		prevc = Address( tau1);
		tnew = Address( tau2);
		bf_multi_copy( maxstore, prevc, tnew);

/*  for each coefficient in 24 term product of next term,
	multiply by every term in previous product and sum
	to shifted location.  */
	
		for( i=1; i <= 24; i++)
		{
			coef = Address( q24) + i;
			shift = k*i;
			if( shift > maxstore) continue;
			for( j=0; j<= k*24; j++)
			{
				if( j + shift > maxstore) continue;
				tsubj = Address( tau1) + j;
				tnew = Address( tau2) + j + shift;
				bf_multiply( tsubj, coef, &bctop);
				bf_add( &bctop, tnew, tnew);
				bf_round( tnew, tnew);
			}
		}
		k++;

/*  now flip things over and go back to tau 1  */

		if( k>maxstore) break;
		prevc = Address( tau2);
		tnew = Address( tau1);
		bf_multi_copy( maxstore, prevc, tnew);
		for( i=1; i<=24; i++)
		{
			coef = Address(q24) + i;
			shift = k*i;
			if( shift > maxstore) continue;
			for( j=0; j<= k*24; j++)
			{
				if( j + shift > maxstore) continue;
				tsubj = Address( tau2) + j;
				tnew = Address( tau1) + j + shift;
				bf_multiply( tsubj, coef, &bctop);
				bf_add( &bctop, tnew, tnew);
				bf_round( tnew, tnew);
			}
		}
		k++;
	}

/*  compute top polynomial of joftau 
	 (1 + 240*sum(sigma3(n)*q^n))^3 */

	bf_multi_dup( sigma3, &joftop);
	coef = Address( joftop);
	bf_int_to_float( 1, coef);
	bf_int_to_float( 240, &bctop);
	for( i=1; i<limit; i++)
	{
		coef = Address( joftop) + i;
		bf_multiply( &bctop, coef, coef);
	}
	bf_power_mul( joftop, joftop, &joftau);  
	bf_power_mul( joftop, joftau, &joftop);

/*  finally compute joftau coefficients */

	bf_power_div( joftop, tau1, &joftau);
/*	for( i=0; i<limit; i++)
	{
		tsubj = Address( joftau) + i;
		printf("i=%d\n", i);
		printfloat("joftau(i) = ", tsubj);
	}
*/
/*  region F is defined as | Re(tau) | < 1/2 and || tau || > 1.
	For each point tau in F, find j(tau).
	save binary data to disk.  Format is (x, y) and (start,
	end) where x and y are integer indexes, and start, end
	are complex points.  Initial start is an arc along
	tau = 1.
*/
	svplot = fopen("joftau.complex", "wb");
	if( !svplot)
	{
		printf( "can't create output file\n");
		exit(0);
	}

/*  create an arc along bottom of F  */

	bf_copy( &P2, &theta);
	theta.expnt += 2;  // create 2PI/3
	bf_int_to_float( 3, &n);
	bf_divide( &theta, &n, &theta);
	bf_copy( &theta, &dtheta);
	dtheta.expnt--;		// create PI/3/511
	bf_int_to_float( gridsize-1, &n);
	bf_divide( &dtheta, &n, &dtheta);
	for( i=0; i<gridsize; i++)
	{
		bf_cosine( &theta, &arc[i].real);
		bf_sine( &theta, &arc[i].imag);
		bf_subtract( &theta, &dtheta, &theta);
	}

/*  Move arc up F, and find j(tau) for each point  */

	bf_int_to_float( 10, &bctop);
	bf_int_to_float( gridsize, &n);
	bf_divide( &bctop, &n, &bctop);  // upper limit of F
	for( i=0; i<gridsize; i++)
	{
		printf("i= %d\n", i);
		bf_int_to_float(i, &n);
		bf_multiply( &bctop, &n, &tau.imag);
		bf_null( &tau.real);
		datablock.y = i;
		for( j=0; j<gridsize; j++)
		{
			datablock.x = j;

			bf_add_cmplx( &tau, &arc[j], &datablock.start);
//			print_cmplx("data block start", &datablock.start);
			
/*  compute j(tau) for this point.  Note power of q = index - 1 */

			bf_firstj( &datablock.start, &q, &jtau);
//			print_cmplx("q = exp(2 i PI tau)", &q);
//			print_cmplx("first terms", &jtau);
			bf_copy_cmplx( &q, &qn);
			for( k=2; k<limit; k++)
			{
				offset = Address( joftau) + k;
				bf_multiply( offset, &qn.real, &temp.real);
				bf_multiply( offset, &qn.imag, &temp.imag);
				bf_add_cmplx( &temp, &jtau, &jtau);
				bf_multiply_cmplx( &q, &qn, &qn);
			}
			
/*  save data point to disk  */

//			print_cmplx("j(tau) = ", &jtau);
			bf_copy_cmplx( &jtau, &datablock.jt);
			if( fwrite(  &datablock, sizeof( datablock), 1, svplot) <= 0)
				printf("can't write to disk\n");
		}
	}
	fclose( svplot);
	printf("all done!\n\n");
}