int mode_filter(int argc, char** argv)
{
uint argo = 0;
uint negative = FALSE;
if (strcmp(argv[2 + argo], "-n") == 0) {
negative = TRUE;
argo++;
}
uint update = FALSE;
if (strcmp(argv[2 + argo], "-u") == 0) {
update = TRUE;
argo++;
}
char* bloom_file = argv[2 + argo];
bf_t* filter = bf_load(bloom_file, BLOOM_HASH_FUNC);
if (!filter) {
printf("File %s has wrong format or doesn't exist.\n", bloom_file);
return 2;
}
char input_buffer[BF_KEY_BUFFER_SIZE] = { 0 };
uint actual_length = 0;
while (gets_skip(input_buffer, BF_KEY_BUFFER_SIZE, &actual_length)) {
uint has = bf_has(filter, (const char*)input_buffer, actual_length);
if (negative) {
if (!has) {
printf("%s\n", input_buffer);
fflush(stdout);
if (update) {
bf_add(filter, input_buffer, actual_length);
}
}
} else {
if (has) {
printf("%s\n", input_buffer);
fflush(stdout);
} else {
if (update) {
bf_add(filter, input_buffer, actual_length);
}
}
}
}
if (update) {
bf_save(filter, bloom_file);
}
bf_destroy(filter);
return 0;
}
/* add two complex numbers.
c = a + b
*/
void bf_add_cmplx( COMPLEX *a, COMPLEX *b, COMPLEX *c)
{
COMPLEX mya, myb;
bf_copy_cmplx( a, &mya);
bf_copy_cmplx( b, &myb);
bf_add( &mya.real, &myb.real, &c->real);
bf_add( &mya.imag, &myb.imag, &c->imag);
}
END_TEST
START_TEST(test_bf_shared_compatible_persist)
{
bloom_filter_params params = {0, 0, 1e6, 1e-4};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bloom_bloomfilter filter;
fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, PERSISTENT, &map) == 0);
fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
fchmod(map.fileno, 0777);
// Check all the keys get added
char buf[100];
int res;
for (int i=0;i<1000;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_add(&filter, (char*)&buf);
fail_unless(res == 1);
}
fail_unless(bf_close(&filter) == 0);
// Test all the keys are contained
fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, SHARED, &map) == 0);
fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
for (int i=0;i<1000;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_contains(&filter, (char*)&buf);
fail_unless(res == 1);
}
unlink("/tmp/shared_compat_persist.mmap");
}
END_TEST
START_TEST(test_bf_fp_prob_extended)
{
bloom_filter_params params = {0, 0, 1e6, 0.001};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bloom_bloomfilter filter;
fail_unless(bitmap_from_file(-1, params.bytes, ANONYMOUS, &map) == 0);
fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0);
// Check all the keys get added
char buf[100];
int res;
int num_wrong = 0;
for (int i=0;i<1e6;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_add(&filter, (char*)&buf);
if (res == 0) num_wrong++;
}
// We added 1M items, with a capacity of 1M and error of 1/1000.
// Technically we should have 1K false positives
fail_unless(num_wrong <= 1000);
}
END_TEST
START_TEST(test_length)
{
bloom_filter_params params = {0, 0, 1e6, 1e-4};
bf_params_for_capacity(¶ms);
bloom_bitmap map;
bitmap_from_file(-1, params.bytes, ANONYMOUS, &map);
bloom_bloomfilter filter;
bf_from_bitmap(&map, params.k_num, 1, &filter);
// Check the size
fail_unless(bf_size(&filter) == 0);
// Check all the keys get added
char buf[100];
int res;
for (int i=0;i<1000;i++) {
snprintf((char*)&buf, 100, "test%d", i);
res = bf_add(&filter, (char*)&buf);
fail_unless(res == 1);
}
// Check the size
fail_unless(bf_size(&filter) == 1000);
}
void bf_magnitude_cmplx( COMPLEX *x, FLOAT *m)
{
FLOAT x2, y2;
bf_multiply( &x->real, &x->real, &x2);
bf_multiply( &x->imag, &x->imag, &y2);
bf_add( &x2, &y2, m);
bf_square_root( m, m);
}
/* multiply two complex numbers.
c = (a.real * b.real - a.imag * b.imag) + i(a.imag * b.real + a.real * b.imag)
*/
void bf_multiply_cmplx( COMPLEX *a, COMPLEX *b, COMPLEX *c)
{
COMPLEX mya, myb;
FLOAT temp1, temp2;
bf_copy_cmplx( a, &mya);
bf_copy_cmplx( b, &myb);
bf_multiply( &mya.real, &myb.real, &temp1);
bf_multiply( &mya.imag, &myb.imag, &temp2);
bf_subtract( &temp1, &temp2, &c->real);
bf_multiply( &mya.real, &myb.imag, &temp1);
bf_multiply( &mya.imag, &myb.real, &temp2);
bf_add( &temp1, &temp2, &c->imag);
}
/* divide two complex numbers.
To keep things in Cartesian form multiply top by
conjugate of bottom. Scale result by magnitude of
bottom.
output is c = ( a * b^*) / |b|
returns 1 if b != 0, 0 if |b| = 0
*/
int bf_divide_cmplx( COMPLEX *a, COMPLEX *b, COMPLEX *c)
{
FLOAT mag1, mag2;
COMPLEX myb;
bf_copy_cmplx( b, &myb);
bf_multiply( &myb.real, &myb.real, &mag1);
bf_multiply( &myb.imag, &myb.imag, &mag2);
bf_add( &mag1, &mag2, &mag1);
bf_negate( &myb.imag);
bf_multiply_cmplx( a, &myb, c);
if( ! bf_divide( &c->real, &mag1, &c->real)) return 0;
bf_divide( &c->imag, &mag1, &c->imag);
return 1;
}
int mode_create(int argc, char** argv)
{
uint argo = 0;
uint progress = FALSE;
if (strcmp(argv[2], "-p") == 0) {
argo++;
progress = TRUE;
}
if (argc - argo < 5) {
show_help(argv[0]);
return 1;
}
char* bloom_file = argv[2 + argo];
double error_rate = atof(argv[3 + argo]);
bf_index_t key_count = atol(argv[4 + argo]);
bf_t* filter = bf_create(error_rate, key_count, BLOOM_HASH_FUNC);
char input_buffer[BF_KEY_BUFFER_SIZE] = { 0 };
bf_index_t key = 0;
uint actual_length;
while (gets_skip(input_buffer, BF_KEY_BUFFER_SIZE, &actual_length)) {
bf_add(filter, (const char*)input_buffer, actual_length);
key = key + 1;
if (progress && key % 1000000 == 0) {
printf("%s: %s <- %ld\n", argv[0], bloom_file, key);
}
}
bf_save(filter, bloom_file);
bf_destroy(filter);
return 0;
}
const char *CmdAddHandler(bloom_filter_t *bloom, const char element[]) {
bf_add(bloom, element);
return added_response;
}
main()
{
FLOAT o1, dcubed, n, *offset;
INDEX i, j, k, limit;
MULTIPOLY sigma3;
MULTIPOLY q24, tau1, tau2;
MULTIPOLY joftop, jofbot, joftau;
FLOAT *coef, *tsubj, *tnew, *prevc;
FLOAT bctop, bcbottom;
int shift, maxstore;
MULTIPOLY cheb[100];
struct
{
ELEMENT x, y;
COMPLEX start, jt;
} datablock;
FILE *svplot;
COMPLEX tau, jtau, arc[512], q, qn;
FLOAT theta, dtheta;
COMPLEX temp;
limit = 50;
maxstore = limit+5;
bf_init_ram_space();
bf_init_float();
/* create table of sigma_3(n) (sum of cube of all factors
of n).
*/
sigma3.degree = limit;
if( !bf_get_space( &sigma3))
{
printf("no space for that much data\n");
exit(0);
}
bf_one( &o1);
bf_null( &n);
for( i=1; i<limit; i++)
{
bf_add( &o1, &n, &n);
bf_multiply( &n, &n, &dcubed);
bf_multiply( &n, &dcubed, &dcubed);
for( j=i; j<limit; j+=i)
{
offset = Address( sigma3) + j;
bf_add( &dcubed, offset, offset);
}
}
/* save to disk here if necessary */
/* for( i=0; i<limit; i++)
{
tsubj = Address( sigma3) + i;
printf("i=%d\n", i);
printfloat("sigma3(i) = ", tsubj);
}
/* compute Ramanujan's tau function to some ridiculous degree.
First step is compute coefficients of (1-x)^24 using
binomial expansion.
*/
q24.degree = 24;
if( !bf_get_space( &q24))
{
printf("no room for binomial coefficients?\n");
exit(0);
}
coef = Address(q24);
bf_int_to_float( 1, coef);
bf_int_to_float( 24, &bctop);
bf_int_to_float( 1, &bcbottom);
for( i=1; i<=24; i++)
{
prevc = coef;
coef = Address(q24) + i;
bf_multiply( prevc, &bctop, coef);
bf_divide( coef, &bcbottom, coef);
bf_negate( coef);
bf_round( coef, coef);
/* printf("i= %d\n", i);
printfloat(" coef =", coef);
/* decrement top and increment bottom for next term
in binomial expansion */
bf_subtract( &bctop, &o1, &bctop);
bf_add( &o1, &bcbottom, &bcbottom);
}
/* now compute Ramanujan's tau.
Keep two versions, a source and a destination.
work back and forth multiplying source by
binomial coefficients and sum to power shifted
location. Seeking coefficients for each power
of q: q*product( 1- q^n)^24 n = 1 to infinity.
*/
tau1.degree = maxstore;
tau2.degree = maxstore;
if( !bf_get_space( &tau1))
{
printf("can't allocate first tau block.\n");
exit(0);
}
if( !bf_get_space( &tau2))
{
printf("can't allocate second tau block.\n");
exit(0);
}
coef = Address( q24);
tnew = Address( tau1);
bf_multi_copy( 25, coef, tnew);
k = 2;
while( k<maxstore ) /* loop over products */
{
/* multiply by 1, first coefficient of each product term */
prevc = Address( tau1);
tnew = Address( tau2);
bf_multi_copy( maxstore, prevc, tnew);
/* for each coefficient in 24 term product of next term,
multiply by every term in previous product and sum
to shifted location. */
for( i=1; i <= 24; i++)
{
coef = Address( q24) + i;
shift = k*i;
if( shift > maxstore) continue;
for( j=0; j<= k*24; j++)
{
if( j + shift > maxstore) continue;
tsubj = Address( tau1) + j;
tnew = Address( tau2) + j + shift;
bf_multiply( tsubj, coef, &bctop);
bf_add( &bctop, tnew, tnew);
bf_round( tnew, tnew);
}
}
k++;
/* now flip things over and go back to tau 1 */
if( k>maxstore) break;
prevc = Address( tau2);
tnew = Address( tau1);
bf_multi_copy( maxstore, prevc, tnew);
for( i=1; i<=24; i++)
{
coef = Address(q24) + i;
shift = k*i;
if( shift > maxstore) continue;
for( j=0; j<= k*24; j++)
{
if( j + shift > maxstore) continue;
tsubj = Address( tau2) + j;
tnew = Address( tau1) + j + shift;
bf_multiply( tsubj, coef, &bctop);
bf_add( &bctop, tnew, tnew);
bf_round( tnew, tnew);
}
}
k++;
}
/* compute top polynomial of joftau
(1 + 240*sum(sigma3(n)*q^n))^3 */
bf_multi_dup( sigma3, &joftop);
coef = Address( joftop);
bf_int_to_float( 1, coef);
bf_int_to_float( 240, &bctop);
for( i=1; i<limit; i++)
{
coef = Address( joftop) + i;
bf_multiply( &bctop, coef, coef);
}
bf_power_mul( joftop, joftop, &joftau);
bf_power_mul( joftop, joftau, &joftop);
/* finally compute joftau coefficients */
bf_power_div( joftop, tau1, &joftau);
/* for( i=0; i<limit; i++)
{
tsubj = Address( joftau) + i;
printf("i=%d\n", i);
printfloat("joftau(i) = ", tsubj);
}
*/
/* region F is defined as | Re(tau) | < 1/2 and || tau || > 1.
For each point tau in F, find j(tau).
save binary data to disk. Format is (x, y) and (start,
end) where x and y are integer indexes, and start, end
are complex points. Initial start is an arc along
tau = 1.
*/
svplot = fopen("joftau.complex", "wb");
if( !svplot)
{
printf( "can't create output file\n");
exit(0);
}
/* create an arc along bottom of F */
bf_copy( &P2, &theta);
theta.expnt += 2; // create 2PI/3
bf_int_to_float( 3, &n);
bf_divide( &theta, &n, &theta);
bf_copy( &theta, &dtheta);
dtheta.expnt--; // create PI/3/511
bf_int_to_float( gridsize-1, &n);
bf_divide( &dtheta, &n, &dtheta);
for( i=0; i<gridsize; i++)
{
bf_cosine( &theta, &arc[i].real);
bf_sine( &theta, &arc[i].imag);
bf_subtract( &theta, &dtheta, &theta);
}
/* Move arc up F, and find j(tau) for each point */
bf_int_to_float( 10, &bctop);
bf_int_to_float( gridsize, &n);
bf_divide( &bctop, &n, &bctop); // upper limit of F
for( i=0; i<gridsize; i++)
{
printf("i= %d\n", i);
bf_int_to_float(i, &n);
bf_multiply( &bctop, &n, &tau.imag);
bf_null( &tau.real);
datablock.y = i;
for( j=0; j<gridsize; j++)
{
datablock.x = j;
bf_add_cmplx( &tau, &arc[j], &datablock.start);
// print_cmplx("data block start", &datablock.start);
/* compute j(tau) for this point. Note power of q = index - 1 */
bf_firstj( &datablock.start, &q, &jtau);
// print_cmplx("q = exp(2 i PI tau)", &q);
// print_cmplx("first terms", &jtau);
bf_copy_cmplx( &q, &qn);
for( k=2; k<limit; k++)
{
offset = Address( joftau) + k;
bf_multiply( offset, &qn.real, &temp.real);
bf_multiply( offset, &qn.imag, &temp.imag);
bf_add_cmplx( &temp, &jtau, &jtau);
bf_multiply_cmplx( &q, &qn, &qn);
}
/* save data point to disk */
// print_cmplx("j(tau) = ", &jtau);
bf_copy_cmplx( &jtau, &datablock.jt);
if( fwrite( &datablock, sizeof( datablock), 1, svplot) <= 0)
printf("can't write to disk\n");
}
}
fclose( svplot);
printf("all done!\n\n");
}