int main(int argc, char **argv) {
int c;
const char *extension = ".txt";
bool verbose = false;
uint64_t data[13];
initializeMemUsageCounter();
while ((c = getopt(argc, argv, "ve:h")) != -1) switch (c) {
case 'e':
extension = optarg;
break;
case 'v':
verbose = true;
break;
case 'h':
printusage(argv[0]);
return 0;
default:
abort();
}
if (optind >= argc) {
printusage(argv[0]);
return -1;
}
char *dirname = argv[optind];
size_t count;
size_t *howmany = NULL;
uint32_t **numbers =
read_all_integer_files(dirname, extension, &howmany, &count);
if (numbers == NULL) {
printf(
"I could not find or load any data file with extension %s in "
"directory %s.\n",
extension, dirname);
return -1;
}
uint32_t maxvalue = 0;
for (size_t i = 0; i < count; i++) {
if( howmany[i] > 0 ) {
if(maxvalue < numbers[i][howmany[i]-1]) {
maxvalue = numbers[i][howmany[i]-1];
}
}
}
uint64_t totalcard = 0;
for (size_t i = 0; i < count; i++) {
totalcard += howmany[i];
}
uint64_t successivecard = 0;
for (size_t i = 1; i < count; i++) {
successivecard += howmany[i-1] + howmany[i];
}
uint64_t cycles_start = 0, cycles_final = 0;
RDTSC_START(cycles_start);
std::vector<vector> bitmaps = create_all_bitmaps(howmany, numbers, count);
RDTSC_FINAL(cycles_final);
if (bitmaps.empty()) return -1;
if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname);
uint64_t totalsize = getMemUsageInBytes();
data[0] = totalsize;
if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize);
uint64_t successive_and = 0;
uint64_t successive_or = 0;
uint64_t total_or = 0;
uint64_t total_count = 0;
uint64_t successive_andnot = 0;
uint64_t successive_xor = 0;
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; ++i) {
vector v;
std::set_intersection(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),std::back_inserter(v));
successive_and += v.size();
}
RDTSC_FINAL(cycles_final);
data[1] = cycles_final - cycles_start;
if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; ++i) {
vector v;
std::set_union(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),std::back_inserter(v));
successive_or += v.size();
}
RDTSC_FINAL(cycles_final);
data[2] = cycles_final - cycles_start;
if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
RDTSC_START(cycles_start);
if(count>1) {
vector v;
std::set_union(bitmaps[0].begin(), bitmaps[0].end(),bitmaps[1].begin(), bitmaps[1].end(),std::back_inserter(v));
for (int i = 2; i < (int)count ; ++i) {
vector newv;
std::set_union(v.begin(), v.end(),bitmaps[i].begin(), bitmaps[i].end(),std::back_inserter(newv));
v.swap(newv);
}
total_or = v.size();
}
RDTSC_FINAL(cycles_final);
data[3] = cycles_final - cycles_start;
if(verbose) printf("Total naive unions on %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
RDTSC_START(cycles_start);
if(count>1) {
const vector ** allofthem = new const vector* [count];
for(int i = 0 ; i < (int) count; ++i) allofthem[i] = & bitmaps[i];
vector totalorbitmap = fast_logicalor(count, allofthem);
total_or = totalorbitmap.size();
delete[] allofthem;
}
RDTSC_FINAL(cycles_final);
data[4] = cycles_final - cycles_start;
if(verbose) printf("Total heap unions on %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
RDTSC_START(cycles_start);
uint64_t quartcount = 0;
for (size_t i = 0; i < count ; ++i) {
if ( std::binary_search(bitmaps[i].begin(),bitmaps[i].end(),maxvalue/4 ) )
quartcount ++;
if ( std::binary_search(bitmaps[i].begin(),bitmaps[i].end(),maxvalue/2 ) )
quartcount ++;
if ( std::binary_search(bitmaps[i].begin(),bitmaps[i].end(),3*maxvalue/4 ) )
quartcount ++;
}
RDTSC_FINAL(cycles_final);
data[5] = cycles_final - cycles_start;
if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount);
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; ++i) {
vector v;
std::set_difference(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),std::back_inserter(v));
successive_andnot += v.size();
}
RDTSC_FINAL(cycles_final);
data[6] = cycles_final - cycles_start;
if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; ++i) {
vector v;
std::set_symmetric_difference(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),std::back_inserter(v));
successive_xor += v.size();
}
RDTSC_FINAL(cycles_final);
data[7] = cycles_final - cycles_start;
if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
RDTSC_START(cycles_start);
for (size_t i = 0; i < count; ++i) {
vector & b = bitmaps[i];
for(auto j = b.begin(); j != b.end() ; j++) {
total_count++;
}
}
RDTSC_FINAL(cycles_final);
data[8] = cycles_final - cycles_start;
assert(total_count == totalcard);
if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
assert(successive_xor + successive_and == successive_or);
/**
* and, or, andnot and xor cardinality
*/
uint64_t successive_andcard = 0;
uint64_t successive_orcard = 0;
uint64_t successive_andnotcard = 0;
uint64_t successive_xorcard = 0;
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; ++i) {
std::set_intersection(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),inserter(successive_andcard));
}
RDTSC_FINAL(cycles_final);
data[9] = cycles_final - cycles_start;
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; ++i) {
std::set_union(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),inserter(successive_orcard));
}
RDTSC_FINAL(cycles_final);
data[10] = cycles_final - cycles_start;
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; ++i) {
std::set_difference(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),inserter(successive_andnotcard));
}
RDTSC_FINAL(cycles_final);
data[11] = cycles_final - cycles_start;
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; ++i) {
std::set_symmetric_difference(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),inserter(successive_xorcard));
}
RDTSC_FINAL(cycles_final);
data[12] = cycles_final - cycles_start;
assert(successive_andcard == successive_and);
assert(successive_orcard == successive_or);
assert(successive_xorcard == successive_xor);
assert(successive_andnotcard == successive_andnot);
/**
* end and, or, andnot and xor cardinality
*/
printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n",
data[0]*8.0/totalcard,
data[1]*1.0/successivecard,
data[2]*1.0/successivecard,
data[3]*1.0/totalcard,
data[4]*1.0/totalcard,
data[5]*1.0/(3*count),
data[6]*1.0/successivecard,
data[7]*1.0/successivecard,
data[8]*1.0/totalcard,
data[9]*1.0/successivecard,
data[10]*1.0/successivecard,
data[11]*1.0/successivecard,
data[12]*1.0/successivecard
);
for (int i = 0; i < (int)count; ++i) {
free(numbers[i]);
numbers[i] = NULL; // paranoid
}
free(howmany);
free(numbers);
return 0;
}
bool loadAndCheckAll(const char *dirname, bool copy_on_write) {
printf("[%s] %s datadir=%s %s\n", __FILE__, __func__, dirname,
copy_on_write ? "copy-on-write" : "hard-copies");
char *extension = ".txt";
size_t count;
size_t *howmany = NULL;
uint32_t **numbers =
read_all_integer_files(dirname, extension, &howmany, &count);
if (numbers == NULL) {
printf(
"I could not find or load any data file with extension %s in "
"directory %s.\n",
extension, dirname);
return false;
}
roaring_bitmap_t **bitmaps =
create_all_bitmaps(howmany, numbers, count, copy_on_write);
for (size_t i = 0; i < count; i++) {
if (!is_bitmap_equal_to_array(bitmaps[i], numbers[i], howmany[i])) {
printf("arrays don't agree with set values\n");
return false;
}
}
roaring_bitmap_t **bitmapswrun = malloc(sizeof(roaring_bitmap_t *) * count);
for (int i = 0; i < (int)count; i++) {
bitmapswrun[i] = roaring_bitmap_copy(bitmaps[i]);
roaring_bitmap_run_optimize(bitmapswrun[i]);
if (roaring_bitmap_get_cardinality(bitmaps[i]) !=
roaring_bitmap_get_cardinality(bitmapswrun[i])) {
printf("cardinality change due to roaring_bitmap_run_optimize\n");
return false;
}
}
for (size_t i = 0; i < count; i++) {
if (!is_bitmap_equal_to_array(bitmapswrun[i], numbers[i], howmany[i])) {
printf("arrays don't agree with set values\n");
return false;
}
}
for (int i = 0; i < (int)count; i++) {
if (!serialize_correctly(bitmaps[i])) {
return false; // memory leaks
}
if (!serialize_correctly(bitmapswrun[i])) {
return false; // memory leaks
}
}
if (!compare_intersections(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_unions(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_wide_unions(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_negations(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_xors(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_andnots(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_wide_xors(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
for (int i = 0; i < (int)count; ++i) {
free(numbers[i]);
numbers[i] = NULL; // paranoid
roaring_bitmap_free(bitmaps[i]);
bitmaps[i] = NULL; // paranoid
roaring_bitmap_free(bitmapswrun[i]);
bitmapswrun[i] = NULL; // paranoid
}
free(bitmapswrun);
free(bitmaps);
free(howmany);
free(numbers);
return true;
}
int main(int argc, char **argv) {
int c;
const char *extension = ".txt";
bool copy_on_write = false;
bool runoptimize = true;
while ((c = getopt(argc, argv, "e:h")) != -1) switch (c) {
case 'e':
extension = optarg;
break;
case 'h':
printusage(argv[0]);
return 0;
default:
abort();
}
if (optind >= argc) {
printusage(argv[0]);
return -1;
}
char *dirname = argv[optind];
size_t count;
size_t *howmany = NULL;
uint32_t **numbers =
read_all_integer_files(dirname, extension, &howmany, &count);
if (numbers == NULL) {
printf(
"I could not find or load any data file with extension %s in "
"directory %s.\n",
extension, dirname);
return -1;
}
uint64_t cycles_start = 0, cycles_final = 0;
RDTSC_START(cycles_start);
roaring_bitmap_t **bitmaps =
create_all_bitmaps(howmany, numbers, count, runoptimize, copy_on_write);
RDTSC_FINAL(cycles_final);
if (bitmaps == NULL) return -1;
printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname);
printf("Creating %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
if(count == 0) return -1;
uint32_t maxvalue = roaring_bitmap_maximum(bitmaps[0]);
for (int i = 1; i < (int)count; i ++) {
uint32_t thismax = roaring_bitmap_maximum(bitmaps[0]);
if(thismax > maxvalue) maxvalue = thismax;
}
const int quartile_test_repetitions = 1000;
uint64_t quartcount;
uint64_t cycles;
STARTBEST(quartile_test_repetitions)
quartcount = 0;
for (size_t i = 0; i < count ; ++i) {
quartcount += roaring_bitmap_contains(bitmaps[i],maxvalue/4);
quartcount += roaring_bitmap_contains(bitmaps[i],maxvalue/2);
quartcount += roaring_bitmap_contains(bitmaps[i],3*maxvalue/4);
}
ENDBEST(cycles)
printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles);
for (int i = 0; i < (int)count; ++i) {
free(numbers[i]);
numbers[i] = NULL; // paranoid
roaring_bitmap_free(bitmaps[i]);
bitmaps[i] = NULL; // paranoid
}
free(bitmaps);
free(howmany);
free(numbers);
return (int) quartcount;
}
int main(int argc, char **argv) {
int c;
char *extension = ".txt";
bool copy_on_write = false;
while ((c = getopt(argc, argv, "e:h")) != -1) switch (c) {
case 'e':
extension = optarg;
break;
case 'h':
printusage(argv[0]);
return 0;
default:
abort();
}
if (optind >= argc) {
printusage(argv[0]);
return -1;
}
char *dirname = argv[optind];
size_t count;
size_t *howmany = NULL;
uint32_t **numbers =
read_all_integer_files(dirname, extension, &howmany, &count);
if (numbers == NULL) {
printf(
"I could not find or load any data file with extension %s in "
"directory %s.\n",
extension, dirname);
return -1;
}
uint64_t cycles_start = 0, cycles_final = 0;
RDTSC_START(cycles_start);
roaring_bitmap_t **bitmaps = create_all_bitmaps(howmany, numbers, count, copy_on_write);
RDTSC_FINAL(cycles_final);
if (bitmaps == NULL) return -1;
printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname);
printf("Creating %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count; i += 2) {
roaring_bitmap_t *CI = roaring_bitmap_copy(
bitmaps[i]); // to test the inplace version we create a copy
roaring_bitmap_free(CI);
}
RDTSC_FINAL(cycles_final);
printf("Copying and freeing %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
uint64_t successive_and = 0;
uint64_t successive_or = 0;
// try ANDing and ORing together consecutive pairs
for (int i = 0; i < (int)count - 1; ++i) {
uint32_t c1 = roaring_bitmap_get_cardinality(bitmaps[i]);
uint32_t c2 = roaring_bitmap_get_cardinality(bitmaps[i + 1]);
RDTSC_START(cycles_start);
roaring_bitmap_t *tempand =
roaring_bitmap_and(bitmaps[i], bitmaps[i + 1]);
RDTSC_FINAL(cycles_final);
successive_and += cycles_final - cycles_start;
uint32_t ci = roaring_bitmap_get_cardinality(tempand);
roaring_bitmap_free(tempand);
RDTSC_START(cycles_start);
roaring_bitmap_t *tempor =
roaring_bitmap_or(bitmaps[i], bitmaps[i + 1]);
RDTSC_FINAL(cycles_final);
successive_or += cycles_final - cycles_start;
uint32_t co = roaring_bitmap_get_cardinality(tempor);
roaring_bitmap_free(tempor);
if (c1 + c2 != co + ci) {
printf(KRED "cardinalities are wrong somehow\n");
printf("c1 = %d, c2 = %d, co = %d, ci = %d\n", c1, c2, co, ci);
return -1;
}
}
printf(" %zu successive bitmaps intersections took %" PRIu64 " cycles\n",
count - 1, successive_and);
printf(" %zu successive bitmaps unions took %" PRIu64 " cycles\n",
count - 1, successive_or);
roaring_bitmap_t **copyofr = malloc(sizeof(roaring_bitmap_t *) * count);
for (int i = 0; i < (int)count; i++) {
copyofr[i] = roaring_bitmap_copy(bitmaps[i]);
}
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; i++) {
roaring_bitmap_and_inplace(copyofr[i], bitmaps[i + 1]);
}
RDTSC_FINAL(cycles_final);
printf(" %zu successive in-place bitmaps intersections took %" PRIu64
" cycles\n",
count - 1, cycles_final - cycles_start);
free(copyofr);
copyofr = malloc(sizeof(roaring_bitmap_t *) * count);
for (int i = 0; i < (int)count; i++) {
copyofr[i] = roaring_bitmap_copy(bitmaps[i]);
}
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; i++) {
roaring_bitmap_or_inplace(copyofr[i], bitmaps[i + 1]);
}
RDTSC_FINAL(cycles_final);
printf(" %zu successive in-place bitmaps unions took %" PRIu64 " cycles\n",
count - 1, cycles_final - cycles_start);
for (int i = 0; i < (int)count; ++i) {
free(numbers[i]);
numbers[i] = NULL; // paranoid
roaring_bitmap_free(bitmaps[i]);
bitmaps[i] = NULL; // paranoid
}
free(bitmaps);
free(howmany);
free(numbers);
return 0;
}