bool is_negation_correct(roaring_bitmap_t *bitmap) {
roaring_statistics_t stats;
bool answer = true;
roaring_bitmap_statistics(bitmap, &stats);
unsigned universe_size = stats.max_value + 1;
roaring_bitmap_t *inverted = roaring_bitmap_flip(bitmap, 0U, universe_size);
roaring_bitmap_t *double_inverted =
roaring_bitmap_flip(inverted, 0U, universe_size);
answer = (roaring_bitmap_get_cardinality(inverted) +
roaring_bitmap_get_cardinality(bitmap) ==
universe_size);
if (answer) answer = roaring_bitmap_equals(bitmap, double_inverted);
if (!answer) {
printf("Bad flip\n\nbitmap1:\n");
show_structure(bitmap->high_low_container); // debug
printf("\n\nflipped:\n");
show_structure(inverted->high_low_container); // debug
}
roaring_bitmap_free(double_inverted);
roaring_bitmap_free(inverted);
return answer;
}
bool slow_bitmap_equals(roaring_bitmap_t *bitmap1, roaring_bitmap_t *bitmap2) {
uint64_t card1, card2;
card1 = roaring_bitmap_get_cardinality(bitmap1);
card2 = roaring_bitmap_get_cardinality(bitmap2);
uint32_t *arr1 = (uint32_t *)malloc(card1 * sizeof(uint32_t));
uint32_t *arr2 = (uint32_t *)malloc(card2 * sizeof(uint32_t));
roaring_bitmap_to_uint32_array(bitmap1, arr1);
roaring_bitmap_to_uint32_array(bitmap2, arr2);
bool answer = array_equals(arr1, card1, arr2, card2);
free(arr1);
free(arr2);
return answer;
}
bool is_intersection_correct(roaring_bitmap_t *bitmap1,
roaring_bitmap_t *bitmap2) {
roaring_bitmap_t *temp = roaring_bitmap_and(bitmap1, bitmap2);
uint64_t card1, card2, card;
card1 = roaring_bitmap_get_cardinality(bitmap1);
card2 = roaring_bitmap_get_cardinality(bitmap2);
card = roaring_bitmap_get_cardinality(temp);
uint32_t *arr1 = (uint32_t *)malloc(card1 * sizeof(uint32_t));
uint32_t *arr2 = (uint32_t *)malloc(card2 * sizeof(uint32_t));
uint32_t *arr = (uint32_t *)malloc(card * sizeof(uint32_t));
if ((arr1 == NULL) || (arr2 == NULL) || (arr == NULL)) {
free(arr1);
free(arr2);
free(arr);
return false;
}
roaring_bitmap_to_uint32_array(bitmap1, arr1);
roaring_bitmap_to_uint32_array(bitmap2, arr2);
roaring_bitmap_to_uint32_array(temp, arr);
uint32_t *buffer = (uint32_t *)malloc(sizeof(uint32_t) * (card1 + card2));
size_t cardtrue = intersection_uint32(arr1, card1, arr2, card2, buffer);
bool answer = array_equals(arr, card, buffer, cardtrue);
if (!answer) {
printf("\n\nbitmap1:\n");
show_structure(bitmap1->high_low_container); // debug
printf("\n\nbitmap2:\n");
show_structure(bitmap2->high_low_container); // debug
printf("\n\nresult:\n");
show_structure(temp->high_low_container); // debug
roaring_bitmap_t *ca = roaring_bitmap_of_ptr(cardtrue, buffer);
printf("\n\ncorrect result:\n");
show_structure(ca->high_low_container); // debug
free(ca);
}
free(buffer);
free(arr1);
free(arr2);
free(arr);
roaring_bitmap_free(temp);
return answer;
}
bool is_bitmap_equal_to_array(roaring_bitmap_t *bitmap, uint32_t *vals,
size_t numbers) {
uint64_t card;
card = roaring_bitmap_get_cardinality(bitmap);
uint32_t *arr = (uint32_t *)malloc(card * sizeof(uint32_t));
roaring_bitmap_to_uint32_array(bitmap, arr);
bool answer = array_equals(arr, card, vals, numbers);
free(arr);
return answer;
}
void test_example(bool copy_on_write) {
// create a new empty bitmap
roaring_bitmap_t *r1 = roaring_bitmap_create();
r1->copy_on_write = copy_on_write;
assert_ptr_not_equal(r1, NULL);
// then we can add values
for (uint32_t i = 100; i < 1000; i++) {
roaring_bitmap_add(r1, i);
}
// check whether a value is contained
assert_true(roaring_bitmap_contains(r1, 500));
// compute how many bits there are:
uint32_t cardinality = roaring_bitmap_get_cardinality(r1);
printf("Cardinality = %d \n", cardinality);
assert_int_equal(900, cardinality);
// if your bitmaps have long runs, you can compress them by calling
// run_optimize
uint32_t size = roaring_bitmap_portable_size_in_bytes(r1);
roaring_bitmap_run_optimize(r1);
uint32_t compact_size = roaring_bitmap_portable_size_in_bytes(r1);
printf("size before run optimize %d bytes, and after %d bytes\n", size,
compact_size);
// create a new bitmap with varargs
roaring_bitmap_t *r2 = roaring_bitmap_of(5, 1, 2, 3, 5, 6);
assert_ptr_not_equal(r2, NULL);
roaring_bitmap_printf(r2);
printf("\n");
// we can also create a bitmap from a pointer to 32-bit integers
const uint32_t values[] = {2, 3, 4};
roaring_bitmap_t *r3 = roaring_bitmap_of_ptr(3, values);
r3->copy_on_write = copy_on_write;
// we can also go in reverse and go from arrays to bitmaps
uint64_t card1 = roaring_bitmap_get_cardinality(r1);
uint32_t *arr1 = new uint32_t[card1];
assert_ptr_not_equal(arr1, NULL);
roaring_bitmap_to_uint32_array(r1, arr1);
roaring_bitmap_t *r1f = roaring_bitmap_of_ptr(card1, arr1);
delete[] arr1;
assert_ptr_not_equal(r1f, NULL);
// bitmaps shall be equal
assert_true(roaring_bitmap_equals(r1, r1f));
roaring_bitmap_free(r1f);
// we can copy and compare bitmaps
roaring_bitmap_t *z = roaring_bitmap_copy(r3);
z->copy_on_write = copy_on_write;
assert_true(roaring_bitmap_equals(r3, z));
roaring_bitmap_free(z);
// we can compute union two-by-two
roaring_bitmap_t *r1_2_3 = roaring_bitmap_or(r1, r2);
r1_2_3->copy_on_write = copy_on_write;
roaring_bitmap_or_inplace(r1_2_3, r3);
// we can compute a big union
const roaring_bitmap_t *allmybitmaps[] = {r1, r2, r3};
roaring_bitmap_t *bigunion = roaring_bitmap_or_many(3, allmybitmaps);
assert_true(roaring_bitmap_equals(r1_2_3, bigunion));
roaring_bitmap_t *bigunionheap =
roaring_bitmap_or_many_heap(3, allmybitmaps);
assert_true(roaring_bitmap_equals(r1_2_3, bigunionheap));
roaring_bitmap_free(r1_2_3);
roaring_bitmap_free(bigunion);
roaring_bitmap_free(bigunionheap);
// we can compute intersection two-by-two
roaring_bitmap_t *i1_2 = roaring_bitmap_and(r1, r2);
roaring_bitmap_free(i1_2);
// we can write a bitmap to a pointer and recover it later
uint32_t expectedsize = roaring_bitmap_portable_size_in_bytes(r1);
char *serializedbytes = (char *)malloc(expectedsize);
roaring_bitmap_portable_serialize(r1, serializedbytes);
roaring_bitmap_t *t = roaring_bitmap_portable_deserialize(serializedbytes);
assert_true(expectedsize == roaring_bitmap_portable_size_in_bytes(t));
assert_true(roaring_bitmap_equals(r1, t));
roaring_bitmap_free(t);
free(serializedbytes);
// we can iterate over all values using custom functions
uint32_t counter = 0;
roaring_iterate(r1, roaring_iterator_sumall, &counter);
/**
* void roaring_iterator_sumall(uint32_t value, void *param) {
* *(uint32_t *) param += value;
* }
*
*/
roaring_bitmap_free(r1);
roaring_bitmap_free(r2);
roaring_bitmap_free(r3);
}
bool loadAndCheckAll(const char *dirname, bool copy_on_write) {
printf("[%s] %s datadir=%s %s\n", __FILE__, __func__, dirname,
copy_on_write ? "copy-on-write" : "hard-copies");
char *extension = ".txt";
size_t count;
size_t *howmany = NULL;
uint32_t **numbers =
read_all_integer_files(dirname, extension, &howmany, &count);
if (numbers == NULL) {
printf(
"I could not find or load any data file with extension %s in "
"directory %s.\n",
extension, dirname);
return false;
}
roaring_bitmap_t **bitmaps =
create_all_bitmaps(howmany, numbers, count, copy_on_write);
for (size_t i = 0; i < count; i++) {
if (!is_bitmap_equal_to_array(bitmaps[i], numbers[i], howmany[i])) {
printf("arrays don't agree with set values\n");
return false;
}
}
roaring_bitmap_t **bitmapswrun = malloc(sizeof(roaring_bitmap_t *) * count);
for (int i = 0; i < (int)count; i++) {
bitmapswrun[i] = roaring_bitmap_copy(bitmaps[i]);
roaring_bitmap_run_optimize(bitmapswrun[i]);
if (roaring_bitmap_get_cardinality(bitmaps[i]) !=
roaring_bitmap_get_cardinality(bitmapswrun[i])) {
printf("cardinality change due to roaring_bitmap_run_optimize\n");
return false;
}
}
for (size_t i = 0; i < count; i++) {
if (!is_bitmap_equal_to_array(bitmapswrun[i], numbers[i], howmany[i])) {
printf("arrays don't agree with set values\n");
return false;
}
}
for (int i = 0; i < (int)count; i++) {
if (!serialize_correctly(bitmaps[i])) {
return false; // memory leaks
}
if (!serialize_correctly(bitmapswrun[i])) {
return false; // memory leaks
}
}
if (!compare_intersections(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_unions(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_wide_unions(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_negations(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_xors(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_andnots(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
if (!compare_wide_xors(bitmaps, bitmapswrun, count)) {
return false; // memory leaks
}
for (int i = 0; i < (int)count; ++i) {
free(numbers[i]);
numbers[i] = NULL; // paranoid
roaring_bitmap_free(bitmaps[i]);
bitmaps[i] = NULL; // paranoid
roaring_bitmap_free(bitmapswrun[i]);
bitmapswrun[i] = NULL; // paranoid
}
free(bitmapswrun);
free(bitmaps);
free(howmany);
free(numbers);
return true;
}
int main(int argc, char **argv) {
int c;
char *extension = ".txt";
bool copy_on_write = false;
while ((c = getopt(argc, argv, "e:h")) != -1) switch (c) {
case 'e':
extension = optarg;
break;
case 'h':
printusage(argv[0]);
return 0;
default:
abort();
}
if (optind >= argc) {
printusage(argv[0]);
return -1;
}
char *dirname = argv[optind];
size_t count;
size_t *howmany = NULL;
uint32_t **numbers =
read_all_integer_files(dirname, extension, &howmany, &count);
if (numbers == NULL) {
printf(
"I could not find or load any data file with extension %s in "
"directory %s.\n",
extension, dirname);
return -1;
}
uint64_t cycles_start = 0, cycles_final = 0;
RDTSC_START(cycles_start);
roaring_bitmap_t **bitmaps = create_all_bitmaps(howmany, numbers, count, copy_on_write);
RDTSC_FINAL(cycles_final);
if (bitmaps == NULL) return -1;
printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname);
printf("Creating %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count; i += 2) {
roaring_bitmap_t *CI = roaring_bitmap_copy(
bitmaps[i]); // to test the inplace version we create a copy
roaring_bitmap_free(CI);
}
RDTSC_FINAL(cycles_final);
printf("Copying and freeing %zu bitmaps took %" PRIu64 " cycles\n", count,
cycles_final - cycles_start);
uint64_t successive_and = 0;
uint64_t successive_or = 0;
// try ANDing and ORing together consecutive pairs
for (int i = 0; i < (int)count - 1; ++i) {
uint32_t c1 = roaring_bitmap_get_cardinality(bitmaps[i]);
uint32_t c2 = roaring_bitmap_get_cardinality(bitmaps[i + 1]);
RDTSC_START(cycles_start);
roaring_bitmap_t *tempand =
roaring_bitmap_and(bitmaps[i], bitmaps[i + 1]);
RDTSC_FINAL(cycles_final);
successive_and += cycles_final - cycles_start;
uint32_t ci = roaring_bitmap_get_cardinality(tempand);
roaring_bitmap_free(tempand);
RDTSC_START(cycles_start);
roaring_bitmap_t *tempor =
roaring_bitmap_or(bitmaps[i], bitmaps[i + 1]);
RDTSC_FINAL(cycles_final);
successive_or += cycles_final - cycles_start;
uint32_t co = roaring_bitmap_get_cardinality(tempor);
roaring_bitmap_free(tempor);
if (c1 + c2 != co + ci) {
printf(KRED "cardinalities are wrong somehow\n");
printf("c1 = %d, c2 = %d, co = %d, ci = %d\n", c1, c2, co, ci);
return -1;
}
}
printf(" %zu successive bitmaps intersections took %" PRIu64 " cycles\n",
count - 1, successive_and);
printf(" %zu successive bitmaps unions took %" PRIu64 " cycles\n",
count - 1, successive_or);
roaring_bitmap_t **copyofr = malloc(sizeof(roaring_bitmap_t *) * count);
for (int i = 0; i < (int)count; i++) {
copyofr[i] = roaring_bitmap_copy(bitmaps[i]);
}
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; i++) {
roaring_bitmap_and_inplace(copyofr[i], bitmaps[i + 1]);
}
RDTSC_FINAL(cycles_final);
printf(" %zu successive in-place bitmaps intersections took %" PRIu64
" cycles\n",
count - 1, cycles_final - cycles_start);
free(copyofr);
copyofr = malloc(sizeof(roaring_bitmap_t *) * count);
for (int i = 0; i < (int)count; i++) {
copyofr[i] = roaring_bitmap_copy(bitmaps[i]);
}
RDTSC_START(cycles_start);
for (int i = 0; i < (int)count - 1; i++) {
roaring_bitmap_or_inplace(copyofr[i], bitmaps[i + 1]);
}
RDTSC_FINAL(cycles_final);
printf(" %zu successive in-place bitmaps unions took %" PRIu64 " cycles\n",
count - 1, cycles_final - cycles_start);
for (int i = 0; i < (int)count; ++i) {
free(numbers[i]);
numbers[i] = NULL; // paranoid
roaring_bitmap_free(bitmaps[i]);
bitmaps[i] = NULL; // paranoid
}
free(bitmaps);
free(howmany);
free(numbers);
return 0;
}
