int main() {
GC_INIT();
randomize_rand();
int fail = 0;
const RRB *rrb = rrb_create();
for (uint32_t i = 0; i < SIZE; i++) {
rrb = rrb_push(rrb, (void *)((intptr_t) rand() % 10000));
}
const RRB **sliced = GC_MALLOC(sizeof(RRB *) * SLICED);
for (uint32_t i = 0; i < SLICED; i++) {
uint32_t from = (uint32_t) rand() % SIZE;
uint32_t to = (uint32_t) (rand() % (SIZE - from)) + from;
sliced[i] = rrb_slice(rrb, from, to);
}
for (uint32_t i = 0; i < CATTED; i++) {
const RRB *multicat = rrb_create(); // Originally empty
uint32_t tot_cats = (uint32_t) rand() % TOT_CATTED;
uint32_t *merged_in = GC_MALLOC_ATOMIC(sizeof(uint32_t) * tot_cats);
for (uint32_t cat_num = 0; cat_num < tot_cats; cat_num++) {
merged_in[cat_num] = (uint32_t) rand() % SLICED;
multicat = rrb_concat(multicat, sliced[merged_in[cat_num]]);
}
// checking consistency here
uint32_t pos = 0;
uint32_t merged_pos = 0;
while (pos < rrb_count(multicat)) {
const RRB *merged = sliced[merged_in[merged_pos]];
for (uint32_t merged_i = 0; merged_i < rrb_count(merged);
merged_i++, pos++) {
intptr_t expected = (intptr_t) rrb_nth(merged, merged_i);
intptr_t actual = (intptr_t) rrb_nth(multicat, pos);
if (expected != actual) {
printf("On multicatted object #%u, at merged element %u:\n",
i, merged_pos);
printf(" Expected val at pos %u (%u in merged) to be %ld, but was %ld\n",
pos, merged_i, expected, actual);
printf(" Size of merged: %u, is at index %u\n", rrb_count(merged),
merged_in[merged_pos]);
printf("Sliced lists:\n");
fail = 1;
return fail;
}
}
merged_pos++;
}
}
return fail;
}
int main() {
GC_INIT();
const RRB *rrb = rrb_create();
for (uintptr_t i = 0; i < 33; i++) {
rrb = rrb_push(rrb, (void *) i);
}
int file_size = rrb_to_dot_file(rrb, "foo.dot");
switch (file_size) {
case -1:
puts("Had trouble either opening or closing \"foo.dot\".");
return 1;
case -2:
puts("Had trouble writing to the file \"foo.dot\".");
return 1;
default:
printf("The file \"foo.dot\" contains an RRB-tree in dot format "
"(%d bytes)\n", file_size);
if (RRB_BITS != 2) {
puts("\n"
"(If you want to use the code for visualising RRB-trees, I'd recommend to use the\n"
" settings `CFLAGS='-Ofast' ./configure --with-branching=2` instead. It is much\n"
" easier to visualise/comprehend with 4 elements per trie node instead of 32.)");
}
return 0;
}
}
int main() {
GC_INIT();
randomize_rand();
int fail = 0;
intptr_t *list = GC_MALLOC_ATOMIC(sizeof(intptr_t) * SIZE);
for (uint32_t i = 0; i < SIZE; i++) {
list[i] = (intptr_t) rand();
}
const RRB *rrb = rrb_create();
for (uint32_t i = 0; i < SIZE; i++) {
rrb = rrb_push(rrb, (void *) list[i]);
intptr_t val = (intptr_t) rrb_peek(rrb);
if (val != list[i]) {
printf("Expected val at pos %d to be %ld, was %ld.\n", i, list[i], val);
fail = 1;
}
}
return fail;
}
int main(int argc, char *argv[]) {
GC_INIT();
// CLI argument parsing
if (argc != 4) {
fprintf(stderr, "Expected 3 arguments, got %d\nExiting...\n", argc - 1);
exit(1);
}
else {
char *end;
uint32_t thread_count = (uint32_t) strtol(argv[1], &end, 10);
if (*end) {
fprintf(stderr, "Error, expects first argument to be a power of two,"
" was '%s'.\n", argv[1]);
exit(1);
}
if (!(thread_count && !(thread_count & (thread_count - 1)))) {
fprintf(stderr, "Error, first argument must be a power of two, not %d.\n",
thread_count);
exit(1);
}
fprintf(stderr, "Looking for '%s' in file %s using %d threads...\n",
argv[2], argv[3], thread_count);
char *search_term = argv[2];
FILE *fp;
long file_size;
char *buffer;
fp = fopen(argv[3], "rb");
if (!fp) {
perror(argv[1]);
exit(1);
}
fseek(fp, 0, SEEK_END);
file_size = ftell(fp);
rewind(fp);
buffer = malloc(file_size + 1);
if (!buffer) {
fclose(fp);
fprintf(stderr, "Cannot allocate buffer for file (probably too large).\n");
exit(1);
}
buffer[file_size] = 0;
if (1 != fread(buffer, file_size, 1, fp)) {
fclose(fp);
free(buffer);
fprintf(stderr, "Entire read failed.\n");
exit(1);
}
else {
fclose(fp);
}
// Find all lines
pthread_t *tid = malloc(thread_count * sizeof(pthread_t));
LineSplitArgs *lsa = malloc(thread_count * sizeof(LineSplitArgs));
const RRB **intervals = GC_MALLOC(thread_count * sizeof(RRB *));
for (uint32_t i = 0; i < thread_count; i++) {
LineSplitArgs arguments =
{ .buffer = buffer, .file_size = (uint32_t) file_size,
.own_tid = i, .thread_count = thread_count,
.intervals = intervals
};
lsa[i] = arguments;
}
for (uint32_t i = 0; i < thread_count; i++) {
pthread_create(&tid[i], NULL, &split_to_lines, (void *) &lsa[i]);
}
for (uint32_t i = 0; i < thread_count; i++) {
pthread_join(tid[i], NULL);
}
free(lsa);
// Concatenate work
ConcatArgs *ca = malloc(thread_count * sizeof(ConcatArgs));
pthread_barrier_t *barriers = malloc(thread_count * sizeof(pthread_barrier_t));
uint32_t *max_concat_size = calloc(sizeof(uint32_t), thread_count);
for (uint32_t i = 0; i < thread_count; i++) {
pthread_barrier_init(&barriers[i], NULL, 2);
ConcatArgs args = {.own_tid = i, .thread_count = thread_count,
.intervals = intervals, .barriers = barriers,
.max_concat_size = max_concat_size
};
ca[i] = args;
}
for (uint32_t i = 0; i < thread_count; i++) {
pthread_create(&tid[i], NULL, &concatenate_rrbs, (void *) &ca[i]);
}
for (uint32_t i = 0; i < thread_count; i++) {
pthread_join(tid[i], NULL);
}
const RRB *lines = intervals[0];
// Found all lines, now onto searching in each list
fprintf(stderr, "%d newlines\n", rrb_count(intervals[0]));
FilterArgs *fa = malloc(thread_count * sizeof(FilterArgs));
// Reuse intervals array
for (uint32_t i = 0; i < thread_count; i++) {
FilterArgs arguments =
{ .buffer = buffer, .search_term = search_term,
.lines = lines, .intervals = intervals,
.own_tid = i, .thread_count = thread_count
};
fa[i] = arguments;
}
for (uint32_t i = 0; i < thread_count; i++) {
pthread_create(&tid[i], NULL, &filter_by_term, (void *) &fa[i]);
}
for (uint32_t i = 0; i < thread_count; i++) {
pthread_join(tid[i], NULL);
}
// find total memory usage
uint32_t total_mem = rrb_memory_usage(intervals, thread_count);
// Concatenate work
// Reuse concat args and barriers
for (uint32_t i = 0; i < thread_count; i++) {
pthread_barrier_init(&barriers[i], NULL, 2);
ConcatArgs args = {.own_tid = i, .thread_count = thread_count,
.intervals = intervals, .barriers = barriers,
.max_concat_size = max_concat_size
};
ca[i] = args;
}
for (uint32_t i = 0; i < thread_count; i++) {
pthread_create(&tid[i], NULL, &concatenate_rrbs, (void *) &ca[i]);
}
for (uint32_t i = 0; i < thread_count; i++) {
pthread_join(tid[i], NULL);
}
uint32_t maxcat = 0;
for (uint32_t i = 0; i < thread_count; i++) {
maxcat = MAX(maxcat, max_concat_size[i]);
}
fprintf(stderr, "%d hits\n", rrb_count(intervals[0]));
printf("%d %d \"%s\"\n", total_mem, maxcat, search_term);
free(barriers);
free(buffer);
free(max_concat_size);
exit(0);
}
}
static void* split_to_lines(void *void_input) {
LineSplitArgs *lsa = (LineSplitArgs *) void_input;
const char *buffer = lsa->buffer;
const uint32_t own_tid = lsa->own_tid;
const uint32_t file_size = lsa->file_size;
const uint32_t thread_count = lsa->thread_count;
const RRB **intervals = lsa->intervals;
// calculate the interval to compute for
uint32_t partition_size = file_size / thread_count;
uint32_t from = partition_size * own_tid;
uint32_t to = partition_size * (own_tid + 1);
if (own_tid + 1 == thread_count) {
to = file_size;
}
// find the lines
const RRB *lines = rrb_create();
// rewind to start of line
uint32_t line_start = from;
while (0 < line_start && lsa->buffer[line_start] != '\n') {
line_start--;
}
// find and collect lines
for (uint32_t i = from; i < to; i++) {
if (buffer[i] == '\n') {
Interval interval = {.from = line_start, .to = i};
lines = rrb_push(lines, (void *) interval_to_uint64_t(interval));
line_start = i + 1;
}
}