static void
init_mtbl(int fd) {
struct mtbl_writer_options *writer_options = mtbl_writer_options_init();
assert(writer_options != NULL);
mtbl_writer_options_set_block_size(writer_options, 1024);
struct mtbl_writer *writer = mtbl_writer_init_fd(fd, writer_options);
assert(writer != NULL);
/* Populate the mtbl with hex(i)->i */
for (uint32_t i = 0; i < NUM_KEYS; i++) {
ubuf *key = ubuf_init(1);
ubuf *value = ubuf_init(1);
ubuf_add_fmt(key, KEY_FMT, i);
ubuf_add_fmt(value, VAL_FMT, i);
assert(mtbl_writer_add(writer, ubuf_data(key), ubuf_size(key), ubuf_data(value), ubuf_size(value)) == mtbl_res_success);
ubuf_destroy(&key);
ubuf_destroy(&value);
}
mtbl_writer_destroy(&writer);
mtbl_writer_options_destroy(&writer_options);
}
static int
test6(void)
{
int ret = 0;
ubuf *v = ubuf_init(16);
ubuf_append(v, (uint8_t *) str_array, sizeof(str_array));
ubuf_clip(v, 17);
if (ubuf_size(v) != 17)
ret |= 1;
ubuf_destroy(&v);
return (ret);
}
struct block_iter *
block_iter_init(struct block *b)
{
assert(b->size >= 2 * sizeof(uint32_t));
struct block_iter *bi = my_calloc(1, sizeof(*bi));
bi->block = b;
bi->data = b->data;
bi->restarts = b->restart_offset;
bi->num_restarts = num_restarts(b);
bi->current = bi->restarts;
bi->restart_index = bi->num_restarts;
assert(bi->num_restarts > 0);
bi->key = ubuf_init(64);
return (bi);
}
static int
test3(void)
{
int ret = 0;
ubuf *v;
v = ubuf_init(16);
ubuf_append(v, (uint8_t *) str_array, sizeof(str_array));
for (unsigned i = 0; i < ubuf_size(v); i++) {
//fprintf(stderr, "v->v[%d] = '%c'\n", i, v->v[i]);
if (ubuf_value(v, i) != str_array[i])
ret |= 1;
}
ubuf_destroy(&v);
return (ret);
}
static void *
thr_producer(void *arg)
{
struct producer *p = (struct producer *) arg;
memset(&p->pstat, 0, sizeof(p->pstat));
for (unsigned i = 0; i < p->num_messages; i++) {
fstrm_res res;
size_t len = 0;
uint8_t *message = NULL;
ubuf *u = ubuf_init(512);
unsigned ndups = (p->pstat.count_generated % 4) + 1;
for (unsigned j = 0; j < ndups; j++)
ubuf_add_cstr(u, test_string);
ubuf_detach(u, &message, &len);
ubuf_destroy(&u);
res = fstrm_iothr_submit(p->iothr, p->ioq,
message, len, fstrm_free_wrapper, NULL);
if (res == fstrm_res_success) {
p->pstat.count_submitted++;
p->pstat.bytes_submitted += len;
} else {
free(message);
}
p->pstat.count_generated++;
p->pstat.bytes_generated += len;
if ((i % 1000) == 0)
poll(NULL, 0, 1);
}
return NULL;
}
static int
test4(void)
{
int ret = 0;
ubuf *v;
v = ubuf_init(0);
for (unsigned i = 0; i < 1024; i++)
ubuf_append(v, (uint8_t *) str_array, sizeof(str_array));
for (unsigned i = 0; i < ubuf_size(v); i++) {
unsigned j = i % sizeof(str_array);
/*
fprintf(stderr, "v->v[%d] = '%c', str_array[%d] = '%c'\n",
i, v->v[i], j, str_array[j]);
*/
if (ubuf_value(v, i) != str_array[j])
ret |= 1;
}
ubuf_destroy(&v);
return (ret);
}
static void
init_dso(void)
{
merge_dso_path = getenv("MTBL_MERGE_DSO");
merge_dso_prefix = getenv("MTBL_MERGE_FUNC_PREFIX");
if (merge_dso_path == NULL) {
fprintf(stderr, "Error: MTBL_MERGE_DSO environment variable not set.\n\n");
usage();
}
if (merge_dso_prefix == NULL) {
fprintf(stderr, "Error: MTBL_MERGE_FUNC_PREFIX environment variable not set.\n\n");
usage();
}
dlerror();
void *handle = dlopen(merge_dso_path, RTLD_NOW);
if (handle == NULL) {
fprintf(stderr, "Error: dlopen() failed: %s\n", dlerror());
exit(EXIT_FAILURE);
}
/* merge func */
ubuf *func_name = ubuf_init(0);
ubuf_append(func_name,
(const uint8_t *) merge_dso_prefix,
strlen(merge_dso_prefix));
ubuf_append(func_name,
(const uint8_t *) "_func",
sizeof("_func"));
user_func_merge = dlsym(handle, (const char *) ubuf_data(func_name));
if (user_func_merge == NULL) {
fprintf(stderr, "Error: user merge function required but not found in DSO.\n\n");
usage();
}
/* init func */
ubuf_clip(func_name, 0);
ubuf_append(func_name,
(const uint8_t *) merge_dso_prefix,
strlen(merge_dso_prefix));
ubuf_append(func_name,
(const uint8_t *) "_init_func",
sizeof("_init_func"));
user_func_init = dlsym(handle, (const char *) ubuf_data(func_name));
if (user_func_init != NULL)
user_clos = user_func_init();
/* free func */
ubuf_clip(func_name, 0);
ubuf_append(func_name,
(const uint8_t *) merge_dso_prefix,
strlen(merge_dso_prefix));
ubuf_append(func_name,
(const uint8_t *) "_free_func",
sizeof("_free_func"));
user_func_free = dlsym(handle, (const char *) ubuf_data(func_name));
/* cleanup */
ubuf_destroy(&func_name);
}
static void
test_iter(struct mtbl_iter *iter)
{
/* Iterate completely through the mtbl */
for (uint32_t i = 0; i < NUM_KEYS; i++) {
ubuf *expected_key = ubuf_init(1);
ubuf_add_fmt(expected_key, KEY_FMT, i);
const uint8_t *key, *value;
size_t len_key, len_value;
assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_success);
assert(bytes_compare(ubuf_data(expected_key), ubuf_size(expected_key), key, len_key) == 0);
ubuf_destroy(&expected_key);
}
/* Ensure that we have completely iterated through the set. */
{
const uint8_t *key, *value;
size_t len_key, len_value;
assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_failure);
}
/* Seek to a key, ensure that we get the one we want and that we go
* all the way to the end. */
for (uint32_t i = NUM_KEYS; i-- > 0; ) {
ubuf *seek_key = ubuf_init(1);
ubuf_add_fmt(seek_key, KEY_FMT, i);
const uint8_t *key, *value;
size_t len_key, len_value;
assert(mtbl_iter_seek(iter, ubuf_data(seek_key), ubuf_size(seek_key)) == mtbl_res_success);
for (uint32_t j = i; j < NUM_KEYS; j++) {
ubuf *expected_key = ubuf_init(1);
ubuf_add_fmt(expected_key, KEY_FMT, j);
assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_success);
assert(bytes_compare(ubuf_data(expected_key), ubuf_size(expected_key), key, len_key) == 0);
ubuf_destroy(&expected_key);
}
assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_failure);
ubuf_destroy(&seek_key);
}
/* Attempt to seek past end of iterator */
ubuf *seek_key = ubuf_init(1);
const uint8_t *key, *value;
size_t len_key, len_value;
ubuf_add_fmt(seek_key, KEY_FMT, NUM_KEYS + 1);
assert(mtbl_iter_seek(iter, ubuf_data(seek_key), ubuf_size(seek_key)) == mtbl_res_success);
assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_failure);
}
void
my_fileset_reload(struct my_fileset *fs)
{
assert(fs != NULL);
struct fileset_entry *ent, **entptr;
entry_vec *new_entries;
FILE *fp;
char *fname, *line = NULL;
size_t len = 0;
ubuf *u;
if (!setfile_updated(fs))
return;
fp = fopen(fs->setfile, "r");
if (fp == NULL)
return;
u = ubuf_init(64);
new_entries = entry_vec_init(1);
while (getline(&line, &len, fp) != -1) {
ubuf_clip(u, 0);
if (line[0] != '/') { /* if not absolute path, prepend fileset file's path */
ubuf_add_cstr(u, fs->setdir);
ubuf_add(u, '/');
}
ubuf_add_cstr(u, line);
ubuf_rstrip(u, '\n');
fname = ubuf_cstr(u);
if (path_exists(fname)) {
entptr = fetch_entry(fs->entries, fname);
if (entptr == NULL) {
ent = my_calloc(1, sizeof(*ent));
ent->fname = my_strdup(fname);
if (fs->load)
ent->ptr = fs->load(fs, fname);
entry_vec_add(new_entries, ent);
} else {
ent = my_calloc(1, sizeof(*ent));
ent->fname = my_strdup(fname);
ent->ptr = (*entptr)->ptr;
(*entptr)->keep = true;
entry_vec_add(new_entries, ent);
}
}
}
free(line);
fclose(fp);
qsort(entry_vec_data(new_entries),
entry_vec_size(new_entries),
sizeof(void *),
cmp_fileset_entry);
for (size_t i = 0; i < entry_vec_size(fs->entries); i++) {
ent = entry_vec_value(fs->entries, i);
assert(ent != NULL);
if (ent->keep == false && fs->unload)
fs->unload(fs, ent->fname, ent->ptr);
free(ent->fname);
free(ent);
}
entry_vec_destroy(&fs->entries);
fs->entries = new_entries;
ubuf_destroy(&u);
}