static void
init_mtbl(int fd) {
struct mtbl_writer_options *writer_options = mtbl_writer_options_init();
assert(writer_options != NULL);
mtbl_writer_options_set_block_size(writer_options, 1024);
struct mtbl_writer *writer = mtbl_writer_init_fd(fd, writer_options);
assert(writer != NULL);
/* Populate the mtbl with hex(i)->i */
for (uint32_t i = 0; i < NUM_KEYS; i++) {
ubuf *key = ubuf_init(1);
ubuf *value = ubuf_init(1);
ubuf_add_fmt(key, KEY_FMT, i);
ubuf_add_fmt(value, VAL_FMT, i);
assert(mtbl_writer_add(writer, ubuf_data(key), ubuf_size(key), ubuf_data(value), ubuf_size(value)) == mtbl_res_success);
ubuf_destroy(&key);
ubuf_destroy(&value);
}
mtbl_writer_destroy(&writer);
mtbl_writer_options_destroy(&writer_options);
}
static void
init_mtbl(void)
{
struct mtbl_merger_options *mopt;
struct mtbl_writer_options *wopt;
mopt = mtbl_merger_options_init();
wopt = mtbl_writer_options_init();
mtbl_merger_options_set_merge_func(mopt, merge_func, user_clos);
mtbl_writer_options_set_compression(wopt, opt_compression_type);
mtbl_writer_options_set_block_size(wopt, opt_block_size);
merger = mtbl_merger_init(mopt);
assert(merger != NULL);
fprintf(stderr, "%s: opening output file %s\n", program_name, mtbl_output_fname);
writer = mtbl_writer_init(mtbl_output_fname, wopt);
if (writer == NULL) {
fprintf(stderr, "Error: mtbl_writer_init() failed.\n\n");
usage();
}
mtbl_merger_options_destroy(&mopt);
mtbl_writer_options_destroy(&wopt);
}
/**
* Write a test key ('A' * 10000) + test value ('B' * 20000) to an mtbl_writer,
* using the given compression type. Then re-open the file with an mtbl_reader
* and verify that we get the exact same key/value entry back.
*
* This tests that data block compression/decompression is functioning
* correctly.
*
* We use mkstemp() to create a temporary file, then immediately unlink() and
* dup() it, which allows us to read from the file after we close it. This
* avoids leaving the temporary file on the filesystem if the test fails.
*/
static int
test_compression(mtbl_compression_type c_type, const char *dirname)
{
mtbl_res res;
/* Initialize test key/value. */
size_t len_key = 10000;
size_t len_val = 20000;
uint8_t *key = my_malloc(len_key);
uint8_t *val = my_malloc(len_val);
memset(key, 'A', len_key);
memset(val, 'B', len_val);
/* Create and open temporary file. */
char fname[strlen(dirname) + 100];
sprintf(fname, "%s/.mtbl." NAME ".%ld.XXXXXX", dirname, (long)getpid());
int fd = mkstemp(fname);
if (fd == -1) {
fprintf(stderr, NAME ": mkstemp() failed: %s\n", strerror(errno));
return 1;
}
/* Unlink the temporary filename. */
int unlink_ret = unlink(fname);
if (unlink_ret == -1) {
fprintf(stderr, NAME ": unlink() on file '%s' failed: %s\n",
fname, strerror(errno));
return 1;
}
/**
* Duplicate the file descriptor.
* Used by the reader, since mtbl_writer_destroy() will close the fd,
* which will delete the file, since it has no links on the filesystem.
*/
int dup_fd = dup(fd);
if (dup_fd == -1) {
fprintf(stderr, NAME ": dup() failed: %s\n", strerror(errno));
return 1;
}
/* Open a writer on the temporary file. */
struct mtbl_writer_options *wopt = mtbl_writer_options_init();
mtbl_writer_options_set_compression(wopt, c_type);
struct mtbl_writer *w = mtbl_writer_init_fd(fd, wopt);
mtbl_writer_options_destroy(&wopt);
if (w == NULL) {
fprintf(stderr, NAME ": mtbl_writer_init_fd() failed\n");
return 1;
}
/* Write the test key/value entry. */
res = mtbl_writer_add(w, key, len_key, val, len_val);
if (res != mtbl_res_success) {
fprintf(stderr, NAME ": mtbl_writer_add() failed\n");
return 1;
}
/* Close the writer. */
mtbl_writer_destroy(&w);
fd = -1;
/* Open the reader on the dup()'d file descriptor. */
struct mtbl_reader *r = mtbl_reader_init_fd(dup_fd, NULL);
if (r == NULL) {
fprintf(stderr, NAME ": mtbl_reader_init_fd() failed\n");
return 1;
}
/**
* Check that the compresison algorithm on the reader was what we set
* on the writer.
*/
const struct mtbl_metadata *m = mtbl_reader_metadata(r);
uint64_t m_c_type = mtbl_metadata_compression_algorithm(m);
if ((mtbl_compression_type) m_c_type != c_type) {
fprintf(stderr, NAME ": mtbl_metadata_compression_algorithm() "
"returned unexpected value %" PRIu64 " (!= %u)\n",
m_c_type, c_type);
return 1;
}
/* Retrieve the test key/value entry. */
const struct mtbl_source *s = mtbl_reader_source(r);
if (s == NULL) {
fprintf(stderr, NAME ": mtbl_reader_source() failed\n");
return 1;
}
struct mtbl_iter *it = mtbl_source_iter(s);
const uint8_t *it_key, *it_val;
size_t len_it_key, len_it_val;
res = mtbl_iter_next(it, &it_key, &len_it_key, &it_val, &len_it_val);
if (res != mtbl_res_success) {
fprintf(stderr, NAME ": mtbl_iter_next() failed\n");
return 1;
}
/* Check the test entry against our original entry. */
if (len_it_key != len_key) {
fprintf(stderr, NAME ": len_it_key != len_key\n");
return 1;
}
if (len_it_val != len_val) {
fprintf(stderr, NAME ": len_it_val != len_val\n");
return 1;
}
if (memcmp(it_key, key, len_key) != 0) {
fprintf(stderr, NAME ": it_key != key\n");
return 1;
}
if (memcmp(it_val, val, len_val) != 0) {
fprintf(stderr, NAME ": it_val != val\n");
return 1;
}
/* Test that there are no more entries. */
res = mtbl_iter_next(it, &it_key, &len_it_key, &it_val, &len_it_val);
if (res != mtbl_res_failure) {
fprintf(stderr, NAME ": mtbl_iter_next() returned an "
"additional unexpected entry\n");
return 1;
}
/* Cleanup. */
mtbl_iter_destroy(&it);
mtbl_reader_destroy(&r);
free(key);
free(val);
return 0;
}
struct chunk *c = my_calloc(1, sizeof(*c));
char template[64];
sprintf(template, "/.mtbl.%ld.XXXXXX", (long)getpid());
ubuf *tmp_fname = ubuf_init(strlen(s->opt.tmp_dname) + strlen(template) + 1);
ubuf_append(tmp_fname, (uint8_t *) s->opt.tmp_dname, strlen(s->opt.tmp_dname));
ubuf_append(tmp_fname, (uint8_t *) template, strlen(template));
ubuf_append(tmp_fname, (const uint8_t *) "\x00", 1);
c->fd = mkstemp((char *) ubuf_data(tmp_fname));
assert(c->fd >= 0);
int unlink_ret = unlink((char *) ubuf_data(tmp_fname));
assert(unlink_ret == 0);
ubuf_destroy(&tmp_fname);
struct mtbl_writer_options *wopt = mtbl_writer_options_init();
mtbl_writer_options_set_compression(wopt, MTBL_COMPRESSION_SNAPPY);
struct mtbl_writer *w = mtbl_writer_init_fd(c->fd, wopt);
mtbl_writer_options_destroy(&wopt);
size_t entries_written = 0;
struct entry **array = entry_vec_data(s->vec);
qsort(array, entry_vec_size(s->vec), sizeof(void *), _mtbl_sorter_compare);
for (unsigned i = 0; i < entry_vec_size(s->vec); i++) {
struct entry *ent = entry_vec_value(s->vec, i);
if (i + 1 < entry_vec_size(s->vec)) {
struct entry *next_ent = entry_vec_value(s->vec, i + 1);
struct entry *merge_ent = NULL;
if (_mtbl_sorter_compare(&ent, &next_ent) == 0) {