Example #1
0
static void
init_mtbl(int fd) {
	struct mtbl_writer_options *writer_options = mtbl_writer_options_init();
	assert(writer_options != NULL);

	mtbl_writer_options_set_block_size(writer_options, 1024);

	struct mtbl_writer *writer = mtbl_writer_init_fd(fd, writer_options);
	assert(writer != NULL);

	/* Populate the mtbl with hex(i)->i */
	for (uint32_t i = 0; i < NUM_KEYS; i++) {
		ubuf *key = ubuf_init(1);
		ubuf *value = ubuf_init(1);

		ubuf_add_fmt(key, KEY_FMT, i);
		ubuf_add_fmt(value, VAL_FMT, i);

		assert(mtbl_writer_add(writer, ubuf_data(key), ubuf_size(key), ubuf_data(value), ubuf_size(value)) == mtbl_res_success);

		ubuf_destroy(&key);
		ubuf_destroy(&value);
	}

	mtbl_writer_destroy(&writer);
	mtbl_writer_options_destroy(&writer_options);
}
Example #2
0
static void
init_mtbl(void)
{
	struct mtbl_merger_options *mopt;
	struct mtbl_writer_options *wopt;

	mopt = mtbl_merger_options_init();
	wopt = mtbl_writer_options_init();

	mtbl_merger_options_set_merge_func(mopt, merge_func, user_clos);
	mtbl_writer_options_set_compression(wopt, opt_compression_type);
	mtbl_writer_options_set_block_size(wopt, opt_block_size);

	merger = mtbl_merger_init(mopt);
	assert(merger != NULL);

	fprintf(stderr, "%s: opening output file %s\n", program_name, mtbl_output_fname);
	writer = mtbl_writer_init(mtbl_output_fname, wopt);
	if (writer == NULL) {
		fprintf(stderr, "Error: mtbl_writer_init() failed.\n\n");
		usage();
	}

	mtbl_merger_options_destroy(&mopt);
	mtbl_writer_options_destroy(&wopt);
}
Example #3
0
/**
 * Write a test key ('A' * 10000) + test value ('B' * 20000) to an mtbl_writer,
 * using the given compression type. Then re-open the file with an mtbl_reader
 * and verify that we get the exact same key/value entry back.
 *
 * This tests that data block compression/decompression is functioning
 * correctly.
 *
 * We use mkstemp() to create a temporary file, then immediately unlink() and
 * dup() it, which allows us to read from the file after we close it. This
 * avoids leaving the temporary file on the filesystem if the test fails.
 */
static int
test_compression(mtbl_compression_type c_type, const char *dirname)
{
	mtbl_res res;

	/* Initialize test key/value. */
	size_t len_key = 10000;
	size_t len_val = 20000;
	uint8_t *key = my_malloc(len_key);
	uint8_t *val = my_malloc(len_val);
	memset(key, 'A', len_key);
	memset(val, 'B', len_val);

	/* Create and open temporary file. */
	char fname[strlen(dirname) + 100];
	sprintf(fname, "%s/.mtbl." NAME ".%ld.XXXXXX", dirname, (long)getpid());
	int fd = mkstemp(fname);
	if (fd == -1) {
		fprintf(stderr, NAME ": mkstemp() failed: %s\n", strerror(errno));
		return 1;
	}

	/* Unlink the temporary filename. */
	int unlink_ret = unlink(fname);
	if (unlink_ret == -1) {
		fprintf(stderr, NAME ": unlink() on file '%s' failed: %s\n",
			fname, strerror(errno));
		return 1;
	}

	/**
	 * Duplicate the file descriptor.
	 * Used by the reader, since mtbl_writer_destroy() will close the fd,
	 * which will delete the file, since it has no links on the filesystem.
	 */
	int dup_fd = dup(fd);
	if (dup_fd == -1) {
		fprintf(stderr, NAME ": dup() failed: %s\n", strerror(errno));
		return 1;
	}

	/* Open a writer on the temporary file. */
	struct mtbl_writer_options *wopt = mtbl_writer_options_init();
	mtbl_writer_options_set_compression(wopt, c_type);
	struct mtbl_writer *w = mtbl_writer_init_fd(fd, wopt);
	mtbl_writer_options_destroy(&wopt);
	if (w == NULL) {
		fprintf(stderr, NAME ": mtbl_writer_init_fd() failed\n");
		return 1;
	}

	/* Write the test key/value entry. */
	res = mtbl_writer_add(w, key, len_key, val, len_val);
	if (res != mtbl_res_success) {
		fprintf(stderr, NAME ": mtbl_writer_add() failed\n");
		return 1;
	}

	/* Close the writer. */
	mtbl_writer_destroy(&w);
	fd = -1;

	/* Open the reader on the dup()'d file descriptor. */
	struct mtbl_reader *r = mtbl_reader_init_fd(dup_fd, NULL);
	if (r == NULL) {
		fprintf(stderr, NAME ": mtbl_reader_init_fd() failed\n");
		return 1;
	}

	/**
	 * Check that the compresison algorithm on the reader was what we set
	 * on the writer.
	 */
	const struct mtbl_metadata *m = mtbl_reader_metadata(r);
	uint64_t m_c_type = mtbl_metadata_compression_algorithm(m);
	if ((mtbl_compression_type) m_c_type != c_type) {
		fprintf(stderr, NAME ": mtbl_metadata_compression_algorithm() "
			"returned unexpected value %" PRIu64 " (!= %u)\n",
			m_c_type, c_type);
		return 1;
	}

	/* Retrieve the test key/value entry. */
	const struct mtbl_source *s = mtbl_reader_source(r);
	if (s == NULL) {
		fprintf(stderr, NAME ": mtbl_reader_source() failed\n");
		return 1;
	}
	struct mtbl_iter *it = mtbl_source_iter(s);
	const uint8_t *it_key, *it_val;
	size_t len_it_key, len_it_val;
	res = mtbl_iter_next(it, &it_key, &len_it_key, &it_val, &len_it_val);
	if (res != mtbl_res_success) {
		fprintf(stderr, NAME ": mtbl_iter_next() failed\n");
		return 1;
	}

	/* Check the test entry against our original entry. */
	if (len_it_key != len_key) {
		fprintf(stderr, NAME ": len_it_key != len_key\n");
		return 1;
	}
	if (len_it_val != len_val) {
		fprintf(stderr, NAME ": len_it_val != len_val\n");
		return 1;
	}
	if (memcmp(it_key, key, len_key) != 0) {
		fprintf(stderr, NAME ": it_key != key\n");
		return 1;
	}
	if (memcmp(it_val, val, len_val) != 0) {
		fprintf(stderr, NAME ": it_val != val\n");
		return 1;
	}

	/* Test that there are no more entries. */
	res = mtbl_iter_next(it, &it_key, &len_it_key, &it_val, &len_it_val);
	if (res != mtbl_res_failure) {
		fprintf(stderr, NAME ": mtbl_iter_next() returned an "
			"additional unexpected entry\n");
		return 1;
	}

	/* Cleanup. */
	mtbl_iter_destroy(&it);
	mtbl_reader_destroy(&r);
	free(key);
	free(val);

	return 0;
}
Example #4
0
	struct chunk *c = my_calloc(1, sizeof(*c));

	char template[64];
	sprintf(template, "/.mtbl.%ld.XXXXXX", (long)getpid());
	ubuf *tmp_fname = ubuf_init(strlen(s->opt.tmp_dname) + strlen(template) + 1);
	ubuf_append(tmp_fname, (uint8_t *) s->opt.tmp_dname, strlen(s->opt.tmp_dname));
	ubuf_append(tmp_fname, (uint8_t *) template, strlen(template));
	ubuf_append(tmp_fname, (const uint8_t *) "\x00", 1);

	c->fd = mkstemp((char *) ubuf_data(tmp_fname));
	assert(c->fd >= 0);
	int unlink_ret = unlink((char *) ubuf_data(tmp_fname));
	assert(unlink_ret == 0);
	ubuf_destroy(&tmp_fname);

	struct mtbl_writer_options *wopt = mtbl_writer_options_init();
	mtbl_writer_options_set_compression(wopt, MTBL_COMPRESSION_SNAPPY);
	struct mtbl_writer *w = mtbl_writer_init_fd(c->fd, wopt);
	mtbl_writer_options_destroy(&wopt);

	size_t entries_written = 0;
	struct entry **array = entry_vec_data(s->vec);
	qsort(array, entry_vec_size(s->vec), sizeof(void *), _mtbl_sorter_compare);
	for (unsigned i = 0; i < entry_vec_size(s->vec); i++) {
		struct entry *ent = entry_vec_value(s->vec, i);

		if (i + 1 < entry_vec_size(s->vec)) {
			struct entry *next_ent = entry_vec_value(s->vec, i + 1);
			struct entry *merge_ent = NULL;

			if (_mtbl_sorter_compare(&ent, &next_ent) == 0) {