static bool dump(const char *fname, const bool silent) { const uint8_t *key, *val; size_t len_key, len_val; struct mtbl_reader *r; struct mtbl_iter *it; r = mtbl_reader_init(fname, NULL); if (r == NULL) { fprintf(stderr, "Error: mtbl_reader_init() on %s failed\n", fname); return (false); } it = mtbl_source_iter(mtbl_reader_source(r)); while (mtbl_iter_next(it, &key, &len_key, &val, &len_val)) { if (silent) continue; print_string(key, len_key, stdout); fputc(' ', stdout); print_string(val, len_val, stdout); fputc('\n', stdout); } mtbl_iter_destroy(&it); mtbl_reader_destroy(&r); return (true); }
static void merge(void) { const uint8_t *key, *val; size_t len_key, len_val; struct mtbl_iter *it; it = mtbl_source_iter(mtbl_merger_source(merger)); assert(it != NULL); while (mtbl_iter_next(it, &key, &len_key, &val, &len_val) == mtbl_res_success) { mtbl_res res = mtbl_writer_add(writer, key, len_key, val, len_val); assert(res == mtbl_res_success); if ((++count % STATS_INTERVAL) == 0) print_stats(); } mtbl_iter_destroy(&it); mtbl_merger_destroy(&merger); mtbl_writer_destroy(&writer); }
/** * Write a test key ('A' * 10000) + test value ('B' * 20000) to an mtbl_writer, * using the given compression type. Then re-open the file with an mtbl_reader * and verify that we get the exact same key/value entry back. * * This tests that data block compression/decompression is functioning * correctly. * * We use mkstemp() to create a temporary file, then immediately unlink() and * dup() it, which allows us to read from the file after we close it. This * avoids leaving the temporary file on the filesystem if the test fails. */ static int test_compression(mtbl_compression_type c_type, const char *dirname) { mtbl_res res; /* Initialize test key/value. */ size_t len_key = 10000; size_t len_val = 20000; uint8_t *key = my_malloc(len_key); uint8_t *val = my_malloc(len_val); memset(key, 'A', len_key); memset(val, 'B', len_val); /* Create and open temporary file. */ char fname[strlen(dirname) + 100]; sprintf(fname, "%s/.mtbl." NAME ".%ld.XXXXXX", dirname, (long)getpid()); int fd = mkstemp(fname); if (fd == -1) { fprintf(stderr, NAME ": mkstemp() failed: %s\n", strerror(errno)); return 1; } /* Unlink the temporary filename. */ int unlink_ret = unlink(fname); if (unlink_ret == -1) { fprintf(stderr, NAME ": unlink() on file '%s' failed: %s\n", fname, strerror(errno)); return 1; } /** * Duplicate the file descriptor. * Used by the reader, since mtbl_writer_destroy() will close the fd, * which will delete the file, since it has no links on the filesystem. */ int dup_fd = dup(fd); if (dup_fd == -1) { fprintf(stderr, NAME ": dup() failed: %s\n", strerror(errno)); return 1; } /* Open a writer on the temporary file. */ struct mtbl_writer_options *wopt = mtbl_writer_options_init(); mtbl_writer_options_set_compression(wopt, c_type); struct mtbl_writer *w = mtbl_writer_init_fd(fd, wopt); mtbl_writer_options_destroy(&wopt); if (w == NULL) { fprintf(stderr, NAME ": mtbl_writer_init_fd() failed\n"); return 1; } /* Write the test key/value entry. */ res = mtbl_writer_add(w, key, len_key, val, len_val); if (res != mtbl_res_success) { fprintf(stderr, NAME ": mtbl_writer_add() failed\n"); return 1; } /* Close the writer. */ mtbl_writer_destroy(&w); fd = -1; /* Open the reader on the dup()'d file descriptor. */ struct mtbl_reader *r = mtbl_reader_init_fd(dup_fd, NULL); if (r == NULL) { fprintf(stderr, NAME ": mtbl_reader_init_fd() failed\n"); return 1; } /** * Check that the compresison algorithm on the reader was what we set * on the writer. */ const struct mtbl_metadata *m = mtbl_reader_metadata(r); uint64_t m_c_type = mtbl_metadata_compression_algorithm(m); if ((mtbl_compression_type) m_c_type != c_type) { fprintf(stderr, NAME ": mtbl_metadata_compression_algorithm() " "returned unexpected value %" PRIu64 " (!= %u)\n", m_c_type, c_type); return 1; } /* Retrieve the test key/value entry. */ const struct mtbl_source *s = mtbl_reader_source(r); if (s == NULL) { fprintf(stderr, NAME ": mtbl_reader_source() failed\n"); return 1; } struct mtbl_iter *it = mtbl_source_iter(s); const uint8_t *it_key, *it_val; size_t len_it_key, len_it_val; res = mtbl_iter_next(it, &it_key, &len_it_key, &it_val, &len_it_val); if (res != mtbl_res_success) { fprintf(stderr, NAME ": mtbl_iter_next() failed\n"); return 1; } /* Check the test entry against our original entry. */ if (len_it_key != len_key) { fprintf(stderr, NAME ": len_it_key != len_key\n"); return 1; } if (len_it_val != len_val) { fprintf(stderr, NAME ": len_it_val != len_val\n"); return 1; } if (memcmp(it_key, key, len_key) != 0) { fprintf(stderr, NAME ": it_key != key\n"); return 1; } if (memcmp(it_val, val, len_val) != 0) { fprintf(stderr, NAME ": it_val != val\n"); return 1; } /* Test that there are no more entries. */ res = mtbl_iter_next(it, &it_key, &len_it_key, &it_val, &len_it_val); if (res != mtbl_res_failure) { fprintf(stderr, NAME ": mtbl_iter_next() returned an " "additional unexpected entry\n"); return 1; } /* Cleanup. */ mtbl_iter_destroy(&it); mtbl_reader_destroy(&r); free(key); free(val); return 0; }
static void test_iter(struct mtbl_iter *iter) { /* Iterate completely through the mtbl */ for (uint32_t i = 0; i < NUM_KEYS; i++) { ubuf *expected_key = ubuf_init(1); ubuf_add_fmt(expected_key, KEY_FMT, i); const uint8_t *key, *value; size_t len_key, len_value; assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_success); assert(bytes_compare(ubuf_data(expected_key), ubuf_size(expected_key), key, len_key) == 0); ubuf_destroy(&expected_key); } /* Ensure that we have completely iterated through the set. */ { const uint8_t *key, *value; size_t len_key, len_value; assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_failure); } /* Seek to a key, ensure that we get the one we want and that we go * all the way to the end. */ for (uint32_t i = NUM_KEYS; i-- > 0; ) { ubuf *seek_key = ubuf_init(1); ubuf_add_fmt(seek_key, KEY_FMT, i); const uint8_t *key, *value; size_t len_key, len_value; assert(mtbl_iter_seek(iter, ubuf_data(seek_key), ubuf_size(seek_key)) == mtbl_res_success); for (uint32_t j = i; j < NUM_KEYS; j++) { ubuf *expected_key = ubuf_init(1); ubuf_add_fmt(expected_key, KEY_FMT, j); assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_success); assert(bytes_compare(ubuf_data(expected_key), ubuf_size(expected_key), key, len_key) == 0); ubuf_destroy(&expected_key); } assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_failure); ubuf_destroy(&seek_key); } /* Attempt to seek past end of iterator */ ubuf *seek_key = ubuf_init(1); const uint8_t *key, *value; size_t len_key, len_value; ubuf_add_fmt(seek_key, KEY_FMT, NUM_KEYS + 1); assert(mtbl_iter_seek(iter, ubuf_data(seek_key), ubuf_size(seek_key)) == mtbl_res_success); assert(mtbl_iter_next(iter, &key, &len_key, &value, &len_value) == mtbl_res_failure); }