int main(int argc, char** argv) {
leveldb_t* db;
leveldb_comparator_t* cmp;
leveldb_cache_t* cache;
leveldb_env_t* env;
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
char* err = NULL;
int run = -1;
CheckCondition(leveldb_major_version() >= 1);
CheckCondition(leveldb_minor_version() >= 1);
snprintf(dbname, sizeof(dbname),
"%s/leveldb_c_test-%di/meta/0",
GetTempDir(),
((int) geteuid()));
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1);
leveldb_options_set_cache(options, cache);
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL);
leveldb_options_set_write_buffer_size(options, 100000);
leveldb_options_set_paranoid_checks(options, 1);
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 1);
leveldb_readoptions_set_fill_cache(roptions, 0);
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, 1);
StartPhase("destroy");
leveldb_destroy_db(options, dbname, &err);
Free(&err);
StartPhase("open");
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
StartPhase("put");
leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
StartPhase("compactall");
leveldb_compact_range(db, NULL, 0, NULL, 0);
CheckGet(db, roptions, "foo", "hello");
StartPhase("compactrange");
leveldb_compact_range(db, "a", 1, "z", 1);
CheckGet(db, roptions, "foo", "hello");
StartPhase("writebatch");
{
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
int pos = 0;
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_destroy(wb);
}
StartPhase("iter");
{
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
StartPhase("approximate_sizes");
{
int i;
int n = 20000;
char keybuf[100];
char valbuf[100];
uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" };
size_t start_len[2] = { 1, 21 };
const char* limit[2] = { "k00000000000000010000", "z" };
size_t limit_len[2] = { 21, 1 };
leveldb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
snprintf(valbuf, sizeof(valbuf), "v%020d", i);
leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
&err);
CheckNoError(err);
}
leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
CheckCondition(sizes[0] > 0);
CheckCondition(sizes[1] > 0);
}
StartPhase("property");
{
char* prop = leveldb_property_value(db, "nosuchprop");
/* CheckCondition(prop == NULL);*/
prop = leveldb_property_value(db, "leveldb.stats");
CheckCondition(prop != NULL);
Free(&prop);
}
StartPhase("snapshot");
{
const leveldb_snapshot_t* snap;
snap = leveldb_create_snapshot(db);
leveldb_delete(db, woptions, "foo", 3, &err);
CheckNoError(err);
leveldb_readoptions_set_snapshot(roptions, snap);
CheckGet(db, roptions, "foo", "hello");
leveldb_readoptions_set_snapshot(roptions, NULL);
CheckGet(db, roptions, "foo", NULL);
leveldb_release_snapshot(db, snap);
}
StartPhase("repair");
{
leveldb_close(db);
leveldb_options_set_error_if_exists(options, 0);
leveldb_repair_db(options, dbname, &err);
CheckNoError(err);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
leveldb_options_set_error_if_exists(options, 1);
}
StartPhase("filter");
for (run = 0; run < 2; run++) {
// First run uses custom filter, second run uses bloom filter
CheckNoError(err);
leveldb_filterpolicy_t* policy;
if (run == 0) {
policy = leveldb_filterpolicy_create(
NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
} else {
policy = leveldb_filterpolicy_create_bloom(10);
}
// Create new database
leveldb_close(db);
leveldb_destroy_db(options, dbname, &err);
leveldb_options_set_filter_policy(options, policy);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
CheckNoError(err);
leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
CheckNoError(err);
leveldb_compact_range(db, NULL, 0, NULL, 0);
fake_filter_result = 1;
CheckGet(db, roptions, "foo", "foovalue");
CheckGet(db, roptions, "bar", "barvalue");
if (phase == 0) {
// Must not find value when custom filter returns false
fake_filter_result = 0;
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
fake_filter_result = 1;
CheckGet(db, roptions, "foo", "foovalue");
CheckGet(db, roptions, "bar", "barvalue");
}
leveldb_options_set_filter_policy(options, NULL);
leveldb_filterpolicy_destroy(policy);
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
return 0;
}
//TODO: add "bool create_if_missing" to this API!
int leveldb_client_start(leveldb **db, const char *name,
size_t write_buffer_size, bool use_compression, bool sync_writes)
{
int namelen;
char *db_name;
char *err = NULL;
leveldb_t* db_t;
leveldb_comparator_t *cmp;
leveldb_cache_t *cache;
leveldb_env_t *env;
leveldb_options_t *options;
leveldb_readoptions_t *roptions;
leveldb_writeoptions_t *woptions;
*db = (leveldb *)malloc(sizeof(leveldb));
if (*db == NULL) {
kp_error("malloc(leveldb) failed!\n");
return -1;
}
/* Check arguments: */
if (strlen(name) < 1) {
kp_error("name has no length!\n");
return -1;
}
if (write_buffer_size == 0) {
write_buffer_size = WRITE_BUFFER_DEFAULT;
}
if (write_buffer_size < WRITE_BUFFER_MIN) {
kp_error("write_buffer_size=%zu too small; min=%u\n",
write_buffer_size, WRITE_BUFFER_MIN);
return -1;
}
kp_debug("got args: name=%s, write_buffer_size=%zu, use_compression=%s, "
"sync_writes=%s\n",
name, write_buffer_size, use_compression ? "true" : "false",
sync_writes ? "true" : "false");
/* Format of db name: LEVELDB_DIR + / + name */
namelen = strlen(name);
db_name = malloc(namelen + 1);
if (!db_name) {
kp_error("malloc(db_name) failed!\n");
return -1;
}
strncpy(db_name, name, namelen+1); //copy null-zero
(*db)->name = db_name;
kp_debug("constructed db->name=[%s]\n", (*db)->name);
/* Set up the database's comparator, environment, cache, etc. According
* to leveldb/include/leveldb/c.h, all functions that can raise an error
* must be passed a "char **errptr" (set to NULL!) as the last argument;
* I guess functions that don't take this pointer can't return errors.
*/
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_compare,
cmp_name); //first arg is "state"
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(CACHE_CAPACITY);
// cache = NULL;
(*db)->cmp = cmp;
(*db)->env = env;
(*db)->cache = cache;
/* Set up the database's various options. Many of these will affect the
* database's performance! (see leveldb/include/leveldb/options.h).
*/
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1); //raise error if db already exists
leveldb_options_set_cache(options, cache);
// leveldb_options_set_cache(options, NULL); //disable cache??
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL); //NULL: write info to file in db's dir
leveldb_options_set_write_buffer_size(options, write_buffer_size);
/* Amount of data to build up in memory (backed by an unsorted log
* on disk) before converting to a sorted on-disk file.
* Larger values increase performance, especially during bulk loads.
* Up to two write buffers may be held in memory at the same time,
* so you may wish to adjust this parameter to control memory usage.
* Also, a larger write buffer will result in a longer recovery time
* the next time the database is opened.
* Default: 4MB (test file uses 100000 bytes)
*/
leveldb_options_set_paranoid_checks(options, 0); //default false; test file uses true
leveldb_options_set_max_open_files(options, MAX_OPEN_FILES);
leveldb_options_set_block_size(options, BLOCK_SIZE);
leveldb_options_set_block_restart_interval(options, BLOCK_RESTART_INTERVAL);
leveldb_options_set_compression(options,
use_compression ? leveldb_snappy_compression : leveldb_no_compression);
(*db)->options = options;
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 0);
/* If true, all data read from underlying storage will be
* verified against corresponding checksums.
* Default false; test file uses true.
*/
leveldb_readoptions_set_fill_cache(roptions, true);
/* Should the data read for this iteration be cached in memory?
* Callers may wish to set this field to false for bulk scans.
* Default true; test file uses false.
*/
(*db)->roptions = roptions;
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, sync_writes ? 1 : 0);
/* If true, the write will be flushed from the operating system
* buffer cache (by calling WritableFile::Sync()) before the write
* is considered complete. If this flag is true, writes will be
* slower.
* If this flag is false, and the machine crashes, some recent
* writes may be lost. Note that if it is just the process that
* crashes (i.e., the machine does not reboot), no writes will be
* lost even if sync==false.
* In other words, a DB write with sync==false has similar
* crash semantics as the "write()" system call. A DB write
* with sync==true has similar crash semantics to a "write()"
* system call followed by "fsync()".
*/
(*db)->woptions = woptions;
kp_debug("destroying previous copy of database, if it exists\n");
leveldb_destroy_db((*db)->options, (*db)->name, &err);
free_err(&err);
kp_debug("opening/creating database [%s]\n", (*db)->name);
db_t = leveldb_open((*db)->options, (*db)->name, &err);
if (err) {
kp_error("opening db returned error: %s\n", err);
return -1;
}
free_err(&err);
(*db)->db = db_t;
kp_debug("successfully started leveldb [%s]\n", (*db)->name);
return 0;
}
int main(int argc, char** argv) {
leveldb_t* db;
leveldb_comparator_t* cmp;
leveldb_cache_t* cache;
leveldb_env_t* env;
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
char* err = NULL;
#if defined(LEVELDB_PLATFORM_WINDOWS)
snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test");
#else
snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d",
((int) geteuid()));
#endif
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1);
leveldb_options_set_cache(options, cache);
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL);
leveldb_options_set_write_buffer_size(options, 100000);
leveldb_options_set_paranoid_checks(options, 1);
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 1);
leveldb_readoptions_set_fill_cache(roptions, 0);
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, 1);
StartPhase("destroy");
leveldb_destroy_db(options, dbname, &err);
leveldb_free(&err);
StartPhase("open_error");
db = leveldb_open(options, dbname, &err);
CheckCondition(err != NULL);
leveldb_free(&err);
StartPhase("open");
leveldb_options_set_create_if_missing(options, 1);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
StartPhase("put");
leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
StartPhase("writebatch");
{
int pos = 0;
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_destroy(wb);
}
StartPhase("iter");
{
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
StartPhase("approximate_sizes");
{
int i;
int n = 20000;
char keybuf[100];
char valbuf[100];
uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" };
size_t start_len[2] = { 1, 21 };
const char* limit[2] = { "k00000000000000010000", "z" };
size_t limit_len[2] = { 21, 1 };
leveldb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
snprintf(valbuf, sizeof(valbuf), "v%020d", i);
leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
&err);
CheckNoError(err);
}
leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
CheckCondition(sizes[0] > 0);
CheckCondition(sizes[1] > 0);
}
StartPhase("property");
{
char* prop = leveldb_property_value(db, "nosuchprop");
CheckCondition(prop == NULL);
prop = leveldb_property_value(db, "leveldb.stats");
CheckCondition(prop != NULL);
leveldb_free(&prop);
}
StartPhase("snapshot");
{
const leveldb_snapshot_t* snap;
snap = leveldb_create_snapshot(db);
leveldb_delete(db, woptions, "foo", 3, &err);
CheckNoError(err);
leveldb_readoptions_set_snapshot(roptions, snap);
CheckGet(db, roptions, "foo", "hello");
leveldb_readoptions_set_snapshot(roptions, NULL);
CheckGet(db, roptions, "foo", NULL);
leveldb_release_snapshot(db, snap);
}
StartPhase("repair");
{
leveldb_close(db);
leveldb_options_set_create_if_missing(options, 0);
leveldb_options_set_error_if_exists(options, 0);
leveldb_repair_db(options, dbname, &err);
CheckNoError(err);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
return 0;
}
int mdhim_leveldb_open(void **dbh, void **dbs, char *path, int flags,
struct mdhim_store_opts_t *mstore_opts) {
leveldb_t *db;
leveldb_options_t *options;
leveldb_options_t *stat_options;
char *err = NULL;
leveldb_comparator_t* cmp = NULL;
char stats_path[PATH_MAX];
leveldb_filterpolicy_t *main_filter;
leveldb_filterpolicy_t *stats_filter;
leveldb_cache_t *main_cache;
leveldb_cache_t *stats_cache;
leveldb_env_t *main_env;
leveldb_env_t *stats_env;
//Create the options for the main database
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
//leveldb_options_set_compression(options, 0);
main_filter = leveldb_filterpolicy_create_bloom(256);
main_cache = leveldb_cache_create_lru(5242880);
main_env = leveldb_create_default_env();
leveldb_options_set_cache(options, main_cache);
leveldb_options_set_filter_policy(options, main_filter);
leveldb_options_set_max_open_files(options, 10000);
leveldb_options_set_write_buffer_size(options, 5242880);
leveldb_options_set_env(options, main_env);
//Create the options for the stat database
stat_options = leveldb_options_create();
leveldb_options_set_create_if_missing(stat_options, 1);
//leveldb_options_set_compression(stat_options, 0);
stats_filter = leveldb_filterpolicy_create_bloom(256);
stats_cache = leveldb_cache_create_lru(1024);
stats_env = leveldb_create_default_env();
leveldb_options_set_cache(stat_options, stats_cache);
leveldb_options_set_filter_policy(stat_options, stats_filter);
leveldb_options_set_write_buffer_size(stat_options, 1024);
leveldb_options_set_env(stat_options, stats_env);
switch(mstore_opts->key_type) {
case MDHIM_INT_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_int_compare, cmp_name);
break;
case MDHIM_LONG_INT_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_lint_compare, cmp_name);
break;
case MDHIM_FLOAT_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_float_compare, cmp_name);
break;
case MDHIM_DOUBLE_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_double_compare, cmp_name);
break;
case MDHIM_STRING_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_string_compare, cmp_name);
break;
default:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_byte_compare, cmp_name);
break;
}
leveldb_options_set_comparator(options, cmp);
//Check to see if the given path + "_stat" and the null char will be more than the max
if (strlen(path) + 6 > PATH_MAX) {
mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database - path provided is too long");
return MDHIM_DB_ERROR;
}
sprintf(stats_path, "%s_stats", path);
//Open the main database
db = leveldb_open(options, path, &err);
//Set the output handle
*((leveldb_t **) dbh) = db;
if (err != NULL) {
mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database");
return MDHIM_DB_ERROR;
}
//Open the stats database
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_int_compare, cmp_name);
leveldb_options_set_comparator(stat_options, cmp);
db = leveldb_open(stat_options, stats_path, &err);
*((leveldb_t **) dbs) = db;
if (err != NULL) {
mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database");
return MDHIM_DB_ERROR;
}
//Set the output comparator
mstore_opts->db_ptr1 = cmp;
//Set the generic pointers to hold the options for the mdhim db
mstore_opts->db_ptr2 = options;
mstore_opts->db_ptr3 = leveldb_readoptions_create();
mstore_opts->db_ptr4 = leveldb_writeoptions_create();
//Set the generic pointers to hold the options for the mdhim stats db
mstore_opts->db_ptr5 = stat_options;
mstore_opts->db_ptr6 = leveldb_readoptions_create();
mstore_opts->db_ptr7 = leveldb_writeoptions_create();
//Set the rest of the options that aren't used until close
mstore_opts->db_ptr8 = main_filter;
mstore_opts->db_ptr9 = stats_filter;
mstore_opts->db_ptr10 = main_cache;
mstore_opts->db_ptr11 = stats_cache;
mstore_opts->db_ptr12 = main_env;
mstore_opts->db_ptr13 = stats_env;
return MDHIM_SUCCESS;
}