/**
* mdhim_leveldb_batch_put
* Stores multiple keys in the data store
*
* @param dbh in pointer to the leveldb handle
* @param keys in void ** to the key to store
* @param key_lens in int * to the lengths of the keys
* @param data in void ** to the values of the keys
* @param data_lens in int * to the lengths of the value data
* @param num_records in int for the number of records to insert
* @param mstore_opts in additional options for the data store layer
*
* @return MDHIM_SUCCESS on success or MDHIM_DB_ERROR on failure
*/
int mdhim_leveldb_batch_put(void *dbh, void **keys, int32_t *key_lens,
void **data, int32_t *data_lens, int num_records,
struct mdhim_store_opts_t *mstore_opts) {
leveldb_writeoptions_t *options;
char *err = NULL;
leveldb_t *db = (leveldb_t *) dbh;
struct timeval start, end;
leveldb_writebatch_t* write_batch;
int i;
gettimeofday(&start, NULL);
write_batch = leveldb_writebatch_create();
options = (leveldb_writeoptions_t *) mstore_opts->db_ptr4;
for (i = 0; i < num_records; i++) {
leveldb_writebatch_put(write_batch, keys[i], key_lens[i],
data[i], data_lens[i]);
}
leveldb_write(db, options, write_batch, &err);
leveldb_writebatch_destroy(write_batch);
if (err != NULL) {
mlog(MDHIM_SERVER_CRIT, "Error in batch put in leveldb");
return MDHIM_DB_ERROR;
}
gettimeofday(&end, NULL);
mlog(MDHIM_SERVER_DBG, "Took: %d seconds to put %d records",
(int) (end.tv_sec - start.tv_sec), num_records);
return MDHIM_SUCCESS;
}
struct _leveldb_stuff *ldb_initialize(char *path)
{
struct _leveldb_stuff *ldbs = NULL;
leveldb_cache_t *cache;
leveldb_filterpolicy_t *policy;
char* err = NULL;
ldbs = (struct _leveldb_stuff *) malloc(sizeof(struct _leveldb_stuff));
memset(ldbs, 0, sizeof(struct _leveldb_stuff));
ldbs->options = leveldb_options_create();
snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s%s%s", LDB_WORK_PATH, "/",
path);
cache = leveldb_cache_create_lru(LDB_CACHE_LRU_SIZE);
policy = leveldb_filterpolicy_create_bloom(LDB_BLOOM_KEY_SIZE);
leveldb_options_set_filter_policy(ldbs->options, policy);
leveldb_options_set_cache(ldbs->options, cache);
leveldb_options_set_block_size(ldbs->options, LDB_BLOCK_SIZE);
leveldb_options_set_write_buffer_size(ldbs->options, LDB_WRITE_BUFFER_SIZE);
#if defined(OPEN_COMPRESSION)
leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression);
#else
leveldb_options_set_compression(ldbs->options, leveldb_no_compression);
#endif
/* Read options */
ldbs->roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1);
leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */
/* W options */
ldbs->woptions = leveldb_writeoptions_create();
#ifdef SYNC_PUT
leveldb_writeoptions_set_sync(ldbs->woptions, 1);
#else
leveldb_writeoptions_set_sync(ldbs->woptions, 0);
#endif
/* Batch write */
ldbs->wbatch = leveldb_writebatch_create();
leveldb_options_set_create_if_missing(ldbs->options, 1);
ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err);
if (err) {
fprintf(stderr, "%s", err);
leveldb_free(err);
err = NULL;
free(ldbs);
return NULL;
} else {
return ldbs;
}
}
/*
* Open the specified database, if no exists, then create it.
* Args:
* 1. name: db name
* 2. block_size: block size
* 3. wb_size: write buffer size
* 4. lru_size: lru cache size
* 5. bloom_size: bloom key size
* Return:
* _leveldb_stuff: leveldb handler.
*/
struct _leveldb_stuff *ldb_initialize(const char *name, size_t block_size, size_t wb_size, size_t lru_size, short bloom_size)
{
struct _leveldb_stuff *ldbs = NULL;
leveldb_cache_t *cache;
leveldb_filterpolicy_t *policy;
char* err = NULL;
ldbs = malloc(sizeof(struct _leveldb_stuff));
memset(ldbs, 0, sizeof(struct _leveldb_stuff));
ldbs->options = leveldb_options_create();
snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s", name);
cache = leveldb_cache_create_lru(lru_size);
policy = leveldb_filterpolicy_create_bloom(bloom_size);
leveldb_options_set_filter_policy(ldbs->options, policy);
leveldb_options_set_cache(ldbs->options, cache);
leveldb_options_set_block_size(ldbs->options, block_size);
leveldb_options_set_write_buffer_size(ldbs->options, wb_size);
#if defined(OPEN_COMPRESSION)
leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression);
#else
leveldb_options_set_compression(ldbs->options, leveldb_no_compression);
#endif
/* R */
ldbs->roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1);
leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */
/* W */
ldbs->woptions = leveldb_writeoptions_create();
#ifdef SYNC_PUT
leveldb_writeoptions_set_sync(ldbs->woptions, 1);
#else
leveldb_writeoptions_set_sync(ldbs->woptions, 0);
#endif
/* B */
ldbs->wbatch = leveldb_writebatch_create();
leveldb_options_set_create_if_missing(ldbs->options, 1);
ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err);
if (err) {
fprintf(stderr, "%s", err);
leveldb_free(err);
err = NULL;
free(ldbs);
return NULL ;
}
return ldbs;
}
void * leveldb_client_start_trans(leveldb *db){
if (!db){
kp_error("got a NULL argument\n");
return NULL;
}
leveldb_writebatch_t* trans = NULL;
trans = leveldb_writebatch_create();
return (void*) trans;
}
/**
* set if not exists.
* return 1 (not exists and saved)
* return 0 (already exists)
*/
void setnxCommand(redisClient *c) {
char *value;
sds keyword;
size_t val_len;
char *err = NULL;
leveldb_writebatch_t *wb;
err = NULL;
val_len = 0;
keyword = c->argv[1]->ptr;
value = leveldb_get(server.ds_db, server.roptions, keyword, sdslen(keyword), &val_len, &err);
if (err != NULL) {
addReplyError(c, err);
leveldb_free(err);
return;
} else if (val_len > 0) {
// already exists
leveldb_free(value);
addReplyLongLong(c, 0);
return;
}
wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, keyword, sdslen(keyword), (char *) c->argv[2]->ptr, sdslen((sds) c->argv[2]->ptr));
leveldb_write(server.ds_db, server.woptions, wb, &err);
if (err != NULL) {
addReplyError(c, err);
leveldb_free(err);
} else {
addReplyLongLong(c, 1);
server.dirty++;
signalModifiedKey(c->db, c->argv[1]);
}
leveldb_writebatch_destroy(wb);
return;
}
int main(int argc, char** argv) {
leveldb_t* db;
leveldb_comparator_t* cmp;
leveldb_cache_t* cache;
leveldb_env_t* env;
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
char* err = NULL;
int run = -1;
CheckCondition(leveldb_major_version() >= 1);
CheckCondition(leveldb_minor_version() >= 1);
snprintf(dbname, sizeof(dbname),
"%s/leveldb_c_test-%di/meta/0",
GetTempDir(),
((int) geteuid()));
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1);
leveldb_options_set_cache(options, cache);
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL);
leveldb_options_set_write_buffer_size(options, 100000);
leveldb_options_set_paranoid_checks(options, 1);
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 1);
leveldb_readoptions_set_fill_cache(roptions, 0);
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, 1);
StartPhase("destroy");
leveldb_destroy_db(options, dbname, &err);
Free(&err);
StartPhase("open");
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
StartPhase("put");
leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
StartPhase("compactall");
leveldb_compact_range(db, NULL, 0, NULL, 0);
CheckGet(db, roptions, "foo", "hello");
StartPhase("compactrange");
leveldb_compact_range(db, "a", 1, "z", 1);
CheckGet(db, roptions, "foo", "hello");
StartPhase("writebatch");
{
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
int pos = 0;
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_destroy(wb);
}
StartPhase("iter");
{
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
StartPhase("approximate_sizes");
{
int i;
int n = 20000;
char keybuf[100];
char valbuf[100];
uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" };
size_t start_len[2] = { 1, 21 };
const char* limit[2] = { "k00000000000000010000", "z" };
size_t limit_len[2] = { 21, 1 };
leveldb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
snprintf(valbuf, sizeof(valbuf), "v%020d", i);
leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
&err);
CheckNoError(err);
}
leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
CheckCondition(sizes[0] > 0);
CheckCondition(sizes[1] > 0);
}
StartPhase("property");
{
char* prop = leveldb_property_value(db, "nosuchprop");
/* CheckCondition(prop == NULL);*/
prop = leveldb_property_value(db, "leveldb.stats");
CheckCondition(prop != NULL);
Free(&prop);
}
StartPhase("snapshot");
{
const leveldb_snapshot_t* snap;
snap = leveldb_create_snapshot(db);
leveldb_delete(db, woptions, "foo", 3, &err);
CheckNoError(err);
leveldb_readoptions_set_snapshot(roptions, snap);
CheckGet(db, roptions, "foo", "hello");
leveldb_readoptions_set_snapshot(roptions, NULL);
CheckGet(db, roptions, "foo", NULL);
leveldb_release_snapshot(db, snap);
}
StartPhase("repair");
{
leveldb_close(db);
leveldb_options_set_error_if_exists(options, 0);
leveldb_repair_db(options, dbname, &err);
CheckNoError(err);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
leveldb_options_set_error_if_exists(options, 1);
}
StartPhase("filter");
for (run = 0; run < 2; run++) {
// First run uses custom filter, second run uses bloom filter
CheckNoError(err);
leveldb_filterpolicy_t* policy;
if (run == 0) {
policy = leveldb_filterpolicy_create(
NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
} else {
policy = leveldb_filterpolicy_create_bloom(10);
}
// Create new database
leveldb_close(db);
leveldb_destroy_db(options, dbname, &err);
leveldb_options_set_filter_policy(options, policy);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
CheckNoError(err);
leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
CheckNoError(err);
leveldb_compact_range(db, NULL, 0, NULL, 0);
fake_filter_result = 1;
CheckGet(db, roptions, "foo", "foovalue");
CheckGet(db, roptions, "bar", "barvalue");
if (phase == 0) {
// Must not find value when custom filter returns false
fake_filter_result = 0;
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
fake_filter_result = 1;
CheckGet(db, roptions, "foo", "foovalue");
CheckGet(db, roptions, "bar", "barvalue");
}
leveldb_options_set_filter_policy(options, NULL);
leveldb_filterpolicy_destroy(policy);
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
return 0;
}
int main(int argc, char** argv) {
leveldb_t* db;
leveldb_comparator_t* cmp;
leveldb_cache_t* cache;
leveldb_env_t* env;
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
char* err = NULL;
#if defined(LEVELDB_PLATFORM_WINDOWS)
snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test");
#else
snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d",
((int) geteuid()));
#endif
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1);
leveldb_options_set_cache(options, cache);
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL);
leveldb_options_set_write_buffer_size(options, 100000);
leveldb_options_set_paranoid_checks(options, 1);
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 1);
leveldb_readoptions_set_fill_cache(roptions, 0);
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, 1);
StartPhase("destroy");
leveldb_destroy_db(options, dbname, &err);
leveldb_free(&err);
StartPhase("open_error");
db = leveldb_open(options, dbname, &err);
CheckCondition(err != NULL);
leveldb_free(&err);
StartPhase("open");
leveldb_options_set_create_if_missing(options, 1);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
StartPhase("put");
leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
StartPhase("writebatch");
{
int pos = 0;
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_destroy(wb);
}
StartPhase("iter");
{
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
StartPhase("approximate_sizes");
{
int i;
int n = 20000;
char keybuf[100];
char valbuf[100];
uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" };
size_t start_len[2] = { 1, 21 };
const char* limit[2] = { "k00000000000000010000", "z" };
size_t limit_len[2] = { 21, 1 };
leveldb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
snprintf(valbuf, sizeof(valbuf), "v%020d", i);
leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
&err);
CheckNoError(err);
}
leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
CheckCondition(sizes[0] > 0);
CheckCondition(sizes[1] > 0);
}
StartPhase("property");
{
char* prop = leveldb_property_value(db, "nosuchprop");
CheckCondition(prop == NULL);
prop = leveldb_property_value(db, "leveldb.stats");
CheckCondition(prop != NULL);
leveldb_free(&prop);
}
StartPhase("snapshot");
{
const leveldb_snapshot_t* snap;
snap = leveldb_create_snapshot(db);
leveldb_delete(db, woptions, "foo", 3, &err);
CheckNoError(err);
leveldb_readoptions_set_snapshot(roptions, snap);
CheckGet(db, roptions, "foo", "hello");
leveldb_readoptions_set_snapshot(roptions, NULL);
CheckGet(db, roptions, "foo", NULL);
leveldb_release_snapshot(db, snap);
}
StartPhase("repair");
{
leveldb_close(db);
leveldb_options_set_create_if_missing(options, 0);
leveldb_options_set_error_if_exists(options, 0);
leveldb_repair_db(options, dbname, &err);
CheckNoError(err);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
return 0;
}
int
main ()
{
unsigned int N_KEYS;
unsigned int N_BATCH;
printf("\n number of keys:");
scanf("%u",&N_KEYS);
printf("\n batch size:");
scanf("%u",&N_BATCH);
tinymt32_t tinymt32;
//initialize random generator
tinymt32_init (&tinymt32, 0);
int iter;
int sec_iter;
int stop;
int counter;
//initialize database
char *errptr = NULL;
leveldb_options_t *options = leveldb_options_create ();
/* initializeOptions */
leveldb_options_set_create_if_missing (options, 1);
leveldb_options_set_write_buffer_size(options,120000000 );
leveldb_options_set_max_open_files(options,800000);
/*open Database */
leveldb_t *db_helper =
leveldb_open (options, "/mnt/database/database_helper", &errptr);
leveldb_t *db = leveldb_open (options, "/mnt/database/database", &errptr);
leveldb_readoptions_t *readoptions = leveldb_readoptions_create ();
leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create ();
leveldb_writebatch_t* batch=leveldb_writebatch_create();
int64_t diff = zclock_time ();
//write into database_helper
iter=0;
while( iter < N_KEYS)
{
sec_iter=0;
while((iter<N_KEYS) && (sec_iter<N_BATCH)){
unsigned int key = tinymt32_generate_uint32 (&tinymt32);
unsigned int val = tinymt32_generate_uint32 (&tinymt32);
leveldb_writebatch_put (batch,
(const char *) &key, sizeof (int),
(const char *) &val, sizeof (int));
sec_iter++;
iter++;
}
leveldb_write(
db_helper,
writeoptions,
batch,
&errptr);
if(errptr!=NULL){
printf("\n%s",errptr);
}
assert (errptr == NULL);
leveldb_writebatch_clear(batch);
}
leveldb_writebatch_destroy(batch);
diff = zclock_time () - diff;
printf ("\nrandom write: %d", diff);
diff = zclock_time ();
//write sequentially into db
leveldb_iterator_t *liter = leveldb_create_iterator (db_helper,
readoptions);
leveldb_iter_seek_to_first (liter);
while (leveldb_iter_valid (liter))
{
size_t length;
char key[4];
memcpy (key, leveldb_iter_key (liter, &length), 4);
char val[4];
memcpy (val, leveldb_iter_value (liter, &length), 4);
leveldb_iter_get_error (liter, &errptr);
assert (errptr == NULL);
leveldb_put (db,
writeoptions,
(const char *) &key, sizeof (int),
(const char *) &val, sizeof (int), &errptr);
assert (errptr == NULL);
leveldb_iter_next (liter);
}
diff = zclock_time () - diff;
leveldb_close (db);
leveldb_close (db_helper);
}
db_writebatch_t *db_writebatch_create(void) {
return leveldb_writebatch_create();
}