/** * mdhim_leveldb_batch_put * Stores multiple keys in the data store * * @param dbh in pointer to the leveldb handle * @param keys in void ** to the key to store * @param key_lens in int * to the lengths of the keys * @param data in void ** to the values of the keys * @param data_lens in int * to the lengths of the value data * @param num_records in int for the number of records to insert * @param mstore_opts in additional options for the data store layer * * @return MDHIM_SUCCESS on success or MDHIM_DB_ERROR on failure */ int mdhim_leveldb_batch_put(void *dbh, void **keys, int32_t *key_lens, void **data, int32_t *data_lens, int num_records, struct mdhim_store_opts_t *mstore_opts) { leveldb_writeoptions_t *options; char *err = NULL; leveldb_t *db = (leveldb_t *) dbh; struct timeval start, end; leveldb_writebatch_t* write_batch; int i; gettimeofday(&start, NULL); write_batch = leveldb_writebatch_create(); options = (leveldb_writeoptions_t *) mstore_opts->db_ptr4; for (i = 0; i < num_records; i++) { leveldb_writebatch_put(write_batch, keys[i], key_lens[i], data[i], data_lens[i]); } leveldb_write(db, options, write_batch, &err); leveldb_writebatch_destroy(write_batch); if (err != NULL) { mlog(MDHIM_SERVER_CRIT, "Error in batch put in leveldb"); return MDHIM_DB_ERROR; } gettimeofday(&end, NULL); mlog(MDHIM_SERVER_DBG, "Took: %d seconds to put %d records", (int) (end.tv_sec - start.tv_sec), num_records); return MDHIM_SUCCESS; }
struct _leveldb_stuff *ldb_initialize(char *path) { struct _leveldb_stuff *ldbs = NULL; leveldb_cache_t *cache; leveldb_filterpolicy_t *policy; char* err = NULL; ldbs = (struct _leveldb_stuff *) malloc(sizeof(struct _leveldb_stuff)); memset(ldbs, 0, sizeof(struct _leveldb_stuff)); ldbs->options = leveldb_options_create(); snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s%s%s", LDB_WORK_PATH, "/", path); cache = leveldb_cache_create_lru(LDB_CACHE_LRU_SIZE); policy = leveldb_filterpolicy_create_bloom(LDB_BLOOM_KEY_SIZE); leveldb_options_set_filter_policy(ldbs->options, policy); leveldb_options_set_cache(ldbs->options, cache); leveldb_options_set_block_size(ldbs->options, LDB_BLOCK_SIZE); leveldb_options_set_write_buffer_size(ldbs->options, LDB_WRITE_BUFFER_SIZE); #if defined(OPEN_COMPRESSION) leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression); #else leveldb_options_set_compression(ldbs->options, leveldb_no_compression); #endif /* Read options */ ldbs->roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1); leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */ /* W options */ ldbs->woptions = leveldb_writeoptions_create(); #ifdef SYNC_PUT leveldb_writeoptions_set_sync(ldbs->woptions, 1); #else leveldb_writeoptions_set_sync(ldbs->woptions, 0); #endif /* Batch write */ ldbs->wbatch = leveldb_writebatch_create(); leveldb_options_set_create_if_missing(ldbs->options, 1); ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err); if (err) { fprintf(stderr, "%s", err); leveldb_free(err); err = NULL; free(ldbs); return NULL; } else { return ldbs; } }
/* * Open the specified database, if no exists, then create it. * Args: * 1. name: db name * 2. block_size: block size * 3. wb_size: write buffer size * 4. lru_size: lru cache size * 5. bloom_size: bloom key size * Return: * _leveldb_stuff: leveldb handler. */ struct _leveldb_stuff *ldb_initialize(const char *name, size_t block_size, size_t wb_size, size_t lru_size, short bloom_size) { struct _leveldb_stuff *ldbs = NULL; leveldb_cache_t *cache; leveldb_filterpolicy_t *policy; char* err = NULL; ldbs = malloc(sizeof(struct _leveldb_stuff)); memset(ldbs, 0, sizeof(struct _leveldb_stuff)); ldbs->options = leveldb_options_create(); snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s", name); cache = leveldb_cache_create_lru(lru_size); policy = leveldb_filterpolicy_create_bloom(bloom_size); leveldb_options_set_filter_policy(ldbs->options, policy); leveldb_options_set_cache(ldbs->options, cache); leveldb_options_set_block_size(ldbs->options, block_size); leveldb_options_set_write_buffer_size(ldbs->options, wb_size); #if defined(OPEN_COMPRESSION) leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression); #else leveldb_options_set_compression(ldbs->options, leveldb_no_compression); #endif /* R */ ldbs->roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1); leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */ /* W */ ldbs->woptions = leveldb_writeoptions_create(); #ifdef SYNC_PUT leveldb_writeoptions_set_sync(ldbs->woptions, 1); #else leveldb_writeoptions_set_sync(ldbs->woptions, 0); #endif /* B */ ldbs->wbatch = leveldb_writebatch_create(); leveldb_options_set_create_if_missing(ldbs->options, 1); ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err); if (err) { fprintf(stderr, "%s", err); leveldb_free(err); err = NULL; free(ldbs); return NULL ; } return ldbs; }
void * leveldb_client_start_trans(leveldb *db){ if (!db){ kp_error("got a NULL argument\n"); return NULL; } leveldb_writebatch_t* trans = NULL; trans = leveldb_writebatch_create(); return (void*) trans; }
/** * set if not exists. * return 1 (not exists and saved) * return 0 (already exists) */ void setnxCommand(redisClient *c) { char *value; sds keyword; size_t val_len; char *err = NULL; leveldb_writebatch_t *wb; err = NULL; val_len = 0; keyword = c->argv[1]->ptr; value = leveldb_get(server.ds_db, server.roptions, keyword, sdslen(keyword), &val_len, &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); return; } else if (val_len > 0) { // already exists leveldb_free(value); addReplyLongLong(c, 0); return; } wb = leveldb_writebatch_create(); leveldb_writebatch_put(wb, keyword, sdslen(keyword), (char *) c->argv[2]->ptr, sdslen((sds) c->argv[2]->ptr)); leveldb_write(server.ds_db, server.woptions, wb, &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); } else { addReplyLongLong(c, 1); server.dirty++; signalModifiedKey(c->db, c->argv[1]); } leveldb_writebatch_destroy(wb); return; }
int main(int argc, char** argv) { leveldb_t* db; leveldb_comparator_t* cmp; leveldb_cache_t* cache; leveldb_env_t* env; leveldb_options_t* options; leveldb_readoptions_t* roptions; leveldb_writeoptions_t* woptions; char* err = NULL; int run = -1; CheckCondition(leveldb_major_version() >= 1); CheckCondition(leveldb_minor_version() >= 1); snprintf(dbname, sizeof(dbname), "%s/leveldb_c_test-%di/meta/0", GetTempDir(), ((int) geteuid())); StartPhase("create_objects"); cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); env = leveldb_create_default_env(); cache = leveldb_cache_create_lru(100000); options = leveldb_options_create(); leveldb_options_set_comparator(options, cmp); leveldb_options_set_error_if_exists(options, 1); leveldb_options_set_cache(options, cache); leveldb_options_set_env(options, env); leveldb_options_set_info_log(options, NULL); leveldb_options_set_write_buffer_size(options, 100000); leveldb_options_set_paranoid_checks(options, 1); leveldb_options_set_max_open_files(options, 10); leveldb_options_set_block_size(options, 1024); leveldb_options_set_block_restart_interval(options, 8); leveldb_options_set_compression(options, leveldb_no_compression); roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(roptions, 1); leveldb_readoptions_set_fill_cache(roptions, 0); woptions = leveldb_writeoptions_create(); leveldb_writeoptions_set_sync(woptions, 1); StartPhase("destroy"); leveldb_destroy_db(options, dbname, &err); Free(&err); StartPhase("open"); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); StartPhase("put"); leveldb_put(db, woptions, "foo", 3, "hello", 5, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); StartPhase("compactall"); leveldb_compact_range(db, NULL, 0, NULL, 0); CheckGet(db, roptions, "foo", "hello"); StartPhase("compactrange"); leveldb_compact_range(db, "a", 1, "z", 1); CheckGet(db, roptions, "foo", "hello"); StartPhase("writebatch"); { leveldb_writebatch_t* wb = leveldb_writebatch_create(); leveldb_writebatch_put(wb, "foo", 3, "a", 1); leveldb_writebatch_clear(wb); leveldb_writebatch_put(wb, "bar", 3, "b", 1); leveldb_writebatch_put(wb, "box", 3, "c", 1); leveldb_writebatch_delete(wb, "bar", 3); leveldb_write(db, woptions, wb, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); int pos = 0; leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); CheckCondition(pos == 3); leveldb_writebatch_destroy(wb); } StartPhase("iter"); { leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_first(iter); CheckCondition(leveldb_iter_valid(iter)); CheckIter(iter, "box", "c"); leveldb_iter_next(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_prev(iter); CheckIter(iter, "box", "c"); leveldb_iter_prev(iter); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_last(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_seek(iter, "b", 1); CheckIter(iter, "box", "c"); leveldb_iter_get_error(iter, &err); CheckNoError(err); leveldb_iter_destroy(iter); } StartPhase("approximate_sizes"); { int i; int n = 20000; char keybuf[100]; char valbuf[100]; uint64_t sizes[2]; const char* start[2] = { "a", "k00000000000000010000" }; size_t start_len[2] = { 1, 21 }; const char* limit[2] = { "k00000000000000010000", "z" }; size_t limit_len[2] = { 21, 1 }; leveldb_writeoptions_set_sync(woptions, 0); for (i = 0; i < n; i++) { snprintf(keybuf, sizeof(keybuf), "k%020d", i); snprintf(valbuf, sizeof(valbuf), "v%020d", i); leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf), &err); CheckNoError(err); } leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes); CheckCondition(sizes[0] > 0); CheckCondition(sizes[1] > 0); } StartPhase("property"); { char* prop = leveldb_property_value(db, "nosuchprop"); /* CheckCondition(prop == NULL);*/ prop = leveldb_property_value(db, "leveldb.stats"); CheckCondition(prop != NULL); Free(&prop); } StartPhase("snapshot"); { const leveldb_snapshot_t* snap; snap = leveldb_create_snapshot(db); leveldb_delete(db, woptions, "foo", 3, &err); CheckNoError(err); leveldb_readoptions_set_snapshot(roptions, snap); CheckGet(db, roptions, "foo", "hello"); leveldb_readoptions_set_snapshot(roptions, NULL); CheckGet(db, roptions, "foo", NULL); leveldb_release_snapshot(db, snap); } StartPhase("repair"); { leveldb_close(db); leveldb_options_set_error_if_exists(options, 0); leveldb_repair_db(options, dbname, &err); CheckNoError(err); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); leveldb_options_set_error_if_exists(options, 1); } StartPhase("filter"); for (run = 0; run < 2; run++) { // First run uses custom filter, second run uses bloom filter CheckNoError(err); leveldb_filterpolicy_t* policy; if (run == 0) { policy = leveldb_filterpolicy_create( NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName); } else { policy = leveldb_filterpolicy_create_bloom(10); } // Create new database leveldb_close(db); leveldb_destroy_db(options, dbname, &err); leveldb_options_set_filter_policy(options, policy); db = leveldb_open(options, dbname, &err); CheckNoError(err); leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err); CheckNoError(err); leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err); CheckNoError(err); leveldb_compact_range(db, NULL, 0, NULL, 0); fake_filter_result = 1; CheckGet(db, roptions, "foo", "foovalue"); CheckGet(db, roptions, "bar", "barvalue"); if (phase == 0) { // Must not find value when custom filter returns false fake_filter_result = 0; CheckGet(db, roptions, "foo", NULL); CheckGet(db, roptions, "bar", NULL); fake_filter_result = 1; CheckGet(db, roptions, "foo", "foovalue"); CheckGet(db, roptions, "bar", "barvalue"); } leveldb_options_set_filter_policy(options, NULL); leveldb_filterpolicy_destroy(policy); } StartPhase("cleanup"); leveldb_close(db); leveldb_options_destroy(options); leveldb_readoptions_destroy(roptions); leveldb_writeoptions_destroy(woptions); leveldb_cache_destroy(cache); leveldb_comparator_destroy(cmp); leveldb_env_destroy(env); fprintf(stderr, "PASS\n"); return 0; }
int main(int argc, char** argv) { leveldb_t* db; leveldb_comparator_t* cmp; leveldb_cache_t* cache; leveldb_env_t* env; leveldb_options_t* options; leveldb_readoptions_t* roptions; leveldb_writeoptions_t* woptions; char* err = NULL; #if defined(LEVELDB_PLATFORM_WINDOWS) snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test"); #else snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d", ((int) geteuid())); #endif StartPhase("create_objects"); cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); env = leveldb_create_default_env(); cache = leveldb_cache_create_lru(100000); options = leveldb_options_create(); leveldb_options_set_comparator(options, cmp); leveldb_options_set_error_if_exists(options, 1); leveldb_options_set_cache(options, cache); leveldb_options_set_env(options, env); leveldb_options_set_info_log(options, NULL); leveldb_options_set_write_buffer_size(options, 100000); leveldb_options_set_paranoid_checks(options, 1); leveldb_options_set_max_open_files(options, 10); leveldb_options_set_block_size(options, 1024); leveldb_options_set_block_restart_interval(options, 8); leveldb_options_set_compression(options, leveldb_no_compression); roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(roptions, 1); leveldb_readoptions_set_fill_cache(roptions, 0); woptions = leveldb_writeoptions_create(); leveldb_writeoptions_set_sync(woptions, 1); StartPhase("destroy"); leveldb_destroy_db(options, dbname, &err); leveldb_free(&err); StartPhase("open_error"); db = leveldb_open(options, dbname, &err); CheckCondition(err != NULL); leveldb_free(&err); StartPhase("open"); leveldb_options_set_create_if_missing(options, 1); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); StartPhase("put"); leveldb_put(db, woptions, "foo", 3, "hello", 5, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); StartPhase("writebatch"); { int pos = 0; leveldb_writebatch_t* wb = leveldb_writebatch_create(); leveldb_writebatch_put(wb, "foo", 3, "a", 1); leveldb_writebatch_clear(wb); leveldb_writebatch_put(wb, "bar", 3, "b", 1); leveldb_writebatch_put(wb, "box", 3, "c", 1); leveldb_writebatch_delete(wb, "bar", 3); leveldb_write(db, woptions, wb, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); CheckCondition(pos == 3); leveldb_writebatch_destroy(wb); } StartPhase("iter"); { leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_first(iter); CheckCondition(leveldb_iter_valid(iter)); CheckIter(iter, "box", "c"); leveldb_iter_next(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_prev(iter); CheckIter(iter, "box", "c"); leveldb_iter_prev(iter); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_last(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_seek(iter, "b", 1); CheckIter(iter, "box", "c"); leveldb_iter_get_error(iter, &err); CheckNoError(err); leveldb_iter_destroy(iter); } StartPhase("approximate_sizes"); { int i; int n = 20000; char keybuf[100]; char valbuf[100]; uint64_t sizes[2]; const char* start[2] = { "a", "k00000000000000010000" }; size_t start_len[2] = { 1, 21 }; const char* limit[2] = { "k00000000000000010000", "z" }; size_t limit_len[2] = { 21, 1 }; leveldb_writeoptions_set_sync(woptions, 0); for (i = 0; i < n; i++) { snprintf(keybuf, sizeof(keybuf), "k%020d", i); snprintf(valbuf, sizeof(valbuf), "v%020d", i); leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf), &err); CheckNoError(err); } leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes); CheckCondition(sizes[0] > 0); CheckCondition(sizes[1] > 0); } StartPhase("property"); { char* prop = leveldb_property_value(db, "nosuchprop"); CheckCondition(prop == NULL); prop = leveldb_property_value(db, "leveldb.stats"); CheckCondition(prop != NULL); leveldb_free(&prop); } StartPhase("snapshot"); { const leveldb_snapshot_t* snap; snap = leveldb_create_snapshot(db); leveldb_delete(db, woptions, "foo", 3, &err); CheckNoError(err); leveldb_readoptions_set_snapshot(roptions, snap); CheckGet(db, roptions, "foo", "hello"); leveldb_readoptions_set_snapshot(roptions, NULL); CheckGet(db, roptions, "foo", NULL); leveldb_release_snapshot(db, snap); } StartPhase("repair"); { leveldb_close(db); leveldb_options_set_create_if_missing(options, 0); leveldb_options_set_error_if_exists(options, 0); leveldb_repair_db(options, dbname, &err); CheckNoError(err); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); } StartPhase("cleanup"); leveldb_close(db); leveldb_options_destroy(options); leveldb_readoptions_destroy(roptions); leveldb_writeoptions_destroy(woptions); leveldb_cache_destroy(cache); leveldb_comparator_destroy(cmp); leveldb_env_destroy(env); fprintf(stderr, "PASS\n"); return 0; }
int main () { unsigned int N_KEYS; unsigned int N_BATCH; printf("\n number of keys:"); scanf("%u",&N_KEYS); printf("\n batch size:"); scanf("%u",&N_BATCH); tinymt32_t tinymt32; //initialize random generator tinymt32_init (&tinymt32, 0); int iter; int sec_iter; int stop; int counter; //initialize database char *errptr = NULL; leveldb_options_t *options = leveldb_options_create (); /* initializeOptions */ leveldb_options_set_create_if_missing (options, 1); leveldb_options_set_write_buffer_size(options,120000000 ); leveldb_options_set_max_open_files(options,800000); /*open Database */ leveldb_t *db_helper = leveldb_open (options, "/mnt/database/database_helper", &errptr); leveldb_t *db = leveldb_open (options, "/mnt/database/database", &errptr); leveldb_readoptions_t *readoptions = leveldb_readoptions_create (); leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create (); leveldb_writebatch_t* batch=leveldb_writebatch_create(); int64_t diff = zclock_time (); //write into database_helper iter=0; while( iter < N_KEYS) { sec_iter=0; while((iter<N_KEYS) && (sec_iter<N_BATCH)){ unsigned int key = tinymt32_generate_uint32 (&tinymt32); unsigned int val = tinymt32_generate_uint32 (&tinymt32); leveldb_writebatch_put (batch, (const char *) &key, sizeof (int), (const char *) &val, sizeof (int)); sec_iter++; iter++; } leveldb_write( db_helper, writeoptions, batch, &errptr); if(errptr!=NULL){ printf("\n%s",errptr); } assert (errptr == NULL); leveldb_writebatch_clear(batch); } leveldb_writebatch_destroy(batch); diff = zclock_time () - diff; printf ("\nrandom write: %d", diff); diff = zclock_time (); //write sequentially into db leveldb_iterator_t *liter = leveldb_create_iterator (db_helper, readoptions); leveldb_iter_seek_to_first (liter); while (leveldb_iter_valid (liter)) { size_t length; char key[4]; memcpy (key, leveldb_iter_key (liter, &length), 4); char val[4]; memcpy (val, leveldb_iter_value (liter, &length), 4); leveldb_iter_get_error (liter, &errptr); assert (errptr == NULL); leveldb_put (db, writeoptions, (const char *) &key, sizeof (int), (const char *) &val, sizeof (int), &errptr); assert (errptr == NULL); leveldb_iter_next (liter); } diff = zclock_time () - diff; leveldb_close (db); leveldb_close (db_helper); }
db_writebatch_t *db_writebatch_create(void) { return leveldb_writebatch_create(); }