/* =============
* Database info extraction methods
* =============
*/
void init_nwdb() {
char path[PATH_MAX];
char *err = NULL;
sprintf(path, "%s/%s", cde_pseudo_pkg_dir, DB_NAME);
if (access(path, R_OK)==-1) {
fprintf(stderr, "Network provenance database does not exist!\n");
exit(-1);
}
netdb = malloc(sizeof(lvldb_t));
netdb->options = leveldb_options_create();
leveldb_options_set_create_if_missing(netdb->options, 0);
netdb->db = leveldb_open(netdb->options, path, &err);
if (err != NULL || netdb == NULL) {
fprintf(stderr, "Leveldb open fail!\n");
exit(-1);
}
assert(netdb->db!=NULL);
netdb->woptions = leveldb_writeoptions_create();
netdb->roptions = leveldb_readoptions_create();
/* reset error var */
leveldb_free(err);
err = NULL;
/* read in root pid */
netdb_root = db_readc(netdb, "meta.root");
assert(netdb_root != NULL);
if (CDE_nw_mode) {
/* create temp db for current execution graph */
sprintf(path, "%s/%s.tempXXXXXX", cde_pseudo_pkg_dir, DB_NAME);
if (mkdtemp(path) == NULL) {
fprintf(stderr, "Cannot create temp db!\n");
exit(-1);
}
currdb = malloc(sizeof(lvldb_t));
currdb->options = leveldb_options_create();
leveldb_options_set_create_if_missing(currdb->options, 1);
currdb->db = leveldb_open(currdb->options, path, &err);
assert(currdb->db!=NULL);
leveldb_free(err);
err = NULL;
currdb->woptions = leveldb_writeoptions_create();
currdb->roptions = leveldb_readoptions_create();
db_write_root(currdb, getpid());
}
}
static int
txdb_open_db(struct txdb *txdb)
{
char *err = NULL;
txdb->db_opts = leveldb_options_create();
leveldb_options_set_create_if_missing(txdb->db_opts, 1);
txdb->db = leveldb_open(txdb->db_opts, txdb->path, &err);
if (err) {
Warning(LGPFX" failed to open db '%s' : %s\n", txdb->path, err);
printf("btc failed to open the leveldb database at '%s' : %s\n",
txdb->path, err);
printf("Could there be another instance of btc running?\n");
free(err);
return 1;
}
txdb->rd_opts = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(txdb->rd_opts, 1);
leveldb_readoptions_set_fill_cache(txdb->rd_opts, 0);
txdb->wr_opts = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(txdb->wr_opts, 1);
return 0;
}
struct db_conn *xleveldb_open(const char *name, int flags)
{
struct db_conn *db;
leveldb_options_t *options;
leveldb_t *level_db;
options = leveldb_options_create();
if (options == NULL)
return NULL;
leveldb_options_set_create_if_missing(options, 1);
level_db = leveldb_open(options, name, NULL);
leveldb_options_destroy(options);
if (level_db == NULL)
return NULL;
db = malloc(sizeof(struct db_conn));
if (db == NULL) {
leveldb_close(level_db);
return NULL;
}
db->conn = level_db;
db->set = xleveldb_set;
db->get = xleveldb_get;
db->del = xleveldb_del;
db->firstkey = xleveldb_firstkey;
db->nextkey = xleveldb_nextkey;
db->close = xleveldb_close;
return db;
}
int stat_cache_open(stat_cache_t **cache, struct stat_cache_supplemental *supplemental, char *cache_path) {
char *error = NULL;
char storage_path[PATH_MAX];
// Check that a directory is set.
if (!cache_path) {
// @TODO: Before public release: Use a mkdtemp-based path.
log_print(LOG_WARNING, "No cache path specified.");
return -EINVAL;
}
snprintf(storage_path, PATH_MAX, "%s/leveldb", cache_path);
supplemental->options = leveldb_options_create();
// Initialize LevelDB's LRU cache.
supplemental->lru = NULL;
//supplemental->lru = leveldb_cache_create_lru(5 * 1048576); // 5MB
//leveldb_options_set_cache(supplemental->options, supplemental->lru);
// Create the database if missing.
leveldb_options_set_create_if_missing(supplemental->options, true);
leveldb_options_set_error_if_exists(supplemental->options, false);
// Use a fusedav logger.
leveldb_options_set_info_log(supplemental->options, NULL);
*cache = leveldb_open(supplemental->options, storage_path, &error);
if (error) {
log_print(LOG_ERR, "ERROR opening db: %s", error);
return -1;
}
return 0;
}
int db_open()
{
char *error = NULL;
char *filename = option_get_str("db_file");
ldb_options = leveldb_options_create();
ldb_cache = leveldb_cache_create_lru(option_get_int("cache_size"));
leveldb_options_set_create_if_missing(ldb_options, option_get_int("create_db_if_missing"));
leveldb_options_set_error_if_exists(ldb_options, option_get_int("error_if_db_exists"));
leveldb_options_set_paranoid_checks(ldb_options, option_get_int("paranoid_checks"));
leveldb_options_set_write_buffer_size(ldb_options, option_get_int("write_buffer_size"));
leveldb_options_set_block_size(ldb_options, option_get_int("block_size"));
leveldb_options_set_cache(ldb_options, ldb_cache);
leveldb_options_set_max_open_files(ldb_options, option_get_int("leveldb_max_open_files"));
leveldb_options_set_block_restart_interval(ldb_options, 8);
leveldb_options_set_compression(ldb_options, option_get_int("compression"));
// leveldb_options_set_env(options, self->_env);
leveldb_options_set_info_log(ldb_options, NULL);
ldb = leveldb_open(ldb_options, filename, &error);
if (error) {
fprintf(stderr, "ERROR opening db:%s\n", error);
return 0;
}
return 1;
}
// Opens the tablet for reading and writing. This should be called directly.
// Only the owning table should open a tablet.
//
// tablet - The tablet.
//
// Returns 0 if successful, otherwise returns -1.
int sky_tablet_open(sky_tablet *tablet)
{
char* errptr = NULL;
leveldb_options_t* options = NULL;
assert(tablet != NULL);
check(tablet->path != NULL, "Tablet path required");
// Close the tablet if it's already open.
sky_tablet_close(tablet);
// Initialize data file.
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, true);
tablet->leveldb_db = leveldb_open(options, bdata(tablet->path), &errptr);
check(errptr == NULL, "LevelDB Error: %s", errptr);
check(tablet->leveldb_db != NULL, "Unable to create LevelDB data file");
leveldb_options_destroy(options);
return 0;
error:
if(errptr) leveldb_free(errptr);
if(options) leveldb_options_destroy(options);
sky_tablet_close(tablet);
return -1;
}
void
localdb_init (localdb_t ** localdb)
{
*localdb = malloc (sizeof (localdb_t));
char *errptr = NULL;
leveldb_options_t *options = leveldb_options_create ();
leveldb_options_set_create_if_missing (options, 1);
(*localdb)->options = options;
leveldb_readoptions_t *readoptions = leveldb_readoptions_create ();
(*localdb)->readoptions = readoptions;
leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create ();
(*localdb)->writeoptions = writeoptions;
leveldb_writeoptions_set_sync (writeoptions, 1);
(*localdb)->db = leveldb_open (options, "/mnt/localdb", &errptr);
if (errptr) {
printf ("\n%s", errptr);
exit (1);
}
}
db_t* open_db(const char* path, database_t type)
{
db_t* db = (db_t*) malloc(sizeof(db_t));
db->type = type;
if(type == LEVELDB)
{
levelDB_t* leveldb = (levelDB_t*) malloc(sizeof(levelDB_t));
leveldb->err = NULL;
leveldb->options = leveldb_options_create();
leveldb->woptions = leveldb_writeoptions_create();
leveldb->roptions = leveldb_readoptions_create();
leveldb_options_set_create_if_missing(leveldb->options, 1);
leveldb->db = leveldb_open(leveldb->options, path, &(leveldb->err));
if(leveldb->err != NULL)
{
fprintf(stderr, "Failed to open database! Error message: %s\n", leveldb->err);
exit(-1);
}
leveldb_free(leveldb->err);
leveldb->err = NULL;
db->database = (void*) leveldb;
}
else
{
kernelDB_t* kerneldb = (kernelDB_t*) malloc(sizeof(kernelDB_t));
kerneldb->client = kr_open(path);
db->database = (void*) kerneldb;
}
return db;
}
db_t *db_open(db_options_t *o, const char *name, char **errptr) {
leveldb_filterpolicy_t *fp = leveldb_filterpolicy_create_bloom(10);
leveldb_options_set_create_if_missing(o, 1);
leveldb_options_set_filter_policy(o, fp);
return leveldb_open(o, name, errptr);
}
void db_leveldb_init()
{
char *errstr = NULL;
leveldb_options = leveldb_options_create();
/* create if missing */
leveldb_options_set_create_if_missing(leveldb_options, 1);
/* lru cache */
leveldb_cache = leveldb_cache_create_lru(conf->leveldb_cache_size);
leveldb_options_set_cache(leveldb_options, leveldb_cache);
/* block size */
leveldb_options_set_block_size(leveldb_options, conf->leveldb_block_size);
/* write buffer size */
leveldb_options_set_write_buffer_size(leveldb_options, conf->leveldb_write_buffer_size);
/* filter policy */
leveldb_filterpolicy = leveldb_filterpolicy_create_bloom(10);
leveldb_options_set_filter_policy(leveldb_options, leveldb_filterpolicy);
/* open db */
leveldb_db = leveldb_open(leveldb_options, conf->db, &errstr);
if (errstr) {
terrx(1, "unable to open db at %s: %s", conf->db, errstr);
}
leveldb_roptions = leveldb_readoptions_create();
leveldb_woptions = leveldb_writeoptions_create();
}
struct _leveldb_stuff *ldb_initialize(char *path)
{
struct _leveldb_stuff *ldbs = NULL;
leveldb_cache_t *cache;
leveldb_filterpolicy_t *policy;
char* err = NULL;
ldbs = (struct _leveldb_stuff *) malloc(sizeof(struct _leveldb_stuff));
memset(ldbs, 0, sizeof(struct _leveldb_stuff));
ldbs->options = leveldb_options_create();
snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s%s%s", LDB_WORK_PATH, "/",
path);
cache = leveldb_cache_create_lru(LDB_CACHE_LRU_SIZE);
policy = leveldb_filterpolicy_create_bloom(LDB_BLOOM_KEY_SIZE);
leveldb_options_set_filter_policy(ldbs->options, policy);
leveldb_options_set_cache(ldbs->options, cache);
leveldb_options_set_block_size(ldbs->options, LDB_BLOCK_SIZE);
leveldb_options_set_write_buffer_size(ldbs->options, LDB_WRITE_BUFFER_SIZE);
#if defined(OPEN_COMPRESSION)
leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression);
#else
leveldb_options_set_compression(ldbs->options, leveldb_no_compression);
#endif
/* Read options */
ldbs->roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1);
leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */
/* W options */
ldbs->woptions = leveldb_writeoptions_create();
#ifdef SYNC_PUT
leveldb_writeoptions_set_sync(ldbs->woptions, 1);
#else
leveldb_writeoptions_set_sync(ldbs->woptions, 0);
#endif
/* Batch write */
ldbs->wbatch = leveldb_writebatch_create();
leveldb_options_set_create_if_missing(ldbs->options, 1);
ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err);
if (err) {
fprintf(stderr, "%s", err);
leveldb_free(err);
err = NULL;
free(ldbs);
return NULL;
} else {
return ldbs;
}
}
/*
* Open the specified database, if no exists, then create it.
* Args:
* 1. name: db name
* 2. block_size: block size
* 3. wb_size: write buffer size
* 4. lru_size: lru cache size
* 5. bloom_size: bloom key size
* Return:
* _leveldb_stuff: leveldb handler.
*/
struct _leveldb_stuff *ldb_initialize(const char *name, size_t block_size, size_t wb_size, size_t lru_size, short bloom_size)
{
struct _leveldb_stuff *ldbs = NULL;
leveldb_cache_t *cache;
leveldb_filterpolicy_t *policy;
char* err = NULL;
ldbs = malloc(sizeof(struct _leveldb_stuff));
memset(ldbs, 0, sizeof(struct _leveldb_stuff));
ldbs->options = leveldb_options_create();
snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s", name);
cache = leveldb_cache_create_lru(lru_size);
policy = leveldb_filterpolicy_create_bloom(bloom_size);
leveldb_options_set_filter_policy(ldbs->options, policy);
leveldb_options_set_cache(ldbs->options, cache);
leveldb_options_set_block_size(ldbs->options, block_size);
leveldb_options_set_write_buffer_size(ldbs->options, wb_size);
#if defined(OPEN_COMPRESSION)
leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression);
#else
leveldb_options_set_compression(ldbs->options, leveldb_no_compression);
#endif
/* R */
ldbs->roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1);
leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */
/* W */
ldbs->woptions = leveldb_writeoptions_create();
#ifdef SYNC_PUT
leveldb_writeoptions_set_sync(ldbs->woptions, 1);
#else
leveldb_writeoptions_set_sync(ldbs->woptions, 0);
#endif
/* B */
ldbs->wbatch = leveldb_writebatch_create();
leveldb_options_set_create_if_missing(ldbs->options, 1);
ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err);
if (err) {
fprintf(stderr, "%s", err);
leveldb_free(err);
err = NULL;
free(ldbs);
return NULL ;
}
return ldbs;
}
RLServer::RLServer(const char *_db_path, const char *_hostaddr, int _port, int dbn):
db_num(dbn), db_path(_db_path), hostaddr(_hostaddr), port(_port),
fd(-1), clients_num(0)
{
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
read_options = leveldb_readoptions_create();
write_options = leveldb_writeoptions_create();
char* err = 0;
if(db_num<1){
db=new leveldb_t*[1];
db[0] = leveldb_open(options, db_path.c_str(), &err);
if(err) {
puts(err);
exit(1);
}
}else{
db=new leveldb_t*[db_num];
char buf[16];
for(int i=0;i<db_num;i++){
int count = sprintf(buf, "/db-%03d", i);
//TODO the db path
db[i] = leveldb_open(options, (db_path+std::string(buf,count)).c_str(), &err);
if(err) {
puts(buf);
puts(err);
exit(1);
}
}
}
loop = ev_loop_new(EVBACKEND);
connection_watcher.data = this;
}
leveldb_t * store_open(char * name){
leveldb_t *db = 0x00;
leveldb_options_t *options;
leveldb_readoptions_t *roptions;
leveldb_writeoptions_t *woptions;
char *err = 0x00;
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
db = leveldb_open(options, name, &err);
if (err) {
fprintf(stderr, "error: %s\n", err);
free(err);
db = 0x00;
}
return db;
}
db_t *
db_open(const char *path, char **errptr) {
db_t *out;
leveldb_options_t *opts;
leveldb_t *db;
opts = leveldb_options_create();
leveldb_options_set_create_if_missing(opts, 1);
db = leveldb_open(opts, path, errptr);
if (*errptr) {
leveldb_options_destroy(opts);
return NULL;
}
out = malloc(sizeof(db_t));
*out = (db_t){ .db=db, .opts=opts };
return out;
}
kvdb_t *kvdb_leveldb_open(const char *dbpath)
{
kvdb_leveldb_t *leveldb = (kvdb_leveldb_t *)zmalloc(sizeof(struct kvdb_leveldb_t));
memset(leveldb, 0, sizeof(kvdb_leveldb_t));
leveldb->kvdb.dbclass = "leveldb";
leveldb->kvdb.db_methods = &leveldb_methods;
leveldb->pOpt = leveldb_options_create();
leveldb_options_set_create_if_missing(leveldb->pOpt, 1);
leveldb->pWriteOpt = leveldb_writeoptions_create();
leveldb->pReadOpt = leveldb_readoptions_create();
char *szErr = NULL;
leveldb->db = leveldb_open(leveldb->pOpt, dbpath, &szErr);
if( szErr ){
kvdb_leveldb_close((kvdb_t*)leveldb);
return NULL;
}
return (kvdb_t*)leveldb;
}
void ds_init() {
char *err = NULL;
server.ds_cache = leveldb_cache_create_lru(server.ds_lru_cache * 1048576);
server.ds_options = leveldb_options_create();
server.policy = leveldb_filterpolicy_create_bloom(10);
//leveldb_options_set_comparator(server.ds_options, cmp);
leveldb_options_set_filter_policy(server.ds_options, server.policy);
leveldb_options_set_create_if_missing(server.ds_options, server.ds_create_if_missing);
leveldb_options_set_error_if_exists(server.ds_options, server.ds_error_if_exists);
leveldb_options_set_cache(server.ds_options, server.ds_cache);
leveldb_options_set_info_log(server.ds_options, NULL);
leveldb_options_set_write_buffer_size(server.ds_options, server.ds_write_buffer_size * 1048576);
leveldb_options_set_paranoid_checks(server.ds_options, server.ds_paranoid_checks);
leveldb_options_set_max_open_files(server.ds_options, server.ds_max_open_files);
leveldb_options_set_block_size(server.ds_options, server.ds_block_size * 1024);
leveldb_options_set_block_restart_interval(server.ds_options, server.ds_block_restart_interval);
leveldb_options_set_compression(server.ds_options, leveldb_snappy_compression);
server.ds_db = leveldb_open(server.ds_options, server.ds_path, &err);
if (err != NULL) {
fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, err);
leveldb_free(err);
exit(1);
}
server.woptions = leveldb_writeoptions_create();
server.roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(server.roptions, 0);
leveldb_readoptions_set_fill_cache(server.roptions, 1);
leveldb_writeoptions_set_sync(server.woptions, 0);
}
int
main ()
{
int64_t diff;
unsigned int N_KEYS;
unsigned int TN_KEYS;
int bool_bench;
int N_THREADS;
printf ("\n number of keys to retrieve:");
scanf ("%u", &N_KEYS);
printf ("\n total number of keys:");
scanf ("%u", &TN_KEYS);
printf("\n benchmark randomly inserted database or sequentially inserted database:(1->rand or 0->seq)");
scanf ("%u", &bool_bench);
printf ("\n number of threads:");
scanf ("%u", &N_THREADS);
//zeromq context
zctx_t *ctx = zctx_new ();
//initialize worker args
worker_args_t worker_args;
//initialize rb_tree;
struct dbkey_rb_t dbkey_rb;
RB_INIT (&dbkey_rb);
//initialize keys
unsigned int factor = (unsigned int) N_KEYS / TN_KEYS;
tinymt32_t tinymt32;
/*initializing random generator with the same seed */
tinymt32_init (&tinymt32, 0);
unsigned int *keys =
(unsigned int *) malloc (sizeof (unsigned int) * N_KEYS);
unsigned int sec_iter;
unsigned int iter;
iter = 0;
while (iter < N_KEYS)
{
keys[iter] = tinymt32_generate_uint32 (&tinymt32);
tinymt32_generate_uint32 (&tinymt32);
for (sec_iter = 1; sec_iter < factor; sec_iter++)
{
tinymt32_generate_uint32 (&tinymt32);
tinymt32_generate_uint32 (&tinymt32);
}
iter++;
}
//initialize database
char *errptr = NULL;
leveldb_options_t *options = leveldb_options_create ();
worker_args.options = &options;
/* initialize Options */
leveldb_options_set_create_if_missing (options, 1);
leveldb_options_set_write_buffer_size (options, 62914560);
leveldb_options_set_max_open_files (options, 800000);
//bloom filter
leveldb_filterpolicy_t *bloom = leveldb_filterpolicy_create_bloom (10);
leveldb_options_set_filter_policy (options, bloom);
leveldb_readoptions_t *readoptions = leveldb_readoptions_create ();
worker_args.readoptions = readoptions;
leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create ();
worker_args.writeoptions = writeoptions;
int rc;
void *push = zsocket_new (ctx, ZMQ_PUSH);
int water = 2000000000;
zsocket_set_hwm (push, water);
rc = zsocket_bind (push, "inproc://bind_point");
//connect
void *pub = zsocket_new (ctx, ZMQ_PUB);
rc = zsocket_bind (pub, "inproc://pub_point");
//connect
void *router = zsocket_new (ctx, ZMQ_ROUTER);
rc = zsocket_bind (router, "inproc://response_point");
//sleep a while
zclock_sleep (1000);
//this assumes some synchronization at the start and end of each new bench
leveldb_t *db_pointer;
worker_args.db = &db_pointer;
//initialize the threads
void *pipe[6];
unsigned char i;
for (i = 0; i < N_THREADS; i++)
{
pipe[i] = zthread_fork (ctx, &worker_fn, (void *) &(worker_args));
}
//sleep a while
zclock_sleep (1000);
if(bool_bench==0){
db_pointer = leveldb_open (options, "/mnt/database/database", &errptr);
benchmark_notree (push, pub, router, N_KEYS, keys,N_THREADS);
printf ("\n benchmark with tree 1st pass");
leveldb_close (db_pointer);
db_pointer = leveldb_open (options, "/mnt/database/database", &errptr);
benchmark_tree (push, pub, router, N_KEYS, keys, dbkey_rb ,N_THREADS);
leveldb_close (db_pointer);
}else{if(bool_bench==1){
db_pointer =
leveldb_open (options, "/mnt/database/database_helper", &errptr);
printf
("\n starting random read without a rb_btree on random inserted database");
benchmark_notree (push, pub, router, N_KEYS, keys,N_THREADS);
leveldb_close (db_pointer);
db_pointer =
leveldb_open (options, "/mnt/database/database_helper", &errptr);
printf ("\n with a tree");
benchmark_tree (push, pub, router, N_KEYS, keys, dbkey_rb,N_THREADS);
leveldb_close (db_pointer);
}
}
}
int main(int argc, char** argv) {
leveldb_t* db;
leveldb_comparator_t* cmp;
leveldb_cache_t* cache;
leveldb_env_t* env;
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
char* err = NULL;
int run = -1;
CheckCondition(leveldb_major_version() >= 1);
CheckCondition(leveldb_minor_version() >= 1);
snprintf(dbname, sizeof(dbname),
"%s/leveldb_c_test-%di/meta/0",
GetTempDir(),
((int) geteuid()));
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1);
leveldb_options_set_cache(options, cache);
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL);
leveldb_options_set_write_buffer_size(options, 100000);
leveldb_options_set_paranoid_checks(options, 1);
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 1);
leveldb_readoptions_set_fill_cache(roptions, 0);
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, 1);
StartPhase("destroy");
leveldb_destroy_db(options, dbname, &err);
Free(&err);
StartPhase("open");
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
StartPhase("put");
leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
StartPhase("compactall");
leveldb_compact_range(db, NULL, 0, NULL, 0);
CheckGet(db, roptions, "foo", "hello");
StartPhase("compactrange");
leveldb_compact_range(db, "a", 1, "z", 1);
CheckGet(db, roptions, "foo", "hello");
StartPhase("writebatch");
{
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
int pos = 0;
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_destroy(wb);
}
StartPhase("iter");
{
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
StartPhase("approximate_sizes");
{
int i;
int n = 20000;
char keybuf[100];
char valbuf[100];
uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" };
size_t start_len[2] = { 1, 21 };
const char* limit[2] = { "k00000000000000010000", "z" };
size_t limit_len[2] = { 21, 1 };
leveldb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
snprintf(valbuf, sizeof(valbuf), "v%020d", i);
leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
&err);
CheckNoError(err);
}
leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
CheckCondition(sizes[0] > 0);
CheckCondition(sizes[1] > 0);
}
StartPhase("property");
{
char* prop = leveldb_property_value(db, "nosuchprop");
/* CheckCondition(prop == NULL);*/
prop = leveldb_property_value(db, "leveldb.stats");
CheckCondition(prop != NULL);
Free(&prop);
}
StartPhase("snapshot");
{
const leveldb_snapshot_t* snap;
snap = leveldb_create_snapshot(db);
leveldb_delete(db, woptions, "foo", 3, &err);
CheckNoError(err);
leveldb_readoptions_set_snapshot(roptions, snap);
CheckGet(db, roptions, "foo", "hello");
leveldb_readoptions_set_snapshot(roptions, NULL);
CheckGet(db, roptions, "foo", NULL);
leveldb_release_snapshot(db, snap);
}
StartPhase("repair");
{
leveldb_close(db);
leveldb_options_set_error_if_exists(options, 0);
leveldb_repair_db(options, dbname, &err);
CheckNoError(err);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
leveldb_options_set_error_if_exists(options, 1);
}
StartPhase("filter");
for (run = 0; run < 2; run++) {
// First run uses custom filter, second run uses bloom filter
CheckNoError(err);
leveldb_filterpolicy_t* policy;
if (run == 0) {
policy = leveldb_filterpolicy_create(
NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
} else {
policy = leveldb_filterpolicy_create_bloom(10);
}
// Create new database
leveldb_close(db);
leveldb_destroy_db(options, dbname, &err);
leveldb_options_set_filter_policy(options, policy);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
CheckNoError(err);
leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
CheckNoError(err);
leveldb_compact_range(db, NULL, 0, NULL, 0);
fake_filter_result = 1;
CheckGet(db, roptions, "foo", "foovalue");
CheckGet(db, roptions, "bar", "barvalue");
if (phase == 0) {
// Must not find value when custom filter returns false
fake_filter_result = 0;
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
fake_filter_result = 1;
CheckGet(db, roptions, "foo", "foovalue");
CheckGet(db, roptions, "bar", "barvalue");
}
leveldb_options_set_filter_policy(options, NULL);
leveldb_filterpolicy_destroy(policy);
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
return 0;
}
int main(int argc, char** argv) {
leveldb_t* db;
leveldb_comparator_t* cmp;
leveldb_cache_t* cache;
leveldb_env_t* env;
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
char* err = NULL;
#if defined(LEVELDB_PLATFORM_WINDOWS)
snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test");
#else
snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d",
((int) geteuid()));
#endif
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1);
leveldb_options_set_cache(options, cache);
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL);
leveldb_options_set_write_buffer_size(options, 100000);
leveldb_options_set_paranoid_checks(options, 1);
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 1);
leveldb_readoptions_set_fill_cache(roptions, 0);
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, 1);
StartPhase("destroy");
leveldb_destroy_db(options, dbname, &err);
leveldb_free(&err);
StartPhase("open_error");
db = leveldb_open(options, dbname, &err);
CheckCondition(err != NULL);
leveldb_free(&err);
StartPhase("open");
leveldb_options_set_create_if_missing(options, 1);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
StartPhase("put");
leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
StartPhase("writebatch");
{
int pos = 0;
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_destroy(wb);
}
StartPhase("iter");
{
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
StartPhase("approximate_sizes");
{
int i;
int n = 20000;
char keybuf[100];
char valbuf[100];
uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" };
size_t start_len[2] = { 1, 21 };
const char* limit[2] = { "k00000000000000010000", "z" };
size_t limit_len[2] = { 21, 1 };
leveldb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
snprintf(valbuf, sizeof(valbuf), "v%020d", i);
leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
&err);
CheckNoError(err);
}
leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
CheckCondition(sizes[0] > 0);
CheckCondition(sizes[1] > 0);
}
StartPhase("property");
{
char* prop = leveldb_property_value(db, "nosuchprop");
CheckCondition(prop == NULL);
prop = leveldb_property_value(db, "leveldb.stats");
CheckCondition(prop != NULL);
leveldb_free(&prop);
}
StartPhase("snapshot");
{
const leveldb_snapshot_t* snap;
snap = leveldb_create_snapshot(db);
leveldb_delete(db, woptions, "foo", 3, &err);
CheckNoError(err);
leveldb_readoptions_set_snapshot(roptions, snap);
CheckGet(db, roptions, "foo", "hello");
leveldb_readoptions_set_snapshot(roptions, NULL);
CheckGet(db, roptions, "foo", NULL);
leveldb_release_snapshot(db, snap);
}
StartPhase("repair");
{
leveldb_close(db);
leveldb_options_set_create_if_missing(options, 0);
leveldb_options_set_error_if_exists(options, 0);
leveldb_repair_db(options, dbname, &err);
CheckNoError(err);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
return 0;
}
int main(int argc, char **argv) {
int goopt;
int listenport=8080;
int numthreads=10;
int tf;
char *dbd=NULL;
char *lpstr=NULL;
char *ntstr=NULL;
char *alfile=NULL;
leveldb_options_t *dbopt;
struct mg_context *ctx= NULL;
char **mgoptions;
signal(SIGINT,handlesig);
signal(SIGTERM,handlesig);
// command line parsing
while ((goopt=getopt (argc, argv, "d:p:n:a:t:vh")) != -1) {
switch (goopt) {
case 'd': // database
dbd=calloc(strlen((char*)optarg)+1,sizeof(char));
strncpy(dbd,(char*)optarg,strlen((char*)optarg));
break;
case 'a': // access log, passed to mongoose
alfile=calloc(strlen((char*)optarg)+1,sizeof(char));
strncpy(alfile,(char*)optarg,strlen((char*)optarg));
break;
case 'p': // port
listenport=atoi(optarg);
break;
case 'n': // number of threads
numthreads=atoi(optarg);
break;
case 'v': // verbose
vlevel++;
break;
case 'h': // help
usage(NULL,EXIT_SUCCESS);
break;
default: // fallthrough
usage(NULL,EXIT_FAILURE);
}
}
LOG_TRACE(vlevel, _("Validation\n"),dbd);
if(dbd==NULL) {
usage("Must give a database dir\n",EXIT_FAILURE);
} else {
LOG_TRACE(vlevel, _("Using database dir: %s\n"),dbd);
}
if(alfile!=NULL) {
if((tf=open(alfile,O_WRONLY|O_CREAT,S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH))==-1) {
LOG_FATAL(vlevel,_("Unable to open access log file: %s: %s\n"), alfile, strerror(errno));
exit(EXIT_FAILURE);
}
close(tf);
}
if(alfile!=NULL && access(alfile, W_OK)!=0) {
LOG_FATAL(vlevel, _("Unable to write to access log file: %s\n"),alfile);
exit(EXIT_FAILURE);
}
if(listenport<0 || listenport>65536) {
LOG_FATAL(vlevel, _("Given port out of bounds: %i\n"),listenport);
exit(EXIT_FAILURE);
}
if(numthreads<0 || numthreads>1024) {
LOG_FATAL(vlevel, _("Given threads out of bounds: %i\n"),numthreads);
exit(EXIT_FAILURE);
}
LOG_TRACE(vlevel, _("Setting up leveldb store in %s\n"),dbd);
dbopt=leveldb_options_create();
leveldb_options_set_create_if_missing(dbopt, 1);
leveldb_options_set_write_buffer_size(dbopt, 8388608);
leveldb_options_set_compression(dbopt,leveldb_no_compression);
dbh=leveldb_open(dbopt,dbd,&errptr);
LOG_TRACE(vlevel, _("Setting leveldb read options\n"));
ropt = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(ropt, 1);
leveldb_readoptions_set_fill_cache(ropt, 0);
LOG_TRACE(vlevel, _("Setting leveldb write options\n"));
wopt = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(wopt, 0);
// set mgoptions - XXX this needs to be handled better
lpstr=calloc(7,sizeof(char));
snprintf(lpstr,6,"%i",listenport);
ntstr=calloc(4,sizeof(char));
snprintf(ntstr,3,"%i",numthreads);
if(alfile!=NULL) {
mgoptions = calloc(9,sizeof(char*));
} else {
mgoptions = calloc(7,sizeof(char*));
}
mgoptions[0]="listening_ports";
mgoptions[1]=lpstr;
mgoptions[2]="document_root";
mgoptions[3]="/dev/null";
mgoptions[4]="num_threads";
mgoptions[5]=ntstr;
if(alfile!=NULL) {
mgoptions[6]="access_log_file";
mgoptions[7]=alfile;
mgoptions[8]=NULL;
} else {
mgoptions[6]=NULL;
}
// main loop
LOG_INFO(vlevel, _("Starting Mongoose HTTP server loop\n"));
ctx = mg_start(&mghandle, NULL, (const char**)mgoptions);
if(ctx!=NULL) {
while(!done) {
// cleaner thread here?
sleep(1);
}
LOG_INFO(vlevel, _("Ending Mongoose HTTP server loop\n"));
mg_stop(ctx);
} else {
LOG_FATAL(vlevel,_("Error in creating Mongoose HTTP server\n"));
}
LOG_TRACE(vlevel, _("Closing database handle\n"));
// close leveldb handle
LOG_TRACE(vlevel, _("Cleaning up leveldb\n"));
leveldb_compact_range(dbh, NULL, 0, NULL, 0);
leveldb_options_destroy(dbopt);
leveldb_readoptions_destroy(ropt);
leveldb_writeoptions_destroy(wopt);
leveldb_close(dbh);
LOG_TRACE(vlevel, _("Cleaning up\n"));
free(dbd);
free(lpstr);
free(ntstr);
free(mgoptions);
return EXIT_SUCCESS;
}
//TODO: add "bool create_if_missing" to this API!
int leveldb_client_start(leveldb **db, const char *name,
size_t write_buffer_size, bool use_compression, bool sync_writes)
{
int namelen;
char *db_name;
char *err = NULL;
leveldb_t* db_t;
leveldb_comparator_t *cmp;
leveldb_cache_t *cache;
leveldb_env_t *env;
leveldb_options_t *options;
leveldb_readoptions_t *roptions;
leveldb_writeoptions_t *woptions;
*db = (leveldb *)malloc(sizeof(leveldb));
if (*db == NULL) {
kp_error("malloc(leveldb) failed!\n");
return -1;
}
/* Check arguments: */
if (strlen(name) < 1) {
kp_error("name has no length!\n");
return -1;
}
if (write_buffer_size == 0) {
write_buffer_size = WRITE_BUFFER_DEFAULT;
}
if (write_buffer_size < WRITE_BUFFER_MIN) {
kp_error("write_buffer_size=%zu too small; min=%u\n",
write_buffer_size, WRITE_BUFFER_MIN);
return -1;
}
kp_debug("got args: name=%s, write_buffer_size=%zu, use_compression=%s, "
"sync_writes=%s\n",
name, write_buffer_size, use_compression ? "true" : "false",
sync_writes ? "true" : "false");
/* Format of db name: LEVELDB_DIR + / + name */
namelen = strlen(name);
db_name = malloc(namelen + 1);
if (!db_name) {
kp_error("malloc(db_name) failed!\n");
return -1;
}
strncpy(db_name, name, namelen+1); //copy null-zero
(*db)->name = db_name;
kp_debug("constructed db->name=[%s]\n", (*db)->name);
/* Set up the database's comparator, environment, cache, etc. According
* to leveldb/include/leveldb/c.h, all functions that can raise an error
* must be passed a "char **errptr" (set to NULL!) as the last argument;
* I guess functions that don't take this pointer can't return errors.
*/
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_compare,
cmp_name); //first arg is "state"
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(CACHE_CAPACITY);
// cache = NULL;
(*db)->cmp = cmp;
(*db)->env = env;
(*db)->cache = cache;
/* Set up the database's various options. Many of these will affect the
* database's performance! (see leveldb/include/leveldb/options.h).
*/
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1); //raise error if db already exists
leveldb_options_set_cache(options, cache);
// leveldb_options_set_cache(options, NULL); //disable cache??
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL); //NULL: write info to file in db's dir
leveldb_options_set_write_buffer_size(options, write_buffer_size);
/* Amount of data to build up in memory (backed by an unsorted log
* on disk) before converting to a sorted on-disk file.
* Larger values increase performance, especially during bulk loads.
* Up to two write buffers may be held in memory at the same time,
* so you may wish to adjust this parameter to control memory usage.
* Also, a larger write buffer will result in a longer recovery time
* the next time the database is opened.
* Default: 4MB (test file uses 100000 bytes)
*/
leveldb_options_set_paranoid_checks(options, 0); //default false; test file uses true
leveldb_options_set_max_open_files(options, MAX_OPEN_FILES);
leveldb_options_set_block_size(options, BLOCK_SIZE);
leveldb_options_set_block_restart_interval(options, BLOCK_RESTART_INTERVAL);
leveldb_options_set_compression(options,
use_compression ? leveldb_snappy_compression : leveldb_no_compression);
(*db)->options = options;
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 0);
/* If true, all data read from underlying storage will be
* verified against corresponding checksums.
* Default false; test file uses true.
*/
leveldb_readoptions_set_fill_cache(roptions, true);
/* Should the data read for this iteration be cached in memory?
* Callers may wish to set this field to false for bulk scans.
* Default true; test file uses false.
*/
(*db)->roptions = roptions;
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, sync_writes ? 1 : 0);
/* If true, the write will be flushed from the operating system
* buffer cache (by calling WritableFile::Sync()) before the write
* is considered complete. If this flag is true, writes will be
* slower.
* If this flag is false, and the machine crashes, some recent
* writes may be lost. Note that if it is just the process that
* crashes (i.e., the machine does not reboot), no writes will be
* lost even if sync==false.
* In other words, a DB write with sync==false has similar
* crash semantics as the "write()" system call. A DB write
* with sync==true has similar crash semantics to a "write()"
* system call followed by "fsync()".
*/
(*db)->woptions = woptions;
kp_debug("destroying previous copy of database, if it exists\n");
leveldb_destroy_db((*db)->options, (*db)->name, &err);
free_err(&err);
kp_debug("opening/creating database [%s]\n", (*db)->name);
db_t = leveldb_open((*db)->options, (*db)->name, &err);
if (err) {
kp_error("opening db returned error: %s\n", err);
return -1;
}
free_err(&err);
(*db)->db = db_t;
kp_debug("successfully started leveldb [%s]\n", (*db)->name);
return 0;
}
int
main ()
{
unsigned int N_KEYS;
unsigned int N_BATCH;
printf("\n number of keys:");
scanf("%u",&N_KEYS);
printf("\n batch size:");
scanf("%u",&N_BATCH);
tinymt32_t tinymt32;
//initialize random generator
tinymt32_init (&tinymt32, 0);
int iter;
int sec_iter;
int stop;
int counter;
//initialize database
char *errptr = NULL;
leveldb_options_t *options = leveldb_options_create ();
/* initializeOptions */
leveldb_options_set_create_if_missing (options, 1);
leveldb_options_set_write_buffer_size(options,120000000 );
leveldb_options_set_max_open_files(options,800000);
/*open Database */
leveldb_t *db_helper =
leveldb_open (options, "/mnt/database/database_helper", &errptr);
leveldb_t *db = leveldb_open (options, "/mnt/database/database", &errptr);
leveldb_readoptions_t *readoptions = leveldb_readoptions_create ();
leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create ();
leveldb_writebatch_t* batch=leveldb_writebatch_create();
int64_t diff = zclock_time ();
//write into database_helper
iter=0;
while( iter < N_KEYS)
{
sec_iter=0;
while((iter<N_KEYS) && (sec_iter<N_BATCH)){
unsigned int key = tinymt32_generate_uint32 (&tinymt32);
unsigned int val = tinymt32_generate_uint32 (&tinymt32);
leveldb_writebatch_put (batch,
(const char *) &key, sizeof (int),
(const char *) &val, sizeof (int));
sec_iter++;
iter++;
}
leveldb_write(
db_helper,
writeoptions,
batch,
&errptr);
if(errptr!=NULL){
printf("\n%s",errptr);
}
assert (errptr == NULL);
leveldb_writebatch_clear(batch);
}
leveldb_writebatch_destroy(batch);
diff = zclock_time () - diff;
printf ("\nrandom write: %d", diff);
diff = zclock_time ();
//write sequentially into db
leveldb_iterator_t *liter = leveldb_create_iterator (db_helper,
readoptions);
leveldb_iter_seek_to_first (liter);
while (leveldb_iter_valid (liter))
{
size_t length;
char key[4];
memcpy (key, leveldb_iter_key (liter, &length), 4);
char val[4];
memcpy (val, leveldb_iter_value (liter, &length), 4);
leveldb_iter_get_error (liter, &errptr);
assert (errptr == NULL);
leveldb_put (db,
writeoptions,
(const char *) &key, sizeof (int),
(const char *) &val, sizeof (int), &errptr);
assert (errptr == NULL);
leveldb_iter_next (liter);
}
diff = zclock_time () - diff;
leveldb_close (db);
leveldb_close (db_helper);
}
int
main(int argc, char *argv[]) {
if (argc != 5) {
fprintf(stderr, "Usage: ./leveldb_index [leveldb_directory] [sample_input] [index_output] [index_time_file]\n");
return -1;
}
if ((file = fopen(argv[2], "r")) == NULL) {
fprintf(stderr, "Failed to open sample file \"%s\" for reading\n", argv[1]);
return -1;
}
for (i = 0; fscanf(file, "%"PRId64"\t", &cnt) != EOF; ++i) {
fgets(buffer, BUFFER_SIZE, file);
for (len = strlen(buffer) - 1; len >= 0 && (buffer[len] == '\r' || buffer[len] == '\n'); buffer[len--] = 0);
++len;
strcpy(request[i].key, buffer);
}
fclose(file);
if ((file = fopen(argv[4], "w")) == NULL) {
fprintf(stderr, "Failed to open time information file \"%s\" for reading\n", argv[4]);
return -1;
}
fprintf(stdout, "Start indexing\n");
srand((unsigned)time(NULL));
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
// default: write_buffer_size: 4MB, block_cache: 8MB, block_size ~ 4KB
leveldb_options_set_write_buffer_size(options, BLK_SIZE * 1024);
leveldb_options_set_cache(options, cache = leveldb_cache_create_lru(BLK_SIZE * 2048));
leveldb_options_set_block_size(options, BLK_SIZE);
leveldb_options_set_compression(options, 0);
db = leveldb_open(options, argv[1], &err);
if (err != NULL) {
fprintf(stderr, "Failed to open leveldb.\n");
return -1;
}
leveldb_free(err);
roptions = leveldb_readoptions_create();
gettimeofday(&start, NULL);
for (i = seekn = tot = 0; i < DISK_INDEX_SIZE; ++i) {
val = leveldb_get(db, roptions, request[i].key, strlen(request[i].key), &read_len, &err);
if (err != NULL) {
fprintf(stderr, "Failed to read from leveldb.\n");
return -1;
}
leveldb_free(err);
if (read_len == 0)
request[i].value = 0;
else {
val[read_len] = 0;
sscanf(val, "%"PRId64"", &request[i].value);
}
leveldb_free(val);
if ((i + 1) % 1000 == 0) {
gettimeofday(&end, NULL);
tot += (int64_t)(end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec;
fprintf(stdout, "\rIndexed %"PRId64" items in %"PRId64"us", i + 1, tot);
fflush(stdout);
fprintf(file, "Indexed %"PRId64" items in %"PRId64"us\n", i + 1, tot);
fflush(file);
gettimeofday(&start, NULL);
}
}
fprintf(stdout, "\rIndexed %"PRId64" items in %"PRId64"us\n", i, tot);
fprintf(file, "Indexed %"PRId64" items in %"PRId64"us\n", i, tot);
fflush(file);
fclose(file);
leveldb_cache_destroy(cache);
leveldb_readoptions_destroy(roptions);
leveldb_options_destroy(options);
leveldb_close(db);
fprintf(stdout, "Used %"PRId64" us and seeked %"PRId64" times to index %"PRId64" keys\n", tot, seekn, i);
if ((file = fopen(argv[3], "w")) == NULL) {
fprintf(stdout, "Failed to open file \"%s\" for storing indexing results\n", argv[3]);
return -1;
}
for (i = 0; i < DISK_INDEX_SIZE; ++i)
fprintf(file, "%"PRId64"\t%s\n", request[i].value, request[i].key);
fflush(file);
fclose(file);
return 0;
}
int mdhim_leveldb_open(void **dbh, void **dbs, char *path, int flags,
struct mdhim_store_opts_t *mstore_opts) {
leveldb_t *db;
leveldb_options_t *options;
leveldb_options_t *stat_options;
char *err = NULL;
leveldb_comparator_t* cmp = NULL;
char stats_path[PATH_MAX];
leveldb_filterpolicy_t *main_filter;
leveldb_filterpolicy_t *stats_filter;
leveldb_cache_t *main_cache;
leveldb_cache_t *stats_cache;
leveldb_env_t *main_env;
leveldb_env_t *stats_env;
//Create the options for the main database
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
//leveldb_options_set_compression(options, 0);
main_filter = leveldb_filterpolicy_create_bloom(256);
main_cache = leveldb_cache_create_lru(5242880);
main_env = leveldb_create_default_env();
leveldb_options_set_cache(options, main_cache);
leveldb_options_set_filter_policy(options, main_filter);
leveldb_options_set_max_open_files(options, 10000);
leveldb_options_set_write_buffer_size(options, 5242880);
leveldb_options_set_env(options, main_env);
//Create the options for the stat database
stat_options = leveldb_options_create();
leveldb_options_set_create_if_missing(stat_options, 1);
//leveldb_options_set_compression(stat_options, 0);
stats_filter = leveldb_filterpolicy_create_bloom(256);
stats_cache = leveldb_cache_create_lru(1024);
stats_env = leveldb_create_default_env();
leveldb_options_set_cache(stat_options, stats_cache);
leveldb_options_set_filter_policy(stat_options, stats_filter);
leveldb_options_set_write_buffer_size(stat_options, 1024);
leveldb_options_set_env(stat_options, stats_env);
switch(mstore_opts->key_type) {
case MDHIM_INT_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_int_compare, cmp_name);
break;
case MDHIM_LONG_INT_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_lint_compare, cmp_name);
break;
case MDHIM_FLOAT_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_float_compare, cmp_name);
break;
case MDHIM_DOUBLE_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_double_compare, cmp_name);
break;
case MDHIM_STRING_KEY:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_string_compare, cmp_name);
break;
default:
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_byte_compare, cmp_name);
break;
}
leveldb_options_set_comparator(options, cmp);
//Check to see if the given path + "_stat" and the null char will be more than the max
if (strlen(path) + 6 > PATH_MAX) {
mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database - path provided is too long");
return MDHIM_DB_ERROR;
}
sprintf(stats_path, "%s_stats", path);
//Open the main database
db = leveldb_open(options, path, &err);
//Set the output handle
*((leveldb_t **) dbh) = db;
if (err != NULL) {
mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database");
return MDHIM_DB_ERROR;
}
//Open the stats database
cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_int_compare, cmp_name);
leveldb_options_set_comparator(stat_options, cmp);
db = leveldb_open(stat_options, stats_path, &err);
*((leveldb_t **) dbs) = db;
if (err != NULL) {
mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database");
return MDHIM_DB_ERROR;
}
//Set the output comparator
mstore_opts->db_ptr1 = cmp;
//Set the generic pointers to hold the options for the mdhim db
mstore_opts->db_ptr2 = options;
mstore_opts->db_ptr3 = leveldb_readoptions_create();
mstore_opts->db_ptr4 = leveldb_writeoptions_create();
//Set the generic pointers to hold the options for the mdhim stats db
mstore_opts->db_ptr5 = stat_options;
mstore_opts->db_ptr6 = leveldb_readoptions_create();
mstore_opts->db_ptr7 = leveldb_writeoptions_create();
//Set the rest of the options that aren't used until close
mstore_opts->db_ptr8 = main_filter;
mstore_opts->db_ptr9 = stats_filter;
mstore_opts->db_ptr10 = main_cache;
mstore_opts->db_ptr11 = stats_cache;
mstore_opts->db_ptr12 = main_env;
mstore_opts->db_ptr13 = stats_env;
return MDHIM_SUCCESS;
}