Beispiel #1
0
/* =============
 * Database info extraction methods
 * =============
 */
void init_nwdb() {
    char path[PATH_MAX];
    char *err = NULL;

    sprintf(path, "%s/%s", cde_pseudo_pkg_dir, DB_NAME);
    if (access(path, R_OK)==-1) {
        fprintf(stderr, "Network provenance database does not exist!\n");
        exit(-1);
    }

    netdb = malloc(sizeof(lvldb_t));
    netdb->options = leveldb_options_create();
    leveldb_options_set_create_if_missing(netdb->options, 0);
    netdb->db = leveldb_open(netdb->options, path, &err);

    if (err != NULL || netdb == NULL) {
        fprintf(stderr, "Leveldb open fail!\n");
        exit(-1);
    }

    assert(netdb->db!=NULL);
    netdb->woptions = leveldb_writeoptions_create();
    netdb->roptions = leveldb_readoptions_create();

    /* reset error var */
    leveldb_free(err);
    err = NULL;

    /* read in root pid */
    netdb_root = db_readc(netdb, "meta.root");
    assert(netdb_root != NULL);

    if (CDE_nw_mode) {
        /* create temp db for current execution graph */
        sprintf(path, "%s/%s.tempXXXXXX", cde_pseudo_pkg_dir, DB_NAME);
        if (mkdtemp(path) == NULL) {
            fprintf(stderr, "Cannot create temp db!\n");
            exit(-1);
        }
        currdb = malloc(sizeof(lvldb_t));
        currdb->options = leveldb_options_create();
        leveldb_options_set_create_if_missing(currdb->options, 1);
        currdb->db = leveldb_open(currdb->options, path, &err);
        assert(currdb->db!=NULL);
        leveldb_free(err);
        err = NULL;
        currdb->woptions = leveldb_writeoptions_create();
        currdb->roptions = leveldb_readoptions_create();

        db_write_root(currdb, getpid());
    }
}
Beispiel #2
0
int xleveldb_firstkey(struct db_conn *db, unsigned char *key, size_t *ksize)
{
	int rv = 0;
	const char *db_key;
	size_t db_ksize;

	leveldb_readoptions_t *options = leveldb_readoptions_create();
	leveldb_iterator_t *it = leveldb_create_iterator(db->conn, options);

	leveldb_iter_seek_to_first(it);
	if (! leveldb_iter_valid(it)) {
		rv = 0;
		goto exit;
	}

	db_key = leveldb_iter_key(it, &db_ksize);

	if (db_key == NULL || db_ksize > *ksize) {
		rv = 0;
		goto exit;
	}

	*ksize = db_ksize;
	memcpy(key, db_key, db_ksize);
	rv = 1;

exit:
	leveldb_readoptions_destroy(options);
	leveldb_iter_destroy(it);
	return rv;
}
Beispiel #3
0
int xleveldb_get(struct db_conn *db, const unsigned char *key, size_t ksize,
		unsigned char *val, size_t *vsize)
{
	int rv;
	char *db_val = NULL;
	size_t db_vsize;
	leveldb_readoptions_t *options = leveldb_readoptions_create();
	char *err, *origerr;

	err = origerr = malloc(1);
	db_val = leveldb_get(db->conn, options,
			(const char *) key, ksize, &db_vsize, &err);

	free(err);
	leveldb_readoptions_destroy(options);

	if (err != origerr || db_val == NULL || db_vsize > *vsize) {
		rv = 0;
		goto exit;
	}

	*vsize = db_vsize;
	memcpy(val, db_val, db_vsize);
	rv = 1;

exit:
	free(db_val);
	return rv;
}
Beispiel #4
0
time_t stat_cache_read_updated_children(stat_cache_t *cache, const char *path) {
    leveldb_readoptions_t *options;
    char *key = NULL;
    char *errptr = NULL;
    time_t *value = NULL;
    time_t r;
    size_t vallen;

    asprintf(&key, "updated_children:%s", path);

    options = leveldb_readoptions_create();
    value = (time_t *) leveldb_get(cache, options, key, strlen(key) + 1, &vallen, &errptr);
    leveldb_readoptions_destroy(options);

    if (errptr != NULL) {
        log_print(LOG_ERR, "leveldb_get error: %s", errptr);
        free(errptr);
        r = 0;
    }

    free(key);

    if (value == NULL) return 0;

    r = *value;

    //log_print(LOG_DEBUG, "Children for directory %s were updated at timestamp %lu.", path, r);

    free(value);
    return r;
}
Beispiel #5
0
void
localdb_init (localdb_t ** localdb)
{
    *localdb = malloc (sizeof (localdb_t));

    char *errptr = NULL;

    leveldb_options_t *options = leveldb_options_create ();
    leveldb_options_set_create_if_missing (options, 1);

    (*localdb)->options = options;


    leveldb_readoptions_t *readoptions = leveldb_readoptions_create ();
    (*localdb)->readoptions = readoptions;
    leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create ();
    (*localdb)->writeoptions = writeoptions;



    leveldb_writeoptions_set_sync (writeoptions, 1);

    (*localdb)->db = leveldb_open (options, "/mnt/localdb", &errptr);
    if (errptr) {
        printf ("\n%s", errptr);
        exit (1);
    }
}
db_t* open_db(const char* path, database_t type)
{
	db_t* db = (db_t*) malloc(sizeof(db_t));
	db->type = type;
	if(type == LEVELDB)
	{
		levelDB_t* leveldb = (levelDB_t*) malloc(sizeof(levelDB_t));
		leveldb->err = NULL;
		leveldb->options = leveldb_options_create();
		leveldb->woptions = leveldb_writeoptions_create();
		leveldb->roptions = leveldb_readoptions_create();
		leveldb_options_set_create_if_missing(leveldb->options, 1);
		leveldb->db = leveldb_open(leveldb->options, path, &(leveldb->err));
		if(leveldb->err != NULL)
		{
			fprintf(stderr, "Failed to open database! Error message: %s\n", leveldb->err);
			exit(-1);
		}
		leveldb_free(leveldb->err);
		leveldb->err = NULL;	
		db->database = (void*) leveldb;
	}
	else
	{
		kernelDB_t* kerneldb = (kernelDB_t*) malloc(sizeof(kernelDB_t));
		kerneldb->client = kr_open(path);
		db->database = (void*) kerneldb;
	}
	return db;
}
Beispiel #7
0
void db_leveldb_init()
{
    char *errstr = NULL;
    leveldb_options = leveldb_options_create();
    /* create if missing */
    leveldb_options_set_create_if_missing(leveldb_options, 1);
    /* lru cache */
    leveldb_cache = leveldb_cache_create_lru(conf->leveldb_cache_size);
    leveldb_options_set_cache(leveldb_options, leveldb_cache);
    /* block size */
    leveldb_options_set_block_size(leveldb_options, conf->leveldb_block_size);
    /* write buffer size */
    leveldb_options_set_write_buffer_size(leveldb_options, conf->leveldb_write_buffer_size);
    /* filter policy */
    leveldb_filterpolicy = leveldb_filterpolicy_create_bloom(10);
    leveldb_options_set_filter_policy(leveldb_options, leveldb_filterpolicy);
    /* open db */
    leveldb_db = leveldb_open(leveldb_options, conf->db, &errstr);

    if (errstr) {
        terrx(1, "unable to open db at %s: %s", conf->db, errstr);
    }

    leveldb_roptions = leveldb_readoptions_create();
    leveldb_woptions = leveldb_writeoptions_create();
}
Beispiel #8
0
static int
txdb_open_db(struct txdb *txdb)
{
    char *err = NULL;

    txdb->db_opts = leveldb_options_create();
    leveldb_options_set_create_if_missing(txdb->db_opts, 1);
    txdb->db = leveldb_open(txdb->db_opts, txdb->path, &err);
    if (err) {
        Warning(LGPFX" failed to open db '%s' : %s\n", txdb->path, err);
        printf("btc failed to open the leveldb database at '%s' : %s\n",
               txdb->path, err);
        printf("Could there be another instance of btc running?\n");
        free(err);
        return 1;
    }

    txdb->rd_opts = leveldb_readoptions_create();
    leveldb_readoptions_set_verify_checksums(txdb->rd_opts, 1);
    leveldb_readoptions_set_fill_cache(txdb->rd_opts, 0);

    txdb->wr_opts = leveldb_writeoptions_create();
    leveldb_writeoptions_set_sync(txdb->wr_opts, 1);

    return 0;
}
Beispiel #9
0
struct _leveldb_stuff *ldb_initialize(char *path)
{

    struct _leveldb_stuff *ldbs = NULL;
    leveldb_cache_t *cache;
    leveldb_filterpolicy_t *policy;
    char* err = NULL;

    ldbs = (struct _leveldb_stuff *) malloc(sizeof(struct _leveldb_stuff));
    memset(ldbs, 0, sizeof(struct _leveldb_stuff));

    ldbs->options = leveldb_options_create();
    snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s%s%s", LDB_WORK_PATH, "/",
            path);

    cache = leveldb_cache_create_lru(LDB_CACHE_LRU_SIZE);
    policy = leveldb_filterpolicy_create_bloom(LDB_BLOOM_KEY_SIZE);

    leveldb_options_set_filter_policy(ldbs->options, policy);
    leveldb_options_set_cache(ldbs->options, cache);
    leveldb_options_set_block_size(ldbs->options, LDB_BLOCK_SIZE);
    leveldb_options_set_write_buffer_size(ldbs->options, LDB_WRITE_BUFFER_SIZE);
#if defined(OPEN_COMPRESSION)
    leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression);
#else
    leveldb_options_set_compression(ldbs->options, leveldb_no_compression);
#endif
    /* Read options */
    ldbs->roptions = leveldb_readoptions_create();
    leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1);
    leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */

    /* W  options */
    ldbs->woptions = leveldb_writeoptions_create();
#ifdef SYNC_PUT
    leveldb_writeoptions_set_sync(ldbs->woptions, 1);
#else
    leveldb_writeoptions_set_sync(ldbs->woptions, 0);
#endif

    /* Batch write */
    ldbs->wbatch = leveldb_writebatch_create();

    leveldb_options_set_create_if_missing(ldbs->options, 1);
    ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err);

    if (err) {
        fprintf(stderr, "%s", err);
        leveldb_free(err);
        err = NULL;
        free(ldbs);
        return NULL;
    } else {
        return ldbs;
    }

}
Beispiel #10
0
/*
 * Open the specified database, if no exists, then create it.
 * Args:
 *      1. name: db name
 *      2. block_size: block size
 *      3. wb_size: write buffer size
 *      4. lru_size: lru cache size
 *      5. bloom_size: bloom key size
 * Return:
 *      _leveldb_stuff: leveldb handler.
 */
struct _leveldb_stuff *ldb_initialize(const char *name, size_t block_size, size_t wb_size, size_t lru_size, short bloom_size)
{

    struct _leveldb_stuff *ldbs = NULL;
    leveldb_cache_t *cache;
    leveldb_filterpolicy_t *policy;
    char* err = NULL;

    ldbs = malloc(sizeof(struct _leveldb_stuff));
    memset(ldbs, 0, sizeof(struct _leveldb_stuff));

    ldbs->options = leveldb_options_create();

    snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s", name);

    cache = leveldb_cache_create_lru(lru_size);
    policy = leveldb_filterpolicy_create_bloom(bloom_size);

    leveldb_options_set_filter_policy(ldbs->options, policy);
    leveldb_options_set_cache(ldbs->options, cache);
    leveldb_options_set_block_size(ldbs->options, block_size);
    leveldb_options_set_write_buffer_size(ldbs->options, wb_size);
#if defined(OPEN_COMPRESSION)
    leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression);
#else
    leveldb_options_set_compression(ldbs->options, leveldb_no_compression);
#endif
    /* R */
    ldbs->roptions = leveldb_readoptions_create();
    leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1);
    leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */

    /* W */
    ldbs->woptions = leveldb_writeoptions_create();
#ifdef SYNC_PUT
    leveldb_writeoptions_set_sync(ldbs->woptions, 1);
#else
    leveldb_writeoptions_set_sync(ldbs->woptions, 0);
#endif

    /* B */
    ldbs->wbatch = leveldb_writebatch_create();

    leveldb_options_set_create_if_missing(ldbs->options, 1);
    ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err);

    if (err) {
        fprintf(stderr, "%s", err);
        leveldb_free(err);
        err = NULL;
        free(ldbs);
        return NULL ;
    }

    return ldbs;
}
Beispiel #11
0
char * store_get(leveldb_t * store, char * key,size_t key_size, size_t *read_len){
	char * err = 0x00;
	leveldb_readoptions_t *read_options=leveldb_readoptions_create();
	char * resp = leveldb_get(store,read_options, key, key_size, read_len, &err);
	leveldb_readoptions_destroy(read_options);
	if(err){
		fprintf(stderr, "error: %s\n", err);
		free(err);
	}
	return resp;
}
Beispiel #12
0
const char *
db_get(db_t *db, const char *key, size_t klen,
       size_t *vlen, char **errptr) {
	const char *val;
	leveldb_readoptions_t *opts;

	opts = leveldb_readoptions_create();
	val = leveldb_get(db->db, opts, key, klen, vlen, errptr);
	leveldb_readoptions_destroy(opts);

	return val;
}
Beispiel #13
0
// Creates a reference to a tablet.
//
// table - The table the tablet belongs to.
//
// Returns a reference to the new tablet if successful. Otherwise returns
// null.
sky_tablet *sky_tablet_create(sky_table *table)
{
    sky_tablet *tablet = NULL;
    assert(table != NULL);
    tablet = calloc(sizeof(sky_tablet), 1); check_mem(tablet);
    tablet->table = table;
    tablet->readoptions = leveldb_readoptions_create();
    tablet->writeoptions = leveldb_writeoptions_create();
    return tablet;
    
error:
    sky_tablet_free(tablet);
    return NULL;
}
Beispiel #14
0
static struct Queue * readoptions (va_list argp) {
	struct Queue * q = malloc(sizeof (struct Queue));
	memset(q, 0, sizeof(struct Queue));

	q->options = leveldb_options_create();
	q->cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
	leveldb_options_set_comparator(q->options, q->cmp);
	leveldb_options_set_create_if_missing(q->options, 1);

	q->rop = leveldb_readoptions_create();
	q->wop = leveldb_writeoptions_create();
	const char * p;
	for (p = va_arg(argp, char *); p != NULL; p = va_arg(argp,char *)) {
		if (0 == strcmp(p, "failIfMissing")) {
			leveldb_options_set_create_if_missing(q->options,0);
		}
		if (0 == strcmp(p,"paranoidChecks")) {
			leveldb_options_set_paranoid_checks(q->options,1);
		}
		if (0 == strcmp(p,"writeBufferSize")) {
			size_t bufferSize= va_arg(argp, size_t);
			leveldb_options_set_write_buffer_size(q->options,bufferSize);
		}
		if (0 == strcmp(p,"blockSize")) {
			size_t blockSize= va_arg(argp, size_t);
			leveldb_options_set_block_size(q->options,blockSize);
		}
		if (0 == strcmp(p,"blockRestartInterval")) {
			size_t blockRestartInterval= va_arg(argp, size_t);
			leveldb_options_set_block_restart_interval(q->options,blockRestartInterval);
		}
		if (0 == strcmp(p,"maxOpenFiles")) {
			int maxOpenFiles= va_arg(argp, int );
			leveldb_options_set_max_open_files(q->options,maxOpenFiles);
		}
		if (0 == strcmp(p,"noCompress")) {
			leveldb_options_set_compression(q->options,0);
		}
		if (0 == strcmp(p,"syncWrte")) {
			leveldb_writeoptions_set_sync(q->wop , 1);
		}
	}
Beispiel #15
0
db_iter_t *
db_iter_seek(db_t *db, const char *key, size_t klen) {
	db_iter_t *it;
	leveldb_readoptions_t *opts;

	it = malloc(sizeof(db_iter_t));
	memset(it, 0, sizeof(db_iter_t));

	it->base_key_len = klen;
	it->base_key = malloc(klen);
	memcpy(it->base_key, key, klen);

	opts = leveldb_readoptions_create();
	/* don't fill cache in iterations */
	leveldb_readoptions_set_fill_cache(opts, 0);
	it->it = leveldb_create_iterator(db->db, opts);
	leveldb_iter_seek(it->it, it->base_key, it->base_key_len);
	it->first = 1;

	return it;
}
Beispiel #16
0
int zset_get(ldb_context_t* context, const ldb_slice_t* name, 
             const ldb_slice_t* key, int64_t* score){ 
  char *val, *errptr = NULL;
  size_t vallen = 0;
  leveldb_readoptions_t* readoptions = leveldb_readoptions_create();
  ldb_slice_t *slice_key = NULL;
  encode_zset_key(ldb_slice_data(name), ldb_slice_size(name), ldb_slice_data(key), ldb_slice_size(key), NULL, &slice_key); 
  val = leveldb_get(context->database_, readoptions, ldb_slice_data(slice_key), ldb_slice_size(slice_key), &vallen, &errptr);
  leveldb_readoptions_destroy(readoptions);
  ldb_slice_destroy(slice_key);
  int retval = 0;
  if(errptr != NULL){
    fprintf(stderr, "leveldb_get fail %s.\n", errptr);
    leveldb_free(errptr);
    retval = LDB_ERR;
    goto end;
  }
  if(val != NULL){
    assert(vallen >= LDB_VAL_META_SIZE);
    uint8_t type = leveldb_decode_fixed8(val);
    if(type & LDB_VALUE_TYPE_VAL){
        if(type & LDB_VALUE_TYPE_LAT){
          retval = LDB_OK_NOT_EXIST;
          goto end;
        }
        *score = leveldb_decode_fixed64(val + LDB_VAL_META_SIZE);
        retval = LDB_OK;
      }else{
        retval = LDB_OK_NOT_EXIST;
      }
  }else{
    retval = LDB_OK_NOT_EXIST;
  }

end:
  if(val != NULL){
    leveldb_free(val); 
  }
  return retval;
}
Beispiel #17
0
int xleveldb_nextkey(struct db_conn *db,
		const unsigned char *key, size_t ksize,
		unsigned char *nextkey, size_t *nksize)
{
	int rv;
	const char *db_nextkey;
	size_t db_nksize = 0;

	leveldb_readoptions_t *options = leveldb_readoptions_create();
	leveldb_iterator_t *it = leveldb_create_iterator(db->conn, options);

	leveldb_iter_seek(it, (const char *) key, ksize);
	if (! leveldb_iter_valid(it)) {
		rv = 0;
		goto exit;
	}

	leveldb_iter_next(it);
	if (! leveldb_iter_valid(it)) {
		rv = 0;
		goto exit;
	}

	db_nextkey = leveldb_iter_key(it, &db_nksize);

	if (db_nextkey == NULL || db_nksize > *nksize) {
		rv = 0;
		goto exit;
	}

	*nksize = db_nksize;
	memcpy(nextkey, db_nextkey, db_nksize);
	rv = 1;

exit:
	leveldb_iter_destroy(it);
	leveldb_readoptions_destroy(options);
	return rv;
}
Beispiel #18
0
RLServer::RLServer(const char *_db_path, const char *_hostaddr, int _port, int dbn):
    db_num(dbn), db_path(_db_path), hostaddr(_hostaddr), port(_port),
    fd(-1), clients_num(0)
{
    options = leveldb_options_create();
    leveldb_options_set_create_if_missing(options, 1);

    read_options = leveldb_readoptions_create();
    write_options = leveldb_writeoptions_create();
    
    char* err = 0;
    
    if(db_num<1){
        db=new leveldb_t*[1];
        db[0] = leveldb_open(options, db_path.c_str(), &err);
        if(err) {
            puts(err);
            exit(1);
        }
    }else{
        db=new leveldb_t*[db_num];
        char buf[16];
        for(int i=0;i<db_num;i++){
            int count = sprintf(buf, "/db-%03d", i);
            //TODO the db path
            db[i] = leveldb_open(options, (db_path+std::string(buf,count)).c_str(), &err);
            if(err) {
                puts(buf);
                puts(err);
                exit(1);
            }
        }
    }

    loop = ev_loop_new(EVBACKEND);
    connection_watcher.data = this;

}
Beispiel #19
0
static struct stat_cache_iterator *stat_cache_iter_init(stat_cache_t *cache, const char *path_prefix) {
    struct stat_cache_iterator *iter = NULL;

    iter = malloc(sizeof(struct stat_cache_iterator));
    iter->key_prefix = path2key(path_prefix, true); // Handles allocating the duplicate.
    iter->key_prefix_len = strlen(iter->key_prefix) + 1;

    //log_print(LOG_DEBUG, "creating leveldb iterator for prefix %s", iter->key_prefix);
    iter->ldb_options = leveldb_readoptions_create();
    iter->ldb_iter = leveldb_create_iterator(cache, iter->ldb_options);

    //log_print(LOG_DEBUG, "checking iterator validity");

    //if (!leveldb_iter_valid(iter->ldb_iter)) {
    //    log_print(LOG_ERR, "Initial LevelDB iterator is not valid.");
    //    return NULL;
    //}

    //log_print(LOG_DEBUG, "seeking");
    leveldb_iter_seek(iter->ldb_iter, iter->key_prefix, iter->key_prefix_len);

    return iter;
}
Beispiel #20
0
kvdb_t *kvdb_leveldb_open(const char *dbpath)
{
    kvdb_leveldb_t *leveldb = (kvdb_leveldb_t *)zmalloc(sizeof(struct kvdb_leveldb_t));
    memset(leveldb, 0, sizeof(kvdb_leveldb_t));

    leveldb->kvdb.dbclass = "leveldb";
    leveldb->kvdb.db_methods = &leveldb_methods;

    leveldb->pOpt = leveldb_options_create();
    leveldb_options_set_create_if_missing(leveldb->pOpt, 1);
    leveldb->pWriteOpt = leveldb_writeoptions_create();
    leveldb->pReadOpt = leveldb_readoptions_create();

    char *szErr = NULL;
    leveldb->db = leveldb_open(leveldb->pOpt, dbpath, &szErr);

    if( szErr ){
        kvdb_leveldb_close((kvdb_t*)leveldb);
        return NULL;
    }

    return (kvdb_t*)leveldb;
}
Beispiel #21
0
void ds_init() {
    char *err = NULL;

    server.ds_cache = leveldb_cache_create_lru(server.ds_lru_cache * 1048576);
    server.ds_options = leveldb_options_create();

    server.policy = leveldb_filterpolicy_create_bloom(10);


    //leveldb_options_set_comparator(server.ds_options, cmp);
    leveldb_options_set_filter_policy(server.ds_options, server.policy);
    leveldb_options_set_create_if_missing(server.ds_options, server.ds_create_if_missing);
    leveldb_options_set_error_if_exists(server.ds_options, server.ds_error_if_exists);
    leveldb_options_set_cache(server.ds_options, server.ds_cache);
    leveldb_options_set_info_log(server.ds_options, NULL);
    leveldb_options_set_write_buffer_size(server.ds_options, server.ds_write_buffer_size * 1048576);
    leveldb_options_set_paranoid_checks(server.ds_options, server.ds_paranoid_checks);
    leveldb_options_set_max_open_files(server.ds_options, server.ds_max_open_files);
    leveldb_options_set_block_size(server.ds_options, server.ds_block_size * 1024);
    leveldb_options_set_block_restart_interval(server.ds_options, server.ds_block_restart_interval);
    leveldb_options_set_compression(server.ds_options, leveldb_snappy_compression);

    server.ds_db = leveldb_open(server.ds_options, server.ds_path, &err);
    if (err != NULL) {
        fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, err);
        leveldb_free(err);
        exit(1);
    }

    server.woptions = leveldb_writeoptions_create();
    server.roptions = leveldb_readoptions_create();
    leveldb_readoptions_set_verify_checksums(server.roptions, 0);
    leveldb_readoptions_set_fill_cache(server.roptions, 1);

    leveldb_writeoptions_set_sync(server.woptions, 0);
}
int
main ()
{

  int64_t diff;
  unsigned int N_KEYS;
  unsigned int TN_KEYS;
  int bool_bench;
  int N_THREADS;

  printf ("\n number of keys to retrieve:");
  scanf ("%u", &N_KEYS);

  printf ("\n total number of keys:");
  scanf ("%u", &TN_KEYS);

  printf("\n benchmark randomly inserted database or sequentially inserted database:(1->rand or 0->seq)");
  scanf ("%u", &bool_bench);

  printf ("\n number of threads:");
  scanf ("%u", &N_THREADS);
//zeromq context
  zctx_t *ctx = zctx_new ();

//initialize worker args
  worker_args_t worker_args;


//initialize rb_tree;
  struct dbkey_rb_t dbkey_rb;
  RB_INIT (&dbkey_rb);






//initialize keys

  unsigned int factor = (unsigned int) N_KEYS / TN_KEYS;
  tinymt32_t tinymt32;
/*initializing random generator with the same seed  */
  tinymt32_init (&tinymt32, 0);

  unsigned int *keys =
    (unsigned int *) malloc (sizeof (unsigned int) * N_KEYS);
  unsigned int sec_iter;
  unsigned int iter;

  iter = 0;
  while (iter < N_KEYS)
    {

      keys[iter] = tinymt32_generate_uint32 (&tinymt32);
      tinymt32_generate_uint32 (&tinymt32);
      for (sec_iter = 1; sec_iter < factor; sec_iter++)
	{
	  tinymt32_generate_uint32 (&tinymt32);
	  tinymt32_generate_uint32 (&tinymt32);
	}
      iter++;
    }





//initialize database
  char *errptr = NULL;

  leveldb_options_t *options = leveldb_options_create ();
  worker_args.options = &options;

/* initialize Options */
  leveldb_options_set_create_if_missing (options, 1);
  leveldb_options_set_write_buffer_size (options, 62914560);
  leveldb_options_set_max_open_files (options, 800000);
//bloom filter
  leveldb_filterpolicy_t *bloom = leveldb_filterpolicy_create_bloom (10);
  leveldb_options_set_filter_policy (options, bloom);




  leveldb_readoptions_t *readoptions = leveldb_readoptions_create ();
  worker_args.readoptions = readoptions;




  leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create ();
  worker_args.writeoptions = writeoptions;


  int rc;
  void *push = zsocket_new (ctx, ZMQ_PUSH);

  int water = 2000000000;
  zsocket_set_hwm (push, water);

  rc = zsocket_bind (push, "inproc://bind_point");

  //connect 
  void *pub = zsocket_new (ctx, ZMQ_PUB);
  rc = zsocket_bind (pub, "inproc://pub_point");

//connect 
  void *router = zsocket_new (ctx, ZMQ_ROUTER);
  rc = zsocket_bind (router, "inproc://response_point");

//sleep a while
  zclock_sleep (1000);

//this assumes some synchronization at the start and end of each new bench
  leveldb_t *db_pointer;
  worker_args.db = &db_pointer;

//initialize the threads
  void *pipe[6];

  unsigned char i;
  for (i = 0; i < N_THREADS; i++)
    {
      pipe[i] = zthread_fork (ctx, &worker_fn, (void *) &(worker_args));
    }

//sleep a while
  zclock_sleep (1000);


  if(bool_bench==0){

  db_pointer = leveldb_open (options, "/mnt/database/database", &errptr);


  benchmark_notree (push, pub, router, N_KEYS, keys,N_THREADS);



  printf ("\n benchmark with tree 1st pass");

  leveldb_close (db_pointer);

  db_pointer = leveldb_open (options, "/mnt/database/database", &errptr);

  benchmark_tree (push, pub, router, N_KEYS, keys, dbkey_rb ,N_THREADS);

  leveldb_close (db_pointer);

  }else{if(bool_bench==1){
  db_pointer =
    leveldb_open (options, "/mnt/database/database_helper", &errptr);


  printf
    ("\n starting random read without a rb_btree on random inserted database");


  benchmark_notree (push, pub, router, N_KEYS, keys,N_THREADS);

  leveldb_close (db_pointer);

  db_pointer =
    leveldb_open (options, "/mnt/database/database_helper", &errptr);

  printf ("\n with a tree");

  benchmark_tree (push, pub, router, N_KEYS, keys, dbkey_rb,N_THREADS);

  leveldb_close (db_pointer);
}
}
}
Beispiel #23
0
int main(int argc, char** argv) {
  leveldb_t* db;
  leveldb_comparator_t* cmp;
  leveldb_cache_t* cache;
  leveldb_env_t* env;
  leveldb_options_t* options;
  leveldb_readoptions_t* roptions;
  leveldb_writeoptions_t* woptions;
  char* err = NULL;
  int run = -1;

  CheckCondition(leveldb_major_version() >= 1);
  CheckCondition(leveldb_minor_version() >= 1);

  snprintf(dbname, sizeof(dbname),
           "%s/leveldb_c_test-%di/meta/0",
           GetTempDir(),
           ((int) geteuid()));

  StartPhase("create_objects");
  cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
  env = leveldb_create_default_env();
  cache = leveldb_cache_create_lru(100000);

  options = leveldb_options_create();
  leveldb_options_set_comparator(options, cmp);
  leveldb_options_set_error_if_exists(options, 1);
  leveldb_options_set_cache(options, cache);
  leveldb_options_set_env(options, env);
  leveldb_options_set_info_log(options, NULL);
  leveldb_options_set_write_buffer_size(options, 100000);
  leveldb_options_set_paranoid_checks(options, 1);
  leveldb_options_set_max_open_files(options, 10);
  leveldb_options_set_block_size(options, 1024);
  leveldb_options_set_block_restart_interval(options, 8);
  leveldb_options_set_compression(options, leveldb_no_compression);

  roptions = leveldb_readoptions_create();
  leveldb_readoptions_set_verify_checksums(roptions, 1);
  leveldb_readoptions_set_fill_cache(roptions, 0);

  woptions = leveldb_writeoptions_create();
  leveldb_writeoptions_set_sync(woptions, 1);

  StartPhase("destroy");
  leveldb_destroy_db(options, dbname, &err);
  Free(&err);

  StartPhase("open");
  db = leveldb_open(options, dbname, &err);
  CheckNoError(err);
  CheckGet(db, roptions, "foo", NULL);

  StartPhase("put");
  leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
  CheckNoError(err);
  CheckGet(db, roptions, "foo", "hello");

  StartPhase("compactall");
  leveldb_compact_range(db, NULL, 0, NULL, 0);
  CheckGet(db, roptions, "foo", "hello");

  StartPhase("compactrange");
  leveldb_compact_range(db, "a", 1, "z", 1);
  CheckGet(db, roptions, "foo", "hello");

  StartPhase("writebatch");
  {
    leveldb_writebatch_t* wb = leveldb_writebatch_create();
    leveldb_writebatch_put(wb, "foo", 3, "a", 1);
    leveldb_writebatch_clear(wb);
    leveldb_writebatch_put(wb, "bar", 3, "b", 1);
    leveldb_writebatch_put(wb, "box", 3, "c", 1);
    leveldb_writebatch_delete(wb, "bar", 3);
    leveldb_write(db, woptions, wb, &err);
    CheckNoError(err);
    CheckGet(db, roptions, "foo", "hello");
    CheckGet(db, roptions, "bar", NULL);
    CheckGet(db, roptions, "box", "c");
    int pos = 0;
    leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
    CheckCondition(pos == 3);
    leveldb_writebatch_destroy(wb);
  }

  StartPhase("iter");
  {
    leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
    CheckCondition(!leveldb_iter_valid(iter));
    leveldb_iter_seek_to_first(iter);
    CheckCondition(leveldb_iter_valid(iter));
    CheckIter(iter, "box", "c");
    leveldb_iter_next(iter);
    CheckIter(iter, "foo", "hello");
    leveldb_iter_prev(iter);
    CheckIter(iter, "box", "c");
    leveldb_iter_prev(iter);
    CheckCondition(!leveldb_iter_valid(iter));
    leveldb_iter_seek_to_last(iter);
    CheckIter(iter, "foo", "hello");
    leveldb_iter_seek(iter, "b", 1);
    CheckIter(iter, "box", "c");
    leveldb_iter_get_error(iter, &err);
    CheckNoError(err);
    leveldb_iter_destroy(iter);
  }

  StartPhase("approximate_sizes");
  {
    int i;
    int n = 20000;
    char keybuf[100];
    char valbuf[100];
    uint64_t sizes[2];
    const char* start[2] = { "a", "k00000000000000010000" };
    size_t start_len[2] = { 1, 21 };
    const char* limit[2] = { "k00000000000000010000", "z" };
    size_t limit_len[2] = { 21, 1 };
    leveldb_writeoptions_set_sync(woptions, 0);
    for (i = 0; i < n; i++) {
      snprintf(keybuf, sizeof(keybuf), "k%020d", i);
      snprintf(valbuf, sizeof(valbuf), "v%020d", i);
      leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
                  &err);
      CheckNoError(err);
    }
    leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
    CheckCondition(sizes[0] > 0);
    CheckCondition(sizes[1] > 0);
  }

  StartPhase("property");
  {
    char* prop = leveldb_property_value(db, "nosuchprop");
/*    CheckCondition(prop == NULL);*/
    prop = leveldb_property_value(db, "leveldb.stats");
    CheckCondition(prop != NULL);
    Free(&prop);
  }

  StartPhase("snapshot");
  {
    const leveldb_snapshot_t* snap;
    snap = leveldb_create_snapshot(db);
    leveldb_delete(db, woptions, "foo", 3, &err);
    CheckNoError(err);
    leveldb_readoptions_set_snapshot(roptions, snap);
    CheckGet(db, roptions, "foo", "hello");
    leveldb_readoptions_set_snapshot(roptions, NULL);
    CheckGet(db, roptions, "foo", NULL);
    leveldb_release_snapshot(db, snap);
  }

  StartPhase("repair");
  {
    leveldb_close(db);
    leveldb_options_set_error_if_exists(options, 0);
    leveldb_repair_db(options, dbname, &err);
    CheckNoError(err);
    db = leveldb_open(options, dbname, &err);
    CheckNoError(err);
    CheckGet(db, roptions, "foo", NULL);
    CheckGet(db, roptions, "bar", NULL);
    CheckGet(db, roptions, "box", "c");
    leveldb_options_set_error_if_exists(options, 1);
  }

  StartPhase("filter");
  for (run = 0; run < 2; run++) {
    // First run uses custom filter, second run uses bloom filter
    CheckNoError(err);
    leveldb_filterpolicy_t* policy;
    if (run == 0) {
      policy = leveldb_filterpolicy_create(
          NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
    } else {
      policy = leveldb_filterpolicy_create_bloom(10);
    }

    // Create new database
    leveldb_close(db);
    leveldb_destroy_db(options, dbname, &err);
    leveldb_options_set_filter_policy(options, policy);
    db = leveldb_open(options, dbname, &err);
    CheckNoError(err);
    leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
    CheckNoError(err);
    leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
    CheckNoError(err);
    leveldb_compact_range(db, NULL, 0, NULL, 0);

    fake_filter_result = 1;
    CheckGet(db, roptions, "foo", "foovalue");
    CheckGet(db, roptions, "bar", "barvalue");
    if (phase == 0) {
      // Must not find value when custom filter returns false
      fake_filter_result = 0;
      CheckGet(db, roptions, "foo", NULL);
      CheckGet(db, roptions, "bar", NULL);
      fake_filter_result = 1;

      CheckGet(db, roptions, "foo", "foovalue");
      CheckGet(db, roptions, "bar", "barvalue");
    }
    leveldb_options_set_filter_policy(options, NULL);
    leveldb_filterpolicy_destroy(policy);
  }

  StartPhase("cleanup");
  leveldb_close(db);
  leveldb_options_destroy(options);
  leveldb_readoptions_destroy(roptions);
  leveldb_writeoptions_destroy(woptions);
  leveldb_cache_destroy(cache);
  leveldb_comparator_destroy(cmp);
  leveldb_env_destroy(env);

  fprintf(stderr, "PASS\n");
  return 0;
}
Beispiel #24
0
int main(int argc, char** argv) {
  leveldb_t* db;
  leveldb_comparator_t* cmp;
  leveldb_cache_t* cache;
  leveldb_env_t* env;
  leveldb_options_t* options;
  leveldb_readoptions_t* roptions;
  leveldb_writeoptions_t* woptions;
  char* err = NULL;

#if defined(LEVELDB_PLATFORM_WINDOWS)
  snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test");
#else
  snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d",
           ((int) geteuid()));
#endif

  StartPhase("create_objects");
  cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
  env = leveldb_create_default_env();
  cache = leveldb_cache_create_lru(100000);

  options = leveldb_options_create();
  leveldb_options_set_comparator(options, cmp);
  leveldb_options_set_error_if_exists(options, 1);
  leveldb_options_set_cache(options, cache);
  leveldb_options_set_env(options, env);
  leveldb_options_set_info_log(options, NULL);
  leveldb_options_set_write_buffer_size(options, 100000);
  leveldb_options_set_paranoid_checks(options, 1);
  leveldb_options_set_max_open_files(options, 10);
  leveldb_options_set_block_size(options, 1024);
  leveldb_options_set_block_restart_interval(options, 8);
  leveldb_options_set_compression(options, leveldb_no_compression);

  roptions = leveldb_readoptions_create();
  leveldb_readoptions_set_verify_checksums(roptions, 1);
  leveldb_readoptions_set_fill_cache(roptions, 0);

  woptions = leveldb_writeoptions_create();
  leveldb_writeoptions_set_sync(woptions, 1);

  StartPhase("destroy");
  leveldb_destroy_db(options, dbname, &err);
  leveldb_free(&err);

  StartPhase("open_error");
  db = leveldb_open(options, dbname, &err);
  CheckCondition(err != NULL);
  leveldb_free(&err);

  StartPhase("open");
  leveldb_options_set_create_if_missing(options, 1);
  db = leveldb_open(options, dbname, &err);
  CheckNoError(err);
  CheckGet(db, roptions, "foo", NULL);

  StartPhase("put");
  leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
  CheckNoError(err);
  CheckGet(db, roptions, "foo", "hello");

  StartPhase("writebatch");
  {
    int pos = 0;
    leveldb_writebatch_t* wb = leveldb_writebatch_create();
    leveldb_writebatch_put(wb, "foo", 3, "a", 1);
    leveldb_writebatch_clear(wb);
    leveldb_writebatch_put(wb, "bar", 3, "b", 1);
    leveldb_writebatch_put(wb, "box", 3, "c", 1);
    leveldb_writebatch_delete(wb, "bar", 3);
    leveldb_write(db, woptions, wb, &err);
    CheckNoError(err);
    CheckGet(db, roptions, "foo", "hello");
    CheckGet(db, roptions, "bar", NULL);
    CheckGet(db, roptions, "box", "c");
    leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
    CheckCondition(pos == 3);
    leveldb_writebatch_destroy(wb);
  }

  StartPhase("iter");
  {
    leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
    CheckCondition(!leveldb_iter_valid(iter));
    leveldb_iter_seek_to_first(iter);
    CheckCondition(leveldb_iter_valid(iter));
    CheckIter(iter, "box", "c");
    leveldb_iter_next(iter);
    CheckIter(iter, "foo", "hello");
    leveldb_iter_prev(iter);
    CheckIter(iter, "box", "c");
    leveldb_iter_prev(iter);
    CheckCondition(!leveldb_iter_valid(iter));
    leveldb_iter_seek_to_last(iter);
    CheckIter(iter, "foo", "hello");
    leveldb_iter_seek(iter, "b", 1);
    CheckIter(iter, "box", "c");
    leveldb_iter_get_error(iter, &err);
    CheckNoError(err);
    leveldb_iter_destroy(iter);
  }

  StartPhase("approximate_sizes");
  {
    int i;
    int n = 20000;
    char keybuf[100];
    char valbuf[100];
    uint64_t sizes[2];
    const char* start[2] = { "a", "k00000000000000010000" };
    size_t start_len[2] = { 1, 21 };
    const char* limit[2] = { "k00000000000000010000", "z" };
    size_t limit_len[2] = { 21, 1 };
    leveldb_writeoptions_set_sync(woptions, 0);
    for (i = 0; i < n; i++) {
      snprintf(keybuf, sizeof(keybuf), "k%020d", i);
      snprintf(valbuf, sizeof(valbuf), "v%020d", i);
      leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
                  &err);
      CheckNoError(err);
    }
    leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
    CheckCondition(sizes[0] > 0);
    CheckCondition(sizes[1] > 0);
  }

  StartPhase("property");
  {
    char* prop = leveldb_property_value(db, "nosuchprop");
    CheckCondition(prop == NULL);
    prop = leveldb_property_value(db, "leveldb.stats");
    CheckCondition(prop != NULL);
    leveldb_free(&prop);
  }

  StartPhase("snapshot");
  {
    const leveldb_snapshot_t* snap;
    snap = leveldb_create_snapshot(db);
    leveldb_delete(db, woptions, "foo", 3, &err);
    CheckNoError(err);
    leveldb_readoptions_set_snapshot(roptions, snap);
    CheckGet(db, roptions, "foo", "hello");
    leveldb_readoptions_set_snapshot(roptions, NULL);
    CheckGet(db, roptions, "foo", NULL);
    leveldb_release_snapshot(db, snap);
  }

  StartPhase("repair");
  {
    leveldb_close(db);
    leveldb_options_set_create_if_missing(options, 0);
    leveldb_options_set_error_if_exists(options, 0);
    leveldb_repair_db(options, dbname, &err);
    CheckNoError(err);
    db = leveldb_open(options, dbname, &err);
    CheckNoError(err);
    CheckGet(db, roptions, "foo", NULL);
    CheckGet(db, roptions, "bar", NULL);
    CheckGet(db, roptions, "box", "c");
  }

  StartPhase("cleanup");
  leveldb_close(db);
  leveldb_options_destroy(options);
  leveldb_readoptions_destroy(roptions);
  leveldb_writeoptions_destroy(woptions);
  leveldb_cache_destroy(cache);
  leveldb_comparator_destroy(cmp);
  leveldb_env_destroy(env);

  fprintf(stderr, "PASS\n");
  return 0;
}
Beispiel #25
0
int main(int argc, char **argv) {
  int goopt;
  int listenport=8080;
  int numthreads=10;
  int tf;
  
  char *dbd=NULL;
  char *lpstr=NULL;
  char *ntstr=NULL;
  char *alfile=NULL;

  leveldb_options_t *dbopt;

  struct mg_context *ctx= NULL; 
  char **mgoptions;
  
  signal(SIGINT,handlesig);
  signal(SIGTERM,handlesig);
  
  // command line parsing
  while ((goopt=getopt (argc, argv, "d:p:n:a:t:vh")) != -1) {
    switch (goopt) {
    case 'd': // database 
      dbd=calloc(strlen((char*)optarg)+1,sizeof(char));
      strncpy(dbd,(char*)optarg,strlen((char*)optarg));
      break;
    case 'a': // access log, passed to mongoose
      alfile=calloc(strlen((char*)optarg)+1,sizeof(char));
      strncpy(alfile,(char*)optarg,strlen((char*)optarg));
      break;
    case 'p': // port
      listenport=atoi(optarg);
      break;
    case 'n': // number of threads
      numthreads=atoi(optarg);
      break;
    case 'v': // verbose
      vlevel++;
      break;
    case 'h': // help
      usage(NULL,EXIT_SUCCESS);
      break;
    default: // fallthrough
      usage(NULL,EXIT_FAILURE);
    }
  }
  
  LOG_TRACE(vlevel, _("Validation\n"),dbd);
  if(dbd==NULL) {
    usage("Must give a database dir\n",EXIT_FAILURE);
  } else {
    LOG_TRACE(vlevel, _("Using database dir: %s\n"),dbd);
  }
  
  if(alfile!=NULL) {
    if((tf=open(alfile,O_WRONLY|O_CREAT,S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH))==-1) {
      LOG_FATAL(vlevel,_("Unable to open access log file: %s: %s\n"), alfile, strerror(errno));
      exit(EXIT_FAILURE);
    }
    close(tf);
  }
  if(alfile!=NULL && access(alfile, W_OK)!=0) {
    LOG_FATAL(vlevel, _("Unable to write to access log file: %s\n"),alfile);
    exit(EXIT_FAILURE);
  }
  
  if(listenport<0 || listenport>65536) {
    LOG_FATAL(vlevel, _("Given port out of bounds: %i\n"),listenport);
    exit(EXIT_FAILURE);
  }
  
  if(numthreads<0 || numthreads>1024) {
    LOG_FATAL(vlevel, _("Given threads out of bounds: %i\n"),numthreads);
    exit(EXIT_FAILURE);
  }

  LOG_TRACE(vlevel, _("Setting up leveldb store in %s\n"),dbd);
  dbopt=leveldb_options_create();
  leveldb_options_set_create_if_missing(dbopt, 1);
  leveldb_options_set_write_buffer_size(dbopt, 8388608);
  leveldb_options_set_compression(dbopt,leveldb_no_compression);
  dbh=leveldb_open(dbopt,dbd,&errptr);

  LOG_TRACE(vlevel, _("Setting leveldb read options\n"));
  ropt = leveldb_readoptions_create();
  leveldb_readoptions_set_verify_checksums(ropt, 1);
  leveldb_readoptions_set_fill_cache(ropt, 0);

  LOG_TRACE(vlevel, _("Setting leveldb write options\n"));
  wopt = leveldb_writeoptions_create();
  leveldb_writeoptions_set_sync(wopt, 0);

  // set mgoptions - XXX this needs to be handled better
  lpstr=calloc(7,sizeof(char));
  snprintf(lpstr,6,"%i",listenport);
  
  ntstr=calloc(4,sizeof(char));
  snprintf(ntstr,3,"%i",numthreads);
  
  if(alfile!=NULL) {
    mgoptions = calloc(9,sizeof(char*));
  } else {
    mgoptions = calloc(7,sizeof(char*));
  }
  mgoptions[0]="listening_ports";
  mgoptions[1]=lpstr;
  mgoptions[2]="document_root";
  mgoptions[3]="/dev/null";
  mgoptions[4]="num_threads";
  mgoptions[5]=ntstr;
  if(alfile!=NULL) {
    mgoptions[6]="access_log_file";
    mgoptions[7]=alfile;
    mgoptions[8]=NULL;
  } else {
    mgoptions[6]=NULL;
  }
  // main loop
  LOG_INFO(vlevel, _("Starting Mongoose HTTP server loop\n"));
  ctx = mg_start(&mghandle, NULL, (const char**)mgoptions);
  if(ctx!=NULL) {
    while(!done) {
      // cleaner thread here?
      sleep(1);
    }
    LOG_INFO(vlevel, _("Ending Mongoose HTTP server loop\n"));
    mg_stop(ctx);
  } else {
    LOG_FATAL(vlevel,_("Error in creating Mongoose HTTP server\n"));
  }
  LOG_TRACE(vlevel, _("Closing database handle\n"));

  // close leveldb handle
  LOG_TRACE(vlevel, _("Cleaning up leveldb\n"));
  leveldb_compact_range(dbh, NULL, 0, NULL, 0);

  leveldb_options_destroy(dbopt);
  leveldb_readoptions_destroy(ropt);
  leveldb_writeoptions_destroy(wopt);
  leveldb_close(dbh);

  LOG_TRACE(vlevel, _("Cleaning up\n"));
  free(dbd);
  free(lpstr);
  free(ntstr);
  free(mgoptions);
  
  return EXIT_SUCCESS;
}
Beispiel #26
0
struct stat_cache_value *stat_cache_value_get(stat_cache_t *cache, const char *path, bool skip_freshness_check) {
    struct stat_cache_value *value = NULL;
    char *key;
    leveldb_readoptions_t *options;
    size_t vallen;
    char *errptr = NULL;
    //void *f;
    time_t current_time;

    key = path2key(path, false);

    log_print(LOG_DEBUG, "CGET: %s", key);

    options = leveldb_readoptions_create();
    value = (struct stat_cache_value *) leveldb_get(cache, options, key, strlen(key) + 1, &vallen, &errptr);
    leveldb_readoptions_destroy(options);
    free(key);

    //log_print(LOG_DEBUG, "Mode: %04o", value->st.st_mode);

    if (errptr != NULL) {
        log_print(LOG_ERR, "leveldb_get error: %s", errptr);
        free(errptr);
        return NULL;
    }

    if (value == NULL) {
        log_print(LOG_DEBUG, "stat_cache_value_get miss on path: %s", path);
        return NULL;
    }

    if (vallen != sizeof(struct stat_cache_value)) {
        log_print(LOG_ERR, "Length %lu is not expected length %lu.", vallen, sizeof(struct stat_cache_value));
    }

    if (!skip_freshness_check) {
        current_time = time(NULL);

        // First, check against the stat item itself.
        //log_print(LOG_DEBUG, "Current time: %lu", current_time);
        if (current_time - value->updated > CACHE_TIMEOUT) {
            char *directory;
            time_t directory_updated;
            int is_dir;

            log_print(LOG_DEBUG, "Stat entry %s is %lu seconds old.", path, current_time - value->updated);

            // If that's too old, check the last update of the directory.
            directory = strip_trailing_slash(ne_path_parent(path), &is_dir);
            directory_updated = stat_cache_read_updated_children(cache, directory);
            //log_print(LOG_DEBUG, "Directory contents for %s are %lu seconds old.", directory, (current_time - directory_updated));
            free(directory);
            if (current_time - directory_updated > CACHE_TIMEOUT) {
                log_print(LOG_DEBUG, "%s is too old.", path);
                free(value);
                return NULL;
            }
        }
    }

    /*
    if ((f = file_cache_get(path))) {
        value->st.st_size = file_cache_get_size(f);
        file_cache_unref(cache, f);
    }
    */

    //print_stat(&value->st, "CGET");

    return value;
}
Beispiel #27
0
//TODO: add "bool create_if_missing" to this API!
int leveldb_client_start(leveldb **db, const char *name,
		size_t write_buffer_size, bool use_compression, bool sync_writes)
{
	int namelen;
	char *db_name;
	char *err = NULL;
	leveldb_t* db_t;
	leveldb_comparator_t *cmp;
	leveldb_cache_t *cache;
	leveldb_env_t *env;
	leveldb_options_t *options;
	leveldb_readoptions_t *roptions;
	leveldb_writeoptions_t *woptions;
	
	*db = (leveldb *)malloc(sizeof(leveldb));
	if (*db == NULL) {
		kp_error("malloc(leveldb) failed!\n");
		return -1;
	}

	/* Check arguments: */
	if (strlen(name) < 1) {
		kp_error("name has no length!\n");
		return -1;
	}
	if (write_buffer_size == 0) {
		write_buffer_size = WRITE_BUFFER_DEFAULT;
	}
	if (write_buffer_size < WRITE_BUFFER_MIN) {
		kp_error("write_buffer_size=%zu too small; min=%u\n",
				write_buffer_size, WRITE_BUFFER_MIN);
		return -1;
	}
	kp_debug("got args: name=%s, write_buffer_size=%zu, use_compression=%s, "
			"sync_writes=%s\n",
			name, write_buffer_size, use_compression ? "true" : "false",
			sync_writes ? "true" : "false");

	/* Format of db name: LEVELDB_DIR + / + name */
	namelen = strlen(name);
	db_name = malloc(namelen + 1);
	if (!db_name) {
		kp_error("malloc(db_name) failed!\n");
		return -1;
	}
	strncpy(db_name, name, namelen+1);          //copy null-zero
	(*db)->name = db_name;
	kp_debug("constructed db->name=[%s]\n", (*db)->name);

	/* Set up the database's comparator, environment, cache, etc. According
	 * to leveldb/include/leveldb/c.h, all functions that can raise an error
	 * must be passed a "char **errptr" (set to NULL!) as the last argument;
	 * I guess functions that don't take this pointer can't return errors.
	 */
	cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_compare,
			cmp_name);  //first arg is "state"
	env = leveldb_create_default_env();
	cache = leveldb_cache_create_lru(CACHE_CAPACITY);
	//	cache = NULL;
	(*db)->cmp = cmp;
	(*db)->env = env;
	(*db)->cache = cache;

	/* Set up the database's various options. Many of these will affect the
	 * database's performance! (see leveldb/include/leveldb/options.h).
	 */
	options = leveldb_options_create();
	leveldb_options_set_create_if_missing(options, 1);
	leveldb_options_set_comparator(options, cmp);
	leveldb_options_set_error_if_exists(options, 1);  //raise error if db already exists
	leveldb_options_set_cache(options, cache);
	//	leveldb_options_set_cache(options, NULL); //disable cache??
	leveldb_options_set_env(options, env);
	leveldb_options_set_info_log(options, NULL);  //NULL: write info to file in db's dir
	leveldb_options_set_write_buffer_size(options, write_buffer_size);
	  /* Amount of data to build up in memory (backed by an unsorted log
	   * on disk) before converting to a sorted on-disk file.
	   * Larger values increase performance, especially during bulk loads.
	   * Up to two write buffers may be held in memory at the same time,
	   * so you may wish to adjust this parameter to control memory usage.
	   * Also, a larger write buffer will result in a longer recovery time
	   * the next time the database is opened.
	   * Default: 4MB  (test file uses 100000 bytes)
	   */
	leveldb_options_set_paranoid_checks(options, 0);  //default false; test file uses true
	leveldb_options_set_max_open_files(options, MAX_OPEN_FILES);
	leveldb_options_set_block_size(options, BLOCK_SIZE);
	leveldb_options_set_block_restart_interval(options, BLOCK_RESTART_INTERVAL);
	leveldb_options_set_compression(options,
			use_compression ? leveldb_snappy_compression : leveldb_no_compression);
	(*db)->options = options;

	roptions = leveldb_readoptions_create();
	leveldb_readoptions_set_verify_checksums(roptions, 0);
	  /* If true, all data read from underlying storage will be
	   * verified against corresponding checksums.
	   * Default false; test file uses true.
	   */
	leveldb_readoptions_set_fill_cache(roptions, true);
	  /* Should the data read for this iteration be cached in memory?
	   * Callers may wish to set this field to false for bulk scans.
	   * Default true; test file uses false.
	   */
	(*db)->roptions = roptions;

	woptions = leveldb_writeoptions_create();
	leveldb_writeoptions_set_sync(woptions, sync_writes ? 1 : 0);
	  /* If true, the write will be flushed from the operating system
	   * buffer cache (by calling WritableFile::Sync()) before the write
	   * is considered complete.  If this flag is true, writes will be
	   * slower.
	   * If this flag is false, and the machine crashes, some recent
	   * writes may be lost.  Note that if it is just the process that
	   * crashes (i.e., the machine does not reboot), no writes will be
	   * lost even if sync==false.
	   * In other words, a DB write with sync==false has similar
	   * crash semantics as the "write()" system call.  A DB write
	   * with sync==true has similar crash semantics to a "write()"
	   * system call followed by "fsync()".
	   */
	(*db)->woptions = woptions;

	kp_debug("destroying previous copy of database, if it exists\n");
	leveldb_destroy_db((*db)->options, (*db)->name, &err);
	free_err(&err);

	kp_debug("opening/creating database [%s]\n", (*db)->name);
	db_t = leveldb_open((*db)->options, (*db)->name, &err);
	if (err) {
		kp_error("opening db returned error: %s\n", err);
		return -1;
	}
	free_err(&err);
	(*db)->db = db_t;

	kp_debug("successfully started leveldb [%s]\n", (*db)->name);
	return 0;
}
int
main ()
{
unsigned int N_KEYS;
unsigned int N_BATCH;

printf("\n number of keys:");
scanf("%u",&N_KEYS);

printf("\n batch size:");
scanf("%u",&N_BATCH);






  tinymt32_t tinymt32;
//initialize random generator
  tinymt32_init (&tinymt32, 0);


  int iter;
  int sec_iter;
  int stop;
  int counter;


//initialize database
  char *errptr = NULL;

  leveldb_options_t *options = leveldb_options_create ();

/* initializeOptions */
  leveldb_options_set_create_if_missing (options, 1);
  leveldb_options_set_write_buffer_size(options,120000000 );
  leveldb_options_set_max_open_files(options,800000);
/*open Database */

  leveldb_t *db_helper =
    leveldb_open (options, "/mnt/database/database_helper", &errptr);
  leveldb_t *db = leveldb_open (options, "/mnt/database/database", &errptr);



  leveldb_readoptions_t *readoptions = leveldb_readoptions_create ();

  leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create ();

leveldb_writebatch_t* batch=leveldb_writebatch_create();


  int64_t diff = zclock_time ();

//write into database_helper
  iter=0;
  while( iter < N_KEYS)
    {
     sec_iter=0;
     while((iter<N_KEYS) && (sec_iter<N_BATCH)){
      unsigned int key = tinymt32_generate_uint32 (&tinymt32);
      unsigned int val = tinymt32_generate_uint32 (&tinymt32);
      leveldb_writebatch_put (batch,
		   (const char *) &key, sizeof (int),
		   (const char *) &val, sizeof (int));
    sec_iter++;
    iter++;
    }
      leveldb_write(
    db_helper,
    writeoptions,
    batch,
    &errptr);

      if(errptr!=NULL){
      printf("\n%s",errptr);
      }
      assert (errptr == NULL);
      leveldb_writebatch_clear(batch);
    }
  
      leveldb_writebatch_destroy(batch);

  diff = zclock_time () - diff;

  printf ("\nrandom write:  %d", diff);


  diff = zclock_time ();
//write sequentially into db

  leveldb_iterator_t *liter = leveldb_create_iterator (db_helper,
						       readoptions);

  leveldb_iter_seek_to_first (liter);

  while (leveldb_iter_valid (liter))
    {
      size_t length;
      char key[4];
      memcpy (key, leveldb_iter_key (liter, &length), 4);
      char val[4];
      memcpy (val, leveldb_iter_value (liter, &length), 4);

      leveldb_iter_get_error (liter, &errptr);
      assert (errptr == NULL);
      leveldb_put (db,
		   writeoptions,
		   (const char *) &key, sizeof (int),
		   (const char *) &val, sizeof (int), &errptr);
      assert (errptr == NULL);
      leveldb_iter_next (liter);
    }


  diff = zclock_time () - diff;




  leveldb_close (db);
  leveldb_close (db_helper);





}
Beispiel #29
0
db_readoptions_t *db_readoptions_create(void) {
	return leveldb_readoptions_create();
}
Beispiel #30
0
int
main(int argc, char *argv[]) {
	if (argc != 5) {
		fprintf(stderr, "Usage: ./leveldb_index [leveldb_directory] [sample_input] [index_output] [index_time_file]\n");
		return -1;
	}
	if ((file = fopen(argv[2], "r")) == NULL) {
		fprintf(stderr, "Failed to open sample file \"%s\" for reading\n", argv[1]);
		return -1;
	}
	for (i = 0; fscanf(file, "%"PRId64"\t", &cnt) != EOF; ++i) {
		fgets(buffer, BUFFER_SIZE, file);
		for (len = strlen(buffer) - 1; len >= 0 && (buffer[len] == '\r' || buffer[len] == '\n'); buffer[len--] = 0);
		++len;
		strcpy(request[i].key, buffer);
	}
	fclose(file);

	if ((file = fopen(argv[4], "w")) == NULL) {
		fprintf(stderr,  "Failed to open time information file \"%s\" for reading\n", argv[4]);
		return -1;
	}
	fprintf(stdout, "Start indexing\n");
	srand((unsigned)time(NULL));
	options = leveldb_options_create();
	leveldb_options_set_create_if_missing(options, 1);
	// default: write_buffer_size: 4MB, block_cache: 8MB, block_size ~ 4KB
	leveldb_options_set_write_buffer_size(options, BLK_SIZE * 1024);
	leveldb_options_set_cache(options, cache = leveldb_cache_create_lru(BLK_SIZE * 2048));
	leveldb_options_set_block_size(options, BLK_SIZE);
	leveldb_options_set_compression(options, 0);
	db = leveldb_open(options, argv[1], &err);
	if (err != NULL) {
		fprintf(stderr, "Failed to open leveldb.\n");
		return -1;
	}
	leveldb_free(err);
	roptions = leveldb_readoptions_create();
	gettimeofday(&start, NULL);
	for (i = seekn = tot = 0; i < DISK_INDEX_SIZE; ++i) {
		val = leveldb_get(db, roptions, request[i].key, strlen(request[i].key), &read_len, &err);
		if (err != NULL) {
			fprintf(stderr, "Failed to read from leveldb.\n");
			return -1;
		}
		leveldb_free(err);
		if (read_len == 0)
			request[i].value = 0;
		else {
			val[read_len] = 0;
			sscanf(val, "%"PRId64"", &request[i].value);
		}
		leveldb_free(val);
		if ((i + 1) % 1000 == 0) {
			gettimeofday(&end, NULL);
			tot += (int64_t)(end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec;
			fprintf(stdout, "\rIndexed %"PRId64" items in %"PRId64"us", i + 1, tot);
			fflush(stdout);
			fprintf(file, "Indexed %"PRId64" items in %"PRId64"us\n", i + 1, tot);
			fflush(file);
			gettimeofday(&start, NULL);
		}
	}
	fprintf(stdout, "\rIndexed %"PRId64" items in %"PRId64"us\n", i, tot);
	fprintf(file, "Indexed %"PRId64" items in %"PRId64"us\n", i, tot);
	fflush(file);
	fclose(file);
	leveldb_cache_destroy(cache);
	leveldb_readoptions_destroy(roptions);
	leveldb_options_destroy(options);
	leveldb_close(db);
	fprintf(stdout, "Used %"PRId64" us and seeked %"PRId64" times to index %"PRId64" keys\n", tot, seekn, i);
	if ((file = fopen(argv[3], "w")) == NULL) {
		fprintf(stdout, "Failed to open file \"%s\" for storing indexing results\n", argv[3]);
		return -1;
	}
	for (i = 0; i < DISK_INDEX_SIZE; ++i)
		fprintf(file, "%"PRId64"\t%s\n", request[i].value, request[i].key);
	fflush(file);
	fclose(file);
	return 0;
}