int stat_cache_value_set(stat_cache_t *cache, const char *path, struct stat_cache_value *value) {
leveldb_writeoptions_t *options;
char *errptr = NULL;
char *key;
int r = 0;
assert(value);
value->updated = time(NULL);
value->local_generation = stat_cache_get_local_generation();
key = path2key(path, false);
log_print(LOG_DEBUG, "CSET: %s (mode %04o)", key, value->st.st_mode);
//print_stat(&value->st, "CSET");
options = leveldb_writeoptions_create();
leveldb_put(cache, options, key, strlen(key) + 1, (char *) value, sizeof(struct stat_cache_value), &errptr);
leveldb_writeoptions_destroy(options);
//log_print(LOG_DEBUG, "Setting key: %s", key);
free(key);
if (errptr != NULL) {
log_print(LOG_ERR, "leveldb_set error: %s", errptr);
free(errptr);
r = -1;
}
return r;
}
int stat_cache_updated_children(stat_cache_t *cache, const char *path, time_t timestamp) {
leveldb_writeoptions_t *options;
char *key = NULL;
char *errptr = NULL;
int r = 0;
asprintf(&key, "updated_children:%s", path);
options = leveldb_writeoptions_create();
if (timestamp == 0)
leveldb_delete(cache, options, key, strlen(key) + 1, &errptr);
else
leveldb_put(cache, options, key, strlen(key) + 1, (char *) ×tamp, sizeof(time_t), &errptr);
leveldb_writeoptions_destroy(options);
free(key);
if (errptr != NULL) {
log_print(LOG_ERR, "leveldb_set error: %s", errptr);
free(errptr);
r = -1;
}
return r;
}
int kvdb_leveldb_put(kvdb_t *kvdb, void *key, uint32_t klen, void *value, uint32_t vlen)
{
kvdb_leveldb_t *leveldb = (kvdb_leveldb_t*)kvdb;
char *szErr = NULL;
leveldb_put(leveldb->db, leveldb->pWriteOpt, (const char*)key, klen, (const char *)value, vlen, &szErr);
if ( szErr ) {
return -1;
} else {
return 0;
}
}
void store_put(leveldb_t * store, char * key, size_t key_size, char * value, size_t value_len){
char * err=0x00;
leveldb_writeoptions_t *write_options=leveldb_writeoptions_create();
//char * resp = leveldb_put(store, leveldb_readoptions_create(), key, key_size, &read_len, &err);
leveldb_put(store,write_options , key, key_size, value, value_len, &err);
leveldb_writeoptions_destroy(write_options);
if(err){
fprintf(stderr, "error: %s\n", err);
free(err);
}
}
void
db_put(db_t *db, const char *key, size_t klen,
const char *val, size_t vlen, char **errptr) {
leveldb_writeoptions_t *opts;
opts = leveldb_writeoptions_create();
/* sync=true flushed buffer */
leveldb_writeoptions_set_sync(opts, 1);
leveldb_put(db->db, opts, key, klen, val, vlen, errptr);
leveldb_writeoptions_destroy(opts);
}
inline int ldb_put(struct _leveldb_stuff *ldbs, const char *key, size_t klen, const char *value, size_t vlen)
{
char *err = NULL;
leveldb_put(ldbs->db, ldbs->woptions, key, klen, value, vlen, &err);
if (err) {
fprintf(stderr, "%s\n", err);
leveldb_free(err);
err = NULL;
return -1;
} else {
return 0;
}
}
int xleveldb_set(struct db_conn *db, const unsigned char *key, size_t ksize,
unsigned char *val, size_t vsize)
{
leveldb_writeoptions_t *options = leveldb_writeoptions_create();
char *err, *origerr;
err = origerr = malloc(1);
leveldb_put(db->conn, options,
(const char *) key, ksize,
(const char *) val, vsize, &err);
free(err);
leveldb_writeoptions_destroy(options);
return err == origerr;
}
void setCommand(redisClient *c) {
char *key, *value;
char *err = NULL;
key = (char *) c->argv[1]->ptr;
value = (char *) c->argv[2]->ptr;
leveldb_put(server.ds_db, server.woptions, key, sdslen((sds) key),
value, sdslen((sds) value), &err);
if (err != NULL) {
addReplyError(c, err);
leveldb_free(err);
return;
}
addReply(c, shared.ok);
server.dirty++;
signalModifiedKey(c->db, c->argv[1]);
return;
}
static int
txdb_save_tx(struct txdb *txdb,
const uint256 *blkHash,
const uint256 *txHash,
mtime_t timestamp,
const uint8 *buf,
size_t len)
{
struct tx_ser_data txdata;
struct buff *bufd;
struct buff *bufk;
char hashStr[80];
char *err;
err = NULL;
memset(&txdata, 0, sizeof txdata);
if (blkHash) {
memcpy(&txdata.blkHash, blkHash, sizeof *blkHash);
}
txdata.buf = (uint8 *)buf;
txdata.len = len;
txdata.timestamp = timestamp;
uint256_snprintf_reverse(hashStr, sizeof hashStr, txHash);
bufk = txdb_serialize_tx_key(txdb->tx_seq, hashStr);
bufd = txdb_serialize_tx_data(&txdata);
leveldb_put(txdb->db, txdb->wr_opts,
buff_base(bufk), buff_curlen(bufk),
buff_base(bufd), buff_curlen(bufd),
&err);
buff_free(bufk);
buff_free(bufd);
if (err) {
Warning(LGPFX" failed to save tx %s: %s\n", hashStr, err);
free(err);
}
return err != NULL;
}
void
localdb_incr_counter (localdb_t * localdb, char *key, unsigned long counter)
{
char *errptr = NULL;
char lkey[18];
sprintf (lkey, "%s0", key);
leveldb_put (localdb->db,
localdb->writeoptions,
lkey, strlen (lkey),
(const char *) &counter, sizeof (unsigned long), &errptr);
if (errptr) {
printf ("\n%s", errptr);
exit (1);
}
}
void put(db_t* db, size_t keySz, void* key, size_t valSz, void* val)
{
if(db->type == LEVELDB)
{
levelDB_t* leveldb = (levelDB_t*)db->database;
leveldb_put(leveldb->db, leveldb->woptions, (char*)key, keySz, (char*)val, valSz, &(leveldb->err));
if(leveldb->err != NULL)
{
fprintf(stderr, "leveldb put failed!\n");
exit(-1);
}
leveldb_free(leveldb->err);
leveldb->err = NULL;
}
else
{
kernelDB_t* kerneldb = (kernelDB_t*) db->database;
kr_put(kerneldb->client, keySz, key, valSz, val);
}
}
void incrDecrCommand(redisClient *c, long long incr) {
sds data;
char *value;
int64_t val, recore;
size_t val_len;
char *err = NULL;
err = NULL;
val_len = 0;
value = leveldb_get(server.ds_db, server.roptions, c->argv[1]->ptr, sdslen((sds) c->argv[1]->ptr), &val_len, &err);
if (err != NULL) {
if (val_len > 0) leveldb_free(value);
addReplyError(c, err);
leveldb_free(err);
return;
} else if (val_len < 1) {
val = 0;
} else {
val = strtoll(value, NULL, 10);
}
err = NULL;
recore = val + incr;
data = sdsfromlonglong(recore);
leveldb_put(server.ds_db, server.woptions, c->argv[1]->ptr, sdslen((sds) c->argv[1]->ptr), data, sdslen(data), &err);
if (err != NULL) {
addReplyError(c, err);
leveldb_free(err);
} else {
addReplyLongLong(c, recore);
server.dirty++;
signalModifiedKey(c->db, c->argv[1]);
}
sdsfree(data);
leveldb_free(value);
return;
}
void
localdb_set_interval (localdb_t * localdb, char *key, int interval)
{
char *errptr = NULL;
char lkey[18];
sprintf (lkey, "%s1", key);
leveldb_put (localdb->db,
localdb->writeoptions,
lkey, strlen (lkey), (const char *) &interval, sizeof (int),
&errptr);
if (errptr) {
printf ("\n%s", errptr);
exit (1);
}
}
/**
* mdhim_leveldb_put
* Stores a single key in the data store
*
* @param dbh in pointer to the leveldb handle
* @param key in void * to the key to store
* @param key_len in length of the key
* @param data in void * to the value of the key
* @param data_len in length of the value data
* @param mstore_opts in additional options for the data store layer
*
* @return MDHIM_SUCCESS on success or MDHIM_DB_ERROR on failure
*/
int mdhim_leveldb_put(void *dbh, void *key, int key_len, void *data, int32_t data_len,
struct mdhim_store_opts_t *mstore_opts) {
leveldb_writeoptions_t *options;
char *err = NULL;
leveldb_t *db = (leveldb_t *) dbh;
struct timeval start, end;
gettimeofday(&start, NULL);
options = (leveldb_writeoptions_t *) mstore_opts->db_ptr4;
leveldb_put(db, options, key, key_len, data, data_len, &err);
if (err != NULL) {
mlog(MDHIM_SERVER_CRIT, "Error putting key/value in leveldb");
return MDHIM_DB_ERROR;
}
gettimeofday(&end, NULL);
mlog(MDHIM_SERVER_DBG, "Took: %d seconds to put the record",
(int) (end.tv_sec - start.tv_sec));
return MDHIM_SUCCESS;
}
int leveldb_client_put(leveldb *db, const char *key, const char *value)
{
char *err = NULL;
size_t keylen, vallen;
if (!db || !key || !value) {
kp_error("got a NULL argument\n");
return -1;
}
/* The lengths that we pass into leveldb_put() should NOT include
* the null-zero!
*/
keylen = strlen(key);
vallen = strlen(value);
if (keylen < 1 || vallen < 1) {
kp_error("got empty string for key (%zu) or val (%zu)\n",
keylen, vallen);
}
kp_debug("putting [%s:%s] into db [%s]; keylen=%zu, vallen=%zu\n",
key, value, db->name, keylen, vallen);
/* leveldb_put() goes to internal leveldb code that will copy the key
* and the value; we do not have to do this here in our leveldb client.
* After our client function returns, the caller will be able to free
* the key and the value. The leveldb code will call Slice() to slice
* off the null-zero from the key and value strings we are passing it,
* and just the character bytes will be stored in the db.
*/
leveldb_put(db->db, db->woptions, key, keylen, value, vallen, &err);
if (err) {
kp_error("leveldb_put() returned error: %s\n", err);
free_err(&err);
return -1;
}
kp_debug("leveldb_put() succeeded\n");
return 0;
}
void
txdb_confirm_one_tx(struct txdb *txdb,
const uint256 *blkHash,
const uint256 *txHash)
{
leveldb_iterator_t *iter;
struct tx_entry *txe;
char bkHashStr[80];
char txHashStr[80];
ASSERT(!uint256_iszero(blkHash));
ASSERT(!uint256_iszero(txHash));
txe = txdb_get_tx_entry(txdb, txHash);
if (txe == NULL) {
return;
}
if (txe->relevant == 0) {
txdb_remove_from_hashtable(txdb, txHash);
NOT_TESTED();
return;
}
if (!uint256_iszero(&txe->blkHash)) {
/*
* It's possible for the ASSERT below to fire if a tx is confirmed in
* a block that is later orphaned. The tx should then be relayed again
* until it finds its way in a new block.
*/
ASSERT(uint256_issame(&txe->blkHash, blkHash));
return;
}
peergroup_stop_broadcast_tx(btc->peerGroup, txHash);
memcpy(&txe->blkHash, blkHash, sizeof *blkHash);
uint256_snprintf_reverse(bkHashStr, sizeof bkHashStr, blkHash);
uint256_snprintf_reverse(txHashStr, sizeof txHashStr, txHash);
Warning(LGPFX" %s confirmed in %s\n", txHashStr, bkHashStr);
NOT_TESTED();
iter = leveldb_create_iterator(txdb->db, txdb->rd_opts);
leveldb_iter_seek_to_first(iter);
while (leveldb_iter_valid(iter)) {
struct tx_ser_key *txkey;
struct tx_ser_data *txdata;
struct buff *buf;
const char *key;
const char *val;
size_t klen;
size_t vlen;
char *err = NULL;
key = leveldb_iter_key(iter, &klen);
txkey = txdb_deserialize_tx_key(key, klen);
if (txkey == NULL || uint256_issame(txHash, &txkey->txHash) == 0) {
free(txkey);
leveldb_iter_next(iter);
continue;
}
NOT_TESTED();
val = leveldb_iter_value(iter, &vlen);
txdata = txdb_deserialize_tx_data(val, vlen);
ASSERT(uint256_iszero(&txdata->blkHash));
ASSERT(txdata->timestamp != 0);
memcpy(&txdata->blkHash, blkHash, sizeof *blkHash);
buf = txdb_serialize_tx_data(txdata);
leveldb_put(txdb->db, txdb->wr_opts, key, klen,
buff_base(buf), buff_curlen(buf), &err);
buff_free(buf);
if (err) {
Warning(LGPFX" failed to write tx entry: %s\n", err);
}
txdb_export_tx_info(txdb);
free(txkey);
free(txdata->buf);
free(txdata);
break;
}
leveldb_iter_destroy(iter);
}
static void *mghandle(enum mg_event event, struct mg_connection *conn) {
const struct mg_request_info *request_info = mg_get_request_info(conn);
if (event == MG_NEW_REQUEST) {
char *req=calloc(URL_STRING_MAX+1,sizeof(char));
struct in_addr saddr;
strncpy(req,request_info->uri, URL_STRING_MAX);
saddr.s_addr = ntohl(request_info->remote_ip);
LOG_DEBUG(vlevel, _("Connection from: %s, request: %s\n"), inet_ntoa(saddr), req);
if(strncmp(req, "/status\0", 8) == 0) {
// XXX how to get more status
mg_printf(conn,
"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"OK\r\n",
4);
} else if(strncmp(req, "/set/", 5) == 0) {
int n=strlen(req);
while(n) {
if(req[n]==':') {
leveldb_put(dbh, wopt, req+5, n-5, req+n+1, strlen(req)-n-1, &errptr);
if(errptr!=NULL) {
LOG_ERROR(vlevel,_("leveldb_put(): %s\n"),errptr);
mg_printf(conn,
"HTTP/1.1 500 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"ERROR: %s\r\n",
9+strlen(errptr), errptr);
} else {
mg_printf(conn,
"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"OK\r\n",
4);
}
break;
}
n--;
}
if(n==0) {
LOG_ERROR(vlevel,_("Malformed request\n"));
mg_printf(conn,
"HTTP/1.1 500 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"MALFORMED\r\n",
11);
}
} else if(strncmp(req, "/get/", 5) == 0) {
size_t rlen=-1;
char *tmp=leveldb_get(dbh, ropt, req+5, strlen(req)-5, &rlen, &errptr);
if(rlen) {
char *val=calloc(rlen+2,sizeof(char));
snprintf(val,rlen+1,"%s",tmp);
LOG_DEBUG(vlevel, _("Found: %s for %s\n"),val,req+5);
mg_printf(conn,
"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s\r\n",
strlen(val)+2, val);
free(val);
} else {
LOG_DEBUG(vlevel, _("Nothing found for %s\n"),req+5);
mg_printf(conn,
"HTTP/1.1 500 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"NOTFOUND\r\n",
10);
}
free(tmp);
} else if(strncmp(req, "/pset/", 6) == 0) {
// XXX
} else if(strncmp(req, "/pget/", 6) == 0) {
// XXX
} else {
// XXX
LOG_ERROR(vlevel,_("Unknown/unhandled request\n"));
}
free(req);
return "";
} else {
return NULL;
}
}
int main(int argc, char** argv) {
leveldb_t* db;
leveldb_comparator_t* cmp;
leveldb_cache_t* cache;
leveldb_env_t* env;
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
char* err = NULL;
int run = -1;
CheckCondition(leveldb_major_version() >= 1);
CheckCondition(leveldb_minor_version() >= 1);
snprintf(dbname, sizeof(dbname),
"%s/leveldb_c_test-%di/meta/0",
GetTempDir(),
((int) geteuid()));
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1);
leveldb_options_set_cache(options, cache);
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL);
leveldb_options_set_write_buffer_size(options, 100000);
leveldb_options_set_paranoid_checks(options, 1);
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 1);
leveldb_readoptions_set_fill_cache(roptions, 0);
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, 1);
StartPhase("destroy");
leveldb_destroy_db(options, dbname, &err);
Free(&err);
StartPhase("open");
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
StartPhase("put");
leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
StartPhase("compactall");
leveldb_compact_range(db, NULL, 0, NULL, 0);
CheckGet(db, roptions, "foo", "hello");
StartPhase("compactrange");
leveldb_compact_range(db, "a", 1, "z", 1);
CheckGet(db, roptions, "foo", "hello");
StartPhase("writebatch");
{
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
int pos = 0;
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_destroy(wb);
}
StartPhase("iter");
{
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
StartPhase("approximate_sizes");
{
int i;
int n = 20000;
char keybuf[100];
char valbuf[100];
uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" };
size_t start_len[2] = { 1, 21 };
const char* limit[2] = { "k00000000000000010000", "z" };
size_t limit_len[2] = { 21, 1 };
leveldb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
snprintf(valbuf, sizeof(valbuf), "v%020d", i);
leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
&err);
CheckNoError(err);
}
leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
CheckCondition(sizes[0] > 0);
CheckCondition(sizes[1] > 0);
}
StartPhase("property");
{
char* prop = leveldb_property_value(db, "nosuchprop");
/* CheckCondition(prop == NULL);*/
prop = leveldb_property_value(db, "leveldb.stats");
CheckCondition(prop != NULL);
Free(&prop);
}
StartPhase("snapshot");
{
const leveldb_snapshot_t* snap;
snap = leveldb_create_snapshot(db);
leveldb_delete(db, woptions, "foo", 3, &err);
CheckNoError(err);
leveldb_readoptions_set_snapshot(roptions, snap);
CheckGet(db, roptions, "foo", "hello");
leveldb_readoptions_set_snapshot(roptions, NULL);
CheckGet(db, roptions, "foo", NULL);
leveldb_release_snapshot(db, snap);
}
StartPhase("repair");
{
leveldb_close(db);
leveldb_options_set_error_if_exists(options, 0);
leveldb_repair_db(options, dbname, &err);
CheckNoError(err);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
leveldb_options_set_error_if_exists(options, 1);
}
StartPhase("filter");
for (run = 0; run < 2; run++) {
// First run uses custom filter, second run uses bloom filter
CheckNoError(err);
leveldb_filterpolicy_t* policy;
if (run == 0) {
policy = leveldb_filterpolicy_create(
NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
} else {
policy = leveldb_filterpolicy_create_bloom(10);
}
// Create new database
leveldb_close(db);
leveldb_destroy_db(options, dbname, &err);
leveldb_options_set_filter_policy(options, policy);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
CheckNoError(err);
leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
CheckNoError(err);
leveldb_compact_range(db, NULL, 0, NULL, 0);
fake_filter_result = 1;
CheckGet(db, roptions, "foo", "foovalue");
CheckGet(db, roptions, "bar", "barvalue");
if (phase == 0) {
// Must not find value when custom filter returns false
fake_filter_result = 0;
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
fake_filter_result = 1;
CheckGet(db, roptions, "foo", "foovalue");
CheckGet(db, roptions, "bar", "barvalue");
}
leveldb_options_set_filter_policy(options, NULL);
leveldb_filterpolicy_destroy(policy);
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
return 0;
}
static int leveldb_backend_write(struct leveldb_backend *s, void *state, struct dnet_cmd *cmd, void *data)
{
struct dnet_ext_list elist;
int err = -EINVAL;
char *error_string = NULL;
struct dnet_io_attr *io = data;
void *read_data = NULL;
dnet_ext_list_init(&elist);
dnet_convert_io_attr(io);
data += sizeof(struct dnet_io_attr);
/* Combine data with empty extension list header */
err = dnet_ext_list_combine(&data, &io->size, &elist);
if (err != 0)
goto err_out_exit;
/*
* key should be locked by elliptics here, so it is safe to run read-modify-write cycle
* if one performs write without lock we do not really care that one write may overwrite another one
*/
if (io->offset || (io->flags & DNET_IO_FLAGS_APPEND)) {
size_t data_size;
size_t offset = io->offset;
read_data = leveldb_get(s->db, s->roptions, (const char *)io->id, DNET_ID_SIZE, &data_size, &error_string);
if (error_string || !read_data) {
free(error_string);
error_string = NULL;
goto plain_write;
}
if (io->flags & DNET_IO_FLAGS_APPEND) {
io->offset = 0;
offset = data_size;
}
/*
* XXX: Account for extended header
*/
if (io->offset > data_size) {
err = -ERANGE;
goto err_out_exit;
}
if (offset + io->size > data_size) {
read_data = realloc(read_data, data_size + io->size);
if (!read_data) {
err = -ENOMEM;
goto err_out_exit;
}
}
memcpy(read_data + offset, data, io->size);
data = read_data;
if (offset + io->size > data_size)
io->size = offset + io->size;
else
io->size = data_size;
}
plain_write:
leveldb_put(s->db, s->woptions, (const char *)cmd->id.id, DNET_ID_SIZE, data, io->size, &error_string);
if (error_string)
goto err_out_free;
if (io->flags & DNET_IO_FLAGS_WRITE_NO_FILE_INFO) {
cmd->flags |= DNET_FLAGS_NEED_ACK;
err = 0;
goto err_out_exit;
}
err = dnet_send_file_info_ts_without_fd(state, cmd, 0, io->size, &elist.timestamp);
if (err < 0)
goto err_out_free;
dnet_backend_log(DNET_LOG_NOTICE, "%s: leveldb: : WRITE: Ok: offset: %llu, size: %llu, ioflags: %x.\n",
dnet_dump_id(&cmd->id), (unsigned long long)io->offset, (unsigned long long)io->size, io->flags);
err_out_free:
free(data);
err_out_exit:
dnet_ext_list_destroy(&elist);
if (err < 0)
dnet_backend_log(DNET_LOG_ERROR, "%s: leveldb: : WRITE: error: %s: %d.\n",
dnet_dump_id(&cmd->id), error_string, err);
free(read_data);
free(error_string);
return err;
}
// Adds an event to the tablet.
//
// tablet - The tablet.
// event - The event to add.
//
// Returns 0 if successful, otherwise returns -1.
int sky_tablet_add_event(sky_tablet *tablet, sky_event *event)
{
int rc;
char *errptr = NULL;
void *new_data = NULL;
void *data = NULL;
sky_data_object *data_object = NULL;
sky_data_descriptor *descriptor = NULL;
sky_cursor cursor; memset(&cursor, 0, sizeof(cursor));
assert(tablet != NULL);
assert(event != NULL);
// Make sure that this event is being added to the correct tablet.
sky_tablet *target_tablet = NULL;
rc = sky_table_get_target_tablet(tablet->table, event->object_id, &target_tablet);
check(rc == 0, "Unable to determine target tablet");
check(tablet == target_tablet, "Event added to invalid tablet; IDX:%d of %d, OID:%d", tablet->index, tablet->table->tablet_count, event->object_id);
// Retrieve the existing value.
size_t data_length;
data = (void*)leveldb_get(tablet->leveldb_db, tablet->readoptions, (const char*)&event->object_id, sizeof(event->object_id), &data_length, &errptr);
check(errptr == NULL, "LevelDB get error: %s", errptr);
// Find the insertion point on the path.
size_t insert_offset = 0;
size_t event_length;
// If the object doesn't exist yet then just set the single event. Easy peasy.
if(data == NULL) {
event_length = sky_event_sizeof(event);
new_data = calloc(1, event_length); check_mem(new_data);
insert_offset = 0;
}
// If the object does exist, we need to find where to insert the event data.
// Also, we need to strip off any state which is redundant at the point of
// insertion.
else {
void *insert_ptr = NULL;
// Initialize data descriptor.
descriptor = sky_data_descriptor_create(); check_mem(descriptor);
rc = sky_data_descriptor_init_with_event(descriptor, event);
check(rc == 0, "Unable to initialize data descriptor for event insert");
// Initialize data object.
data_object = calloc(1, descriptor->data_sz); check_mem(data);
// Attach data & descriptor to the cursor.
cursor.data_descriptor = descriptor;
cursor.data = (void*)data_object;
// Initialize the cursor.
rc = sky_cursor_set_ptr(&cursor, data, data_length);
check(rc == 0, "Unable to set pointer on cursor");
// Loop over cursor until we reach the event insertion point.
while(!cursor.eof) {
// Retrieve event insertion pointer once the timestamp is reached.
if(data_object->ts >= event->timestamp) {
insert_ptr = cursor.ptr;
break;
}
// Move to next event.
check(sky_cursor_next_event(&cursor) == 0, "Unable to move to next event");
}
// If no insertion point was found then append the event to the
// end of the path.
if(insert_ptr == NULL) {
insert_ptr = data + data_length;
}
insert_offset = insert_ptr - data;
// Clear off any object data on the event that matches
// what is the current state of the event in the database.
uint32_t i;
for(i=0; i<event->data_count; i++) {
// Ignore any action properties.
if(event->data[i]->key > 0) {
sky_data_property_descriptor *property_descriptor = &descriptor->property_zero_descriptor[event->data[i]->key];
// If the values match then splice this from the array.
// Compare strings.
void *a = &event->data[i]->value;
void *b = ((void*)data_object)+property_descriptor->offset;
size_t n = sky_data_type_sizeof(event->data[i]->data_type);
bool is_equal = false;
if(event->data[i]->data_type == SKY_DATA_TYPE_STRING) {
is_equal = sky_string_bequals((sky_string*)b, event->data[i]->string_value);
}
// Compare other types.
else if(memcmp(a, b, n) == 0) {
is_equal = true;
}
// If the data is equal then remove it.
if(is_equal) {
sky_event_data_free(event->data[i]);
if(i < event->data_count - 1) {
memmove(&event->data[i], &event->data[i+1], (event->data_count-i-1) * sizeof(*event->data));
}
i--;
event->data_count--;
}
}
}
// Determine the serialized size of the event. If the event is
// completely redundant (e.g. it is a data-only event and the event
// matches the current object state) then don't allocate space for a
// new path value.
event_length = sky_event_sizeof(event);
if(event_length > 0) {
// Allocate space for the existing data plus the new data.
new_data = calloc(1, data_length + event_length); check_mem(new_data);
// Copy in data before event.
if(insert_offset > 0) {
memmove(new_data, data, insert_ptr-data);
}
// Copy in data after event.
if(insert_offset < data_length) {
event_length = sky_event_sizeof(event);
memmove(new_data+insert_offset+event_length, data+insert_offset, data_length-insert_offset);
}
}
}
// If no space was allocated then it means the event is redundant and
// should be ignored.
if(new_data != NULL) {
// If the object doesn't exist then just set the event as the data.
size_t event_sz;
rc = sky_event_pack(event, new_data + insert_offset, &event_sz);
check(rc == 0, "Unable to pack event");
check(event_sz == event_length, "Expected event size (%ld) does not match actual event size (%ld)", event_length, event_sz);
leveldb_put(tablet->leveldb_db, tablet->writeoptions, (const char*)&event->object_id, sizeof(event->object_id), new_data, data_length + event_length, &errptr);
check(errptr == NULL, "LevelDB put error: %s", errptr);
}
free(data_object);
sky_data_descriptor_free(descriptor);
free(data);
free(new_data);
return 0;
error:
if(errptr) leveldb_free(errptr);
sky_data_descriptor_free(descriptor);
if(data) free(data);
if(new_data) free(new_data);
if(data_object) free(data_object);
return -1;
}
int
main ()
{
unsigned int N_KEYS;
unsigned int N_BATCH;
printf("\n number of keys:");
scanf("%u",&N_KEYS);
printf("\n batch size:");
scanf("%u",&N_BATCH);
tinymt32_t tinymt32;
//initialize random generator
tinymt32_init (&tinymt32, 0);
int iter;
int sec_iter;
int stop;
int counter;
//initialize database
char *errptr = NULL;
leveldb_options_t *options = leveldb_options_create ();
/* initializeOptions */
leveldb_options_set_create_if_missing (options, 1);
leveldb_options_set_write_buffer_size(options,120000000 );
leveldb_options_set_max_open_files(options,800000);
/*open Database */
leveldb_t *db_helper =
leveldb_open (options, "/mnt/database/database_helper", &errptr);
leveldb_t *db = leveldb_open (options, "/mnt/database/database", &errptr);
leveldb_readoptions_t *readoptions = leveldb_readoptions_create ();
leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create ();
leveldb_writebatch_t* batch=leveldb_writebatch_create();
int64_t diff = zclock_time ();
//write into database_helper
iter=0;
while( iter < N_KEYS)
{
sec_iter=0;
while((iter<N_KEYS) && (sec_iter<N_BATCH)){
unsigned int key = tinymt32_generate_uint32 (&tinymt32);
unsigned int val = tinymt32_generate_uint32 (&tinymt32);
leveldb_writebatch_put (batch,
(const char *) &key, sizeof (int),
(const char *) &val, sizeof (int));
sec_iter++;
iter++;
}
leveldb_write(
db_helper,
writeoptions,
batch,
&errptr);
if(errptr!=NULL){
printf("\n%s",errptr);
}
assert (errptr == NULL);
leveldb_writebatch_clear(batch);
}
leveldb_writebatch_destroy(batch);
diff = zclock_time () - diff;
printf ("\nrandom write: %d", diff);
diff = zclock_time ();
//write sequentially into db
leveldb_iterator_t *liter = leveldb_create_iterator (db_helper,
readoptions);
leveldb_iter_seek_to_first (liter);
while (leveldb_iter_valid (liter))
{
size_t length;
char key[4];
memcpy (key, leveldb_iter_key (liter, &length), 4);
char val[4];
memcpy (val, leveldb_iter_value (liter, &length), 4);
leveldb_iter_get_error (liter, &errptr);
assert (errptr == NULL);
leveldb_put (db,
writeoptions,
(const char *) &key, sizeof (int),
(const char *) &val, sizeof (int), &errptr);
assert (errptr == NULL);
leveldb_iter_next (liter);
}
diff = zclock_time () - diff;
leveldb_close (db);
leveldb_close (db_helper);
}
int main(int argc, char** argv) {
leveldb_t* db;
leveldb_comparator_t* cmp;
leveldb_cache_t* cache;
leveldb_env_t* env;
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
char* err = NULL;
#if defined(LEVELDB_PLATFORM_WINDOWS)
snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test");
#else
snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d",
((int) geteuid()));
#endif
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
leveldb_options_set_error_if_exists(options, 1);
leveldb_options_set_cache(options, cache);
leveldb_options_set_env(options, env);
leveldb_options_set_info_log(options, NULL);
leveldb_options_set_write_buffer_size(options, 100000);
leveldb_options_set_paranoid_checks(options, 1);
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
leveldb_readoptions_set_verify_checksums(roptions, 1);
leveldb_readoptions_set_fill_cache(roptions, 0);
woptions = leveldb_writeoptions_create();
leveldb_writeoptions_set_sync(woptions, 1);
StartPhase("destroy");
leveldb_destroy_db(options, dbname, &err);
leveldb_free(&err);
StartPhase("open_error");
db = leveldb_open(options, dbname, &err);
CheckCondition(err != NULL);
leveldb_free(&err);
StartPhase("open");
leveldb_options_set_create_if_missing(options, 1);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
StartPhase("put");
leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
StartPhase("writebatch");
{
int pos = 0;
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_destroy(wb);
}
StartPhase("iter");
{
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
StartPhase("approximate_sizes");
{
int i;
int n = 20000;
char keybuf[100];
char valbuf[100];
uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" };
size_t start_len[2] = { 1, 21 };
const char* limit[2] = { "k00000000000000010000", "z" };
size_t limit_len[2] = { 21, 1 };
leveldb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i);
snprintf(valbuf, sizeof(valbuf), "v%020d", i);
leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
&err);
CheckNoError(err);
}
leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
CheckCondition(sizes[0] > 0);
CheckCondition(sizes[1] > 0);
}
StartPhase("property");
{
char* prop = leveldb_property_value(db, "nosuchprop");
CheckCondition(prop == NULL);
prop = leveldb_property_value(db, "leveldb.stats");
CheckCondition(prop != NULL);
leveldb_free(&prop);
}
StartPhase("snapshot");
{
const leveldb_snapshot_t* snap;
snap = leveldb_create_snapshot(db);
leveldb_delete(db, woptions, "foo", 3, &err);
CheckNoError(err);
leveldb_readoptions_set_snapshot(roptions, snap);
CheckGet(db, roptions, "foo", "hello");
leveldb_readoptions_set_snapshot(roptions, NULL);
CheckGet(db, roptions, "foo", NULL);
leveldb_release_snapshot(db, snap);
}
StartPhase("repair");
{
leveldb_close(db);
leveldb_options_set_create_if_missing(options, 0);
leveldb_options_set_error_if_exists(options, 0);
leveldb_repair_db(options, dbname, &err);
CheckNoError(err);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", NULL);
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
return 0;
}
int read_csv()
{
leveldb_writeoptions_t *write_options;
char *input_filename = option_get_str("input_file");
const char *key_ptr, *val_ptr;
size_t key_len, val_len;
uint64_t input_row = 0;
int stats_every = option_get_int("stats_every");
char *err = NULL;
char *buffer = NULL;
char *input_deliminator = option_get_str("input_deliminator");
int quiet = option_get_int("quiet");
const char *comma_ptr;
size_t buffer_len;
FILE *input_file = stdin;
if (input_filename) {
input_file = fopen(input_filename, "r");
}
write_options = leveldb_writeoptions_create();
while ((buffer_len = simplehttp_get_line(input_file, &buffer))) {
input_row++;
key_ptr = buffer;
if (!quiet && (input_row > 0) && ((input_row % stats_every) == 0)) {
fprintf(stderr, "line #%"PRIu64"\r", input_row);
}
// find the index of the first deliminator
if ((comma_ptr = simplehttp_strnchr(buffer, buffer_len, *input_deliminator)) == NULL) {
fprintf(stderr, "%s ERROR: LINE %"PRIu64" DELIMINATOR NOT FOUND %s\n", NAME, input_row, buffer);
return 0;
}
key_len = comma_ptr - buffer;
if (!key_len) {
fprintf(stderr, "%s WARNING: SKIPPING EMPTY KEY, LINE %"PRIu64" %s\n", NAME, input_row, buffer);
continue;
}
val_ptr = comma_ptr;
val_ptr += strlen(input_deliminator);
val_len = (buffer + buffer_len) - val_ptr;
if (*(val_ptr + val_len - 1) == '\n') {
val_len--;
}
leveldb_put(ldb, write_options, key_ptr, key_len, val_ptr, val_len, &err);
if (err) {
break;
}
}
leveldb_writeoptions_destroy(write_options);
if (err) {
fprintf(stderr, "Error (LINE %"PRIu64"): %s\n", input_row, err);
return 0;
}
return 1;
}
void db_put(db_t *db, const db_writeoptions_t *wo,
const char *key, size_t klen, const char *value, size_t vlen, char **errptr) {
leveldb_put(db, wo, key, klen, value, vlen, errptr);
}
void db_leveldb_put(dbi_t *key, dbi_t *val)
{
leveldb_put(leveldb_db, leveldb_woptions, key->data, key->len, val->data, val->len, NULL);
}
