char * store_get(leveldb_t * store, char * key,size_t key_size, size_t *read_len){
char * err = 0x00;
leveldb_readoptions_t *read_options=leveldb_readoptions_create();
char * resp = leveldb_get(store,read_options, key, key_size, read_len, &err);
leveldb_readoptions_destroy(read_options);
if(err){
fprintf(stderr, "error: %s\n", err);
free(err);
}
return resp;
}
const char *
db_get(db_t *db, const char *key, size_t klen,
size_t *vlen, char **errptr) {
const char *val;
leveldb_readoptions_t *opts;
opts = leveldb_readoptions_create();
val = leveldb_get(db->db, opts, key, klen, vlen, errptr);
leveldb_readoptions_destroy(opts);
return val;
}
int leveldb_client_get(leveldb *db, const char *key, char **value,
bool add_null_zero)
{
char *err = NULL;
size_t keylen, vallen;
char *re_value;
if (!db || !key) {
kp_error("got a NULL argument\n");
return -1;
}
keylen = strlen(key);
if (keylen < 1) {
kp_error("got empty string for key (%zu)\n", keylen);
}
kp_debug("calling leveldb_get() with key=%s, keylen=%zu\n", key, keylen);
*value = leveldb_get(db->db, db->roptions, key, keylen, &vallen, &err);
if (err) {
kp_error("leveldb_get() returned error: %s\n", err);
free_err(&err);
return -1;
}
/* leveldb_get() will return NULL if the key is not found. */
if (*value == NULL) {
kp_debug("did not find key=%s in db; returning 1\n", key);
return 1;
}
/* If we got a value back from the db, then it is not null-zero-
* terminated. If the caller asks us to, we can allocate another
* string that is null-terminated.
*/
kp_debug("get(%s) returned vallen=%zu, not-null-terminated-value=[%s]\n",
key, vallen, *value);
if (add_null_zero) {
re_value = (char *)malloc(vallen + 1);
strncpy(re_value, *value, vallen);
re_value[vallen] = '\0';
free(*value);
*value = re_value;
kp_debug("created null-terminated value=%s; vallen=%zu, strlen=%zu\n",
*value, vallen, strlen(*value));
} else {
kp_debug("add_null_zero is false, so value returned will not end "
"in a null-zero\n");
}
kp_debug("got a value, returning 0\n");
return 0;
}
static void CheckGet(
leveldb_t* db,
const leveldb_readoptions_t* options,
const char* key,
const char* expected) {
char* err = NULL;
size_t val_len;
char* val;
val = leveldb_get(db, options, key, strlen(key), &val_len, &err);
CheckNoError(err);
CheckEqual(expected, val, val_len);
Free(&val);
}
int ldb_exists(struct _leveldb_stuff *ldbs, const char *key, size_t klen)
{
char *err = NULL;
char *val = NULL;
size_t vlen = 0;
val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, &vlen, &err);
if (err) {
fprintf(stderr, "\n%s\n", err);
leveldb_free(err);
return -1;
}
leveldb_free(val);
return vlen == 0 ? 0 : 1;
}
static int leveldb_backend_read(struct leveldb_backend *s, void *state, struct dnet_cmd *cmd, void *iodata, int last)
{
struct dnet_io_attr *io = iodata;
struct dnet_ext_list elist;
char *data;
size_t data_size;
int err = -EINVAL;
char *error_string = NULL;
dnet_ext_list_init(&elist);
dnet_convert_io_attr(io);
if (io->size || io->offset) {
err = -ERANGE;
goto err_out_exit;
}
data = leveldb_get(s->db, s->roptions, (const char *)io->id, DNET_ID_SIZE, &data_size, &error_string);
if (error_string || !data) {
if (!data)
err = -ENOENT;
goto err_out_exit;
}
/* Extract original data and extension list from &data */
err = dnet_ext_list_extract((void *)&data, (uint64_t *)&data_size,
&elist, DNET_EXT_FREE_ON_DESTROY);
if (err != 0)
goto err_out_exit;
dnet_ext_list_to_io(&elist, io);
io->size = data_size;
if (data_size && data && last)
cmd->flags &= ~DNET_FLAGS_NEED_ACK;
err = dnet_send_read_data(state, cmd, io, data, -1, io->offset, 0);
if (err < 0)
goto err_out_exit;
dnet_backend_log(DNET_LOG_NOTICE, "%s: leveldb: : READ: Ok: size: %llu.\n",
dnet_dump_id(&cmd->id), (unsigned long long)io->size);
err_out_exit:
dnet_ext_list_destroy(&elist);
if (err < 0)
dnet_backend_log(DNET_LOG_ERROR, "%s: leveldb: READ: error: %s: %d\n",
dnet_dump_id(&cmd->id), error_string, err);
free(error_string);
return err;
}
char *ldb_get(struct _leveldb_stuff *ldbs, const char *key, size_t klen, int *vlen)
{
char *err = NULL;
char *val = NULL;
val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, (size_t *) vlen, &err);
if (err) {
fprintf(stderr, "%s\n", err);
leveldb_free(err);
err = NULL;
*vlen = -1;
return NULL ;
} else {
return val;
}
}
// Retrieves a path for an object in the tablet.
//
// tablet - The tablet.
// object_id - The object identifier for the path.
// data - A pointer to where the path data should be returned.
// data_length - A pointer to where the length of the path data should be returned.
//
//
// Returns 0 if successful, otherwise returns -1.
int sky_tablet_get_path(sky_tablet *tablet, sky_object_id_t object_id,
void **data, size_t *data_length)
{
char *errptr = NULL;
assert(tablet != NULL);
// Retrieve the existing value.
*data = (void*)leveldb_get(tablet->leveldb_db, tablet->readoptions, (const char*)&object_id, sizeof(object_id), data_length, &errptr);
check(errptr == NULL, "LevelDB get path error: %s", errptr);
return 0;
error:
*data = NULL;
*data_length = 0;
return -1;
}
int ldb_exists(struct _leveldb_stuff *ldbs, const char *key, size_t klen)
{
char *err = NULL;
char *val = NULL;
size_t vlen = 0;
val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, (size_t *) &vlen, &err);
if (err) {
log_error("%s", err);
leveldb_free(err);
err = NULL;
return -1;
}
leveldb_free(val);
return vlen ? 1 : 0;
}
void get(db_t* db, size_t keySz, void* key, size_t valSz, void** val)
{
if(db->type == LEVELDB)
{
levelDB_t* leveldb = (levelDB_t*) db->database;
*((char**)val) = leveldb_get(leveldb->db, leveldb->roptions, (char*)key, keySz, &valSz, &(leveldb->err));
if(leveldb->err != NULL)
{
fprintf(stderr, "leveldb get failed!\n");
exit(-1);
}
leveldb_free(leveldb->err);
leveldb->err = NULL;
}
else
{
kernelDB_t* kerneldb = (kernelDB_t*) db->database;
kr_get(kerneldb->client, keySz, key, &valSz, val);
}
}
void incrDecrCommand(redisClient *c, long long incr) {
sds data;
char *value;
int64_t val, recore;
size_t val_len;
char *err = NULL;
err = NULL;
val_len = 0;
value = leveldb_get(server.ds_db, server.roptions, c->argv[1]->ptr, sdslen((sds) c->argv[1]->ptr), &val_len, &err);
if (err != NULL) {
if (val_len > 0) leveldb_free(value);
addReplyError(c, err);
leveldb_free(err);
return;
} else if (val_len < 1) {
val = 0;
} else {
val = strtoll(value, NULL, 10);
}
err = NULL;
recore = val + incr;
data = sdsfromlonglong(recore);
leveldb_put(server.ds_db, server.woptions, c->argv[1]->ptr, sdslen((sds) c->argv[1]->ptr), data, sdslen(data), &err);
if (err != NULL) {
addReplyError(c, err);
leveldb_free(err);
} else {
addReplyLongLong(c, recore);
server.dirty++;
signalModifiedKey(c->db, c->argv[1]);
}
sdsfree(data);
leveldb_free(value);
return;
}
static int leveldb_backend_lookup(struct leveldb_backend *s, void *state, struct dnet_cmd *cmd)
{
char *data = NULL;
size_t data_size;
int err = -EINVAL;
char *error_string = NULL;
struct dnet_ext_list elist;
static const size_t ehdr_size = sizeof(struct dnet_ext_list_hdr);
struct dnet_ext_list_hdr *ehdr = NULL;
dnet_ext_list_init(&elist);
data = leveldb_get(s->db, s->roptions, (const char *)cmd->id.id, DNET_ID_SIZE, &data_size, &error_string);
if (error_string || !data) {
if (!data)
err = -ENOENT;
goto err_out_exit;
}
if (data_size < ehdr_size) {
err = -ERANGE;
goto err_out_exit;
}
ehdr = (struct dnet_ext_list_hdr *)data;
dnet_ext_hdr_to_list(ehdr, &elist);
data_size -= ehdr_size;
data += ehdr_size;
err = dnet_send_file_info_ts_without_fd(state, cmd, data, data_size, &elist.timestamp);
if (err < 0)
goto err_out_free;
err_out_free:
free(data);
err_out_exit:
free(error_string);
dnet_ext_list_destroy(&elist);
return err;
}
int zset_get(ldb_context_t* context, const ldb_slice_t* name,
const ldb_slice_t* key, int64_t* score){
char *val, *errptr = NULL;
size_t vallen = 0;
leveldb_readoptions_t* readoptions = leveldb_readoptions_create();
ldb_slice_t *slice_key = NULL;
encode_zset_key(ldb_slice_data(name), ldb_slice_size(name), ldb_slice_data(key), ldb_slice_size(key), NULL, &slice_key);
val = leveldb_get(context->database_, readoptions, ldb_slice_data(slice_key), ldb_slice_size(slice_key), &vallen, &errptr);
leveldb_readoptions_destroy(readoptions);
ldb_slice_destroy(slice_key);
int retval = 0;
if(errptr != NULL){
fprintf(stderr, "leveldb_get fail %s.\n", errptr);
leveldb_free(errptr);
retval = LDB_ERR;
goto end;
}
if(val != NULL){
assert(vallen >= LDB_VAL_META_SIZE);
uint8_t type = leveldb_decode_fixed8(val);
if(type & LDB_VALUE_TYPE_VAL){
if(type & LDB_VALUE_TYPE_LAT){
retval = LDB_OK_NOT_EXIST;
goto end;
}
*score = leveldb_decode_fixed64(val + LDB_VAL_META_SIZE);
retval = LDB_OK;
}else{
retval = LDB_OK_NOT_EXIST;
}
}else{
retval = LDB_OK_NOT_EXIST;
}
end:
if(val != NULL){
leveldb_free(val);
}
return retval;
}
/**
* set if not exists.
* return 1 (not exists and saved)
* return 0 (already exists)
*/
void setnxCommand(redisClient *c) {
char *value;
sds keyword;
size_t val_len;
char *err = NULL;
leveldb_writebatch_t *wb;
err = NULL;
val_len = 0;
keyword = c->argv[1]->ptr;
value = leveldb_get(server.ds_db, server.roptions, keyword, sdslen(keyword), &val_len, &err);
if (err != NULL) {
addReplyError(c, err);
leveldb_free(err);
return;
} else if (val_len > 0) {
// already exists
leveldb_free(value);
addReplyLongLong(c, 0);
return;
}
wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, keyword, sdslen(keyword), (char *) c->argv[2]->ptr, sdslen((sds) c->argv[2]->ptr));
leveldb_write(server.ds_db, server.woptions, wb, &err);
if (err != NULL) {
addReplyError(c, err);
leveldb_free(err);
} else {
addReplyLongLong(c, 1);
server.dirty++;
signalModifiedKey(c->db, c->argv[1]);
}
leveldb_writebatch_destroy(wb);
return;
}
int ldb_pull(struct _leveldb_stuff *ldbs, const char *key, size_t klen, char *value, size_t vlen)
{
size_t len = 0;
char *err = NULL;
char *val = NULL;
val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, &len, &err);
if (err) {
fprintf(stderr, "%s\n", err);
leveldb_free(err);
err = NULL;
return -1;
} else {
if (!val){
return -1;
}
int ok = (vlen == len) ? 0 : -1;
if (!ok){
memcpy(value, val, vlen);
}
leveldb_free(val);
return ok;
}
}
void existsCommand(redisClient *c) {
char *err;
size_t val_len;
char *key = NULL;
char *value = NULL;
err = NULL;
key = (char *) c->argv[1]->ptr;
value = leveldb_get(server.ds_db, server.roptions,
key, sdslen((sds) key), &val_len, &err);
if (err != NULL) {
addReplyError(c, err);
leveldb_free(err);
return;
} else if (value == NULL) {
addReply(c, shared.czero);
return;
}
addReply(c, shared.cone);
leveldb_free(value);
return;
}
int kvdb_leveldb_get(kvdb_t *kvdb, void *key, uint32_t klen, void **ppVal, uint32_t *pnVal)
{
kvdb_leveldb_t *leveldb = (kvdb_leveldb_t*)kvdb;
char *szErr = NULL;
size_t nVal = 0;
char *pVal = leveldb_get(leveldb->db, leveldb->pReadOpt, (const char *)key, klen, &nVal, &szErr);
if( pVal == NULL ){
*pnVal = 0;
return -1;
}else{
*pnVal = nVal;
}
if ( szErr ) {
return -1;
} else {
char *result = (char*)zmalloc(nVal);
memcpy(result, pVal, nVal);
*ppVal = result;
return 0;
}
}
void
worker_fn (void *arg, zctx_t * ctx, void *pipe)
{
int rc;
worker_args_t *args = (worker_args_t *) arg;
//connect
void *pull = zsocket_new (ctx, ZMQ_PULL);
rc = zsocket_connect (pull, "inproc://bind_point");
//connect
void *sub = zsocket_new (ctx, ZMQ_SUB);
rc = zsocket_connect (sub, "inproc://pub_point");
//connect
void *dealer = zsocket_new (ctx, ZMQ_DEALER);
rc = zsocket_connect (dealer, "inproc://response_point");
//subscribing to all messages
zsocket_set_subscribe (sub, "");
unsigned int counter = 0;
size_t vallen;
char *errptr;
int stop = 0;
int timeout = 0;
zmq_pollitem_t pollitem[] = {
{pull, 0, ZMQ_POLLIN, 0},
{sub, 0, ZMQ_POLLIN, 0},
};
while (1)
{
rc = zmq_poll (pollitem, 2, timeout);
assert (rc != -1);
if (pollitem[0].revents & ZMQ_POLLIN)
{
unsigned int key;
zframe_t *frame = zframe_recv_nowait (pull);
memcpy (&key, zframe_data (frame), 4);
zframe_destroy (&frame);
free (leveldb_get (*(args->db),
args->readoptions,
(const char *) &key, 4, &vallen, &errptr));
counter++;
}
else
{
if (stop)
{
zframe_t *frame = zframe_new (&counter, 4);
zframe_send (&frame, dealer, 0);
counter = 0;
stop = 0;
}
else
{
//request more
zframe_t *frame = zframe_new ("m", strlen ("m"));
zframe_send (&frame, dealer, 0);
zclock_sleep(1);
}
}
if (pollitem[1].revents & ZMQ_POLLIN)
{
zframe_t *frame = zframe_recv_nowait (sub);
memcpy (&stop, zframe_data (frame), 4);
zframe_destroy (&frame);
}
}
}
static void *mghandle(enum mg_event event, struct mg_connection *conn) {
const struct mg_request_info *request_info = mg_get_request_info(conn);
if (event == MG_NEW_REQUEST) {
char *req=calloc(URL_STRING_MAX+1,sizeof(char));
struct in_addr saddr;
strncpy(req,request_info->uri, URL_STRING_MAX);
saddr.s_addr = ntohl(request_info->remote_ip);
LOG_DEBUG(vlevel, _("Connection from: %s, request: %s\n"), inet_ntoa(saddr), req);
if(strncmp(req, "/status\0", 8) == 0) {
// XXX how to get more status
mg_printf(conn,
"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"OK\r\n",
4);
} else if(strncmp(req, "/set/", 5) == 0) {
int n=strlen(req);
while(n) {
if(req[n]==':') {
leveldb_put(dbh, wopt, req+5, n-5, req+n+1, strlen(req)-n-1, &errptr);
if(errptr!=NULL) {
LOG_ERROR(vlevel,_("leveldb_put(): %s\n"),errptr);
mg_printf(conn,
"HTTP/1.1 500 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"ERROR: %s\r\n",
9+strlen(errptr), errptr);
} else {
mg_printf(conn,
"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"OK\r\n",
4);
}
break;
}
n--;
}
if(n==0) {
LOG_ERROR(vlevel,_("Malformed request\n"));
mg_printf(conn,
"HTTP/1.1 500 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"MALFORMED\r\n",
11);
}
} else if(strncmp(req, "/get/", 5) == 0) {
size_t rlen=-1;
char *tmp=leveldb_get(dbh, ropt, req+5, strlen(req)-5, &rlen, &errptr);
if(rlen) {
char *val=calloc(rlen+2,sizeof(char));
snprintf(val,rlen+1,"%s",tmp);
LOG_DEBUG(vlevel, _("Found: %s for %s\n"),val,req+5);
mg_printf(conn,
"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s\r\n",
strlen(val)+2, val);
free(val);
} else {
LOG_DEBUG(vlevel, _("Nothing found for %s\n"),req+5);
mg_printf(conn,
"HTTP/1.1 500 OK\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: %d\r\n"
"\r\n"
"NOTFOUND\r\n",
10);
}
free(tmp);
} else if(strncmp(req, "/pset/", 6) == 0) {
// XXX
} else if(strncmp(req, "/pget/", 6) == 0) {
// XXX
} else {
// XXX
LOG_ERROR(vlevel,_("Unknown/unhandled request\n"));
}
free(req);
return "";
} else {
return NULL;
}
}
struct stat_cache_value *stat_cache_value_get(stat_cache_t *cache, const char *path, bool skip_freshness_check) {
struct stat_cache_value *value = NULL;
char *key;
leveldb_readoptions_t *options;
size_t vallen;
char *errptr = NULL;
//void *f;
time_t current_time;
key = path2key(path, false);
log_print(LOG_DEBUG, "CGET: %s", key);
options = leveldb_readoptions_create();
value = (struct stat_cache_value *) leveldb_get(cache, options, key, strlen(key) + 1, &vallen, &errptr);
leveldb_readoptions_destroy(options);
free(key);
//log_print(LOG_DEBUG, "Mode: %04o", value->st.st_mode);
if (errptr != NULL) {
log_print(LOG_ERR, "leveldb_get error: %s", errptr);
free(errptr);
return NULL;
}
if (value == NULL) {
log_print(LOG_DEBUG, "stat_cache_value_get miss on path: %s", path);
return NULL;
}
if (vallen != sizeof(struct stat_cache_value)) {
log_print(LOG_ERR, "Length %lu is not expected length %lu.", vallen, sizeof(struct stat_cache_value));
}
if (!skip_freshness_check) {
current_time = time(NULL);
// First, check against the stat item itself.
//log_print(LOG_DEBUG, "Current time: %lu", current_time);
if (current_time - value->updated > CACHE_TIMEOUT) {
char *directory;
time_t directory_updated;
int is_dir;
log_print(LOG_DEBUG, "Stat entry %s is %lu seconds old.", path, current_time - value->updated);
// If that's too old, check the last update of the directory.
directory = strip_trailing_slash(ne_path_parent(path), &is_dir);
directory_updated = stat_cache_read_updated_children(cache, directory);
//log_print(LOG_DEBUG, "Directory contents for %s are %lu seconds old.", directory, (current_time - directory_updated));
free(directory);
if (current_time - directory_updated > CACHE_TIMEOUT) {
log_print(LOG_DEBUG, "%s is too old.", path);
free(value);
return NULL;
}
}
}
/*
if ((f = file_cache_get(path))) {
value->st.st_size = file_cache_get_size(f);
file_cache_unref(cache, f);
}
*/
//print_stat(&value->st, "CGET");
return value;
}
static int leveldb_backend_write(struct leveldb_backend *s, void *state, struct dnet_cmd *cmd, void *data)
{
struct dnet_ext_list elist;
int err = -EINVAL;
char *error_string = NULL;
struct dnet_io_attr *io = data;
void *read_data = NULL;
dnet_ext_list_init(&elist);
dnet_convert_io_attr(io);
data += sizeof(struct dnet_io_attr);
/* Combine data with empty extension list header */
err = dnet_ext_list_combine(&data, &io->size, &elist);
if (err != 0)
goto err_out_exit;
/*
* key should be locked by elliptics here, so it is safe to run read-modify-write cycle
* if one performs write without lock we do not really care that one write may overwrite another one
*/
if (io->offset || (io->flags & DNET_IO_FLAGS_APPEND)) {
size_t data_size;
size_t offset = io->offset;
read_data = leveldb_get(s->db, s->roptions, (const char *)io->id, DNET_ID_SIZE, &data_size, &error_string);
if (error_string || !read_data) {
free(error_string);
error_string = NULL;
goto plain_write;
}
if (io->flags & DNET_IO_FLAGS_APPEND) {
io->offset = 0;
offset = data_size;
}
/*
* XXX: Account for extended header
*/
if (io->offset > data_size) {
err = -ERANGE;
goto err_out_exit;
}
if (offset + io->size > data_size) {
read_data = realloc(read_data, data_size + io->size);
if (!read_data) {
err = -ENOMEM;
goto err_out_exit;
}
}
memcpy(read_data + offset, data, io->size);
data = read_data;
if (offset + io->size > data_size)
io->size = offset + io->size;
else
io->size = data_size;
}
plain_write:
leveldb_put(s->db, s->woptions, (const char *)cmd->id.id, DNET_ID_SIZE, data, io->size, &error_string);
if (error_string)
goto err_out_free;
if (io->flags & DNET_IO_FLAGS_WRITE_NO_FILE_INFO) {
cmd->flags |= DNET_FLAGS_NEED_ACK;
err = 0;
goto err_out_exit;
}
err = dnet_send_file_info_ts_without_fd(state, cmd, 0, io->size, &elist.timestamp);
if (err < 0)
goto err_out_free;
dnet_backend_log(DNET_LOG_NOTICE, "%s: leveldb: : WRITE: Ok: offset: %llu, size: %llu, ioflags: %x.\n",
dnet_dump_id(&cmd->id), (unsigned long long)io->offset, (unsigned long long)io->size, io->flags);
err_out_free:
free(data);
err_out_exit:
dnet_ext_list_destroy(&elist);
if (err < 0)
dnet_backend_log(DNET_LOG_ERROR, "%s: leveldb: : WRITE: error: %s: %d.\n",
dnet_dump_id(&cmd->id), error_string, err);
free(read_data);
free(error_string);
return err;
}
// Adds an event to the tablet.
//
// tablet - The tablet.
// event - The event to add.
//
// Returns 0 if successful, otherwise returns -1.
int sky_tablet_add_event(sky_tablet *tablet, sky_event *event)
{
int rc;
char *errptr = NULL;
void *new_data = NULL;
void *data = NULL;
sky_data_object *data_object = NULL;
sky_data_descriptor *descriptor = NULL;
sky_cursor cursor; memset(&cursor, 0, sizeof(cursor));
assert(tablet != NULL);
assert(event != NULL);
// Make sure that this event is being added to the correct tablet.
sky_tablet *target_tablet = NULL;
rc = sky_table_get_target_tablet(tablet->table, event->object_id, &target_tablet);
check(rc == 0, "Unable to determine target tablet");
check(tablet == target_tablet, "Event added to invalid tablet; IDX:%d of %d, OID:%d", tablet->index, tablet->table->tablet_count, event->object_id);
// Retrieve the existing value.
size_t data_length;
data = (void*)leveldb_get(tablet->leveldb_db, tablet->readoptions, (const char*)&event->object_id, sizeof(event->object_id), &data_length, &errptr);
check(errptr == NULL, "LevelDB get error: %s", errptr);
// Find the insertion point on the path.
size_t insert_offset = 0;
size_t event_length;
// If the object doesn't exist yet then just set the single event. Easy peasy.
if(data == NULL) {
event_length = sky_event_sizeof(event);
new_data = calloc(1, event_length); check_mem(new_data);
insert_offset = 0;
}
// If the object does exist, we need to find where to insert the event data.
// Also, we need to strip off any state which is redundant at the point of
// insertion.
else {
void *insert_ptr = NULL;
// Initialize data descriptor.
descriptor = sky_data_descriptor_create(); check_mem(descriptor);
rc = sky_data_descriptor_init_with_event(descriptor, event);
check(rc == 0, "Unable to initialize data descriptor for event insert");
// Initialize data object.
data_object = calloc(1, descriptor->data_sz); check_mem(data);
// Attach data & descriptor to the cursor.
cursor.data_descriptor = descriptor;
cursor.data = (void*)data_object;
// Initialize the cursor.
rc = sky_cursor_set_ptr(&cursor, data, data_length);
check(rc == 0, "Unable to set pointer on cursor");
// Loop over cursor until we reach the event insertion point.
while(!cursor.eof) {
// Retrieve event insertion pointer once the timestamp is reached.
if(data_object->ts >= event->timestamp) {
insert_ptr = cursor.ptr;
break;
}
// Move to next event.
check(sky_cursor_next_event(&cursor) == 0, "Unable to move to next event");
}
// If no insertion point was found then append the event to the
// end of the path.
if(insert_ptr == NULL) {
insert_ptr = data + data_length;
}
insert_offset = insert_ptr - data;
// Clear off any object data on the event that matches
// what is the current state of the event in the database.
uint32_t i;
for(i=0; i<event->data_count; i++) {
// Ignore any action properties.
if(event->data[i]->key > 0) {
sky_data_property_descriptor *property_descriptor = &descriptor->property_zero_descriptor[event->data[i]->key];
// If the values match then splice this from the array.
// Compare strings.
void *a = &event->data[i]->value;
void *b = ((void*)data_object)+property_descriptor->offset;
size_t n = sky_data_type_sizeof(event->data[i]->data_type);
bool is_equal = false;
if(event->data[i]->data_type == SKY_DATA_TYPE_STRING) {
is_equal = sky_string_bequals((sky_string*)b, event->data[i]->string_value);
}
// Compare other types.
else if(memcmp(a, b, n) == 0) {
is_equal = true;
}
// If the data is equal then remove it.
if(is_equal) {
sky_event_data_free(event->data[i]);
if(i < event->data_count - 1) {
memmove(&event->data[i], &event->data[i+1], (event->data_count-i-1) * sizeof(*event->data));
}
i--;
event->data_count--;
}
}
}
// Determine the serialized size of the event. If the event is
// completely redundant (e.g. it is a data-only event and the event
// matches the current object state) then don't allocate space for a
// new path value.
event_length = sky_event_sizeof(event);
if(event_length > 0) {
// Allocate space for the existing data plus the new data.
new_data = calloc(1, data_length + event_length); check_mem(new_data);
// Copy in data before event.
if(insert_offset > 0) {
memmove(new_data, data, insert_ptr-data);
}
// Copy in data after event.
if(insert_offset < data_length) {
event_length = sky_event_sizeof(event);
memmove(new_data+insert_offset+event_length, data+insert_offset, data_length-insert_offset);
}
}
}
// If no space was allocated then it means the event is redundant and
// should be ignored.
if(new_data != NULL) {
// If the object doesn't exist then just set the event as the data.
size_t event_sz;
rc = sky_event_pack(event, new_data + insert_offset, &event_sz);
check(rc == 0, "Unable to pack event");
check(event_sz == event_length, "Expected event size (%ld) does not match actual event size (%ld)", event_length, event_sz);
leveldb_put(tablet->leveldb_db, tablet->writeoptions, (const char*)&event->object_id, sizeof(event->object_id), new_data, data_length + event_length, &errptr);
check(errptr == NULL, "LevelDB put error: %s", errptr);
}
free(data_object);
sky_data_descriptor_free(descriptor);
free(data);
free(new_data);
return 0;
error:
if(errptr) leveldb_free(errptr);
sky_data_descriptor_free(descriptor);
if(data) free(data);
if(new_data) free(new_data);
if(data_object) free(data_object);
return -1;
}
int
main(int argc, char *argv[]) {
if (argc != 5) {
fprintf(stderr, "Usage: ./leveldb_index [leveldb_directory] [sample_input] [index_output] [index_time_file]\n");
return -1;
}
if ((file = fopen(argv[2], "r")) == NULL) {
fprintf(stderr, "Failed to open sample file \"%s\" for reading\n", argv[1]);
return -1;
}
for (i = 0; fscanf(file, "%"PRId64"\t", &cnt) != EOF; ++i) {
fgets(buffer, BUFFER_SIZE, file);
for (len = strlen(buffer) - 1; len >= 0 && (buffer[len] == '\r' || buffer[len] == '\n'); buffer[len--] = 0);
++len;
strcpy(request[i].key, buffer);
}
fclose(file);
if ((file = fopen(argv[4], "w")) == NULL) {
fprintf(stderr, "Failed to open time information file \"%s\" for reading\n", argv[4]);
return -1;
}
fprintf(stdout, "Start indexing\n");
srand((unsigned)time(NULL));
options = leveldb_options_create();
leveldb_options_set_create_if_missing(options, 1);
// default: write_buffer_size: 4MB, block_cache: 8MB, block_size ~ 4KB
leveldb_options_set_write_buffer_size(options, BLK_SIZE * 1024);
leveldb_options_set_cache(options, cache = leveldb_cache_create_lru(BLK_SIZE * 2048));
leveldb_options_set_block_size(options, BLK_SIZE);
leveldb_options_set_compression(options, 0);
db = leveldb_open(options, argv[1], &err);
if (err != NULL) {
fprintf(stderr, "Failed to open leveldb.\n");
return -1;
}
leveldb_free(err);
roptions = leveldb_readoptions_create();
gettimeofday(&start, NULL);
for (i = seekn = tot = 0; i < DISK_INDEX_SIZE; ++i) {
val = leveldb_get(db, roptions, request[i].key, strlen(request[i].key), &read_len, &err);
if (err != NULL) {
fprintf(stderr, "Failed to read from leveldb.\n");
return -1;
}
leveldb_free(err);
if (read_len == 0)
request[i].value = 0;
else {
val[read_len] = 0;
sscanf(val, "%"PRId64"", &request[i].value);
}
leveldb_free(val);
if ((i + 1) % 1000 == 0) {
gettimeofday(&end, NULL);
tot += (int64_t)(end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec;
fprintf(stdout, "\rIndexed %"PRId64" items in %"PRId64"us", i + 1, tot);
fflush(stdout);
fprintf(file, "Indexed %"PRId64" items in %"PRId64"us\n", i + 1, tot);
fflush(file);
gettimeofday(&start, NULL);
}
}
fprintf(stdout, "\rIndexed %"PRId64" items in %"PRId64"us\n", i, tot);
fprintf(file, "Indexed %"PRId64" items in %"PRId64"us\n", i, tot);
fflush(file);
fclose(file);
leveldb_cache_destroy(cache);
leveldb_readoptions_destroy(roptions);
leveldb_options_destroy(options);
leveldb_close(db);
fprintf(stdout, "Used %"PRId64" us and seeked %"PRId64" times to index %"PRId64" keys\n", tot, seekn, i);
if ((file = fopen(argv[3], "w")) == NULL) {
fprintf(stdout, "Failed to open file \"%s\" for storing indexing results\n", argv[3]);
return -1;
}
for (i = 0; i < DISK_INDEX_SIZE; ++i)
fprintf(file, "%"PRId64"\t%s\n", request[i].value, request[i].key);
fflush(file);
fclose(file);
return 0;
}
dbi_t *db_leveldb_get(dbi_t *key)
{
dbi_t *item = dbi_new();
item->data = leveldb_get(leveldb_db, leveldb_roptions, key->data, key->len, &item->len, &item->err);
return item;
}
//---------------------------------------------------------
char* db_get(db_t *db, const db_readoptions_t *ro, const char *key, size_t klen, size_t *vlen, char **errptr)
{
leveldb_get(db, ro, key, klen, vlen, errptr);
}
