inline int ldb_delete(struct _leveldb_stuff *ldbs, const char *key, size_t klen) { char* err = NULL; char *val = NULL; size_t vlen = 0; val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, (size_t *) &vlen, &err); if (err) { fprintf(stderr, "%s\n", err); leveldb_free(err); err = NULL; return -1; } leveldb_free(val); /* if not found, then return 0. */ if (vlen == 0) { return 0; } /* if found, delete it, then return 1. */ leveldb_delete(ldbs->db, ldbs->woptions, key, klen, &err); if (err) { fprintf(stderr, "%s\n", err); leveldb_free(err); err = NULL; return -1; } return 1; }
void getCommand(redisClient *c) { char *err; size_t val_len; char *key = NULL; char *value = NULL; err = NULL; key = (char *) c->argv[1]->ptr; value = leveldb_get(server.ds_db, server.roptions, key, sdslen((sds) key), &val_len, &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); leveldb_free(value); return; } else if (value == NULL) { addReply(c, shared.nullbulk); return; } addReplyBulkCBuffer(c, value, val_len); leveldb_free(value); return; }
/* ============= * Database info extraction methods * ============= */ void init_nwdb() { char path[PATH_MAX]; char *err = NULL; sprintf(path, "%s/%s", cde_pseudo_pkg_dir, DB_NAME); if (access(path, R_OK)==-1) { fprintf(stderr, "Network provenance database does not exist!\n"); exit(-1); } netdb = malloc(sizeof(lvldb_t)); netdb->options = leveldb_options_create(); leveldb_options_set_create_if_missing(netdb->options, 0); netdb->db = leveldb_open(netdb->options, path, &err); if (err != NULL || netdb == NULL) { fprintf(stderr, "Leveldb open fail!\n"); exit(-1); } assert(netdb->db!=NULL); netdb->woptions = leveldb_writeoptions_create(); netdb->roptions = leveldb_readoptions_create(); /* reset error var */ leveldb_free(err); err = NULL; /* read in root pid */ netdb_root = db_readc(netdb, "meta.root"); assert(netdb_root != NULL); if (CDE_nw_mode) { /* create temp db for current execution graph */ sprintf(path, "%s/%s.tempXXXXXX", cde_pseudo_pkg_dir, DB_NAME); if (mkdtemp(path) == NULL) { fprintf(stderr, "Cannot create temp db!\n"); exit(-1); } currdb = malloc(sizeof(lvldb_t)); currdb->options = leveldb_options_create(); leveldb_options_set_create_if_missing(currdb->options, 1); currdb->db = leveldb_open(currdb->options, path, &err); assert(currdb->db!=NULL); leveldb_free(err); err = NULL; currdb->woptions = leveldb_writeoptions_create(); currdb->roptions = leveldb_readoptions_create(); db_write_root(currdb, getpid()); } }
int ldb_exists(struct _leveldb_stuff *ldbs, const char *key, size_t klen) { char *err = NULL; char *val = NULL; size_t vlen = 0; val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, &vlen, &err); if (err) { fprintf(stderr, "\n%s\n", err); leveldb_free(err); return -1; } leveldb_free(val); return vlen == 0 ? 0 : 1; }
db_t* open_db(const char* path, database_t type) { db_t* db = (db_t*) malloc(sizeof(db_t)); db->type = type; if(type == LEVELDB) { levelDB_t* leveldb = (levelDB_t*) malloc(sizeof(levelDB_t)); leveldb->err = NULL; leveldb->options = leveldb_options_create(); leveldb->woptions = leveldb_writeoptions_create(); leveldb->roptions = leveldb_readoptions_create(); leveldb_options_set_create_if_missing(leveldb->options, 1); leveldb->db = leveldb_open(leveldb->options, path, &(leveldb->err)); if(leveldb->err != NULL) { fprintf(stderr, "Failed to open database! Error message: %s\n", leveldb->err); exit(-1); } leveldb_free(leveldb->err); leveldb->err = NULL; db->database = (void*) leveldb; } else { kernelDB_t* kerneldb = (kernelDB_t*) malloc(sizeof(kernelDB_t)); kerneldb->client = kr_open(path); db->database = (void*) kerneldb; } return db; }
int zset_del(ldb_context_t* context, const ldb_slice_t* name, const ldb_slice_t* key, const ldb_meta_t* meta){ int ret = zdel_one(context, name, key, meta); int retval = LDB_OK; if(ret >= 0){ if(ret > 0){ if(zset_incr_size(context, name, -ret) == -1){ retval = LDB_ERR; goto end; } } char* errptr = NULL; ldb_context_writebatch_commit(context, &errptr); if( errptr != NULL ){ fprintf(stderr, "write writebatch fail %s.\n", errptr); leveldb_free(errptr); retval = LDB_ERR; goto end; } retval = LDB_OK; }else{ retval = LDB_ERR; } end: return retval; }
// Opens the tablet for reading and writing. This should be called directly. // Only the owning table should open a tablet. // // tablet - The tablet. // // Returns 0 if successful, otherwise returns -1. int sky_tablet_open(sky_tablet *tablet) { char* errptr = NULL; leveldb_options_t* options = NULL; assert(tablet != NULL); check(tablet->path != NULL, "Tablet path required"); // Close the tablet if it's already open. sky_tablet_close(tablet); // Initialize data file. options = leveldb_options_create(); leveldb_options_set_create_if_missing(options, true); tablet->leveldb_db = leveldb_open(options, bdata(tablet->path), &errptr); check(errptr == NULL, "LevelDB Error: %s", errptr); check(tablet->leveldb_db != NULL, "Unable to create LevelDB data file"); leveldb_options_destroy(options); return 0; error: if(errptr) leveldb_free(errptr); if(options) leveldb_options_destroy(options); sky_tablet_close(tablet); return -1; }
int ldb_exists(struct _leveldb_stuff *ldbs, const char *key, size_t klen) { char *err = NULL; char *val = NULL; size_t vlen = 0; val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, (size_t *) &vlen, &err); if (err) { log_error("%s", err); leveldb_free(err); err = NULL; return -1; } leveldb_free(val); return vlen ? 1 : 0; }
struct _leveldb_stuff *ldb_initialize(char *path) { struct _leveldb_stuff *ldbs = NULL; leveldb_cache_t *cache; leveldb_filterpolicy_t *policy; char* err = NULL; ldbs = (struct _leveldb_stuff *) malloc(sizeof(struct _leveldb_stuff)); memset(ldbs, 0, sizeof(struct _leveldb_stuff)); ldbs->options = leveldb_options_create(); snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s%s%s", LDB_WORK_PATH, "/", path); cache = leveldb_cache_create_lru(LDB_CACHE_LRU_SIZE); policy = leveldb_filterpolicy_create_bloom(LDB_BLOOM_KEY_SIZE); leveldb_options_set_filter_policy(ldbs->options, policy); leveldb_options_set_cache(ldbs->options, cache); leveldb_options_set_block_size(ldbs->options, LDB_BLOCK_SIZE); leveldb_options_set_write_buffer_size(ldbs->options, LDB_WRITE_BUFFER_SIZE); #if defined(OPEN_COMPRESSION) leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression); #else leveldb_options_set_compression(ldbs->options, leveldb_no_compression); #endif /* Read options */ ldbs->roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1); leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */ /* W options */ ldbs->woptions = leveldb_writeoptions_create(); #ifdef SYNC_PUT leveldb_writeoptions_set_sync(ldbs->woptions, 1); #else leveldb_writeoptions_set_sync(ldbs->woptions, 0); #endif /* Batch write */ ldbs->wbatch = leveldb_writebatch_create(); leveldb_options_set_create_if_missing(ldbs->options, 1); ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err); if (err) { fprintf(stderr, "%s", err); leveldb_free(err); err = NULL; free(ldbs); return NULL; } else { return ldbs; } }
/* * Open the specified database, if no exists, then create it. * Args: * 1. name: db name * 2. block_size: block size * 3. wb_size: write buffer size * 4. lru_size: lru cache size * 5. bloom_size: bloom key size * Return: * _leveldb_stuff: leveldb handler. */ struct _leveldb_stuff *ldb_initialize(const char *name, size_t block_size, size_t wb_size, size_t lru_size, short bloom_size) { struct _leveldb_stuff *ldbs = NULL; leveldb_cache_t *cache; leveldb_filterpolicy_t *policy; char* err = NULL; ldbs = malloc(sizeof(struct _leveldb_stuff)); memset(ldbs, 0, sizeof(struct _leveldb_stuff)); ldbs->options = leveldb_options_create(); snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s", name); cache = leveldb_cache_create_lru(lru_size); policy = leveldb_filterpolicy_create_bloom(bloom_size); leveldb_options_set_filter_policy(ldbs->options, policy); leveldb_options_set_cache(ldbs->options, cache); leveldb_options_set_block_size(ldbs->options, block_size); leveldb_options_set_write_buffer_size(ldbs->options, wb_size); #if defined(OPEN_COMPRESSION) leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression); #else leveldb_options_set_compression(ldbs->options, leveldb_no_compression); #endif /* R */ ldbs->roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1); leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */ /* W */ ldbs->woptions = leveldb_writeoptions_create(); #ifdef SYNC_PUT leveldb_writeoptions_set_sync(ldbs->woptions, 1); #else leveldb_writeoptions_set_sync(ldbs->woptions, 0); #endif /* B */ ldbs->wbatch = leveldb_writebatch_create(); leveldb_options_set_create_if_missing(ldbs->options, 1); ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err); if (err) { fprintf(stderr, "%s", err); leveldb_free(err); err = NULL; free(ldbs); return NULL ; } return ldbs; }
void incrDecrCommand(redisClient *c, long long incr) { sds data; char *value; int64_t val, recore; size_t val_len; char *err = NULL; err = NULL; val_len = 0; value = leveldb_get(server.ds_db, server.roptions, c->argv[1]->ptr, sdslen((sds) c->argv[1]->ptr), &val_len, &err); if (err != NULL) { if (val_len > 0) leveldb_free(value); addReplyError(c, err); leveldb_free(err); return; } else if (val_len < 1) { val = 0; } else { val = strtoll(value, NULL, 10); } err = NULL; recore = val + incr; data = sdsfromlonglong(recore); leveldb_put(server.ds_db, server.woptions, c->argv[1]->ptr, sdslen((sds) c->argv[1]->ptr), data, sdslen(data), &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); } else { addReplyLongLong(c, recore); server.dirty++; signalModifiedKey(c->db, c->argv[1]); } sdsfree(data); leveldb_free(value); return; }
int zset_get(ldb_context_t* context, const ldb_slice_t* name, const ldb_slice_t* key, int64_t* score){ char *val, *errptr = NULL; size_t vallen = 0; leveldb_readoptions_t* readoptions = leveldb_readoptions_create(); ldb_slice_t *slice_key = NULL; encode_zset_key(ldb_slice_data(name), ldb_slice_size(name), ldb_slice_data(key), ldb_slice_size(key), NULL, &slice_key); val = leveldb_get(context->database_, readoptions, ldb_slice_data(slice_key), ldb_slice_size(slice_key), &vallen, &errptr); leveldb_readoptions_destroy(readoptions); ldb_slice_destroy(slice_key); int retval = 0; if(errptr != NULL){ fprintf(stderr, "leveldb_get fail %s.\n", errptr); leveldb_free(errptr); retval = LDB_ERR; goto end; } if(val != NULL){ assert(vallen >= LDB_VAL_META_SIZE); uint8_t type = leveldb_decode_fixed8(val); if(type & LDB_VALUE_TYPE_VAL){ if(type & LDB_VALUE_TYPE_LAT){ retval = LDB_OK_NOT_EXIST; goto end; } *score = leveldb_decode_fixed64(val + LDB_VAL_META_SIZE); retval = LDB_OK; }else{ retval = LDB_OK_NOT_EXIST; } }else{ retval = LDB_OK_NOT_EXIST; } end: if(val != NULL){ leveldb_free(val); } return retval; }
static void CheckGet( leveldb_t* db, const leveldb_readoptions_t* options, const char* key, const char* expected) { char* err = NULL; size_t val_len; char* val; val = leveldb_get(db, options, key, strlen(key), &val_len, &err); CheckNoError(err); CheckEqual(expected, val, val_len); leveldb_free(&val); }
inline int ldb_put(struct _leveldb_stuff *ldbs, const char *key, size_t klen, const char *value, size_t vlen) { char *err = NULL; leveldb_put(ldbs->db, ldbs->woptions, key, klen, value, vlen, &err); if (err) { fprintf(stderr, "%s\n", err); leveldb_free(err); err = NULL; return -1; } else { return 0; } }
/** * set if not exists. * return 1 (not exists and saved) * return 0 (already exists) */ void setnxCommand(redisClient *c) { char *value; sds keyword; size_t val_len; char *err = NULL; leveldb_writebatch_t *wb; err = NULL; val_len = 0; keyword = c->argv[1]->ptr; value = leveldb_get(server.ds_db, server.roptions, keyword, sdslen(keyword), &val_len, &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); return; } else if (val_len > 0) { // already exists leveldb_free(value); addReplyLongLong(c, 0); return; } wb = leveldb_writebatch_create(); leveldb_writebatch_put(wb, keyword, sdslen(keyword), (char *) c->argv[2]->ptr, sdslen((sds) c->argv[2]->ptr)); leveldb_write(server.ds_db, server.woptions, wb, &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); } else { addReplyLongLong(c, 1); server.dirty++; signalModifiedKey(c->db, c->argv[1]); } leveldb_writebatch_destroy(wb); return; }
int ldb_pull(struct _leveldb_stuff *ldbs, const char *key, size_t klen, char *value, size_t vlen) { size_t len = 0; char *err = NULL; char *val = NULL; val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, &len, &err); if (err) { fprintf(stderr, "%s\n", err); leveldb_free(err); err = NULL; return -1; } else { if (!val){ return -1; } int ok = (vlen == len) ? 0 : -1; if (!ok){ memcpy(value, val, vlen); } leveldb_free(val); return ok; } }
inline int ldb_batch_commit(struct _leveldb_stuff *ldbs) { char *err = NULL; leveldb_write(ldbs->db, ldbs->woptions, ldbs->wbatch, &err); leveldb_writebatch_clear(ldbs->wbatch); if (err) { fprintf(stderr, "%s\n", err); leveldb_free(err); err = NULL; return -1; } else { return 0; } }
char *ldb_get(struct _leveldb_stuff *ldbs, const char *key, size_t klen, int *vlen) { char *err = NULL; char *val = NULL; val = leveldb_get(ldbs->db, ldbs->roptions, key, klen, (size_t *) vlen, &err); if (err) { fprintf(stderr, "%s\n", err); leveldb_free(err); err = NULL; *vlen = -1; return NULL ; } else { return val; } }
/* * Destroy the ldb. */ void ldb_destroy(struct _leveldb_stuff *ldbs) { char* err = NULL; leveldb_close(ldbs->db); leveldb_destroy_db(ldbs->options, ldbs->dbname, &err); if (err) { fprintf(stderr, "%s\n", err); leveldb_free(err); err = NULL; } leveldb_options_destroy(ldbs->options); leveldb_readoptions_destroy(ldbs->roptions); leveldb_writeoptions_destroy(ldbs->woptions); leveldb_writebatch_destroy(ldbs->wbatch); free(ldbs); }
void setCommand(redisClient *c) { char *key, *value; char *err = NULL; key = (char *) c->argv[1]->ptr; value = (char *) c->argv[2]->ptr; leveldb_put(server.ds_db, server.woptions, key, sdslen((sds) key), value, sdslen((sds) value), &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); return; } addReply(c, shared.ok); server.dirty++; signalModifiedKey(c->db, c->argv[1]); return; }
void get(db_t* db, size_t keySz, void* key, size_t valSz, void** val) { if(db->type == LEVELDB) { levelDB_t* leveldb = (levelDB_t*) db->database; *((char**)val) = leveldb_get(leveldb->db, leveldb->roptions, (char*)key, keySz, &valSz, &(leveldb->err)); if(leveldb->err != NULL) { fprintf(stderr, "leveldb get failed!\n"); exit(-1); } leveldb_free(leveldb->err); leveldb->err = NULL; } else { kernelDB_t* kerneldb = (kernelDB_t*) db->database; kr_get(kerneldb->client, keySz, key, &valSz, val); } }
void delCommand(redisClient *c) { int i; char *key; char *err = NULL; int deleted = 0; for (i = 1; i < c->argc; i++) { key = (char *) c->argv[i]->ptr; leveldb_delete(server.ds_db, server.woptions, key, sdslen((sds) key), &err); if (err != NULL) { leveldb_free(err); continue; } signalModifiedKey(c->db, c->argv[i]); server.dirty++; deleted++; } addReplyLongLong(c, deleted); return; }
void clos(db_t* db) { if(db->type == LEVELDB) { levelDB_t* leveldb = (levelDB_t*) db->database; leveldb_close(leveldb->db); leveldb_destroy_db(leveldb->options, "leveldb_test", &(leveldb->err)); if(leveldb->err != NULL) { fprintf(stderr, "Failed to destroy the database!\n"); exit(-1); } leveldb_free(leveldb->err); leveldb->err = NULL; } else { kernelDB_t* kerneldb = (kernelDB_t*)db->database; kr_close(kerneldb->client); } free(db->database); free(db); }
int zset_incr(ldb_context_t* context, const ldb_slice_t* name, const ldb_slice_t* key, const ldb_meta_t* meta, int64_t by, int64_t* val){ int64_t old_score = 0; int ret = zset_get(context, name, key, &old_score); int retval = LDB_OK; if(ret == LDB_OK){ *val = old_score + by; }else if(ret == LDB_OK_NOT_EXIST){ *val = by; }else{ retval = ret; goto end; } ret = zset_one(context, name, key, meta, *val); if(ret >= 0){ if(ret > 0){ if(zset_incr_size(context, name, ret) == -1){ retval = LDB_ERR; goto end; } } char* errptr = NULL; ldb_context_writebatch_commit(context, &errptr); if(errptr!=NULL){ fprintf(stderr, "leveldb_write fail %s.\n", errptr); leveldb_free(errptr); retval = LDB_ERR; goto end; } retval = LDB_OK; }else{ retval = LDB_ERR; } end: return retval; }
int ldb_recovery_rec(ldb_context_t* context, ldb_recovery_t* recovery){ int retval = 0; ldb_slice_t *slice_key = NULL; size_t vlen = 0; const char *val = ldb_recov_iterator_val_raw(recovery->iter_, &vlen); if(vlen < LDB_VAL_META_SIZE){ fprintf(stderr, "%s iterator encountered invalid value length %lu.\n", __func__, vlen); retval = 0; goto end; } uint64_t version = leveldb_decode_fixed64(val + LDB_VAL_TYPE_SIZE); if(version == 0){ retval = 0; goto end; } size_t klen = 0; const char *key = ldb_recov_iterator_key_raw(recovery->iter_, &klen); ldb_meta_t *meta = ldb_meta_create(0 , 0, version); slice_key = ldb_meta_slice_create(meta); ldb_meta_destroy(meta); ldb_slice_push_back(slice_key, key, klen); char *errptr = NULL; leveldb_put_meta(context->database_, ldb_slice_data(slice_key), ldb_slice_size(slice_key), &errptr); if(errptr!=NULL){ fprintf(stderr, "%s leveldb_put_meta failed %s.\n", __func__, errptr); leveldb_free(errptr); retval = -1; goto end; } retval = 0; end: ldb_slice_destroy(slice_key); return retval; }
void ds_init() { char *err = NULL; server.ds_cache = leveldb_cache_create_lru(server.ds_lru_cache * 1048576); server.ds_options = leveldb_options_create(); server.policy = leveldb_filterpolicy_create_bloom(10); //leveldb_options_set_comparator(server.ds_options, cmp); leveldb_options_set_filter_policy(server.ds_options, server.policy); leveldb_options_set_create_if_missing(server.ds_options, server.ds_create_if_missing); leveldb_options_set_error_if_exists(server.ds_options, server.ds_error_if_exists); leveldb_options_set_cache(server.ds_options, server.ds_cache); leveldb_options_set_info_log(server.ds_options, NULL); leveldb_options_set_write_buffer_size(server.ds_options, server.ds_write_buffer_size * 1048576); leveldb_options_set_paranoid_checks(server.ds_options, server.ds_paranoid_checks); leveldb_options_set_max_open_files(server.ds_options, server.ds_max_open_files); leveldb_options_set_block_size(server.ds_options, server.ds_block_size * 1024); leveldb_options_set_block_restart_interval(server.ds_options, server.ds_block_restart_interval); leveldb_options_set_compression(server.ds_options, leveldb_snappy_compression); server.ds_db = leveldb_open(server.ds_options, server.ds_path, &err); if (err != NULL) { fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, err); leveldb_free(err); exit(1); } server.woptions = leveldb_writeoptions_create(); server.roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(server.roptions, 0); leveldb_readoptions_set_fill_cache(server.roptions, 1); leveldb_writeoptions_set_sync(server.woptions, 0); }
// Adds an event to the tablet. // // tablet - The tablet. // event - The event to add. // // Returns 0 if successful, otherwise returns -1. int sky_tablet_add_event(sky_tablet *tablet, sky_event *event) { int rc; char *errptr = NULL; void *new_data = NULL; void *data = NULL; sky_data_object *data_object = NULL; sky_data_descriptor *descriptor = NULL; sky_cursor cursor; memset(&cursor, 0, sizeof(cursor)); assert(tablet != NULL); assert(event != NULL); // Make sure that this event is being added to the correct tablet. sky_tablet *target_tablet = NULL; rc = sky_table_get_target_tablet(tablet->table, event->object_id, &target_tablet); check(rc == 0, "Unable to determine target tablet"); check(tablet == target_tablet, "Event added to invalid tablet; IDX:%d of %d, OID:%d", tablet->index, tablet->table->tablet_count, event->object_id); // Retrieve the existing value. size_t data_length; data = (void*)leveldb_get(tablet->leveldb_db, tablet->readoptions, (const char*)&event->object_id, sizeof(event->object_id), &data_length, &errptr); check(errptr == NULL, "LevelDB get error: %s", errptr); // Find the insertion point on the path. size_t insert_offset = 0; size_t event_length; // If the object doesn't exist yet then just set the single event. Easy peasy. if(data == NULL) { event_length = sky_event_sizeof(event); new_data = calloc(1, event_length); check_mem(new_data); insert_offset = 0; } // If the object does exist, we need to find where to insert the event data. // Also, we need to strip off any state which is redundant at the point of // insertion. else { void *insert_ptr = NULL; // Initialize data descriptor. descriptor = sky_data_descriptor_create(); check_mem(descriptor); rc = sky_data_descriptor_init_with_event(descriptor, event); check(rc == 0, "Unable to initialize data descriptor for event insert"); // Initialize data object. data_object = calloc(1, descriptor->data_sz); check_mem(data); // Attach data & descriptor to the cursor. cursor.data_descriptor = descriptor; cursor.data = (void*)data_object; // Initialize the cursor. rc = sky_cursor_set_ptr(&cursor, data, data_length); check(rc == 0, "Unable to set pointer on cursor"); // Loop over cursor until we reach the event insertion point. while(!cursor.eof) { // Retrieve event insertion pointer once the timestamp is reached. if(data_object->ts >= event->timestamp) { insert_ptr = cursor.ptr; break; } // Move to next event. check(sky_cursor_next_event(&cursor) == 0, "Unable to move to next event"); } // If no insertion point was found then append the event to the // end of the path. if(insert_ptr == NULL) { insert_ptr = data + data_length; } insert_offset = insert_ptr - data; // Clear off any object data on the event that matches // what is the current state of the event in the database. uint32_t i; for(i=0; i<event->data_count; i++) { // Ignore any action properties. if(event->data[i]->key > 0) { sky_data_property_descriptor *property_descriptor = &descriptor->property_zero_descriptor[event->data[i]->key]; // If the values match then splice this from the array. // Compare strings. void *a = &event->data[i]->value; void *b = ((void*)data_object)+property_descriptor->offset; size_t n = sky_data_type_sizeof(event->data[i]->data_type); bool is_equal = false; if(event->data[i]->data_type == SKY_DATA_TYPE_STRING) { is_equal = sky_string_bequals((sky_string*)b, event->data[i]->string_value); } // Compare other types. else if(memcmp(a, b, n) == 0) { is_equal = true; } // If the data is equal then remove it. if(is_equal) { sky_event_data_free(event->data[i]); if(i < event->data_count - 1) { memmove(&event->data[i], &event->data[i+1], (event->data_count-i-1) * sizeof(*event->data)); } i--; event->data_count--; } } } // Determine the serialized size of the event. If the event is // completely redundant (e.g. it is a data-only event and the event // matches the current object state) then don't allocate space for a // new path value. event_length = sky_event_sizeof(event); if(event_length > 0) { // Allocate space for the existing data plus the new data. new_data = calloc(1, data_length + event_length); check_mem(new_data); // Copy in data before event. if(insert_offset > 0) { memmove(new_data, data, insert_ptr-data); } // Copy in data after event. if(insert_offset < data_length) { event_length = sky_event_sizeof(event); memmove(new_data+insert_offset+event_length, data+insert_offset, data_length-insert_offset); } } } // If no space was allocated then it means the event is redundant and // should be ignored. if(new_data != NULL) { // If the object doesn't exist then just set the event as the data. size_t event_sz; rc = sky_event_pack(event, new_data + insert_offset, &event_sz); check(rc == 0, "Unable to pack event"); check(event_sz == event_length, "Expected event size (%ld) does not match actual event size (%ld)", event_length, event_sz); leveldb_put(tablet->leveldb_db, tablet->writeoptions, (const char*)&event->object_id, sizeof(event->object_id), new_data, data_length + event_length, &errptr); check(errptr == NULL, "LevelDB put error: %s", errptr); } free(data_object); sky_data_descriptor_free(descriptor); free(data); free(new_data); return 0; error: if(errptr) leveldb_free(errptr); sky_data_descriptor_free(descriptor); if(data) free(data); if(new_data) free(new_data); if(data_object) free(data_object); return -1; }
int ctl_cmd_parse(struct data_node *p_node) { int ret = 0; unsigned int loop = 0; int ok = 0; unsigned int i, j = 0; int offset = 0; size_t mlen = 0; char *p_new = NULL; char *p_old = NULL; char *result = NULL; char ret_number[16] = { 0 }; int size = 0; int sum = 0; if (p_node->status == X_MALLOC_FAILED) { add_send_node(p_node, OPT_NO_MEMORY, strlen(OPT_NO_MEMORY)); return X_MALLOC_FAILED; } if (p_node->gtlen > MAX_CMD_LEN) { add_send_node(p_node, OPT_DATA_TOO_LARGE, strlen(OPT_DATA_TOO_LARGE)); return X_DATA_TOO_LARGE; } /* parse cmd */ char *data = p_node->svbf; log_debug("%s", data); if (data[0] == '*') { // 1. read the arg count: if (p_node->kvs == 0) { p_new = p_old = &data[1]; CHECK_BUFFER(p_new, 0); p_node->kvs = atoi(p_old); p_node->doptr = p_new - data; } // 2. read the request name if (p_node->cmd == 0) { if (data[p_node->doptr] != '$') { goto E_OUT_PUT; } p_new = p_old = &data[p_node->doptr + 1]; CHECK_BUFFER(p_new, 0); mlen = atoi(p_old); p_old = p_new; CHECK_BUFFER(p_new, mlen); for (i = 0; i < mlen; i++) { if (*(p_old + i) > 'Z') { *(p_old + i) -= 32;/* 'A' - 'a' */ } p_node->cmd = p_node->cmd * 26 + (*(p_old + i) - 'A');/* A~Z is 26 numbers */ } p_node->doptr = p_new - data; }log_debug("%d", p_node->cmd); // 3. read a arg switch (p_node->cmd) { case LDB_SET: if (LDB_READONLY_SWITCH == 1 || p_node->kvs != LDB_SET_CNT) { goto E_OUT_PUT; } PARSE_LEN(p_node->klen); PARSE_MEMBER(p_node->key, p_node->klen); PARSE_LEN(p_node->vlen); PARSE_MEMBER(p_node->val, p_node->vlen); // x_out_time(&g_dbg_time); ok = ldb_put(g_ldb, &data[p_node->key], p_node->klen, &data[p_node->val], p_node->vlen); // x_out_time(&g_dbg_time); goto W_OUT_PUT; case LDB_DEL: if (LDB_READONLY_SWITCH == 1 || p_node->kvs < LDB_DEL_MIN) { goto E_OUT_PUT; } loop = p_node->kvs - 1; sum = 0; for (j = 0; j < loop; ++j) { PARSE_LEN(p_node->klen); PARSE_MEMBER(p_node->key, p_node->klen); ok = ldb_delete(g_ldb, &data[p_node->key], p_node->klen); if (ok < 0) { goto E_OUT_PUT; } sum += ok; p_node->kvs -= 1; p_node->key = 0; p_node->klen = 0; } goto NUM_OUT_PUT; case LDB_MSET:/* use one thread to write, otherwise (one thread should use one leveldb_writebatch_t) or (add a mutex lock) */ if (LDB_READONLY_SWITCH == 1) { goto E_OUT_PUT; } loop = (p_node->kvs - 1) / 2; if (loop < 1) { goto E_OUT_PUT; } for (j = 0; j < loop; ++j) { PARSE_LEN(p_node->klen); PARSE_MEMBER(p_node->key, p_node->klen); PARSE_LEN(p_node->vlen); PARSE_MEMBER(p_node->val, p_node->vlen); ldb_batch_put(g_ldb, &data[p_node->key], p_node->klen, &data[p_node->val], p_node->vlen); p_node->kvs -= 2; p_node->klen = 0; p_node->key = 0; p_node->vlen = 0; p_node->val = 0; } ok = ldb_batch_commit(g_ldb); goto W_OUT_PUT; case LDB_GET: if (p_node->kvs != LDB_GET_CNT) { goto E_OUT_PUT; } PARSE_LEN(p_node->klen); PARSE_MEMBER(p_node->key, p_node->klen); log_debug("key = %s", &data[ p_node->key ]); result = ldb_get(g_ldb, &data[p_node->key], p_node->klen, &size); log_debug("val = %s", result); goto R_BULK_OUT_PUT; case LDB_LRANGE: if (p_node->kvs != LDB_LRANGE_CNT) { goto E_OUT_PUT; } PARSE_LEN(p_node->klen); PARSE_MEMBER(p_node->key, p_node->klen); /* look as prefix key */ PARSE_LEN(p_node->vlen); PARSE_MEMBER(p_node->val, p_node->vlen); /* look as start time */ PARSE_LEN(p_node->vlen2); PARSE_MEMBER(p_node->val2, p_node->vlen2); /* look as end time */ log_debug("prefix key = %s", &data[ p_node->key ]); log_debug("start = %s", &data[ p_node->val ]); log_debug("end = %s", &data[ p_node->val2 ]); result = ldb_lrangeget(g_ldb, &data[p_node->key], p_node->klen, &data[p_node->val], p_node->vlen, &data[p_node->val2], p_node->vlen2, &size); goto R_MULTI_OUT_PUT; case LDB_KEYS: if (p_node->kvs != LDB_KEYS_CNT) { goto E_OUT_PUT; } PARSE_LEN(p_node->klen); PARSE_MEMBER(p_node->key, p_node->klen); result = ldb_keys(g_ldb, &data[p_node->key], p_node->klen, &size); log_debug("size = %d", size); goto R_MULTI_OUT_PUT; case LDB_INFO: if (p_node->kvs != LDB_INFO_CNT) { goto E_OUT_PUT; } result = ldb_info(g_ldb, &size); log_debug("size = %d\n", size); goto R_MULTI_OUT_PUT; case LDB_PING: if (p_node->kvs != LDB_PING_CNT) { goto E_OUT_PUT; } add_send_node(p_node, OPT_PONG, strlen(OPT_PONG)); return X_DATA_IS_ALL; case LDB_EXISTS: if (p_node->kvs != LDB_EXISTS_CNT) { goto E_OUT_PUT; } PARSE_LEN(p_node->klen); PARSE_MEMBER(p_node->key, p_node->klen); ok = ldb_exists(g_ldb, &data[p_node->key], p_node->klen); if (ok == -1) { goto E_OUT_PUT; } sum = ok; goto NUM_OUT_PUT; case LDB_QUIT: log_debug("-----------------------------------------------------"); return X_CLIENT_QUIT; default: goto E_OUT_PUT; } E_OUT_PUT: add_send_node(p_node, OPT_CMD_ERROR, strlen(OPT_CMD_ERROR)); return X_ERROR_CMD; W_OUT_PUT: if (ok == 0) { log_debug("------------------------------------------"); add_send_node(p_node, OPT_OK, strlen(OPT_OK)); return X_DATA_IS_ALL; } else { add_send_node(p_node, OPT_LDB_ERROR, strlen(OPT_LDB_ERROR)); return X_ERROR_LDB; } NUM_OUT_PUT: sprintf(ret_number, OPT_NUMBER, sum); log_debug("%s", ret_number); add_send_node(p_node, ret_number, strlen(ret_number)); return X_DATA_IS_ALL; R_BULK_OUT_PUT: if (result) { char tmp[32] = { 0 }; sprintf(tmp, "$%d\r\n", size); ret = add_send_node(p_node, tmp, strlen(tmp)); ret = add_send_node(p_node, result, size); leveldb_free(result); if (ret == X_MALLOC_FAILED) { clean_send_node(p_node); add_send_node(p_node, OPT_NO_MEMORY, strlen(OPT_NO_MEMORY)); return X_MALLOC_FAILED; } ret = add_send_node(p_node, "\r\n", 2); return X_DATA_IS_ALL; } else { if (size == 0) { add_send_node(p_node, OPT_NULL, strlen(OPT_NULL)); return X_DATA_IS_ALL; } else { add_send_node(p_node, OPT_LDB_ERROR, strlen(OPT_LDB_ERROR)); return X_ERROR_LDB; } } R_MULTI_OUT_PUT: if (result) { set_send_node(p_node, result, size); return X_DATA_IS_ALL; } else { if (size == 0) { add_send_node(p_node, OPT_NULL, strlen(OPT_NULL)); return X_DATA_IS_ALL; } else { add_send_node(p_node, OPT_LDB_ERROR, strlen(OPT_LDB_ERROR)); return X_ERROR_LDB; } } } else if (data[0] == '$') { #if 0 // 1. read the request name if (p_node->cmd == 0) { p_new = p_old = &data[1]; CHECK_BUFFER(p_new, 0); mlen = atoi( p_old ); p_old = p_new; CHECK_BUFFER(p_new, mlen); for (i = 0; i < mlen; i++) { p_node->cmd = p_node->cmd * 26 + ( *(p_old + i) - 'A' );/* A~Z is 26 numbers */ } p_node->doptr = p_new - data; } log_debug("%d\n", p_node->cmd); // 2. read a arg switch (p_node->cmd) { case LDB_QUIT: return CLIENT_QUIT; default: log_debug("-----------------------------------------------------\n"); return -1; } //TODO #endif add_send_node(p_node, OPT_CMD_ERROR, strlen(OPT_CMD_ERROR)); return X_ERROR_CMD; } else { add_send_node(p_node, OPT_CMD_ERROR, strlen(OPT_CMD_ERROR)); return X_ERROR_CMD; } return X_DATA_IS_ALL; }
int main(int argc, char** argv) { leveldb_t* db; leveldb_comparator_t* cmp; leveldb_cache_t* cache; leveldb_env_t* env; leveldb_options_t* options; leveldb_readoptions_t* roptions; leveldb_writeoptions_t* woptions; char* err = NULL; #if defined(LEVELDB_PLATFORM_WINDOWS) snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test"); #else snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d", ((int) geteuid())); #endif StartPhase("create_objects"); cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); env = leveldb_create_default_env(); cache = leveldb_cache_create_lru(100000); options = leveldb_options_create(); leveldb_options_set_comparator(options, cmp); leveldb_options_set_error_if_exists(options, 1); leveldb_options_set_cache(options, cache); leveldb_options_set_env(options, env); leveldb_options_set_info_log(options, NULL); leveldb_options_set_write_buffer_size(options, 100000); leveldb_options_set_paranoid_checks(options, 1); leveldb_options_set_max_open_files(options, 10); leveldb_options_set_block_size(options, 1024); leveldb_options_set_block_restart_interval(options, 8); leveldb_options_set_compression(options, leveldb_no_compression); roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(roptions, 1); leveldb_readoptions_set_fill_cache(roptions, 0); woptions = leveldb_writeoptions_create(); leveldb_writeoptions_set_sync(woptions, 1); StartPhase("destroy"); leveldb_destroy_db(options, dbname, &err); leveldb_free(&err); StartPhase("open_error"); db = leveldb_open(options, dbname, &err); CheckCondition(err != NULL); leveldb_free(&err); StartPhase("open"); leveldb_options_set_create_if_missing(options, 1); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); StartPhase("put"); leveldb_put(db, woptions, "foo", 3, "hello", 5, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); StartPhase("writebatch"); { int pos = 0; leveldb_writebatch_t* wb = leveldb_writebatch_create(); leveldb_writebatch_put(wb, "foo", 3, "a", 1); leveldb_writebatch_clear(wb); leveldb_writebatch_put(wb, "bar", 3, "b", 1); leveldb_writebatch_put(wb, "box", 3, "c", 1); leveldb_writebatch_delete(wb, "bar", 3); leveldb_write(db, woptions, wb, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); CheckCondition(pos == 3); leveldb_writebatch_destroy(wb); } StartPhase("iter"); { leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_first(iter); CheckCondition(leveldb_iter_valid(iter)); CheckIter(iter, "box", "c"); leveldb_iter_next(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_prev(iter); CheckIter(iter, "box", "c"); leveldb_iter_prev(iter); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_last(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_seek(iter, "b", 1); CheckIter(iter, "box", "c"); leveldb_iter_get_error(iter, &err); CheckNoError(err); leveldb_iter_destroy(iter); } StartPhase("approximate_sizes"); { int i; int n = 20000; char keybuf[100]; char valbuf[100]; uint64_t sizes[2]; const char* start[2] = { "a", "k00000000000000010000" }; size_t start_len[2] = { 1, 21 }; const char* limit[2] = { "k00000000000000010000", "z" }; size_t limit_len[2] = { 21, 1 }; leveldb_writeoptions_set_sync(woptions, 0); for (i = 0; i < n; i++) { snprintf(keybuf, sizeof(keybuf), "k%020d", i); snprintf(valbuf, sizeof(valbuf), "v%020d", i); leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf), &err); CheckNoError(err); } leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes); CheckCondition(sizes[0] > 0); CheckCondition(sizes[1] > 0); } StartPhase("property"); { char* prop = leveldb_property_value(db, "nosuchprop"); CheckCondition(prop == NULL); prop = leveldb_property_value(db, "leveldb.stats"); CheckCondition(prop != NULL); leveldb_free(&prop); } StartPhase("snapshot"); { const leveldb_snapshot_t* snap; snap = leveldb_create_snapshot(db); leveldb_delete(db, woptions, "foo", 3, &err); CheckNoError(err); leveldb_readoptions_set_snapshot(roptions, snap); CheckGet(db, roptions, "foo", "hello"); leveldb_readoptions_set_snapshot(roptions, NULL); CheckGet(db, roptions, "foo", NULL); leveldb_release_snapshot(db, snap); } StartPhase("repair"); { leveldb_close(db); leveldb_options_set_create_if_missing(options, 0); leveldb_options_set_error_if_exists(options, 0); leveldb_repair_db(options, dbname, &err); CheckNoError(err); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); } StartPhase("cleanup"); leveldb_close(db); leveldb_options_destroy(options); leveldb_readoptions_destroy(roptions); leveldb_writeoptions_destroy(woptions); leveldb_cache_destroy(cache); leveldb_comparator_destroy(cmp); leveldb_env_destroy(env); fprintf(stderr, "PASS\n"); return 0; }
char *ldb_values(struct _leveldb_stuff *ldbs, const char *ptn, size_t ptn_len, int *size) { char *err = NULL; char *result = NULL; char *p_dst = NULL; char *p_key = NULL; char *p_value = NULL; size_t klen = 0; size_t vlen = 0; int index = 0; int i, j, z = 0; struct kv_list list = { 0 }; struct some_kv *p_new, *p_old, *p_tmp = NULL; leveldb_iterator_t* iter = leveldb_create_iterator(ldbs->db, ldbs->roptions); leveldb_iterator_t* iter_save = iter; if (!!leveldb_iter_valid(iter)) {/* first use it is invalid */ fprintf(stderr, "%s:%d: this iter is valid already!\n", __FILE__, __LINE__); *size = -1; return NULL ; } leveldb_iter_seek_to_first(iter); p_old = p_new = &list.head; while (leveldb_iter_valid(iter)) { /* parse kv */ p_key = (char *) leveldb_iter_key(iter, &klen); log_debug("%p iter key = %s, klen = %ld\n", p_key, p_key, klen); p_value = (char *) leveldb_iter_value(iter, &vlen); leveldb_iter_get_error(iter, &err); if (err) { goto FAIL_ITER_PARSE; } if (!string_match_len(ptn, ptn_len, p_value, vlen, 0)) { leveldb_iter_next(iter); continue; } /* save parse */ list.count++;/* kv counts */ list.klens += klen; list.knubs += get_number_len(klen); index = list.count % SOME_KV_NODES_COUNT; if ((list.count / SOME_KV_NODES_COUNT >= 1) && (index == 1)) { /* new store */ p_new = malloc(sizeof(struct some_kv)); if (p_new == NULL ) { /* free store */ index = GET_NEED_COUNT( list.count, SOME_KV_NODES_COUNT ); p_tmp = &list.head; for (i = 0, z = list.count - 1; (i < index) && (p_tmp != NULL ); i++ ) { for (j = 0; (j < SOME_KV_NODES_COUNT) && (z > 0); j++, z--) { free(p_tmp->nodes[j].key); } p_old = p_tmp; p_tmp = p_tmp->next; if (p_old != &list.head) { free(p_old); } } goto FAIL_MEMORY; } memset(p_new, 0, sizeof(struct some_kv)); p_old->next = p_new; p_new->prev = p_old; p_old = p_new; } /* * fix bug: index is error if list.count = n * SOME_KV_NODES_COUNT(1024), * SOME_KV_NODES_COUNT = 1024, n > 0. */ if (index == 0) { index = SOME_KV_NODES_COUNT; } /* save key */ p_new->nodes[index - 1].klen = klen; p_new->nodes[index - 1].key = malloc(GET_NEED_COUNT( klen, G_PAGE_SIZE ) * G_PAGE_SIZE); memcpy(p_new->nodes[index - 1].key, p_key, klen); /* find next */ leveldb_iter_next(iter); } /* create result */ if (list.count > 0) { /* has members */ /* *2\r\n$5\r\nmykey\r\n$5\r\nmyval\r\n */ *size = strlen("*\r\n") + get_number_len(list.count) + strlen("$\r\n\r\n") * (list.count) + list.knubs + list.klens; index = GET_NEED_COUNT( *size, G_PAGE_SIZE ) * G_PAGE_SIZE; result = (char *) malloc(index); if (result == NULL ) goto FAIL_MEMORY; memset(result, 0, index); log_debug("----->>>ALL SIZE IS %d, BUFF %p : LEN IS %d\n", *size, result, index); /* split piece */ index = GET_NEED_COUNT( list.count, SOME_KV_NODES_COUNT ); p_tmp = &list.head; sprintf(result, "*%d\r\n", list.count); p_dst = result + strlen(result); for (i = 0, z = list.count; (i < index) && (p_tmp != NULL ); i++ ) { for (j = 0; (j < SOME_KV_NODES_COUNT) && (z > 0); j++, z--) { p_dst = set_bulk(p_dst, p_tmp->nodes[j].key, p_tmp->nodes[j].klen); free(p_tmp->nodes[j].key); } p_old = p_tmp; p_tmp = p_tmp->next; if (p_old != &list.head) { free(p_old); } } } else { /* no members */ *size = 0; } leveldb_iter_destroy(iter_save); return result; FAIL_ITER_PARSE: fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, (err)); leveldb_free(err); err = NULL; leveldb_iter_destroy(iter); *size = -1; return NULL ; FAIL_MEMORY: fprintf(stderr, "%s:%d: FAILED MALLOC !\n", __FILE__, __LINE__); leveldb_iter_destroy(iter); *size = -1; return NULL ; }