static void CheckIter(leveldb_iterator_t* iter, const char* key, const char* val) { size_t len; const char* str; str = leveldb_iter_key(iter, &len); CheckEqual(key, str, len); str = leveldb_iter_value(iter, &len); CheckEqual(val, str, len); }
rstatus_t cursor_next(cursor_t *cursor, sds *key, sds *val, uint64_t *expire) { size_t len; const char *str; if (!leveldb_iter_valid(cursor->iter)) { return NC_ERROR; } str = leveldb_iter_key(cursor->iter, &len); *key = sdsnewlen(str, len); ASSERT(*key != NULL); /* TODO: check mem */ str = leveldb_iter_value(cursor->iter, &len); store_decode_val(str, len, val, expire); leveldb_iter_next(cursor->iter); return NC_OK; }
static void listmdb(struct MetaDB* mdb) { leveldb_iterator_t* iter = leveldb_create_iterator(mdb->db, mdb->scan_options); leveldb_iter_seek_to_first(iter); while (leveldb_iter_valid(iter)) { size_t klen; const char* kstr = leveldb_iter_key(iter, &klen); const metadb_key_t* key = (const metadb_key_t *) kstr; printf("%ld %ld %s\n", key->parent_id, key->partition_id, key->name_hash); size_t vlen; const char* vstr = leveldb_iter_value(iter, &vlen); (void) vstr; leveldb_iter_next(iter); } leveldb_iter_destroy(iter); }
static struct stat_cache_entry *stat_cache_iter_current(struct stat_cache_iterator *iter) { struct stat_cache_entry *entry; const struct stat_cache_value *value; const char *key; size_t klen, vlen; assert(iter); //log_print(LOG_DEBUG, "checking iterator validity"); // If we've gone beyond the end of the dataset, quit. if (!leveldb_iter_valid(iter->ldb_iter)) { return NULL; } //log_print(LOG_DEBUG, "fetching the key"); key = leveldb_iter_key(iter->ldb_iter, &klen); // log_print(LOG_DEBUG, "fetched key: %s", key); //log_print(LOG_DEBUG, "fetched the key"); // If we've gone beyond the end of the prefix range, quit. // Use (iter->key_prefix_len - 1) to exclude the NULL at the prefix end. if (strncmp(key, iter->key_prefix, iter->key_prefix_len - 1) != 0) { log_print(LOG_DEBUG, "Key %s does not match prefix %s for %lu characters. Ending iteration.", key, iter->key_prefix, iter->key_prefix_len); return NULL; } //log_print(LOG_DEBUG, "fetching the value"); value = (const struct stat_cache_value *) leveldb_iter_value(iter->ldb_iter, &vlen); entry = malloc(sizeof(struct stat_cache_entry)); entry->key = key; entry->value = value; // log_print(LOG_DEBUG, "iter_current: key = %s; value = %s", key, value); return entry; }
char *ldb_values(struct _leveldb_stuff *ldbs, const char *ptn, size_t ptn_len, int *size) { char *err = NULL; char *result = NULL; char *p_dst = NULL; char *p_key = NULL; char *p_value = NULL; size_t klen = 0; size_t vlen = 0; int index = 0; int i, j, z = 0; struct kv_list list = { 0 }; struct some_kv *p_new, *p_old, *p_tmp = NULL; leveldb_iterator_t* iter = leveldb_create_iterator(ldbs->db, ldbs->roptions); leveldb_iterator_t* iter_save = iter; if (!!leveldb_iter_valid(iter)) {/* first use it is invalid */ fprintf(stderr, "%s:%d: this iter is valid already!\n", __FILE__, __LINE__); *size = -1; return NULL ; } leveldb_iter_seek_to_first(iter); p_old = p_new = &list.head; while (leveldb_iter_valid(iter)) { /* parse kv */ p_key = (char *) leveldb_iter_key(iter, &klen); log_debug("%p iter key = %s, klen = %ld\n", p_key, p_key, klen); p_value = (char *) leveldb_iter_value(iter, &vlen); leveldb_iter_get_error(iter, &err); if (err) { goto FAIL_ITER_PARSE; } if (!string_match_len(ptn, ptn_len, p_value, vlen, 0)) { leveldb_iter_next(iter); continue; } /* save parse */ list.count++;/* kv counts */ list.klens += klen; list.knubs += get_number_len(klen); index = list.count % SOME_KV_NODES_COUNT; if ((list.count / SOME_KV_NODES_COUNT >= 1) && (index == 1)) { /* new store */ p_new = malloc(sizeof(struct some_kv)); if (p_new == NULL ) { /* free store */ index = GET_NEED_COUNT( list.count, SOME_KV_NODES_COUNT ); p_tmp = &list.head; for (i = 0, z = list.count - 1; (i < index) && (p_tmp != NULL ); i++ ) { for (j = 0; (j < SOME_KV_NODES_COUNT) && (z > 0); j++, z--) { free(p_tmp->nodes[j].key); } p_old = p_tmp; p_tmp = p_tmp->next; if (p_old != &list.head) { free(p_old); } } goto FAIL_MEMORY; } memset(p_new, 0, sizeof(struct some_kv)); p_old->next = p_new; p_new->prev = p_old; p_old = p_new; } /* * fix bug: index is error if list.count = n * SOME_KV_NODES_COUNT(1024), * SOME_KV_NODES_COUNT = 1024, n > 0. */ if (index == 0) { index = SOME_KV_NODES_COUNT; } /* save key */ p_new->nodes[index - 1].klen = klen; p_new->nodes[index - 1].key = malloc(GET_NEED_COUNT( klen, G_PAGE_SIZE ) * G_PAGE_SIZE); memcpy(p_new->nodes[index - 1].key, p_key, klen); /* find next */ leveldb_iter_next(iter); } /* create result */ if (list.count > 0) { /* has members */ /* *2\r\n$5\r\nmykey\r\n$5\r\nmyval\r\n */ *size = strlen("*\r\n") + get_number_len(list.count) + strlen("$\r\n\r\n") * (list.count) + list.knubs + list.klens; index = GET_NEED_COUNT( *size, G_PAGE_SIZE ) * G_PAGE_SIZE; result = (char *) malloc(index); if (result == NULL ) goto FAIL_MEMORY; memset(result, 0, index); log_debug("----->>>ALL SIZE IS %d, BUFF %p : LEN IS %d\n", *size, result, index); /* split piece */ index = GET_NEED_COUNT( list.count, SOME_KV_NODES_COUNT ); p_tmp = &list.head; sprintf(result, "*%d\r\n", list.count); p_dst = result + strlen(result); for (i = 0, z = list.count; (i < index) && (p_tmp != NULL ); i++ ) { for (j = 0; (j < SOME_KV_NODES_COUNT) && (z > 0); j++, z--) { p_dst = set_bulk(p_dst, p_tmp->nodes[j].key, p_tmp->nodes[j].klen); free(p_tmp->nodes[j].key); } p_old = p_tmp; p_tmp = p_tmp->next; if (p_old != &list.head) { free(p_old); } } } else { /* no members */ *size = 0; } leveldb_iter_destroy(iter_save); return result; FAIL_ITER_PARSE: fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, (err)); leveldb_free(err); err = NULL; leveldb_iter_destroy(iter); *size = -1; return NULL ; FAIL_MEMORY: fprintf(stderr, "%s:%d: FAILED MALLOC !\n", __FILE__, __LINE__); leveldb_iter_destroy(iter); *size = -1; return NULL ; }
int txdb_open(struct config *config, char **errStr, struct txdb **out) { leveldb_iterator_t* iter; struct txdb *txdb; int res; txdb = safe_calloc(1, sizeof *txdb); txdb->hash_txo = hashtable_create(); /* index all interesting txos */ txdb->hash_tx = hashtable_create(); /* all TX brought to our attention */ txdb->path = txdb_get_db_path(config); txdb->tx_seq = 0; theTxdb = txdb; if (!file_exists(txdb->path)) { Log(LGPFX" txdb DB '%s' does not exist. Creating..\n", txdb->path); } res = txdb_open_db(txdb); if (res) { *errStr = "failed to open tx DB"; goto error; } res = file_chmod(txdb->path, 0700); if (res) { Log(LGPFX" Failed to chmod txdb to 0700: %s\n", strerror(res)); goto error; } *out = txdb; iter = leveldb_create_iterator(txdb->db, txdb->rd_opts); leveldb_iter_seek_to_first(iter); while (leveldb_iter_valid(iter) && btc->stop == 0) { const char *key; const char *val; size_t klen; size_t vlen; key = leveldb_iter_key(iter, &klen); val = leveldb_iter_value(iter, &vlen); Log(LGPFX" found entry \"%s\" klen=%zu vlen=%zu\n", key, klen, vlen); if (klen > 4 && strncmp(key, "/tx/", 4) == 0) { res = txdb_load_tx(txdb, key, klen, val, vlen); ASSERT(res == 0); } leveldb_iter_next(iter); } leveldb_iter_destroy(iter); txdb_export_tx_info(txdb); txdb_print_coins(txdb, 1); exit: return res; error: txdb_close(txdb); theTxdb = NULL; *out = NULL; goto exit; }
void txdb_confirm_one_tx(struct txdb *txdb, const uint256 *blkHash, const uint256 *txHash) { leveldb_iterator_t *iter; struct tx_entry *txe; char bkHashStr[80]; char txHashStr[80]; ASSERT(!uint256_iszero(blkHash)); ASSERT(!uint256_iszero(txHash)); txe = txdb_get_tx_entry(txdb, txHash); if (txe == NULL) { return; } if (txe->relevant == 0) { txdb_remove_from_hashtable(txdb, txHash); NOT_TESTED(); return; } if (!uint256_iszero(&txe->blkHash)) { /* * It's possible for the ASSERT below to fire if a tx is confirmed in * a block that is later orphaned. The tx should then be relayed again * until it finds its way in a new block. */ ASSERT(uint256_issame(&txe->blkHash, blkHash)); return; } peergroup_stop_broadcast_tx(btc->peerGroup, txHash); memcpy(&txe->blkHash, blkHash, sizeof *blkHash); uint256_snprintf_reverse(bkHashStr, sizeof bkHashStr, blkHash); uint256_snprintf_reverse(txHashStr, sizeof txHashStr, txHash); Warning(LGPFX" %s confirmed in %s\n", txHashStr, bkHashStr); NOT_TESTED(); iter = leveldb_create_iterator(txdb->db, txdb->rd_opts); leveldb_iter_seek_to_first(iter); while (leveldb_iter_valid(iter)) { struct tx_ser_key *txkey; struct tx_ser_data *txdata; struct buff *buf; const char *key; const char *val; size_t klen; size_t vlen; char *err = NULL; key = leveldb_iter_key(iter, &klen); txkey = txdb_deserialize_tx_key(key, klen); if (txkey == NULL || uint256_issame(txHash, &txkey->txHash) == 0) { free(txkey); leveldb_iter_next(iter); continue; } NOT_TESTED(); val = leveldb_iter_value(iter, &vlen); txdata = txdb_deserialize_tx_data(val, vlen); ASSERT(uint256_iszero(&txdata->blkHash)); ASSERT(txdata->timestamp != 0); memcpy(&txdata->blkHash, blkHash, sizeof *blkHash); buf = txdb_serialize_tx_data(txdata); leveldb_put(txdb->db, txdb->wr_opts, key, klen, buff_base(buf), buff_curlen(buf), &err); buff_free(buf); if (err) { Warning(LGPFX" failed to write tx entry: %s\n", err); } txdb_export_tx_info(txdb); free(txkey); free(txdata->buf); free(txdata); break; } leveldb_iter_destroy(iter); }
int main () { unsigned int N_KEYS; unsigned int N_BATCH; printf("\n number of keys:"); scanf("%u",&N_KEYS); printf("\n batch size:"); scanf("%u",&N_BATCH); tinymt32_t tinymt32; //initialize random generator tinymt32_init (&tinymt32, 0); int iter; int sec_iter; int stop; int counter; //initialize database char *errptr = NULL; leveldb_options_t *options = leveldb_options_create (); /* initializeOptions */ leveldb_options_set_create_if_missing (options, 1); leveldb_options_set_write_buffer_size(options,120000000 ); leveldb_options_set_max_open_files(options,800000); /*open Database */ leveldb_t *db_helper = leveldb_open (options, "/mnt/database/database_helper", &errptr); leveldb_t *db = leveldb_open (options, "/mnt/database/database", &errptr); leveldb_readoptions_t *readoptions = leveldb_readoptions_create (); leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create (); leveldb_writebatch_t* batch=leveldb_writebatch_create(); int64_t diff = zclock_time (); //write into database_helper iter=0; while( iter < N_KEYS) { sec_iter=0; while((iter<N_KEYS) && (sec_iter<N_BATCH)){ unsigned int key = tinymt32_generate_uint32 (&tinymt32); unsigned int val = tinymt32_generate_uint32 (&tinymt32); leveldb_writebatch_put (batch, (const char *) &key, sizeof (int), (const char *) &val, sizeof (int)); sec_iter++; iter++; } leveldb_write( db_helper, writeoptions, batch, &errptr); if(errptr!=NULL){ printf("\n%s",errptr); } assert (errptr == NULL); leveldb_writebatch_clear(batch); } leveldb_writebatch_destroy(batch); diff = zclock_time () - diff; printf ("\nrandom write: %d", diff); diff = zclock_time (); //write sequentially into db leveldb_iterator_t *liter = leveldb_create_iterator (db_helper, readoptions); leveldb_iter_seek_to_first (liter); while (leveldb_iter_valid (liter)) { size_t length; char key[4]; memcpy (key, leveldb_iter_key (liter, &length), 4); char val[4]; memcpy (val, leveldb_iter_value (liter, &length), 4); leveldb_iter_get_error (liter, &errptr); assert (errptr == NULL); leveldb_put (db, writeoptions, (const char *) &key, sizeof (int), (const char *) &val, sizeof (int), &errptr); assert (errptr == NULL); leveldb_iter_next (liter); } diff = zclock_time () - diff; leveldb_close (db); leveldb_close (db_helper); }
const char *db_iterator_value(db_iterator_t *it, size_t *vlen) { return leveldb_iter_value(it, vlen); }
static PyObject* RangeIterator_next(RangeIterator *iter) { VERBOSE("%s", "Entering RangeIterator_next..."); if (iter->_db == 0 || !leveldb_iter_valid(iter->_iterator)) { RangeIterator_clean(iter); return 0; } if (iter->_to != NULL) { const char *tkey; /* c style key */ size_t key_len; /* key length */ if (iter->_iterator != NULL) { tkey = leveldb_iter_key(iter->_iterator, &key_len); } else return NULL; int limit_len = (strlen(iter->_to) > key_len) ? key_len: strlen(iter->_to); int c = memcmp(iter->_to, tkey, limit_len); if (!(0 <= c)) { VERBOSE("%s", "Invalid iterator................................"); RangeIterator_clean(iter); return 0; } } // get key and (optional) value const char *ckey; /* c style key */ size_t key_len; /* key length */ if (iter->_iterator != NULL) { ckey = leveldb_iter_key(iter->_iterator, &key_len); } else return NULL; VERBOSE("The key is %s, length is %d", ckey, key_len); PyObject* key = PyString_FromStringAndSize(ckey, key_len); PyObject* value = 0; PyObject* ret = key; if (key == 0) return 0; const char *cvalue; /* c style value */ size_t value_len; /* value length */ if (iter->_iterator != NULL) { cvalue = leveldb_iter_value(iter->_iterator, &value_len); } else return NULL; if (iter->_include_value) { value = PyString_FromStringAndSize(cvalue, value_len); if (value == 0) { Py_XDECREF(key); return 0; } } // key/value pairs are returned as 2-tuples if (value != NULL) { ret = PyTuple_New(2); if (ret == 0) { Py_DECREF(key); Py_XDECREF(value); return 0; } PyTuple_SET_ITEM(ret, 0, key); PyTuple_SET_ITEM(ret, 1, value); } // get next value leveldb_iter_next(iter->_iterator); // return k/v pair or single key return ret; }
// get the corresponding pidkey from traced execution // some graph algorithm is needed here // @input current pidkey // @return pidkey from netdb // TODO char* getMappedPid(char* pidkey) { // TODO: need a cache for this operation as well char key[KEYLEN], *p; const char *read; size_t read_len; ull_t childid; int n=0, i; ull_t idlist[100]; // enough? p = pidkey; while (p != NULL) { sprintf(key, "prv.pid.%s.childid", p); vbp(3, "%s\n", key); db_read_ull(currdb, key, &childid); idlist[n++] = childid; sprintf(key, "prv.pid.%s.parent", p); if (p != pidkey && p != NULL) free(p); p = db_readc(currdb, key); } vbp(2, "%s -> [%d] [", pidkey, n); if (CDE_verbose_mode >= 2) { for (i=0; i<n; i++) { fprintf(stderr, "%llu, ", idlist[i]); } fprintf(stderr, "]\n"); } if (PIDKEY != NULL) { p = strdup(PIDKEY); n -= 2; } else { p = db_readc(netdb, "meta.root"); sprintf(key, "prv.pid.%s.exec.", p); //~ sprintf(key, "prv.pid.%s.actualexec.", p); free(p); leveldb_iterator_t *it = leveldb_create_iterator(netdb->db, netdb->roptions); leveldb_iter_seek(it, key, strlen(key)); read = leveldb_iter_value(it, &read_len); p = malloc(read_len + 1); memcpy(p, read, read_len); p[read_len] = '\0'; n-=2; // skip the first root -> child that I just did } while (n>0) { sprintf(key, "prv.pid.%s.child.%llu", p, idlist[n-1]); n--; free(p); p = db_readc(netdb, key); vbp(3, "%s\n", key); } vbp(2, "return %s\n", p); return p; //~ char *value; //~ const char *read; //~ size_t read_len; //~ //~ sprintf(key, "prv.pid.%s.parent.", netdb_root); //~ leveldb_iterator_t *it = leveldb_create_iterator(netdb->db, netdb->roptions); //~ leveldb_iter_seek(it, key, strlen(key)); //~ //~ read = leveldb_iter_value(it, &read_len); //~ value = malloc(read_len + 1); //~ memcpy(value, read, read_len); //~ value[read_len] = '\0'; //~ return value; }
int mdhim_leveldb_get_prev(void *dbh, void **key, int *key_len, void **data, int32_t *data_len, struct mdhim_store_opts_t *mstore_opts) { leveldb_readoptions_t *options; leveldb_t *db = (leveldb_t *) dbh; int ret = MDHIM_SUCCESS; leveldb_iterator_t *iter; const char *res; int len = 0; void *old_key; int old_key_len; struct timeval start, end; //Init the data to return *data = NULL; *data_len = 0; gettimeofday(&start, NULL); //Create the options and iterator options = (leveldb_readoptions_t *) mstore_opts->db_ptr3; old_key = *key; old_key_len = *key_len; *key = NULL; *key_len = 0; iter = leveldb_create_iterator(db, options); //If the user didn't supply a key, then seek to the last if (!old_key || old_key_len == 0) { leveldb_iter_seek_to_last(iter); } else { leveldb_iter_seek(iter, old_key, old_key_len); if (!leveldb_iter_valid(iter)) { mlog(MDHIM_SERVER_DBG2, "Could not get a valid iterator in leveldb"); goto error; } leveldb_iter_prev(iter); } if (!leveldb_iter_valid(iter)) { goto error; } res = leveldb_iter_value(iter, (size_t *) &len); if (res) { *data = malloc(len); memcpy(*data, res, len); *data_len = len; } else { *data = NULL; *data_len = 0; } res = leveldb_iter_key(iter, (size_t *) key_len); if (res) { *key = malloc(*key_len); memcpy(*key, res, *key_len); } else { *key = NULL; *key_len = 0; } if (!*data) { goto error; } //Destroy iterator leveldb_iter_destroy(iter); gettimeofday(&end, NULL); mlog(MDHIM_SERVER_DBG, "Took: %d seconds to get the prev record", (int) (end.tv_sec - start.tv_sec)); return ret; error: //Destroy iterator leveldb_iter_destroy(iter); *key = NULL; *key_len = 0; *data = NULL; *data_len = 0; return MDHIM_DB_ERROR; }