int stat_cache_value_set(stat_cache_t *cache, const char *path, struct stat_cache_value *value) { leveldb_writeoptions_t *options; char *errptr = NULL; char *key; int r = 0; assert(value); value->updated = time(NULL); value->local_generation = stat_cache_get_local_generation(); key = path2key(path, false); log_print(LOG_DEBUG, "CSET: %s (mode %04o)", key, value->st.st_mode); //print_stat(&value->st, "CSET"); options = leveldb_writeoptions_create(); leveldb_put(cache, options, key, strlen(key) + 1, (char *) value, sizeof(struct stat_cache_value), &errptr); leveldb_writeoptions_destroy(options); //log_print(LOG_DEBUG, "Setting key: %s", key); free(key); if (errptr != NULL) { log_print(LOG_ERR, "leveldb_set error: %s", errptr); free(errptr); r = -1; } return r; }
int stat_cache_updated_children(stat_cache_t *cache, const char *path, time_t timestamp) { leveldb_writeoptions_t *options; char *key = NULL; char *errptr = NULL; int r = 0; asprintf(&key, "updated_children:%s", path); options = leveldb_writeoptions_create(); if (timestamp == 0) leveldb_delete(cache, options, key, strlen(key) + 1, &errptr); else leveldb_put(cache, options, key, strlen(key) + 1, (char *) ×tamp, sizeof(time_t), &errptr); leveldb_writeoptions_destroy(options); free(key); if (errptr != NULL) { log_print(LOG_ERR, "leveldb_set error: %s", errptr); free(errptr); r = -1; } return r; }
int kvdb_leveldb_put(kvdb_t *kvdb, void *key, uint32_t klen, void *value, uint32_t vlen) { kvdb_leveldb_t *leveldb = (kvdb_leveldb_t*)kvdb; char *szErr = NULL; leveldb_put(leveldb->db, leveldb->pWriteOpt, (const char*)key, klen, (const char *)value, vlen, &szErr); if ( szErr ) { return -1; } else { return 0; } }
void store_put(leveldb_t * store, char * key, size_t key_size, char * value, size_t value_len){ char * err=0x00; leveldb_writeoptions_t *write_options=leveldb_writeoptions_create(); //char * resp = leveldb_put(store, leveldb_readoptions_create(), key, key_size, &read_len, &err); leveldb_put(store,write_options , key, key_size, value, value_len, &err); leveldb_writeoptions_destroy(write_options); if(err){ fprintf(stderr, "error: %s\n", err); free(err); } }
void db_put(db_t *db, const char *key, size_t klen, const char *val, size_t vlen, char **errptr) { leveldb_writeoptions_t *opts; opts = leveldb_writeoptions_create(); /* sync=true flushed buffer */ leveldb_writeoptions_set_sync(opts, 1); leveldb_put(db->db, opts, key, klen, val, vlen, errptr); leveldb_writeoptions_destroy(opts); }
inline int ldb_put(struct _leveldb_stuff *ldbs, const char *key, size_t klen, const char *value, size_t vlen) { char *err = NULL; leveldb_put(ldbs->db, ldbs->woptions, key, klen, value, vlen, &err); if (err) { fprintf(stderr, "%s\n", err); leveldb_free(err); err = NULL; return -1; } else { return 0; } }
int xleveldb_set(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t vsize) { leveldb_writeoptions_t *options = leveldb_writeoptions_create(); char *err, *origerr; err = origerr = malloc(1); leveldb_put(db->conn, options, (const char *) key, ksize, (const char *) val, vsize, &err); free(err); leveldb_writeoptions_destroy(options); return err == origerr; }
void setCommand(redisClient *c) { char *key, *value; char *err = NULL; key = (char *) c->argv[1]->ptr; value = (char *) c->argv[2]->ptr; leveldb_put(server.ds_db, server.woptions, key, sdslen((sds) key), value, sdslen((sds) value), &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); return; } addReply(c, shared.ok); server.dirty++; signalModifiedKey(c->db, c->argv[1]); return; }
static int txdb_save_tx(struct txdb *txdb, const uint256 *blkHash, const uint256 *txHash, mtime_t timestamp, const uint8 *buf, size_t len) { struct tx_ser_data txdata; struct buff *bufd; struct buff *bufk; char hashStr[80]; char *err; err = NULL; memset(&txdata, 0, sizeof txdata); if (blkHash) { memcpy(&txdata.blkHash, blkHash, sizeof *blkHash); } txdata.buf = (uint8 *)buf; txdata.len = len; txdata.timestamp = timestamp; uint256_snprintf_reverse(hashStr, sizeof hashStr, txHash); bufk = txdb_serialize_tx_key(txdb->tx_seq, hashStr); bufd = txdb_serialize_tx_data(&txdata); leveldb_put(txdb->db, txdb->wr_opts, buff_base(bufk), buff_curlen(bufk), buff_base(bufd), buff_curlen(bufd), &err); buff_free(bufk); buff_free(bufd); if (err) { Warning(LGPFX" failed to save tx %s: %s\n", hashStr, err); free(err); } return err != NULL; }
void localdb_incr_counter (localdb_t * localdb, char *key, unsigned long counter) { char *errptr = NULL; char lkey[18]; sprintf (lkey, "%s0", key); leveldb_put (localdb->db, localdb->writeoptions, lkey, strlen (lkey), (const char *) &counter, sizeof (unsigned long), &errptr); if (errptr) { printf ("\n%s", errptr); exit (1); } }
void put(db_t* db, size_t keySz, void* key, size_t valSz, void* val) { if(db->type == LEVELDB) { levelDB_t* leveldb = (levelDB_t*)db->database; leveldb_put(leveldb->db, leveldb->woptions, (char*)key, keySz, (char*)val, valSz, &(leveldb->err)); if(leveldb->err != NULL) { fprintf(stderr, "leveldb put failed!\n"); exit(-1); } leveldb_free(leveldb->err); leveldb->err = NULL; } else { kernelDB_t* kerneldb = (kernelDB_t*) db->database; kr_put(kerneldb->client, keySz, key, valSz, val); } }
void incrDecrCommand(redisClient *c, long long incr) { sds data; char *value; int64_t val, recore; size_t val_len; char *err = NULL; err = NULL; val_len = 0; value = leveldb_get(server.ds_db, server.roptions, c->argv[1]->ptr, sdslen((sds) c->argv[1]->ptr), &val_len, &err); if (err != NULL) { if (val_len > 0) leveldb_free(value); addReplyError(c, err); leveldb_free(err); return; } else if (val_len < 1) { val = 0; } else { val = strtoll(value, NULL, 10); } err = NULL; recore = val + incr; data = sdsfromlonglong(recore); leveldb_put(server.ds_db, server.woptions, c->argv[1]->ptr, sdslen((sds) c->argv[1]->ptr), data, sdslen(data), &err); if (err != NULL) { addReplyError(c, err); leveldb_free(err); } else { addReplyLongLong(c, recore); server.dirty++; signalModifiedKey(c->db, c->argv[1]); } sdsfree(data); leveldb_free(value); return; }
void localdb_set_interval (localdb_t * localdb, char *key, int interval) { char *errptr = NULL; char lkey[18]; sprintf (lkey, "%s1", key); leveldb_put (localdb->db, localdb->writeoptions, lkey, strlen (lkey), (const char *) &interval, sizeof (int), &errptr); if (errptr) { printf ("\n%s", errptr); exit (1); } }
/** * mdhim_leveldb_put * Stores a single key in the data store * * @param dbh in pointer to the leveldb handle * @param key in void * to the key to store * @param key_len in length of the key * @param data in void * to the value of the key * @param data_len in length of the value data * @param mstore_opts in additional options for the data store layer * * @return MDHIM_SUCCESS on success or MDHIM_DB_ERROR on failure */ int mdhim_leveldb_put(void *dbh, void *key, int key_len, void *data, int32_t data_len, struct mdhim_store_opts_t *mstore_opts) { leveldb_writeoptions_t *options; char *err = NULL; leveldb_t *db = (leveldb_t *) dbh; struct timeval start, end; gettimeofday(&start, NULL); options = (leveldb_writeoptions_t *) mstore_opts->db_ptr4; leveldb_put(db, options, key, key_len, data, data_len, &err); if (err != NULL) { mlog(MDHIM_SERVER_CRIT, "Error putting key/value in leveldb"); return MDHIM_DB_ERROR; } gettimeofday(&end, NULL); mlog(MDHIM_SERVER_DBG, "Took: %d seconds to put the record", (int) (end.tv_sec - start.tv_sec)); return MDHIM_SUCCESS; }
int leveldb_client_put(leveldb *db, const char *key, const char *value) { char *err = NULL; size_t keylen, vallen; if (!db || !key || !value) { kp_error("got a NULL argument\n"); return -1; } /* The lengths that we pass into leveldb_put() should NOT include * the null-zero! */ keylen = strlen(key); vallen = strlen(value); if (keylen < 1 || vallen < 1) { kp_error("got empty string for key (%zu) or val (%zu)\n", keylen, vallen); } kp_debug("putting [%s:%s] into db [%s]; keylen=%zu, vallen=%zu\n", key, value, db->name, keylen, vallen); /* leveldb_put() goes to internal leveldb code that will copy the key * and the value; we do not have to do this here in our leveldb client. * After our client function returns, the caller will be able to free * the key and the value. The leveldb code will call Slice() to slice * off the null-zero from the key and value strings we are passing it, * and just the character bytes will be stored in the db. */ leveldb_put(db->db, db->woptions, key, keylen, value, vallen, &err); if (err) { kp_error("leveldb_put() returned error: %s\n", err); free_err(&err); return -1; } kp_debug("leveldb_put() succeeded\n"); return 0; }
void txdb_confirm_one_tx(struct txdb *txdb, const uint256 *blkHash, const uint256 *txHash) { leveldb_iterator_t *iter; struct tx_entry *txe; char bkHashStr[80]; char txHashStr[80]; ASSERT(!uint256_iszero(blkHash)); ASSERT(!uint256_iszero(txHash)); txe = txdb_get_tx_entry(txdb, txHash); if (txe == NULL) { return; } if (txe->relevant == 0) { txdb_remove_from_hashtable(txdb, txHash); NOT_TESTED(); return; } if (!uint256_iszero(&txe->blkHash)) { /* * It's possible for the ASSERT below to fire if a tx is confirmed in * a block that is later orphaned. The tx should then be relayed again * until it finds its way in a new block. */ ASSERT(uint256_issame(&txe->blkHash, blkHash)); return; } peergroup_stop_broadcast_tx(btc->peerGroup, txHash); memcpy(&txe->blkHash, blkHash, sizeof *blkHash); uint256_snprintf_reverse(bkHashStr, sizeof bkHashStr, blkHash); uint256_snprintf_reverse(txHashStr, sizeof txHashStr, txHash); Warning(LGPFX" %s confirmed in %s\n", txHashStr, bkHashStr); NOT_TESTED(); iter = leveldb_create_iterator(txdb->db, txdb->rd_opts); leveldb_iter_seek_to_first(iter); while (leveldb_iter_valid(iter)) { struct tx_ser_key *txkey; struct tx_ser_data *txdata; struct buff *buf; const char *key; const char *val; size_t klen; size_t vlen; char *err = NULL; key = leveldb_iter_key(iter, &klen); txkey = txdb_deserialize_tx_key(key, klen); if (txkey == NULL || uint256_issame(txHash, &txkey->txHash) == 0) { free(txkey); leveldb_iter_next(iter); continue; } NOT_TESTED(); val = leveldb_iter_value(iter, &vlen); txdata = txdb_deserialize_tx_data(val, vlen); ASSERT(uint256_iszero(&txdata->blkHash)); ASSERT(txdata->timestamp != 0); memcpy(&txdata->blkHash, blkHash, sizeof *blkHash); buf = txdb_serialize_tx_data(txdata); leveldb_put(txdb->db, txdb->wr_opts, key, klen, buff_base(buf), buff_curlen(buf), &err); buff_free(buf); if (err) { Warning(LGPFX" failed to write tx entry: %s\n", err); } txdb_export_tx_info(txdb); free(txkey); free(txdata->buf); free(txdata); break; } leveldb_iter_destroy(iter); }
static void *mghandle(enum mg_event event, struct mg_connection *conn) { const struct mg_request_info *request_info = mg_get_request_info(conn); if (event == MG_NEW_REQUEST) { char *req=calloc(URL_STRING_MAX+1,sizeof(char)); struct in_addr saddr; strncpy(req,request_info->uri, URL_STRING_MAX); saddr.s_addr = ntohl(request_info->remote_ip); LOG_DEBUG(vlevel, _("Connection from: %s, request: %s\n"), inet_ntoa(saddr), req); if(strncmp(req, "/status\0", 8) == 0) { // XXX how to get more status mg_printf(conn, "HTTP/1.1 200 OK\r\n" "Content-Type: text/plain\r\n" "Content-Length: %d\r\n" "\r\n" "OK\r\n", 4); } else if(strncmp(req, "/set/", 5) == 0) { int n=strlen(req); while(n) { if(req[n]==':') { leveldb_put(dbh, wopt, req+5, n-5, req+n+1, strlen(req)-n-1, &errptr); if(errptr!=NULL) { LOG_ERROR(vlevel,_("leveldb_put(): %s\n"),errptr); mg_printf(conn, "HTTP/1.1 500 OK\r\n" "Content-Type: text/plain\r\n" "Content-Length: %d\r\n" "\r\n" "ERROR: %s\r\n", 9+strlen(errptr), errptr); } else { mg_printf(conn, "HTTP/1.1 200 OK\r\n" "Content-Type: text/plain\r\n" "Content-Length: %d\r\n" "\r\n" "OK\r\n", 4); } break; } n--; } if(n==0) { LOG_ERROR(vlevel,_("Malformed request\n")); mg_printf(conn, "HTTP/1.1 500 OK\r\n" "Content-Type: text/plain\r\n" "Content-Length: %d\r\n" "\r\n" "MALFORMED\r\n", 11); } } else if(strncmp(req, "/get/", 5) == 0) { size_t rlen=-1; char *tmp=leveldb_get(dbh, ropt, req+5, strlen(req)-5, &rlen, &errptr); if(rlen) { char *val=calloc(rlen+2,sizeof(char)); snprintf(val,rlen+1,"%s",tmp); LOG_DEBUG(vlevel, _("Found: %s for %s\n"),val,req+5); mg_printf(conn, "HTTP/1.1 200 OK\r\n" "Content-Type: text/plain\r\n" "Content-Length: %d\r\n" "\r\n" "%s\r\n", strlen(val)+2, val); free(val); } else { LOG_DEBUG(vlevel, _("Nothing found for %s\n"),req+5); mg_printf(conn, "HTTP/1.1 500 OK\r\n" "Content-Type: text/plain\r\n" "Content-Length: %d\r\n" "\r\n" "NOTFOUND\r\n", 10); } free(tmp); } else if(strncmp(req, "/pset/", 6) == 0) { // XXX } else if(strncmp(req, "/pget/", 6) == 0) { // XXX } else { // XXX LOG_ERROR(vlevel,_("Unknown/unhandled request\n")); } free(req); return ""; } else { return NULL; } }
int main(int argc, char** argv) { leveldb_t* db; leveldb_comparator_t* cmp; leveldb_cache_t* cache; leveldb_env_t* env; leveldb_options_t* options; leveldb_readoptions_t* roptions; leveldb_writeoptions_t* woptions; char* err = NULL; int run = -1; CheckCondition(leveldb_major_version() >= 1); CheckCondition(leveldb_minor_version() >= 1); snprintf(dbname, sizeof(dbname), "%s/leveldb_c_test-%di/meta/0", GetTempDir(), ((int) geteuid())); StartPhase("create_objects"); cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); env = leveldb_create_default_env(); cache = leveldb_cache_create_lru(100000); options = leveldb_options_create(); leveldb_options_set_comparator(options, cmp); leveldb_options_set_error_if_exists(options, 1); leveldb_options_set_cache(options, cache); leveldb_options_set_env(options, env); leveldb_options_set_info_log(options, NULL); leveldb_options_set_write_buffer_size(options, 100000); leveldb_options_set_paranoid_checks(options, 1); leveldb_options_set_max_open_files(options, 10); leveldb_options_set_block_size(options, 1024); leveldb_options_set_block_restart_interval(options, 8); leveldb_options_set_compression(options, leveldb_no_compression); roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(roptions, 1); leveldb_readoptions_set_fill_cache(roptions, 0); woptions = leveldb_writeoptions_create(); leveldb_writeoptions_set_sync(woptions, 1); StartPhase("destroy"); leveldb_destroy_db(options, dbname, &err); Free(&err); StartPhase("open"); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); StartPhase("put"); leveldb_put(db, woptions, "foo", 3, "hello", 5, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); StartPhase("compactall"); leveldb_compact_range(db, NULL, 0, NULL, 0); CheckGet(db, roptions, "foo", "hello"); StartPhase("compactrange"); leveldb_compact_range(db, "a", 1, "z", 1); CheckGet(db, roptions, "foo", "hello"); StartPhase("writebatch"); { leveldb_writebatch_t* wb = leveldb_writebatch_create(); leveldb_writebatch_put(wb, "foo", 3, "a", 1); leveldb_writebatch_clear(wb); leveldb_writebatch_put(wb, "bar", 3, "b", 1); leveldb_writebatch_put(wb, "box", 3, "c", 1); leveldb_writebatch_delete(wb, "bar", 3); leveldb_write(db, woptions, wb, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); int pos = 0; leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); CheckCondition(pos == 3); leveldb_writebatch_destroy(wb); } StartPhase("iter"); { leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_first(iter); CheckCondition(leveldb_iter_valid(iter)); CheckIter(iter, "box", "c"); leveldb_iter_next(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_prev(iter); CheckIter(iter, "box", "c"); leveldb_iter_prev(iter); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_last(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_seek(iter, "b", 1); CheckIter(iter, "box", "c"); leveldb_iter_get_error(iter, &err); CheckNoError(err); leveldb_iter_destroy(iter); } StartPhase("approximate_sizes"); { int i; int n = 20000; char keybuf[100]; char valbuf[100]; uint64_t sizes[2]; const char* start[2] = { "a", "k00000000000000010000" }; size_t start_len[2] = { 1, 21 }; const char* limit[2] = { "k00000000000000010000", "z" }; size_t limit_len[2] = { 21, 1 }; leveldb_writeoptions_set_sync(woptions, 0); for (i = 0; i < n; i++) { snprintf(keybuf, sizeof(keybuf), "k%020d", i); snprintf(valbuf, sizeof(valbuf), "v%020d", i); leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf), &err); CheckNoError(err); } leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes); CheckCondition(sizes[0] > 0); CheckCondition(sizes[1] > 0); } StartPhase("property"); { char* prop = leveldb_property_value(db, "nosuchprop"); /* CheckCondition(prop == NULL);*/ prop = leveldb_property_value(db, "leveldb.stats"); CheckCondition(prop != NULL); Free(&prop); } StartPhase("snapshot"); { const leveldb_snapshot_t* snap; snap = leveldb_create_snapshot(db); leveldb_delete(db, woptions, "foo", 3, &err); CheckNoError(err); leveldb_readoptions_set_snapshot(roptions, snap); CheckGet(db, roptions, "foo", "hello"); leveldb_readoptions_set_snapshot(roptions, NULL); CheckGet(db, roptions, "foo", NULL); leveldb_release_snapshot(db, snap); } StartPhase("repair"); { leveldb_close(db); leveldb_options_set_error_if_exists(options, 0); leveldb_repair_db(options, dbname, &err); CheckNoError(err); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); leveldb_options_set_error_if_exists(options, 1); } StartPhase("filter"); for (run = 0; run < 2; run++) { // First run uses custom filter, second run uses bloom filter CheckNoError(err); leveldb_filterpolicy_t* policy; if (run == 0) { policy = leveldb_filterpolicy_create( NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName); } else { policy = leveldb_filterpolicy_create_bloom(10); } // Create new database leveldb_close(db); leveldb_destroy_db(options, dbname, &err); leveldb_options_set_filter_policy(options, policy); db = leveldb_open(options, dbname, &err); CheckNoError(err); leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err); CheckNoError(err); leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err); CheckNoError(err); leveldb_compact_range(db, NULL, 0, NULL, 0); fake_filter_result = 1; CheckGet(db, roptions, "foo", "foovalue"); CheckGet(db, roptions, "bar", "barvalue"); if (phase == 0) { // Must not find value when custom filter returns false fake_filter_result = 0; CheckGet(db, roptions, "foo", NULL); CheckGet(db, roptions, "bar", NULL); fake_filter_result = 1; CheckGet(db, roptions, "foo", "foovalue"); CheckGet(db, roptions, "bar", "barvalue"); } leveldb_options_set_filter_policy(options, NULL); leveldb_filterpolicy_destroy(policy); } StartPhase("cleanup"); leveldb_close(db); leveldb_options_destroy(options); leveldb_readoptions_destroy(roptions); leveldb_writeoptions_destroy(woptions); leveldb_cache_destroy(cache); leveldb_comparator_destroy(cmp); leveldb_env_destroy(env); fprintf(stderr, "PASS\n"); return 0; }
static int leveldb_backend_write(struct leveldb_backend *s, void *state, struct dnet_cmd *cmd, void *data) { struct dnet_ext_list elist; int err = -EINVAL; char *error_string = NULL; struct dnet_io_attr *io = data; void *read_data = NULL; dnet_ext_list_init(&elist); dnet_convert_io_attr(io); data += sizeof(struct dnet_io_attr); /* Combine data with empty extension list header */ err = dnet_ext_list_combine(&data, &io->size, &elist); if (err != 0) goto err_out_exit; /* * key should be locked by elliptics here, so it is safe to run read-modify-write cycle * if one performs write without lock we do not really care that one write may overwrite another one */ if (io->offset || (io->flags & DNET_IO_FLAGS_APPEND)) { size_t data_size; size_t offset = io->offset; read_data = leveldb_get(s->db, s->roptions, (const char *)io->id, DNET_ID_SIZE, &data_size, &error_string); if (error_string || !read_data) { free(error_string); error_string = NULL; goto plain_write; } if (io->flags & DNET_IO_FLAGS_APPEND) { io->offset = 0; offset = data_size; } /* * XXX: Account for extended header */ if (io->offset > data_size) { err = -ERANGE; goto err_out_exit; } if (offset + io->size > data_size) { read_data = realloc(read_data, data_size + io->size); if (!read_data) { err = -ENOMEM; goto err_out_exit; } } memcpy(read_data + offset, data, io->size); data = read_data; if (offset + io->size > data_size) io->size = offset + io->size; else io->size = data_size; } plain_write: leveldb_put(s->db, s->woptions, (const char *)cmd->id.id, DNET_ID_SIZE, data, io->size, &error_string); if (error_string) goto err_out_free; if (io->flags & DNET_IO_FLAGS_WRITE_NO_FILE_INFO) { cmd->flags |= DNET_FLAGS_NEED_ACK; err = 0; goto err_out_exit; } err = dnet_send_file_info_ts_without_fd(state, cmd, 0, io->size, &elist.timestamp); if (err < 0) goto err_out_free; dnet_backend_log(DNET_LOG_NOTICE, "%s: leveldb: : WRITE: Ok: offset: %llu, size: %llu, ioflags: %x.\n", dnet_dump_id(&cmd->id), (unsigned long long)io->offset, (unsigned long long)io->size, io->flags); err_out_free: free(data); err_out_exit: dnet_ext_list_destroy(&elist); if (err < 0) dnet_backend_log(DNET_LOG_ERROR, "%s: leveldb: : WRITE: error: %s: %d.\n", dnet_dump_id(&cmd->id), error_string, err); free(read_data); free(error_string); return err; }
// Adds an event to the tablet. // // tablet - The tablet. // event - The event to add. // // Returns 0 if successful, otherwise returns -1. int sky_tablet_add_event(sky_tablet *tablet, sky_event *event) { int rc; char *errptr = NULL; void *new_data = NULL; void *data = NULL; sky_data_object *data_object = NULL; sky_data_descriptor *descriptor = NULL; sky_cursor cursor; memset(&cursor, 0, sizeof(cursor)); assert(tablet != NULL); assert(event != NULL); // Make sure that this event is being added to the correct tablet. sky_tablet *target_tablet = NULL; rc = sky_table_get_target_tablet(tablet->table, event->object_id, &target_tablet); check(rc == 0, "Unable to determine target tablet"); check(tablet == target_tablet, "Event added to invalid tablet; IDX:%d of %d, OID:%d", tablet->index, tablet->table->tablet_count, event->object_id); // Retrieve the existing value. size_t data_length; data = (void*)leveldb_get(tablet->leveldb_db, tablet->readoptions, (const char*)&event->object_id, sizeof(event->object_id), &data_length, &errptr); check(errptr == NULL, "LevelDB get error: %s", errptr); // Find the insertion point on the path. size_t insert_offset = 0; size_t event_length; // If the object doesn't exist yet then just set the single event. Easy peasy. if(data == NULL) { event_length = sky_event_sizeof(event); new_data = calloc(1, event_length); check_mem(new_data); insert_offset = 0; } // If the object does exist, we need to find where to insert the event data. // Also, we need to strip off any state which is redundant at the point of // insertion. else { void *insert_ptr = NULL; // Initialize data descriptor. descriptor = sky_data_descriptor_create(); check_mem(descriptor); rc = sky_data_descriptor_init_with_event(descriptor, event); check(rc == 0, "Unable to initialize data descriptor for event insert"); // Initialize data object. data_object = calloc(1, descriptor->data_sz); check_mem(data); // Attach data & descriptor to the cursor. cursor.data_descriptor = descriptor; cursor.data = (void*)data_object; // Initialize the cursor. rc = sky_cursor_set_ptr(&cursor, data, data_length); check(rc == 0, "Unable to set pointer on cursor"); // Loop over cursor until we reach the event insertion point. while(!cursor.eof) { // Retrieve event insertion pointer once the timestamp is reached. if(data_object->ts >= event->timestamp) { insert_ptr = cursor.ptr; break; } // Move to next event. check(sky_cursor_next_event(&cursor) == 0, "Unable to move to next event"); } // If no insertion point was found then append the event to the // end of the path. if(insert_ptr == NULL) { insert_ptr = data + data_length; } insert_offset = insert_ptr - data; // Clear off any object data on the event that matches // what is the current state of the event in the database. uint32_t i; for(i=0; i<event->data_count; i++) { // Ignore any action properties. if(event->data[i]->key > 0) { sky_data_property_descriptor *property_descriptor = &descriptor->property_zero_descriptor[event->data[i]->key]; // If the values match then splice this from the array. // Compare strings. void *a = &event->data[i]->value; void *b = ((void*)data_object)+property_descriptor->offset; size_t n = sky_data_type_sizeof(event->data[i]->data_type); bool is_equal = false; if(event->data[i]->data_type == SKY_DATA_TYPE_STRING) { is_equal = sky_string_bequals((sky_string*)b, event->data[i]->string_value); } // Compare other types. else if(memcmp(a, b, n) == 0) { is_equal = true; } // If the data is equal then remove it. if(is_equal) { sky_event_data_free(event->data[i]); if(i < event->data_count - 1) { memmove(&event->data[i], &event->data[i+1], (event->data_count-i-1) * sizeof(*event->data)); } i--; event->data_count--; } } } // Determine the serialized size of the event. If the event is // completely redundant (e.g. it is a data-only event and the event // matches the current object state) then don't allocate space for a // new path value. event_length = sky_event_sizeof(event); if(event_length > 0) { // Allocate space for the existing data plus the new data. new_data = calloc(1, data_length + event_length); check_mem(new_data); // Copy in data before event. if(insert_offset > 0) { memmove(new_data, data, insert_ptr-data); } // Copy in data after event. if(insert_offset < data_length) { event_length = sky_event_sizeof(event); memmove(new_data+insert_offset+event_length, data+insert_offset, data_length-insert_offset); } } } // If no space was allocated then it means the event is redundant and // should be ignored. if(new_data != NULL) { // If the object doesn't exist then just set the event as the data. size_t event_sz; rc = sky_event_pack(event, new_data + insert_offset, &event_sz); check(rc == 0, "Unable to pack event"); check(event_sz == event_length, "Expected event size (%ld) does not match actual event size (%ld)", event_length, event_sz); leveldb_put(tablet->leveldb_db, tablet->writeoptions, (const char*)&event->object_id, sizeof(event->object_id), new_data, data_length + event_length, &errptr); check(errptr == NULL, "LevelDB put error: %s", errptr); } free(data_object); sky_data_descriptor_free(descriptor); free(data); free(new_data); return 0; error: if(errptr) leveldb_free(errptr); sky_data_descriptor_free(descriptor); if(data) free(data); if(new_data) free(new_data); if(data_object) free(data_object); return -1; }
int main () { unsigned int N_KEYS; unsigned int N_BATCH; printf("\n number of keys:"); scanf("%u",&N_KEYS); printf("\n batch size:"); scanf("%u",&N_BATCH); tinymt32_t tinymt32; //initialize random generator tinymt32_init (&tinymt32, 0); int iter; int sec_iter; int stop; int counter; //initialize database char *errptr = NULL; leveldb_options_t *options = leveldb_options_create (); /* initializeOptions */ leveldb_options_set_create_if_missing (options, 1); leveldb_options_set_write_buffer_size(options,120000000 ); leveldb_options_set_max_open_files(options,800000); /*open Database */ leveldb_t *db_helper = leveldb_open (options, "/mnt/database/database_helper", &errptr); leveldb_t *db = leveldb_open (options, "/mnt/database/database", &errptr); leveldb_readoptions_t *readoptions = leveldb_readoptions_create (); leveldb_writeoptions_t *writeoptions = leveldb_writeoptions_create (); leveldb_writebatch_t* batch=leveldb_writebatch_create(); int64_t diff = zclock_time (); //write into database_helper iter=0; while( iter < N_KEYS) { sec_iter=0; while((iter<N_KEYS) && (sec_iter<N_BATCH)){ unsigned int key = tinymt32_generate_uint32 (&tinymt32); unsigned int val = tinymt32_generate_uint32 (&tinymt32); leveldb_writebatch_put (batch, (const char *) &key, sizeof (int), (const char *) &val, sizeof (int)); sec_iter++; iter++; } leveldb_write( db_helper, writeoptions, batch, &errptr); if(errptr!=NULL){ printf("\n%s",errptr); } assert (errptr == NULL); leveldb_writebatch_clear(batch); } leveldb_writebatch_destroy(batch); diff = zclock_time () - diff; printf ("\nrandom write: %d", diff); diff = zclock_time (); //write sequentially into db leveldb_iterator_t *liter = leveldb_create_iterator (db_helper, readoptions); leveldb_iter_seek_to_first (liter); while (leveldb_iter_valid (liter)) { size_t length; char key[4]; memcpy (key, leveldb_iter_key (liter, &length), 4); char val[4]; memcpy (val, leveldb_iter_value (liter, &length), 4); leveldb_iter_get_error (liter, &errptr); assert (errptr == NULL); leveldb_put (db, writeoptions, (const char *) &key, sizeof (int), (const char *) &val, sizeof (int), &errptr); assert (errptr == NULL); leveldb_iter_next (liter); } diff = zclock_time () - diff; leveldb_close (db); leveldb_close (db_helper); }
int main(int argc, char** argv) { leveldb_t* db; leveldb_comparator_t* cmp; leveldb_cache_t* cache; leveldb_env_t* env; leveldb_options_t* options; leveldb_readoptions_t* roptions; leveldb_writeoptions_t* woptions; char* err = NULL; #if defined(LEVELDB_PLATFORM_WINDOWS) snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test"); #else snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d", ((int) geteuid())); #endif StartPhase("create_objects"); cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); env = leveldb_create_default_env(); cache = leveldb_cache_create_lru(100000); options = leveldb_options_create(); leveldb_options_set_comparator(options, cmp); leveldb_options_set_error_if_exists(options, 1); leveldb_options_set_cache(options, cache); leveldb_options_set_env(options, env); leveldb_options_set_info_log(options, NULL); leveldb_options_set_write_buffer_size(options, 100000); leveldb_options_set_paranoid_checks(options, 1); leveldb_options_set_max_open_files(options, 10); leveldb_options_set_block_size(options, 1024); leveldb_options_set_block_restart_interval(options, 8); leveldb_options_set_compression(options, leveldb_no_compression); roptions = leveldb_readoptions_create(); leveldb_readoptions_set_verify_checksums(roptions, 1); leveldb_readoptions_set_fill_cache(roptions, 0); woptions = leveldb_writeoptions_create(); leveldb_writeoptions_set_sync(woptions, 1); StartPhase("destroy"); leveldb_destroy_db(options, dbname, &err); leveldb_free(&err); StartPhase("open_error"); db = leveldb_open(options, dbname, &err); CheckCondition(err != NULL); leveldb_free(&err); StartPhase("open"); leveldb_options_set_create_if_missing(options, 1); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); StartPhase("put"); leveldb_put(db, woptions, "foo", 3, "hello", 5, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); StartPhase("writebatch"); { int pos = 0; leveldb_writebatch_t* wb = leveldb_writebatch_create(); leveldb_writebatch_put(wb, "foo", 3, "a", 1); leveldb_writebatch_clear(wb); leveldb_writebatch_put(wb, "bar", 3, "b", 1); leveldb_writebatch_put(wb, "box", 3, "c", 1); leveldb_writebatch_delete(wb, "bar", 3); leveldb_write(db, woptions, wb, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); CheckCondition(pos == 3); leveldb_writebatch_destroy(wb); } StartPhase("iter"); { leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_first(iter); CheckCondition(leveldb_iter_valid(iter)); CheckIter(iter, "box", "c"); leveldb_iter_next(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_prev(iter); CheckIter(iter, "box", "c"); leveldb_iter_prev(iter); CheckCondition(!leveldb_iter_valid(iter)); leveldb_iter_seek_to_last(iter); CheckIter(iter, "foo", "hello"); leveldb_iter_seek(iter, "b", 1); CheckIter(iter, "box", "c"); leveldb_iter_get_error(iter, &err); CheckNoError(err); leveldb_iter_destroy(iter); } StartPhase("approximate_sizes"); { int i; int n = 20000; char keybuf[100]; char valbuf[100]; uint64_t sizes[2]; const char* start[2] = { "a", "k00000000000000010000" }; size_t start_len[2] = { 1, 21 }; const char* limit[2] = { "k00000000000000010000", "z" }; size_t limit_len[2] = { 21, 1 }; leveldb_writeoptions_set_sync(woptions, 0); for (i = 0; i < n; i++) { snprintf(keybuf, sizeof(keybuf), "k%020d", i); snprintf(valbuf, sizeof(valbuf), "v%020d", i); leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf), &err); CheckNoError(err); } leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes); CheckCondition(sizes[0] > 0); CheckCondition(sizes[1] > 0); } StartPhase("property"); { char* prop = leveldb_property_value(db, "nosuchprop"); CheckCondition(prop == NULL); prop = leveldb_property_value(db, "leveldb.stats"); CheckCondition(prop != NULL); leveldb_free(&prop); } StartPhase("snapshot"); { const leveldb_snapshot_t* snap; snap = leveldb_create_snapshot(db); leveldb_delete(db, woptions, "foo", 3, &err); CheckNoError(err); leveldb_readoptions_set_snapshot(roptions, snap); CheckGet(db, roptions, "foo", "hello"); leveldb_readoptions_set_snapshot(roptions, NULL); CheckGet(db, roptions, "foo", NULL); leveldb_release_snapshot(db, snap); } StartPhase("repair"); { leveldb_close(db); leveldb_options_set_create_if_missing(options, 0); leveldb_options_set_error_if_exists(options, 0); leveldb_repair_db(options, dbname, &err); CheckNoError(err); db = leveldb_open(options, dbname, &err); CheckNoError(err); CheckGet(db, roptions, "foo", NULL); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); } StartPhase("cleanup"); leveldb_close(db); leveldb_options_destroy(options); leveldb_readoptions_destroy(roptions); leveldb_writeoptions_destroy(woptions); leveldb_cache_destroy(cache); leveldb_comparator_destroy(cmp); leveldb_env_destroy(env); fprintf(stderr, "PASS\n"); return 0; }
int read_csv() { leveldb_writeoptions_t *write_options; char *input_filename = option_get_str("input_file"); const char *key_ptr, *val_ptr; size_t key_len, val_len; uint64_t input_row = 0; int stats_every = option_get_int("stats_every"); char *err = NULL; char *buffer = NULL; char *input_deliminator = option_get_str("input_deliminator"); int quiet = option_get_int("quiet"); const char *comma_ptr; size_t buffer_len; FILE *input_file = stdin; if (input_filename) { input_file = fopen(input_filename, "r"); } write_options = leveldb_writeoptions_create(); while ((buffer_len = simplehttp_get_line(input_file, &buffer))) { input_row++; key_ptr = buffer; if (!quiet && (input_row > 0) && ((input_row % stats_every) == 0)) { fprintf(stderr, "line #%"PRIu64"\r", input_row); } // find the index of the first deliminator if ((comma_ptr = simplehttp_strnchr(buffer, buffer_len, *input_deliminator)) == NULL) { fprintf(stderr, "%s ERROR: LINE %"PRIu64" DELIMINATOR NOT FOUND %s\n", NAME, input_row, buffer); return 0; } key_len = comma_ptr - buffer; if (!key_len) { fprintf(stderr, "%s WARNING: SKIPPING EMPTY KEY, LINE %"PRIu64" %s\n", NAME, input_row, buffer); continue; } val_ptr = comma_ptr; val_ptr += strlen(input_deliminator); val_len = (buffer + buffer_len) - val_ptr; if (*(val_ptr + val_len - 1) == '\n') { val_len--; } leveldb_put(ldb, write_options, key_ptr, key_len, val_ptr, val_len, &err); if (err) { break; } } leveldb_writeoptions_destroy(write_options); if (err) { fprintf(stderr, "Error (LINE %"PRIu64"): %s\n", input_row, err); return 0; } return 1; }
void db_put(db_t *db, const db_writeoptions_t *wo, const char *key, size_t klen, const char *value, size_t vlen, char **errptr) { leveldb_put(db, wo, key, klen, value, vlen, errptr); }
void db_leveldb_put(dbi_t *key, dbi_t *val) { leveldb_put(leveldb_db, leveldb_woptions, key->data, key->len, val->data, val->len, NULL); }