counter_t *counter_init(const char *path, uint64_t readers) { counter_t *lc = NULL; if((lc = calloc(1, sizeof(counter_t))) == NULL) { perror("calloc"); return NULL; } // Setup and open the lmdb enviornment MDB_CHECK(mdb_env_create(&lc->env), MDB_SUCCESS, NULL); MDB_CHECK(mdb_env_set_maxreaders(lc->env, readers), MDB_SUCCESS, NULL); MDB_CHECK(mdb_env_set_mapsize(lc->env, MDB_MAPSIZE), MDB_SUCCESS, NULL); MDB_CHECK(mdb_env_open(lc->env, path, MDB_WRITEMAP | MDB_MAPASYNC | MDB_NOSUBDIR, 0664), MDB_SUCCESS, NULL); MDB_txn *txn = NULL; MDB_CHECK(mdb_txn_begin(lc->env, NULL, 0, &txn), MDB_SUCCESS, NULL); if((lc->dbi = calloc(1, sizeof(MDB_dbi))) == NULL) { perror("calloc"); return NULL; } MDB_CHECK(mdb_dbi_open(txn, NULL, 0, lc->dbi), MDB_SUCCESS, NULL); mdb_txn_commit(txn); return lc; }
int main(void) { check(mdb_env_create(&env)); check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL)); check(mdb_env_set_maxreaders(env, 126)); check(mdb_env_set_maxdbs(env, 1)); if(! access(DB_PATH, X_OK)) { system("rm -rf " DB_PATH); } check(mkdir(DB_PATH, 0777)); check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644)); new_txn(); check(mdb_dbi_open(txn, NULL, 0, &dbi)); put("a"); put("b"); put("baa"); put("d"); new_txn(); check(mdb_cursor_open(txn, dbi, &c1)); check(mdb_cursor_get(c1, &keyv, &valv, MDB_LAST)); check(mdb_cursor_del(c1, 0)); check(mdb_cursor_del(c1, 0)); new_txn(); }
static inline int _txn_begin(db_t db, MDB_txn *parent, unsigned int flags, MDB_txn **txn){ int r; // here's the deal - we need to grow the mapsize if: // * we are a write transaction, and there is less than 1gb overhead // * txn fails with MDB_MAP_RESIZED // However, mapsize may only be altered if there are no active txns, including // snapshots. // Additional pthread_mutex_lock(&db->mutex); size_t newsize = parent ? 0 : _env_newsize(db, flags); if(!newsize) r = _mdb_txn_begin(db->env, parent, flags, txn); if(!parent && (newsize || MDB_MAP_RESIZED == r)){ while(db->txns) pthread_cond_wait(&db->cond, &db->mutex); do{ mdb_env_set_mapsize(db->env, newsize); newsize = _env_newsize(db, flags); r = _mdb_txn_begin(db->env, parent, flags, txn); }while(newsize || MDB_MAP_RESIZED == r); } if(MDB_SUCCESS == r) db->txns++; else if(0 == db->txns) pthread_cond_signal(&db->cond); pthread_mutex_unlock(&db->mutex); return r; }
static void db_connect() { MDB_dbi dbi_session; MDB_dbi dbi_session_id; MDB_dbi dbi_event; MDB_dbi dbi_ip; LMDB_CHECK(mdb_env_create(&env)); LMDB_CHECK(mdb_env_set_mapsize(env, 300000L * 4096L)); LMDB_CHECK(mdb_env_set_maxdbs(env, 30)); #if defined(MDBX_LIFORECLAIM) LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664)); #else LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664)); #endif MDB_txn *txn; // transaction init LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn)); // open database in read-write mode LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); // transaction commit LMDB_CHECK(mdb_txn_commit(txn)); printf("Connection open\n"); }
DB(const char* db_path, bool use_leveldb_) : use_leveldb(use_leveldb_), count(0), batch(NULL) { // Open db if (use_leveldb) { // leveldb LOG(INFO) << "Opening leveldb " << db_path; leveldb::Options options; options.error_if_exists = true; options.create_if_missing = true; options.write_buffer_size = 268435456; leveldb::Status status = leveldb::DB::Open(options, db_path, &db); CHECK(status.ok()) << "Failed to open leveldb " << db_path << ". Is it already existing?"; batch = new leveldb::WriteBatch(); } else { // lmdb LOG(INFO) << "Opening lmdb " << db_path; CHECK_EQ(mkdir(db_path, 0744), 0) << "mkdir " << db_path << "failed"; CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB << "mdb_env_set_mapsize failed"; CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS) << "mdb_env_open failed"; CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS) << "mdb_txn_begin failed"; CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS) << "mdb_open failed. Does the lmdb already exist? "; } }
void LMDB::Open(const string& source, Mode mode) { MDB_CHECK(mdb_env_create(&mdb_env_)); MDB_CHECK(mdb_env_set_mapsize(mdb_env_, LMDB_MAP_SIZE)); if (mode == NEW) { CHECK_EQ(mkdir(source.c_str(), 0744), 0) << "mkdir " << source << " failed"; } int flags = 0; if (mode == READ) { flags = MDB_RDONLY | MDB_NOTLS; } int rc = mdb_env_open(mdb_env_, source.c_str(), flags, 0664); #ifndef ALLOW_LMDB_NOLOCK MDB_CHECK(rc); #else if (rc == EACCES) { LOG(WARNING) << "Permission denied. Trying with MDB_NOLOCK ..."; // Close and re-open environment handle mdb_env_close(mdb_env_); MDB_CHECK(mdb_env_create(&mdb_env_)); // Try again with MDB_NOLOCK flags |= MDB_NOLOCK; MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664)); } else { MDB_CHECK(rc); } #endif LOG(INFO) << "Opened lmdb " << source; }
int mail_cache_db_open(const char * filename, struct mail_cache_db ** pcache_db) { int r; struct mail_cache_db * cache_db; MDB_env *env; r = mdb_env_create(&env); if (r != 0) return -1; r = mdb_env_set_mapsize(env, 512*1024*1024 /*max mmap and file size*/); if (r != 0) return -1; r = mdb_env_open(env, filename, MDB_NOSUBDIR, 0660); if (r != 0) goto close_db; cache_db = mail_cache_lmdb_new(env); if (cache_db == NULL) goto close_db; * pcache_db = cache_db; return 0; close_db: mdb_env_close(env); return -1; }
// will fail if this process has active txns/snapshots // supplied mapsize must be a multiple of the OS pagesize int db_set_mapsize(db_t db, size_t mapsize){ int r = -1; pthread_mutex_lock(&db->mutex); if(0 == db->txns) r = mdb_env_set_mapsize(db->env, mapsize); pthread_mutex_unlock(&db->mutex); return r; }
/* * Class: jmdb_DatabaseWrapper * Method: envSetMapSize * Signature: (JJ)V */ JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_envSetMapSize(JNIEnv *vm, jclass clazz, jlong envL, jlong size) { MDB_env *envC = (MDB_env*) envL; int code = mdb_env_set_mapsize(envC, size); if (code) { throwDatabaseException(vm, code); } }
LMDBEngineFactory::LMDBEngineFactory(const Properties& props) : m_env(NULL), m_env_opened(false) { ParseConfig(props, m_cfg); int rc = mdb_env_create(&m_env); DEBUG_LOG("Create env %d", rc); mdb_env_set_mapsize(m_env, 10000000 * 100 * 32L / 10); }
kvenv_t *kvenv_new_lmdb(const char *dbpath, uint64_t max_dbsize, uint32_t max_dbs) { kvenv_lmdb_t *kvenv_lmdb = (kvenv_lmdb_t*)zmalloc(sizeof(kvenv_lmdb_t)); memset(kvenv_lmdb, 0, sizeof(kvenv_lmdb_t)); kvenv_lmdb->kvenv.dbclass = "lmdb"; kvenv_lmdb->kvenv.max_dbsize = max_dbsize; kvenv_lmdb->kvenv.max_dbs = max_dbs; int rc = mdb_env_create(&kvenv_lmdb->env); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_create() failed."); return NULL; } int maxreaders = 256; if ( max_dbs == 0 ){ max_dbs = 256; } if ( max_dbsize == 0 ) { max_dbsize = 1024L * 1024L * 1024L * 4L; } rc = mdb_env_set_mapsize(kvenv_lmdb->env, max_dbsize); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_set_mapsize() failed."); return NULL; } rc = mdb_env_set_maxreaders(kvenv_lmdb->env, maxreaders); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_set_maxreaders() failed."); return NULL; } rc = mdb_env_set_maxdbs(kvenv_lmdb->env, max_dbs); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_set_maxdbs() failed."); return NULL; } /*rc = mdb_env_open(lmdb->env, dbpath, MDB_FIXEDMAP | MDB_NOSYNC, 0640); */ /*rc = mdb_env_open(lmdb->env, dbpath, MDB_MAPASYNC | MDB_WRITEMAP | MDB_NOTLS , 0640); */ /*rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_MAPASYNC | MDB_WRITEMAP, 0640); */ /*rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_NOMETASYNC, 0640); */ rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_NOSYNC, 0640); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_open() failed. dbpath=%s error: %s", dbpath, mdb_strerror(rc)); return NULL; } return (kvenv_t*)kvenv_lmdb; }
// Open a LMDB database. MDB_env *database_open(const char *path, size_t mapsize, unsigned int nbr_threads) { MDB_env *env = NULL; mdb_env_create(&env); mdb_env_set_mapsize(env, mapsize); mdb_env_set_maxreaders(env, nbr_threads); mdb_env_open(env, path, 0, 0664); return (env); }
/* {{{ proto lmDB\\Env::set_maxdbs(int) Set the maximum number of named databases for the environment.*/ PHP_METHOD(lmdb_env, set_maxdbs) { int dbs = 0; int rtr; lmdb_env_object * intern = (lmdb_env_object *)zend_object_store_get_object(getThis() TSRMLS_CC); if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "l", &dbs) == FAILURE) { return; } rtr = mdb_env_set_mapsize(intern->env, dbs); RETURN_LONG(rtr); }
static void db_connect() { LMDB_CHECK(mdb_env_create(&env)); LMDB_CHECK(mdb_env_set_mapsize(env, 3L * 1024L * 1024L * 1024L)); LMDB_CHECK(mdb_env_set_maxdbs(env, 30)); #if defined(MDBX_LIFORECLAIM) LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664)); #else LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664)); #endif printf("Connection open\n"); }
void db_remap(db_t db){ MDB_envinfo info; int r; pthread_mutex_lock(&db->mutex); while(db->txns) pthread_cond_wait(&db->cond, &db->mutex); r = mdb_env_info(db->env, &info); assert(MDB_SUCCESS == r); mdb_env_set_mapsize(db->env, info.me_mapsize); pthread_mutex_unlock(&db->mutex); }
bool CDbxMdb::Remap() { m_dwFileSize += 0x100000; mdb_env_set_mapsize(m_pMdbEnv, m_dwFileSize); int mode = MDB_NOSYNC | MDB_NOSUBDIR; if (m_bReadOnly) mode += MDB_RDONLY; else mode += MDB_WRITEMAP; return mdb_env_open(m_pMdbEnv, _T2A(m_tszProfileName), mode, 0664) == MDB_SUCCESS; }
void LMDB::Open(const string& source, Mode mode) { MDB_CHECK(mdb_env_create(&mdb_env_)); MDB_CHECK(mdb_env_set_mapsize(mdb_env_, LMDB_MAP_SIZE)); if (mode == NEW) { CHECK_EQ(mkdir(source.c_str(), 0744), 0) << "mkdir " << source << "failed"; } int flags = 0; if (mode == READ) { flags = MDB_RDONLY | MDB_NOTLS; } MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664)); LOG(INFO) << "Opened lmdb " << source; }
int LMDBStore::open(const char *path, bool durable) { int rc; rc = mdb_env_create(&env); rc = mdb_env_set_mapsize(env, size_t(1048576000)); rc = mdb_env_set_maxdbs(env, 1024); rc = mdb_env_open(env, path, MDB_NOSYNC | MDB_WRITEMAP, 0664); MDB_txn *txn; rc = mdb_txn_begin(env, NULL, 0, &txn); rc = mdb_open(txn, NULL, 0, &dbi); rc = mdb_txn_commit(txn); return HCAT_SUCCESS; }
static int dbase_open(struct cache *cache, const char *handle) { long page_size = sysconf(_SC_PAGESIZE); if (page_size <= 0) { return KNOT_EINVAL; } int ret = mdb_env_create(&cache->env); if (ret != 0) { return ret; } size_t map_size = (LMDB_MAPSIZE / page_size) * page_size; ret = mdb_env_set_mapsize(cache->env, map_size); if (ret != 0) { mdb_env_close(cache->env); return ret; } ret = mdb_env_open(cache->env, handle, 0, 0644); if (ret != 0) { mdb_env_close(cache->env); return ret; } MDB_txn *txn = NULL; ret = mdb_txn_begin(cache->env, NULL, 0, &txn); if (ret != 0) { mdb_env_close(cache->env); return ret; } ret = mdb_open(txn, NULL, MDB_DUPSORT, &cache->dbi); if (ret != 0) { mdb_txn_abort(txn); mdb_env_close(cache->env); return ret; } ret = mdb_txn_commit(txn); if (ret != 0) { mdb_env_close(cache->env); return ret; } return 0; }
int main(int argc,char * argv[]) { int rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_cursor *cursor; char sval[MAX_DATA_ALLOCATE_SIZE], kval[MAX_KEY_ALLOCATE_SIZE]; /* Note: Most error checking omitted for simplicity */ rc = mdb_env_create(&env); mdb_env_set_mapsize(env, MAX_DB_SIZE); mdb_env_set_maxdbs(env, (MDB_dbi)10); rc = mdb_env_open(env, "./demoDB", 0, 0664); rc = mdb_txn_begin(env, NULL, 0, &txn); rc = mdb_dbi_open(txn, "what", 0, &dbi); // rc = mdb_put(txn, dbi, &key, &data, 0); rc = mdb_txn_commit(txn); if (rc) { fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc)); goto leave; } fprintf(stderr, "print out old data:\n"); rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); rc = mdb_cursor_open(txn, dbi, &cursor); key.mv_size = sizeof(kval); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; memset(kval, argv[1][0], sizeof(kval)); rc = mdb_cursor_get(cursor, &key, &data, MDB_SET); fprintf(stderr, "key: %s, data: %s\n",(char *) key.mv_data,(char *) data.mv_data); mdb_cursor_close(cursor); mdb_txn_abort(txn); leave: mdb_dbi_close(env, dbi); mdb_env_close(env); return 0; }
int db_create(DB **database, DB_ENV *database_env, uint32_t flags) { printf("%s\n", __FUNCTION__); if (database == NULL) return EINVAL; DB *db = calloc(1, sizeof(DB)); *database = db; db->_internal = calloc(1, sizeof(struct _PRKL_DATA)); mdb_env_create(&(db->_internal->env)); mdb_env_set_mapsize(db->_internal->env, 10485760); db->open = _db_open; db->close = _db_close; db->err = _db_err; db->put = _db_put; db->get = _db_get; db->del = _db_del; return 0; }
KeyValueEngine* LMDBEngineFactory::CreateDB(const std::string& name) { if (!m_env_opened) { mdb_env_create(&m_env); int page_size = sysconf(_SC_PAGE_SIZE); int rc = mdb_env_set_mapsize(m_env, (m_cfg.max_db_size / page_size) * page_size); if (rc != MDB_SUCCESS) { ERROR_LOG("Invalid db size:%llu for reason:%s", m_cfg.max_db_size, mdb_strerror(rc)); return NULL; } char tmp[m_cfg.path.size() + name.size() + 10]; sprintf(tmp, "%s/%s", m_cfg.path.c_str(), name.c_str()); m_cfg.path = tmp; make_dir(m_cfg.path); int env_opt = MDB_NOSYNC | MDB_NOMETASYNC | MDB_WRITEMAP | MDB_MAPASYNC ; if(!m_cfg.readahead) { env_opt |= MDB_NORDAHEAD; } rc = mdb_env_open(m_env, m_cfg.path.c_str(), env_opt, 0664); if (rc != MDB_SUCCESS) { ERROR_LOG("Failed to open mdb:%s", mdb_strerror(rc)); return NULL; } m_env_opened = true; } LMDBEngine* engine = new LMDBEngine(); LMDBConfig cfg = m_cfg; if (engine->Init(cfg, m_env, name) != 0) { DELETE(engine); return NULL; } DEBUG_LOG("Create DB:%s at path:%s success", name.c_str(), cfg.path.c_str()); return engine; }
int db_snapshot_to_fd(db_t db, int fd, int compact){ int r; pthread_mutex_lock(&db->mutex); db->txns++; pthread_mutex_unlock(&db->mutex); r = mdb_env_copyfd2(db->env, fd, compact ? MDB_CP_COMPACT : 0); while(MDB_MAP_RESIZED == r){ pthread_mutex_lock(&db->mutex); while(db->txns > 1) pthread_cond_wait(&db->cond, &db->mutex); mdb_env_set_mapsize(db->env, 0); pthread_mutex_unlock(&db->mutex); r = mdb_env_copyfd2(db->env, fd, compact ? MDB_CP_COMPACT : 0); } pthread_mutex_lock(&db->mutex); if(1 == db->txns--) pthread_cond_signal(&db->cond); pthread_mutex_unlock(&db->mutex); return r; }
bool Context::Open(bool setSize, size_t dbMegabytes,size_t osPageSize){ size_t mbbytes = 10485760; if ((dbMegabytes*mbbytes % osPageSize) != 0){ return false; } else if (envCreateReturn!=0){ return false; } else { MDB_dbi dbs = 1; int maxdbsReturn = mdb_env_set_maxdbs(envHandle,dbs); if (maxdbsReturn !=0){ Error(maxdbsReturn); return false; } else{ int mapsizeReturn=0; if (setSize==true){ size_t bytes = dbMegabytes*mbbytes; mapsizeReturn = mdb_env_set_mapsize(envHandle,bytes); } if(setSize==true && mapsizeReturn!=0){ Error(mapsizeReturn); return false; } envOpenReturn = mdb_env_open(envHandle, "/root/mydb",0,0664); if (envOpenReturn!=0){ Error(envOpenReturn); mdb_env_close(envHandle); envHandle=nullptr; return false; } else return true; } } }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32]; char kval[sizeof(int)]; srand(time(NULL)); memset(sval, 0, sizeof(sval)); count = (rand()%384) + 64; values = (int *)je_malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "/tmp/testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi)); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { if (!(i & 0x0f)) sprintf(kval, "%03x", values[i]); sprintf(sval, "%03x %d foo bar", values[i], values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(kval, "%03x", values[i & ~0x0f]); sprintf(sval, "%03x %d foo bar", values[i], values[i]); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } je_free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_close(env, dbi); mdb_txn_abort(txn); mdb_env_close(env); return 0; }
bool B_ACCURATE_LMDB::init(JCR *jcr, uint32_t nbfile) { int result; MDB_env *env; size_t mapsize = 10485760; if (!m_db_env) { result = mdb_env_create(&env); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to create MDB environment: %s\n"), mdb_strerror(result)); return false; } if ((nbfile * AVG_NR_BYTES_PER_ENTRY) > mapsize) { size_t pagesize; #ifdef HAVE_GETPAGESIZE pagesize = getpagesize(); #else pagesize = B_PAGE_SIZE; #endif mapsize = (((nbfile * AVG_NR_BYTES_PER_ENTRY) / pagesize) + 1) * pagesize; } result = mdb_env_set_mapsize(env, mapsize); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to set MDB mapsize: %s\n"), mdb_strerror(result)); goto bail_out; } /* * Explicitly set the number of readers to 1. */ result = mdb_env_set_maxreaders(env, 1); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to set MDB maxreaders: %s\n"), mdb_strerror(result)); goto bail_out; } Mmsg(m_lmdb_name, "%s/.accurate_lmdb.%d", me->working_directory, jcr->JobId); result = mdb_env_open(env, m_lmdb_name, MDB_NOSUBDIR | MDB_NOLOCK | MDB_NOSYNC, 0600); if (result) { Jmsg2(jcr, M_FATAL, 0, _("Unable create LDMD database %s: %s\n"), m_lmdb_name, mdb_strerror(result)); goto bail_out; } result = mdb_txn_begin(env, NULL, 0, &m_db_rw_txn); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to start a write transaction: %s\n"), mdb_strerror(result)); goto bail_out; } result = mdb_dbi_open(m_db_rw_txn, NULL, MDB_CREATE, &m_db_dbi); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to open LMDB internal database: %s\n"), mdb_strerror(result)); mdb_txn_abort(m_db_rw_txn); m_db_rw_txn = NULL; goto bail_out; } m_db_env = env; } if (!m_pay_load) { m_pay_load = get_pool_memory(PM_MESSAGE); } if (!m_lmdb_name) { m_pay_load = get_pool_memory(PM_FNAME); } if (!m_seen_bitmap) { m_seen_bitmap = (char *)malloc(nbytes_for_bits(nbfile)); clear_all_bits(nbfile, m_seen_bitmap); } return true; bail_out: if (env) { mdb_env_close(env); } return false; }
int main(int argc,char * argv[]) { int rc; MDB_env *env; MDB_txn *txn; MDB_dbi db; rc = mdb_env_create(&env); if (rc != 0) { return rc; } rc = mdb_env_set_mapsize(env, 107374182400); //100G if (rc != 0) { return rc; } rc = mdb_env_set_maxdbs(env, 27); if (rc != 0) { return rc; } rc = mdb_env_open(env, "/home/pieter/Downloads/thundergraph/src/main/native/testdb", MDB_NOSYNC, 0664); if (rc != 0) { return rc; } rc = mdb_txn_begin(env, NULL, 0, &txn); if (rc != 0) { mdb_txn_abort(txn); return rc; } rc = mdb_open(txn, "testdb", MDB_CREATE | MDB_INTEGERKEY, &db); if (rc != 0) { mdb_txn_abort(txn); return rc; } rc = mdb_set_compare(txn, db, compareTestDbId); if (rc != 0) { mdb_txn_abort(txn); return rc; } rc = mdb_txn_commit(txn); if (rc != 0) { mdb_txn_abort(txn); return rc; } rc = mdb_txn_begin(env, NULL, 0, &txn); if (rc != 0) { mdb_txn_abort(txn); return rc; } int i= 0; for (i = 0; i < 1000; i++) { MDB_cursor *cursor; rc = mdb_cursor_open(txn, db, &cursor); if (rc != 0) { printf("open cursor failure = %i!\n", rc); goto fail; } //Create a core vertex MDB_val key, data; TestDbId testDbId; testDbId.testId = i; testDbId.coreOrPropertyEnum = 0; testDbId.labelId = -1; testDbId.propertykeyId = -1; testDbId.edgeId = -1LL; key.mv_size = sizeof(TestDbId); key.mv_data = &testDbId; data.mv_size = 50; char *value = malloc(5); char v[] = "12345"; memcpy(value, v, 5); data.mv_data = &v; mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE); free(value); //Create a vertex property MDB_val propertyKey, propertyData; TestDbId propertyTestDbId; propertyTestDbId.testId = i; propertyTestDbId.coreOrPropertyEnum = 1; propertyTestDbId.labelId = -1; propertyTestDbId.propertykeyId = -1; propertyTestDbId.edgeId = -1LL; propertyKey.mv_size = sizeof(TestDbId); propertyKey.mv_data = &propertyTestDbId; data.mv_size = 50; char *propertyValue = malloc(5); char propertyV[] = "12345"; memcpy(propertyValue, propertyV, 5); data.mv_data = &propertyV; mdb_cursor_put(cursor, &propertyKey, &propertyData, MDB_NOOVERWRITE); free(propertyValue); //Create a vertex in edge MDB_val inEdgeKey, inEdgeData; TestDbId inEdgeTestDbId; inEdgeTestDbId.testId = i; inEdgeTestDbId.coreOrPropertyEnum = 2; inEdgeTestDbId.labelId = -1; inEdgeTestDbId.propertykeyId = -1; inEdgeTestDbId.edgeId = -1LL; inEdgeKey.mv_size = sizeof(TestDbId); inEdgeKey.mv_data = &inEdgeTestDbId; data.mv_size = 50; char *inEdgeValue = malloc(5); char inEdgeV[] = "12345"; memcpy(inEdgeValue, inEdgeV, 5); data.mv_data = &inEdgeV; mdb_cursor_put(cursor, &inEdgeKey, &inEdgeData, MDB_NOOVERWRITE); mdb_cursor_close(cursor); free(inEdgeValue); } MDB_cursor *cursor; mdb_cursor_close(cursor); rc = mdb_cursor_open(txn, db, &cursor); if (rc != 0) { printf("open cursor failure = %i!\n", rc); goto fail; } MDB_val key, data; for (i = 0; i < 1000; i++) { TestDbId testDbId; testDbId.testId = i; testDbId.coreOrPropertyEnum = 0; testDbId.labelId = -1; testDbId.propertykeyId = -1; testDbId.edgeId = -1LL; key.mv_size = sizeof(TestDbId); key.mv_data = &testDbId; rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY); printf("mdb_cursor_get index = %i result = %i\n", i, rc); MDB_val keyDelete, dataDelete; TestDbId deleteTestDbId; deleteTestDbId.testId = i; deleteTestDbId.coreOrPropertyEnum = 0; deleteTestDbId.labelId = -1; deleteTestDbId.propertykeyId = -1; deleteTestDbId.edgeId = -1LL; keyDelete.mv_size = sizeof(TestDbId); keyDelete.mv_data = &deleteTestDbId; rc = mdb_del(txn, db, &keyDelete, &dataDelete); printf("mdb_del = %i\n", rc); MDB_val propertyKeyDelete, propertyDataDelete; TestDbId propertyDeleteTestDbId; propertyDeleteTestDbId.testId = i; propertyDeleteTestDbId.coreOrPropertyEnum = 1; propertyDeleteTestDbId.labelId = -1; propertyDeleteTestDbId.propertykeyId = -1; propertyDeleteTestDbId.edgeId = -1LL; propertyKeyDelete.mv_size = sizeof(TestDbId); propertyKeyDelete.mv_data = &propertyDeleteTestDbId; rc = mdb_del(txn, db, &propertyKeyDelete, &propertyDataDelete); printf("mdb_del = %i\n", rc); MDB_val inEdgeKeyDelete, inEdgeDataDelete; TestDbId inEdgeDeleteTestDbId; inEdgeDeleteTestDbId.testId = i; inEdgeDeleteTestDbId.coreOrPropertyEnum = 2; inEdgeDeleteTestDbId.labelId = -1; inEdgeDeleteTestDbId.propertykeyId = -1; inEdgeDeleteTestDbId.edgeId = -1LL; inEdgeKeyDelete.mv_size = sizeof(TestDbId); inEdgeKeyDelete.mv_data = &inEdgeDeleteTestDbId; rc = mdb_del(txn, db, &inEdgeKeyDelete, &inEdgeDataDelete); printf("mdb_del = %i\n", rc); } mdb_cursor_close(cursor); rc = mdb_cursor_open(txn, db, &cursor); if (rc != 0) { printf("open cursor failure = %i!\n", rc); goto fail; } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { TestDbId testDbId = *((TestDbId *) (key.mv_data)); printf("keyId = %llu\n", testDbId.testId); } mdb_cursor_close(cursor); fail: mdb_close(env, db); mdb_env_close(env); printf("closing graph!\n"); return 0; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor, *cur2; MDB_cursor_op op; int count; int *values; char sval[32] = ""; srandom(time(NULL)); count = (random()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = random()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_open(txn, NULL, 0, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) { j++; data.mv_size = sizeof(sval); data.mv_data = sval; } } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; key.mv_data = sval; for (i= count - 1; i > -1; i-= (random()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor last\n"); E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor last/prev\n"); E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); E(mdb_cursor_get(cursor, &key, &data, MDB_PREV)); printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); mdb_txn_abort(txn); printf("Deleting with cursor\n"); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_cursor_open(txn, dbi, &cur2)); for (i=0; i<50; i++) { if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT))) break; printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); E(mdb_del(txn, dbi, &key, NULL)); } printf("Restarting cursor in txn\n"); for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op))) break; printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } mdb_cursor_close(cur2); E(mdb_txn_commit(txn)); printf("Restarting cursor outside txn\n"); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op))) break; printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } mdb_cursor_close(cursor); mdb_close(env, dbi); mdb_txn_abort(txn); mdb_env_close(env); return 0; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[8]; char kval[sizeof(int)]; memset(sval, 0, sizeof(sval)); count = 510; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = i*5; } E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi)); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); strcpy(kval, "001"); for (i=0;i<count;i++) { sprintf(sval, "%07x", values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); /* there should be one full page of dups now. */ E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); /* test all 3 branches of split code: * 1: new key in lower half * 2: new key at split point * 3: new key in upper half */ key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; sprintf(sval, "%07x", values[3]+1); E(mdb_txn_begin(env, NULL, 0, &txn)); (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); mdb_txn_abort(txn); sprintf(sval, "%07x", values[255]+1); E(mdb_txn_begin(env, NULL, 0, &txn)); (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); mdb_txn_abort(txn); sprintf(sval, "%07x", values[500]+1); E(mdb_txn_begin(env, NULL, 0, &txn)); (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); E(mdb_txn_commit(txn)); /* Try MDB_NEXT_MULTIPLE */ E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; for (i= count - 1; i > -1; i-= (rand()%3)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%07x", values[i]); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_close(env, dbi); mdb_txn_abort(txn); mdb_env_close(env); return 0; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data, sdata; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; long kval; char *sval; srand(time(NULL)); E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi)); E(mdb_cursor_open(txn, dbi, &cursor)); E(mdb_stat(txn, dbi, &mst)); sval = calloc(1, mst.ms_psize / 4); key.mv_size = sizeof(long); key.mv_data = &kval; sdata.mv_size = mst.ms_psize / 4 - 30; sdata.mv_data = sval; printf("Adding 12 values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5; sprintf(sval, "%08x", kval); data = sdata; (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } printf("Adding 12 more values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5+4; sprintf(sval, "%08x", kval); data = sdata; (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } printf("Adding 12 more values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5+1; sprintf(sval, "%08x", kval); data = sdata; (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST)); do { printf("key: %p %s, data: %p %.*s\n", key.mv_data, mdb_dkey(&key, dkbuf), data.mv_data, (int) data.mv_size, (char *) data.mv_data); } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_commit(txn); #if 0 j=0; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(kval, "%03x", values[i & ~0x0f]); sprintf(sval, "%03x %d foo bar", values[i], values[i]); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); #endif mdb_env_close(env); return 0; }