counter_t *counter_init(const char *path, uint64_t readers) { counter_t *lc = NULL; if((lc = calloc(1, sizeof(counter_t))) == NULL) { perror("calloc"); return NULL; } // Setup and open the lmdb enviornment MDB_CHECK(mdb_env_create(&lc->env), MDB_SUCCESS, NULL); MDB_CHECK(mdb_env_set_maxreaders(lc->env, readers), MDB_SUCCESS, NULL); MDB_CHECK(mdb_env_set_mapsize(lc->env, MDB_MAPSIZE), MDB_SUCCESS, NULL); MDB_CHECK(mdb_env_open(lc->env, path, MDB_WRITEMAP | MDB_MAPASYNC | MDB_NOSUBDIR, 0664), MDB_SUCCESS, NULL); MDB_txn *txn = NULL; MDB_CHECK(mdb_txn_begin(lc->env, NULL, 0, &txn), MDB_SUCCESS, NULL); if((lc->dbi = calloc(1, sizeof(MDB_dbi))) == NULL) { perror("calloc"); return NULL; } MDB_CHECK(mdb_dbi_open(txn, NULL, 0, lc->dbi), MDB_SUCCESS, NULL); mdb_txn_commit(txn); return lc; }
int main(void) { check(mdb_env_create(&env)); check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL)); check(mdb_env_set_maxreaders(env, 126)); check(mdb_env_set_maxdbs(env, 1)); if(! access(DB_PATH, X_OK)) { system("rm -rf " DB_PATH); } check(mkdir(DB_PATH, 0777)); check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644)); new_txn(); check(mdb_dbi_open(txn, NULL, 0, &dbi)); put("a"); put("b"); put("baa"); put("d"); new_txn(); check(mdb_cursor_open(txn, dbi, &c1)); check(mdb_cursor_get(c1, &keyv, &valv, MDB_LAST)); check(mdb_cursor_del(c1, 0)); check(mdb_cursor_del(c1, 0)); new_txn(); }
/* * Class: jmdb_DatabaseWrapper * Method: envSetMaxReaders * Signature: (JI)V */ JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_envSetMaxReaders(JNIEnv *vm, jclass clazz, jlong envL, jint readers) { MDB_env *envC = (MDB_env*) envL; int code = mdb_env_set_maxreaders(envC, readers); if (code) { throwDatabaseException(vm, code); } }
kvenv_t *kvenv_new_lmdb(const char *dbpath, uint64_t max_dbsize, uint32_t max_dbs) { kvenv_lmdb_t *kvenv_lmdb = (kvenv_lmdb_t*)zmalloc(sizeof(kvenv_lmdb_t)); memset(kvenv_lmdb, 0, sizeof(kvenv_lmdb_t)); kvenv_lmdb->kvenv.dbclass = "lmdb"; kvenv_lmdb->kvenv.max_dbsize = max_dbsize; kvenv_lmdb->kvenv.max_dbs = max_dbs; int rc = mdb_env_create(&kvenv_lmdb->env); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_create() failed."); return NULL; } int maxreaders = 256; if ( max_dbs == 0 ){ max_dbs = 256; } if ( max_dbsize == 0 ) { max_dbsize = 1024L * 1024L * 1024L * 4L; } rc = mdb_env_set_mapsize(kvenv_lmdb->env, max_dbsize); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_set_mapsize() failed."); return NULL; } rc = mdb_env_set_maxreaders(kvenv_lmdb->env, maxreaders); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_set_maxreaders() failed."); return NULL; } rc = mdb_env_set_maxdbs(kvenv_lmdb->env, max_dbs); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_set_maxdbs() failed."); return NULL; } /*rc = mdb_env_open(lmdb->env, dbpath, MDB_FIXEDMAP | MDB_NOSYNC, 0640); */ /*rc = mdb_env_open(lmdb->env, dbpath, MDB_MAPASYNC | MDB_WRITEMAP | MDB_NOTLS , 0640); */ /*rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_MAPASYNC | MDB_WRITEMAP, 0640); */ /*rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_NOMETASYNC, 0640); */ rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_NOSYNC, 0640); if ( rc != 0 ) { zfree(kvenv_lmdb); error_log("mdb_env_open() failed. dbpath=%s error: %s", dbpath, mdb_strerror(rc)); return NULL; } return (kvenv_t*)kvenv_lmdb; }
// Open a LMDB database. MDB_env *database_open(const char *path, size_t mapsize, unsigned int nbr_threads) { MDB_env *env = NULL; mdb_env_create(&env); mdb_env_set_mapsize(env, mapsize); mdb_env_set_maxreaders(env, nbr_threads); mdb_env_open(env, path, 0, 0664); return (env); }
void LMDB::Open(const string& source, Mode mode) { MDB_CHECK(mdb_env_create(&mdb_env_)); MDB_CHECK(mdb_env_set_mapsize(mdb_env_, LMDB_MAP_SIZE)); MDB_CHECK(mdb_env_set_maxreaders(mdb_env_, 100000)); if (mode == NEW) { CHECK_EQ(mkdir(source.c_str(), 0744), 0) << "mkdir " << source << "failed"; } int flags = 0; if (mode == READ) { flags = MDB_RDONLY | MDB_NOTLS; } MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664)); LOG(INFO) << "Opened lmdb " << source; }
static int mdb_db_open( BackendDB *be, ConfigReply *cr ) { int rc, i; struct mdb_info *mdb = (struct mdb_info *) be->be_private; struct stat stat1; uint32_t flags; char *dbhome; MDB_txn *txn; if ( be->be_suffix == NULL ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": need suffix.\n", 1, 0, 0 ); return -1; } Debug( LDAP_DEBUG_ARGS, LDAP_XSTRING(mdb_db_open) ": \"%s\"\n", be->be_suffix[0].bv_val, 0, 0 ); /* Check existence of dbenv_home. Any error means trouble */ rc = stat( mdb->mi_dbenv_home, &stat1 ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "cannot access database directory \"%s\" (%d).\n", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno ); return -1; } /* mdb is always clean */ be->be_flags |= SLAP_DBFLAG_CLEAN; rc = mdb_env_create( &mdb->mi_dbenv ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_create failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } if ( mdb->mi_readers ) { rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_maxreaders failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } } rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_mapsize failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_maxdbs failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } #ifdef HAVE_EBCDIC strcpy( path, mdb->mi_dbenv_home ); __atoe( path ); dbhome = path; #else dbhome = mdb->mi_dbenv_home; #endif Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "dbenv_open(%s).\n", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0); flags = mdb->mi_dbenv_flags; if ( slapMode & SLAP_TOOL_QUICK ) flags |= MDB_NOSYNC|MDB_WRITEMAP; if ( slapMode & SLAP_TOOL_READONLY) flags |= MDB_RDONLY; rc = mdb_env_open( mdb->mi_dbenv, dbhome, flags, mdb->mi_dbenv_mode ); if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). " "Restore from backup!\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn ); if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). " "Restore from backup!\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } /* open (and create) main databases */ for( i = 0; mdmi_databases[i].bv_val; i++ ) { flags = MDB_INTEGERKEY; if( i == MDB_ID2ENTRY ) { if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) )) flags |= MDB_CREATE; } else { if ( i == MDB_DN2ID ) flags |= MDB_DUPSORT; if ( !(slapMode & SLAP_TOOL_READONLY) ) flags |= MDB_CREATE; } rc = mdb_dbi_open( txn, mdmi_databases[i].bv_val, flags, &mdb->mi_dbis[i] ); if ( rc != 0 ) { snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": " "mdb_dbi_open(%s/%s) failed: %s (%d).", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, mdmi_databases[i].bv_val, mdb_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg, 0, 0 ); goto fail; } if ( i == MDB_ID2ENTRY ) mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare ); else if ( i == MDB_DN2ID ) { MDB_cursor *mc; MDB_val key, data; ID id; mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare ); /* check for old dn2id format */ rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc ); /* first record is always ID 0 */ rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST ); if ( rc == 0 ) { rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT ); if ( rc == 0 ) { int len; unsigned char *ptr; ptr = data.mv_data; len = (ptr[0] & 0x7f) << 8 | ptr[1]; if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) { snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": DN index needs upgrade, " "run \"slapindex entryDN\".", be->be_suffix[0].bv_val ); Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg, 0, 0 ); if ( !(slapMode & SLAP_TOOL_READMAIN )) rc = LDAP_OTHER; mdb->mi_flags |= MDB_NEED_UPGRADE; } } } mdb_cursor_close( mc ); if ( rc == LDAP_OTHER ) goto fail; } } rc = mdb_ad_read( mdb, txn ); if ( rc ) { mdb_txn_abort( txn ); goto fail; } /* slapcat doesn't need indexes. avoid a failure if * a configured index wasn't created yet. */ if ( !(slapMode & SLAP_TOOL_READONLY) ) { rc = mdb_attr_dbs_open( be, txn, cr ); if ( rc ) { mdb_txn_abort( txn ); goto fail; } } rc = mdb_txn_commit(txn); if ( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database %s: " "txn_commit failed: %s (%d)\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } /* monitor setup */ rc = mdb_monitor_db_open( be ); if ( rc != 0 ) { goto fail; } mdb->mi_flags |= MDB_IS_OPEN; return 0; fail: mdb_db_close( be, NULL ); return rc; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32] = ""; int env_oflags; struct stat db_stat, exe_stat; (void) argc; (void) argv; srand(time(NULL)); count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_maxreaders(env, 1)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(stat("/proc/self/exe", &exe_stat)?errno:0); E(stat(DBPATH "/.", &db_stat)?errno:0); env_oflags = MDB_FIXEDMAP | MDB_NOSYNC; if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { /* LY: Assume running inside a CI-environment: * 1) don't use FIXEDMAP to avoid EBUSY in case collision, * which could be inspired by address space randomisation feature. * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage. */ env_oflags = 0; } /* LY: especially here we always needs MDB_NOSYNC * for testing mdbx_env_close_ex() and "redo-to-steady" on open. */ env_oflags |= MDB_NOSYNC; E(mdb_env_open(env, DBPATH, env_oflags, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS) E(mdb_drop(txn, dbi, 1)); E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); printf("check-preset-a\n"); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_a = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_a; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_a == count - j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_env_sync(env, 1); int deleted = 0; key.mv_data = sval; for (i = count - 1; i > -1; i -= (rand()%5)) { txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); deleted++; } } free(values); printf("Deleted %d values\n", deleted); printf("check-preset-b.cursor-next\n"); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_b = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_b; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_b == present_a - deleted, "mismatch"); printf("check-preset-b.cursor-prev\n"); j = 1; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++j; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_b == j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); /********************* LY: kept DB dirty ****************/ mdbx_env_close_ex(env, 1); E(mdb_env_create(&env)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, DBPATH, env_oflags, 0664)); printf("check-preset-c.cursor-next\n"); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_dbi_open(txn, "id1", 0, &dbi)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_c = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_c; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Rolled back %d deletion(s)\n", present_c - (present_a - deleted)); CHECK(present_c > present_a - deleted, "mismatch"); printf("check-preset-d.cursor-prev\n"); j = 1; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++j; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_c == j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); mdbx_env_close_ex(env, 0); return 0; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32] = ""; srand(time(NULL)); count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_maxreaders(env, 1)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS) E(mdb_drop(txn, dbi, 1)); E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); printf("check-preset-a\n"); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_a = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_a; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_a == count - j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_env_sync(env, 1); j=0; key.mv_data = sval; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); printf("check-preset-b.cursor-next\n"); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_b = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_b; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_b == present_a - j, "mismatch"); printf("check-preset-b.cursor-prev\n"); j = 1; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++j; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_b == j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); /********************* LY: kept DB dirty ****************/ mdb_env_close_ex(env, 1); E(mdb_env_create(&env)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); printf("check-preset-c.cursor-next\n"); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_dbi_open(txn, "id1", 0, &dbi)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_c = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_c; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_c == present_a, "mismatch"); printf("check-preset-d.cursor-prev\n"); j = 1; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++j; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_c == j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); mdb_env_close_ex(env, 0); return 0; }
bool LMDBFileIndex::create_env() { int rc; if(env==NULL) { rc = mdb_env_create(&env); if(rc) { Server->Log("LMDB: Failed to create LMDB env ("+(std::string)mdb_strerror(rc)+")", LL_ERROR); return false; } { std::auto_ptr<IFile> lmdb_f(Server->openFile("urbackup/fileindex/backup_server_files_index.lmdb", MODE_READ)); if(lmdb_f.get()!=NULL) { while(lmdb_f->Size()>static_cast<_i64>(map_size)) { map_size*=2; } } } rc = mdb_env_set_maxreaders(env, 4094); if (rc) { Server->Log("LMDB: Failed to set max readers (" + (std::string)mdb_strerror(rc) + ")", LL_ERROR); return false; } rc = mdb_env_set_mapsize(env, map_size); if(rc) { Server->Log("LMDB: Failed to set map size ("+(std::string)mdb_strerror(rc)+")", LL_ERROR); return false; } os_create_dir("urbackup/fileindex"); unsigned int flags = MDB_NOSUBDIR|MDB_NOMETASYNC; if(no_sync) { flags|=MDB_NOSYNC; } rc = mdb_env_open(env, "urbackup/fileindex/backup_server_files_index.lmdb", flags, 0664); if(rc) { Server->Log("LMDB: Failed to open LMDB database file ("+(std::string)mdb_strerror(rc)+")", LL_ERROR); return false; } MDB_txn* l_txn; rc = mdb_txn_begin(env, NULL, flags, &l_txn); if (rc) { Server->Log("LMDB: Failed to open transaction handle for dbi open (" + (std::string)mdb_strerror(rc) + ")", LL_ERROR); return false; } rc = mdb_dbi_open(l_txn, NULL, 0, &dbi); if (rc) { Server->Log("LMDB: Failed to open database (" + (std::string)mdb_strerror(rc) + ")", LL_ERROR); return false; } rc = mdb_txn_commit(l_txn); if (rc) { Server->Log("LMDB: Failed to commit txn for dbi handle (" + (std::string)mdb_strerror(rc) + ")", LL_ERROR); return false; } return true; } else { return true; } }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32] = ""; int env_oflags; struct stat db_stat, exe_stat; srand(time(NULL)); count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_maxreaders(env, 1)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(stat("/proc/self/exe", &exe_stat)?errno:0); E(stat(DBPATH "/.", &db_stat)?errno:0); env_oflags = MDB_FIXEDMAP | MDB_NOSYNC; if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { /* LY: Assume running inside a CI-environment: * 1) don't use FIXEDMAP to avoid EBUSY in case collision, * which could be inspired by address space randomisation feature. * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage. */ env_oflags = 0; } E(mdb_env_open(env, DBPATH, env_oflags, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); if (mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi) == MDB_SUCCESS) E(mdb_drop(txn, dbi, 1)); E(mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; key.mv_data = sval; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); mdb_env_close(env); return 0; }
/* * Initialize MDB db * (reference openldap 2.4.31 back-mdb/init.c) */ DWORD VmDirMDBInitializeDB( BOOLEAN bMainDB, const char *dbHomeDir, PVDIR_DB_HANDLE *phHandle ) { DWORD dwError = 0; unsigned int envFlags = 0; mdb_mode_t mode = 0; uint64_t db_max_mapsize = BE_MDB_ENV_MAX_MEM_MAPSIZE; DWORD db_max_size_mb = 0; DWORD db_chkpt_interval = 0; BOOLEAN bMdbWalEnable = TRUE; PVDIR_MDB_DB pDB = NULL; VmDirLog( LDAP_DEBUG_TRACE, "MDBInitializeDB: Begin, DB Home Dir = %s", dbHomeDir ); //Make a db entry for path. fail if already exist dwError = _VmDirMDBInitializeDBEntry(dbHomeDir, &pDB); BAIL_ON_VMDIR_ERROR( dwError ); if (bMainDB) { pDB->bIsMainDB = TRUE; } else { pDB->bIsMainDB = FALSE; } dwError = (sizeof(ENTRYID) == sizeof(VDIR_DB_SEQ_T)) ? 0 : ERROR_BACKEND_ERROR; BAIL_ON_VMDIR_ERROR( dwError ); dwError = MDBInitConfig(pDB); BAIL_ON_VMDIR_ERROR( dwError ); /* Create the environment */ dwError = mdb_env_create ( &pDB->mdbEnv ); BAIL_ON_VMDIR_ERROR( dwError ); dwError = mdb_env_set_maxreaders( pDB->mdbEnv, BE_MDB_ENV_MAX_READERS ); BAIL_ON_VMDIR_ERROR( dwError ); /* FROM mdb.h * The size should be a multiple of the OS page size. The default is * 10485760 bytes. The size of the memory map is also the maximum size * of the database. The value should be chosen as large as possible, * to accommodate future growth of the database. * * // TODO, this is also the max size of database (per logical mdb db or the total dbs) */ dwError = VmDirGetMaxDbSizeMb(&db_max_size_mb); if (dwError != 0) { VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "Use default max-database-size %llu", BE_MDB_ENV_MAX_MEM_MAPSIZE); } else { db_max_mapsize = (uint64_t)(db_max_size_mb)*1024*1024; if (db_max_mapsize < BE_MDB_ENV_MAX_MEM_MAPSIZE) { db_max_mapsize = BE_MDB_ENV_MAX_MEM_MAPSIZE; VMDIR_LOG_WARNING(VMDIR_LOG_MASK_ALL, "RegKey %s value (%u) is too small. Use default max-database-size %llu", VMDIR_REG_KEY_MAXIMUM_DB_SIZE_MB, db_max_size_mb, db_max_mapsize); } else { VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "max-database-size is set to %llu per RegKey %s", db_max_mapsize, VMDIR_REG_KEY_MAXIMUM_DB_SIZE_MB); } } dwError = mdb_env_set_mapsize( pDB->mdbEnv, db_max_mapsize); BAIL_ON_VMDIR_ERROR( dwError ); dwError = mdb_env_set_maxdbs ( pDB->mdbEnv, BE_MDB_ENV_MAX_DBS ); BAIL_ON_VMDIR_ERROR( dwError ); dwError = VmDirGetMdbChkptInterval(&db_chkpt_interval); if (dwError) { db_chkpt_interval = VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_DEFAULT; dwError = 0; } VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: %s is set to %d", __func__, VMDIR_REG_KEY_MDB_CHKPT_INTERVAL, db_chkpt_interval); dwError = mdb_env_set_chkpt_interval(pDB->mdbEnv, db_chkpt_interval); BAIL_ON_VMDIR_ERROR( dwError ); if(bMainDB) { mdb_set_raft_prepare_commit_func(pDB->mdbEnv, VmDirRaftPrepareCommit); mdb_set_raft_post_commit_func(pDB->mdbEnv, VmDirRaftPostCommit); mdb_set_raft_commit_fail_func(pDB->mdbEnv, VmDirRaftCommitFail); } #ifdef MDB_NOTLS envFlags = MDB_NOTLS; // Required for versions of mdb which have this flag #endif // this is experimental from mdb.h comments //envFlags = MDB_FIXEDMAP; /* use a fixed address for the mmap region */ //envFlags |= MDB_NOSYNC need sync for durability //envFlags |= MDB_RDONLY need to open for read and write /* Open the environment. */ #ifndef _WIN32 mode = S_IRUSR | S_IWUSR; #else mode = GENERIC_READ|GENERIC_WRITE; #endif //MDB WAL is the default mode and can be turned off with reg key MdbEnableWal set to 0 dwError = VmDirGetMdbWalEnable(&bMdbWalEnable); if (dwError) { bMdbWalEnable = TRUE; dwError = 0; } VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: %s is set to %s", __func__, VMDIR_REG_KEY_MDB_ENABLE_WAL, bMdbWalEnable?"True":"False"); if (bMdbWalEnable) { envFlags |= MDB_WAL; } dwError = mdb_env_open ( pDB->mdbEnv, dbHomeDir, envFlags, mode ); //TODO, what if open failed? how to recover?? BAIL_ON_VMDIR_ERROR( dwError ); mdb_set_error_log_func(&VmDirMdbErrorLog); /* Open main database. */ dwError = MDBOpenMainDB(pDB); BAIL_ON_VMDIR_ERROR( dwError ); /* Open sequences */ dwError = MDBOpenSequence(pDB); BAIL_ON_VMDIR_ERROR( dwError ); /* Open generic */ dwError = MDBOpenGeneric(pDB); BAIL_ON_VMDIR_ERROR( dwError ); /* Initialize indices */ dwError = VmDirMDBInitializeIndexDB(pDB); BAIL_ON_VMDIR_ERROR( dwError ); dwError = VmDirInitMdbStateGlobals(); BAIL_ON_VMDIR_ERROR(dwError); VmDirLogDBStats(pDB); /* TODO: require per db txn stats for raft groups */ if (bMainDB) { dwError = _VmDirWtxnStatsInit(); BAIL_ON_VMDIR_ERROR( dwError ); } *phHandle = pDB; VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: succeeded with dbHomeDir %s", __func__, dbHomeDir); cleanup: VmDirLog( LDAP_DEBUG_TRACE, "MDBInitializeDB: End" ); return dwError; error: VmDirLog( LDAP_DEBUG_ANY, "MDBInitializeDB failed with error code: %d, error string: %s", dwError, mdb_strerror(dwError) ); //TODO, should shutdown here or caller will do that? //gVdirMdbGlobals.mdbEnv = NULL; goto cleanup; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor, *cur2; MDB_cursor_op op; int count; int *values; char sval[32] = ""; srand(time(NULL)); count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_maxreaders(env, 1)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_dbi_open(txn, NULL, 0, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); /* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */ data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) { j++; data.mv_size = sizeof(sval); data.mv_data = sval; } } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; key.mv_data = sval; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor last\n"); E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor last/prev\n"); E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); E(mdb_cursor_get(cursor, &key, &data, MDB_PREV)); printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); mdb_cursor_close(cursor); mdb_txn_abort(txn); printf("Deleting with cursor\n"); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_cursor_open(txn, dbi, &cur2)); for (i=0; i<50; i++) { if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT))) break; printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); E(mdb_del(txn, dbi, &key, NULL)); } printf("Restarting cursor in txn\n"); for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op))) break; printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } mdb_cursor_close(cur2); E(mdb_txn_commit(txn)); printf("Restarting cursor outside txn\n"); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op))) break; printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); mdb_env_close(env); return 0; }
static int mdb_db_open( BackendDB *be, ConfigReply *cr ) { int rc, i; struct mdb_info *mdb = (struct mdb_info *) be->be_private; struct stat stat1; uint32_t flags; char *dbhome; MDB_txn *txn; if ( be->be_suffix == NULL ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": need suffix.\n", 1, 0, 0 ); return -1; } Debug( LDAP_DEBUG_ARGS, LDAP_XSTRING(mdb_db_open) ": \"%s\"\n", be->be_suffix[0].bv_val, 0, 0 ); /* Check existence of dbenv_home. Any error means trouble */ rc = stat( mdb->mi_dbenv_home, &stat1 ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "cannot access database directory \"%s\" (%d).\n", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno ); return -1; } /* mdb is always clean */ be->be_flags |= SLAP_DBFLAG_CLEAN; rc = mdb_env_create( &mdb->mi_dbenv ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_create failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } if ( mdb->mi_readers ) { rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_maxreaders failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } } rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_mapsize failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_maxdbs failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } #ifdef HAVE_EBCDIC strcpy( path, mdb->mi_dbenv_home ); __atoe( path ); dbhome = path; #else dbhome = mdb->mi_dbenv_home; #endif Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "dbenv_open(%s).\n", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0); flags = mdb->mi_dbenv_flags; if ( slapMode & SLAP_TOOL_QUICK ) flags |= MDB_NOSYNC; if ( slapMode & SLAP_TOOL_READONLY) flags |= MDB_RDONLY; rc = mdb_env_open( mdb->mi_dbenv, dbhome, flags, mdb->mi_dbenv_mode ); if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. " "Restore from backup!\n", be->be_suffix[0].bv_val, rc, 0 ); goto fail; } rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn ); if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. " "Restore from backup!\n", be->be_suffix[0].bv_val, rc, 0 ); goto fail; } /* open (and create) main databases */ for( i = 0; mdmi_databases[i].bv_val; i++ ) { flags = MDB_INTEGERKEY; if( i == MDB_ID2ENTRY ) { if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) )) flags |= MDB_CREATE; } else { if ( i == MDB_DN2ID ) flags |= MDB_DUPSORT; if ( !(slapMode & SLAP_TOOL_READONLY) ) flags |= MDB_CREATE; } rc = mdb_open( txn, mdmi_databases[i].bv_val, flags, &mdb->mi_dbis[i] ); if ( rc != 0 ) { snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": " "mdb_open(%s/%s) failed: %s (%d).", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, mdmi_databases[i].bv_val, mdb_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg, 0, 0 ); goto fail; } if ( i == MDB_ID2ENTRY ) mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare ); else if ( i == MDB_DN2ID ) mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare ); } rc = mdb_ad_read( mdb, txn ); if ( rc ) { mdb_txn_abort( txn ); goto fail; } rc = mdb_attr_dbs_open( be, txn, cr ); if ( rc ) { mdb_txn_abort( txn ); goto fail; } rc = mdb_txn_commit(txn); if ( rc != 0 ) { goto fail; } /* monitor setup */ rc = mdb_monitor_db_open( be ); if ( rc != 0 ) { goto fail; } mdb->mi_flags |= MDB_IS_OPEN; return 0; fail: mdb_db_close( be, NULL ); return rc; }
int main(int argc, char *argv[]) { int i, rc; MDB_env *env; MDB_txn *txn; MDB_cursor *mc; MDB_dbi dbi; char *envname; int envflags = 0, putflags = 0; int dohdr = 0; prog = argv[0]; if (argc < 2) { usage(); } /* -f: load file instead of stdin * -n: use NOSUBDIR flag on env_open * -s: load into named subDB * -N: use NOOVERWRITE on puts * -T: read plaintext * -V: print version and exit */ while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) { switch(i) { case 'V': printf("%s\n", MDB_VERSION_STRING); exit(0); break; case 'f': if (freopen(optarg, "r", stdin) == NULL) { fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno)); exit(EXIT_FAILURE); } break; case 'n': envflags |= MDB_NOSUBDIR; break; case 's': subname = strdup(optarg); break; case 'N': putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; break; case 'T': mode |= NOHDR | PRINT; break; default: usage(); } } if (optind != argc - 1) usage(); dbuf.mv_size = 4096; dbuf.mv_data = malloc(dbuf.mv_size); if (!(mode & NOHDR)) readhdr(); envname = argv[optind]; rc = mdb_env_create(&env); if (rc) { fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); return EXIT_FAILURE; } mdb_env_set_maxdbs(env, 2); if (info.me_maxreaders) mdb_env_set_maxreaders(env, info.me_maxreaders); if (info.me_mapsize) mdb_env_set_mapsize(env, info.me_mapsize); if (info.me_mapaddr) envflags |= MDB_FIXEDMAP; rc = mdb_env_open(env, envname, envflags, 0664); if (rc) { fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; kbuf.mv_data = malloc(kbuf.mv_size); while(!Eof) { MDB_val key, data; int batch = 0; flags = 0; if (!dohdr) { dohdr = 1; } else if (!(mode & NOHDR)) readhdr(); rc = mdb_txn_begin(env, NULL, 0, &txn); if (rc) { fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi); if (rc) { fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } rc = mdb_cursor_open(txn, dbi, &mc); if (rc) { fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } while(1) { rc = readline(&key, &kbuf); if (rc) /* rc == EOF */ break; rc = readline(&data, &dbuf); if (rc) { fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno); goto txn_abort; } rc = mdb_cursor_put(mc, &key, &data, putflags); if (rc == MDB_KEYEXIST && putflags) continue; if (rc) { fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } batch++; if (batch == 100) { rc = mdb_txn_commit(txn); if (rc) { fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; } rc = mdb_txn_begin(env, NULL, 0, &txn); if (rc) { fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } rc = mdb_cursor_open(txn, dbi, &mc); if (rc) { fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } batch = 0; } } rc = mdb_txn_commit(txn); txn = NULL; if (rc) { fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; } mdb_dbi_close(env, dbi); } txn_abort: mdb_txn_abort(txn); env_close: mdb_env_close(env); return rc ? EXIT_FAILURE : EXIT_SUCCESS; }
static int mdb_db_open( BackendDB *be, ConfigReply *cr ) { int rc, i; struct mdb_info *mdb = (struct mdb_info *) be->be_private; struct stat stat1; uint32_t flags; char *dbhome; MDB_txn *txn; if ( be->be_suffix == NULL ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": need suffix.\n" ); return -1; } Debug( LDAP_DEBUG_ARGS, LDAP_XSTRING(mdb_db_open) ": \"%s\"\n", be->be_suffix[0].bv_val ); /* Check existence of dbenv_home. Any error means trouble */ rc = stat( mdb->mi_dbenv_home, &stat1 ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "cannot access database directory \"%s\" (%d).\n", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno ); return -1; } /* mdb is always clean */ be->be_flags |= SLAP_DBFLAG_CLEAN; rc = mdb_env_create( &mdb->mi_dbenv ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_create failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } mdb_env_set_userctx(mdb->mi_dbenv, mdb); if ( mdb->mi_readers ) { rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_maxreaders failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } } rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_mapsize failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_maxdbs failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } #ifdef MDBX_LIFORECLAIM #if MDBX_MODE_ENABLED rc = mdbx_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul); #else rc = mdb_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul); #endif /* MDBX_MODE_ENABLED */ if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_sync_threshold failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } #if MDBX_MODE_ENABLED mdbx_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler ); #else mdb_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler ); #endif /* MDBX_MODE_ENABLED */ if ( (slapMode & SLAP_SERVER_MODE) && SLAP_MULTIMASTER(be) && ((MDBX_OOM_YIELD & mdb->mi_oom_flags) == 0 || mdb->mi_renew_lag == 0)) { snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": " "for properly operation in multi-master mode" " 'oom-handler yield' and 'dreamcatcher' are recommended", be->be_suffix[0].bv_val ); Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg ); } #endif /* MDBX_LIFORECLAIM */ dbhome = mdb->mi_dbenv_home; Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "dbenv_open(%s).\n", be->be_suffix[0].bv_val, mdb->mi_dbenv_home); flags = mdb->mi_dbenv_flags; #ifdef MDBX_PAGEPERTURB if (reopenldap_mode_check()) flags |= MDBX_PAGEPERTURB; #endif if ( slapMode & SLAP_TOOL_QUICK ) flags |= MDB_NOSYNC|MDB_WRITEMAP; if ( slapMode & SLAP_TOOL_READONLY) flags |= MDB_RDONLY; rc = mdb_env_open( mdb->mi_dbenv, dbhome, flags, mdb->mi_dbenv_mode ); if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). " "Restore from backup!\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn ); if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). " "Restore from backup!\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } /* open (and create) main databases */ for( i = 0; mdmi_databases[i].bv_val; i++ ) { flags = MDB_INTEGERKEY; if( i == MDB_ID2ENTRY ) { if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) )) flags |= MDB_CREATE; } else { if ( i == MDB_DN2ID ) flags |= MDB_DUPSORT; if ( !(slapMode & SLAP_TOOL_READONLY) ) flags |= MDB_CREATE; } rc = mdb_dbi_open( txn, mdmi_databases[i].bv_val, flags, &mdb->mi_dbis[i] ); if ( rc != 0 ) { snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": " "mdb_dbi_open(%s/%s) failed: %s (%d).", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, mdmi_databases[i].bv_val, mdb_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg ); goto fail; } if ( i == MDB_ID2ENTRY ) mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare ); else if ( i == MDB_DN2ID ) { MDB_cursor *mc; MDB_val key, data; mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare ); /* check for old dn2id format */ rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc ); /* first record is always ID 0 */ rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST ); if ( rc == 0 ) { rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT ); if ( rc == 0 ) { int len; unsigned char *ptr; ptr = data.mv_data; len = (ptr[0] & 0x7f) << 8 | ptr[1]; if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) { snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": DN index needs upgrade, " "run \"slapindex entryDN\".", be->be_suffix[0].bv_val ); Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg ); if ( !(slapMode & SLAP_TOOL_READMAIN )) rc = LDAP_OTHER; mdb->mi_flags |= MDB_NEED_UPGRADE; } } } mdb_cursor_close( mc ); if ( rc == LDAP_OTHER ) goto fail; } } rc = mdb_ad_read( mdb, txn ); if ( rc ) { mdb_txn_abort( txn ); goto fail; } /* slapcat doesn't need indexes. avoid a failure if * a configured index wasn't created yet. */ if ( !(slapMode & SLAP_TOOL_READONLY) ) { rc = mdb_attr_dbs_open( be, txn, cr ); if ( rc ) { mdb_txn_abort( txn ); goto fail; } } rc = mdb_txn_commit(txn); if ( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database %s: " "txn_commit failed: %s (%d)\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } /* monitor setup */ rc = mdb_monitor_db_open( be ); if ( rc != 0 ) { goto fail; } mdb->mi_flags |= MDB_IS_OPEN; return 0; fail: mdb_db_close( be, NULL ); return rc; }
/* * Open the environment */ static int do_open(const sasl_utils_t *utils, sasl_conn_t *conn, int rdwr, MDB_txn **mtxn) { const char *path = SASL_DB_PATH; void *cntxt; MDB_env *env; MDB_txn *txn; sasl_getopt_t *getopt; size_t mapsize = 0; int readers = 0; int ret; int flags; if (!db_env) { if (utils->getcallback(conn, SASL_CB_GETOPT, (sasl_callback_ft *)&getopt, &cntxt) == SASL_OK) { const char *p; if (getopt(cntxt, NULL, "sasldb_path", &p, NULL) == SASL_OK && p != NULL && *p != 0) { path = p; } if (getopt(cntxt, NULL, "sasldb_maxreaders", &p, NULL) == SASL_OK && p != NULL && *p != 0) { readers = atoi(p); } if (getopt(cntxt, NULL, "sasldb_mapsize", &p, NULL) == SASL_OK && p != NULL && *p != 0) { mapsize = atoi(p); mapsize *= KILO; } } ret = mdb_env_create(&env); if (ret) { utils->log(conn, SASL_LOG_ERR, "unable to create MDB environment: %s", mdb_strerror(ret)); utils->seterror(conn, SASL_NOLOG, "Unable to create MDB environment"); return SASL_FAIL; } if (readers) { ret = mdb_env_set_maxreaders(env, readers); if (ret) { utils->log(conn, SASL_LOG_ERR, "unable to set MDB maxreaders: %s", mdb_strerror(ret)); utils->seterror(conn, SASL_NOLOG, "Unable to set MDB maxreaders"); return SASL_FAIL; } } if (mapsize) { ret = mdb_env_set_mapsize(env, mapsize); if (ret) { utils->log(conn, SASL_LOG_ERR, "unable to set MDB mapsize: %s", mdb_strerror(ret)); utils->seterror(conn, SASL_NOLOG, "Unable to set MDB mapsize"); return SASL_FAIL; } } flags = MDB_NOSUBDIR; if (!rdwr) flags |= MDB_RDONLY; ret = mdb_env_open(env, path, flags, 0660); if (ret) { mdb_env_close(env); if (!rdwr && ret == ENOENT) { /* File not found and we are only reading the data. Treat as SASL_NOUSER. */ return SASL_NOUSER; } utils->log(conn, SASL_LOG_ERR, "unable to open MDB environment %s: %s", path, mdb_strerror(ret)); utils->seterror(conn, SASL_NOLOG, "Unable to open MDB environment"); return SASL_FAIL; } } else { env = db_env; } ret = mdb_txn_begin(env, NULL, rdwr ? 0 : MDB_RDONLY, &txn); if (ret) { mdb_env_close(env); utils->log(conn, SASL_LOG_ERR, "unable to open MDB transaction: %s", mdb_strerror(ret)); utils->seterror(conn, SASL_NOLOG, "Unable to open MDB transaction"); return SASL_FAIL; } if (!db_dbi) { ret = mdb_open(txn, NULL, 0, &db_dbi); if (ret) { mdb_txn_abort(txn); mdb_env_close(env); utils->log(conn, SASL_LOG_ERR, "unable to open MDB database: %s", mdb_strerror(ret)); utils->seterror(conn, SASL_NOLOG, "Unable to open MDB database"); return SASL_FAIL; } } if (!db_env) db_env = env; *mtxn = txn; return SASL_OK; }
bool B_ACCURATE_LMDB::init(JCR *jcr, uint32_t nbfile) { int result; MDB_env *env; size_t mapsize = 10485760; if (!m_db_env) { result = mdb_env_create(&env); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to create MDB environment: %s\n"), mdb_strerror(result)); return false; } if ((nbfile * AVG_NR_BYTES_PER_ENTRY) > mapsize) { size_t pagesize; #ifdef HAVE_GETPAGESIZE pagesize = getpagesize(); #else pagesize = B_PAGE_SIZE; #endif mapsize = (((nbfile * AVG_NR_BYTES_PER_ENTRY) / pagesize) + 1) * pagesize; } result = mdb_env_set_mapsize(env, mapsize); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to set MDB mapsize: %s\n"), mdb_strerror(result)); goto bail_out; } /* * Explicitly set the number of readers to 1. */ result = mdb_env_set_maxreaders(env, 1); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to set MDB maxreaders: %s\n"), mdb_strerror(result)); goto bail_out; } Mmsg(m_lmdb_name, "%s/.accurate_lmdb.%d", me->working_directory, jcr->JobId); result = mdb_env_open(env, m_lmdb_name, MDB_NOSUBDIR | MDB_NOLOCK | MDB_NOSYNC, 0600); if (result) { Jmsg2(jcr, M_FATAL, 0, _("Unable create LDMD database %s: %s\n"), m_lmdb_name, mdb_strerror(result)); goto bail_out; } result = mdb_txn_begin(env, NULL, 0, &m_db_rw_txn); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to start a write transaction: %s\n"), mdb_strerror(result)); goto bail_out; } result = mdb_dbi_open(m_db_rw_txn, NULL, MDB_CREATE, &m_db_dbi); if (result) { Jmsg1(jcr, M_FATAL, 0, _("Unable to open LMDB internal database: %s\n"), mdb_strerror(result)); mdb_txn_abort(m_db_rw_txn); m_db_rw_txn = NULL; goto bail_out; } m_db_env = env; } if (!m_pay_load) { m_pay_load = get_pool_memory(PM_MESSAGE); } if (!m_lmdb_name) { m_pay_load = get_pool_memory(PM_FNAME); } if (!m_seen_bitmap) { m_seen_bitmap = (char *)malloc(nbytes_for_bits(nbfile)); clear_all_bits(nbfile, m_seen_bitmap); } return true; bail_out: if (env) { mdb_env_close(env); } return false; }
DBPriv *DBPrivOpenDB(const char *dbpath, dbid id) { DBPriv *db = xcalloc(1, sizeof(DBPriv)); MDB_txn *txn = NULL; int rc; rc = pthread_key_create(&db->txn_key, &DestroyTransaction); if (rc) { Log(LOG_LEVEL_ERR, "Could not create transaction key. (pthread_key_create: '%s')", GetErrorStrFromCode(rc)); free(db); return NULL; } rc = mdb_env_create(&db->env); if (rc) { Log(LOG_LEVEL_ERR, "Could not create handle for database %s: %s", dbpath, mdb_strerror(rc)); goto err; } rc = mdb_env_set_mapsize(db->env, LMDB_MAXSIZE); if (rc) { Log(LOG_LEVEL_ERR, "Could not set mapsize for database %s: %s", dbpath, mdb_strerror(rc)); goto err; } if (DB_MAX_READERS > 0) { rc = mdb_env_set_maxreaders(db->env, DB_MAX_READERS); if (rc) { Log(LOG_LEVEL_ERR, "Could not set maxreaders for database %s: %s", dbpath, mdb_strerror(rc)); goto err; } } unsigned int open_flags = MDB_NOSUBDIR; if (id == dbid_locks || (GetAmPolicyHub() && id == dbid_lastseen)) { open_flags |= MDB_NOSYNC; } #ifdef __hpux /* * On HP-UX, a unified file cache was not introduced until version 11.31. * This means that on 11.23 there are separate file caches for mmap()'ed * files and open()'ed files. When these two are mixed, changes made using * one mode won't be immediately seen by the other mode, which is an * assumption LMDB is relying on. The MDB_WRITEMAP flag causes LMDB to use * mmap() only, so that we stay within one file cache. */ open_flags |= MDB_WRITEMAP; #endif rc = mdb_env_open(db->env, dbpath, open_flags, 0644); if (rc) { Log(LOG_LEVEL_ERR, "Could not open database %s: %s", dbpath, mdb_strerror(rc)); goto err; } if (DB_MAX_READERS > 0) { int max_readers; rc = mdb_env_get_maxreaders(db->env, &max_readers); if (rc) { Log(LOG_LEVEL_ERR, "Could not get maxreaders for database %s: %s", dbpath, mdb_strerror(rc)); goto err; } if (max_readers < DB_MAX_READERS) { // LMDB will only reinitialize maxreaders if no database handles are // open, including in other processes, which is how we might end up // here. Log(LOG_LEVEL_VERBOSE, "Failed to set LMDB max reader limit on database '%s', " "consider restarting CFEngine", dbpath); } } rc = mdb_txn_begin(db->env, NULL, MDB_RDONLY, &txn); if (rc) { Log(LOG_LEVEL_ERR, "Could not open database txn %s: %s", dbpath, mdb_strerror(rc)); goto err; } rc = mdb_open(txn, NULL, 0, &db->dbi); if (rc) { Log(LOG_LEVEL_ERR, "Could not open database dbi %s: %s", dbpath, mdb_strerror(rc)); mdb_txn_abort(txn); goto err; } rc = mdb_txn_commit(txn); if (rc) { Log(LOG_LEVEL_ERR, "Could not commit database dbi %s: %s", dbpath, mdb_strerror(rc)); goto err; } return db; err: if (db->env) { mdb_env_close(db->env); } pthread_key_delete(db->txn_key); free(db); if (rc == MDB_INVALID) { return DB_PRIV_DATABASE_BROKEN; } return NULL; }