Пример #1
0
counter_t *counter_init(const char *path, uint64_t readers) {
    counter_t *lc = NULL;
    if((lc = calloc(1, sizeof(counter_t))) == NULL) {
        perror("calloc");
        return NULL;
    }

    // Setup and open the lmdb enviornment
    MDB_CHECK(mdb_env_create(&lc->env), MDB_SUCCESS, NULL);
    MDB_CHECK(mdb_env_set_maxreaders(lc->env, readers), MDB_SUCCESS, NULL);
    MDB_CHECK(mdb_env_set_mapsize(lc->env, MDB_MAPSIZE), MDB_SUCCESS, NULL);
    MDB_CHECK(mdb_env_open(lc->env, path, MDB_WRITEMAP | MDB_MAPASYNC | MDB_NOSUBDIR, 0664), MDB_SUCCESS, NULL);

    MDB_txn *txn = NULL;
    MDB_CHECK(mdb_txn_begin(lc->env, NULL, 0, &txn), MDB_SUCCESS, NULL);

    if((lc->dbi = calloc(1, sizeof(MDB_dbi))) == NULL) {
        perror("calloc");
        return NULL;
    }

    MDB_CHECK(mdb_dbi_open(txn, NULL, 0, lc->dbi), MDB_SUCCESS, NULL);
    mdb_txn_commit(txn);
    return lc;
}
Пример #2
0
int main(void)
{
    check(mdb_env_create(&env));
    check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL));
    check(mdb_env_set_maxreaders(env, 126));
    check(mdb_env_set_maxdbs(env, 1));
    if(! access(DB_PATH, X_OK)) {
        system("rm -rf " DB_PATH);
    }
    check(mkdir(DB_PATH, 0777));
    check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644));
    new_txn();
    check(mdb_dbi_open(txn, NULL, 0, &dbi));

    put("a");
    put("b");
    put("baa");
    put("d");

    new_txn();

    check(mdb_cursor_open(txn, dbi, &c1));
    check(mdb_cursor_get(c1, &keyv, &valv, MDB_LAST));
    check(mdb_cursor_del(c1, 0));
    check(mdb_cursor_del(c1, 0));
    new_txn();
}
Пример #3
0
/*
 * Class:     jmdb_DatabaseWrapper
 * Method:    envSetMaxReaders
 * Signature: (JI)V
 */
JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_envSetMaxReaders(JNIEnv *vm,
		jclass clazz, jlong envL, jint readers) {
	MDB_env *envC = (MDB_env*) envL;
	int code = mdb_env_set_maxreaders(envC, readers);
	if (code) {
		throwDatabaseException(vm, code);
	}
}
Пример #4
0
kvenv_t *kvenv_new_lmdb(const char *dbpath, uint64_t max_dbsize, uint32_t max_dbs)
{
    kvenv_lmdb_t *kvenv_lmdb = (kvenv_lmdb_t*)zmalloc(sizeof(kvenv_lmdb_t));
    memset(kvenv_lmdb, 0, sizeof(kvenv_lmdb_t));
    kvenv_lmdb->kvenv.dbclass = "lmdb";
    kvenv_lmdb->kvenv.max_dbsize = max_dbsize;
    kvenv_lmdb->kvenv.max_dbs = max_dbs;

    int rc = mdb_env_create(&kvenv_lmdb->env);
    if ( rc != 0 ) {
        zfree(kvenv_lmdb);
        error_log("mdb_env_create() failed.");
        return NULL;
    }

    int maxreaders = 256;
    if ( max_dbs == 0 ){
        max_dbs = 256;
    }
    if ( max_dbsize == 0 ) {
        max_dbsize = 1024L * 1024L * 1024L * 4L;
    }

    rc = mdb_env_set_mapsize(kvenv_lmdb->env, max_dbsize);
    if ( rc != 0 ) {
        zfree(kvenv_lmdb);
        error_log("mdb_env_set_mapsize() failed.");
        return NULL;
    }

    rc = mdb_env_set_maxreaders(kvenv_lmdb->env, maxreaders); 
    if ( rc != 0 ) {
        zfree(kvenv_lmdb);
        error_log("mdb_env_set_maxreaders() failed.");
        return NULL;
    }

    rc = mdb_env_set_maxdbs(kvenv_lmdb->env, max_dbs); 
    if ( rc != 0 ) {
        zfree(kvenv_lmdb);
        error_log("mdb_env_set_maxdbs() failed.");
        return NULL;
    }

    /*rc = mdb_env_open(lmdb->env, dbpath, MDB_FIXEDMAP | MDB_NOSYNC, 0640); */

    /*rc = mdb_env_open(lmdb->env, dbpath, MDB_MAPASYNC | MDB_WRITEMAP | MDB_NOTLS , 0640); */
    /*rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_MAPASYNC | MDB_WRITEMAP, 0640); */
    /*rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_NOMETASYNC, 0640); */
    rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_NOSYNC, 0640); 
    if ( rc != 0 ) {
        zfree(kvenv_lmdb);
        error_log("mdb_env_open() failed. dbpath=%s error: %s", dbpath, mdb_strerror(rc));
        return NULL;
    }

    return (kvenv_t*)kvenv_lmdb;
}
Пример #5
0
// Open a LMDB database.
MDB_env *database_open(const char *path, size_t mapsize, unsigned int nbr_threads) {
	MDB_env *env = NULL;

	mdb_env_create(&env);
	mdb_env_set_mapsize(env, mapsize);
	mdb_env_set_maxreaders(env, nbr_threads);
	mdb_env_open(env, path, 0, 0664);
	return (env);
}
Пример #6
0
void LMDB::Open(const string& source, Mode mode) {
  MDB_CHECK(mdb_env_create(&mdb_env_));
  MDB_CHECK(mdb_env_set_mapsize(mdb_env_, LMDB_MAP_SIZE));
  MDB_CHECK(mdb_env_set_maxreaders(mdb_env_, 100000));
  if (mode == NEW) {
    CHECK_EQ(mkdir(source.c_str(), 0744), 0) << "mkdir " << source << "failed";
  }
  int flags = 0;
  if (mode == READ) {
    flags = MDB_RDONLY | MDB_NOTLS;
  }
  MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664));
  LOG(INFO) << "Opened lmdb " << source;
}
Пример #7
0
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
	int rc, i;
	struct mdb_info *mdb = (struct mdb_info *) be->be_private;
	struct stat stat1;
	uint32_t flags;
	char *dbhome;
	MDB_txn *txn;

	if ( be->be_suffix == NULL ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": need suffix.\n",
			1, 0, 0 );
		return -1;
	}

	Debug( LDAP_DEBUG_ARGS,
		LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
		be->be_suffix[0].bv_val, 0, 0 );

	/* Check existence of dbenv_home. Any error means trouble */
	rc = stat( mdb->mi_dbenv_home, &stat1 );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"cannot access database directory \"%s\" (%d).\n",
			be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
		return -1;
	}

	/* mdb is always clean */
	be->be_flags |= SLAP_DBFLAG_CLEAN;

	rc = mdb_env_create( &mdb->mi_dbenv );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_create failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	if ( mdb->mi_readers ) {
		rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
		if( rc != 0 ) {
			Debug( LDAP_DEBUG_ANY,
				LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
				"mdb_env_set_maxreaders failed: %s (%d).\n",
				be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
			goto fail;
		}
	}

	rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_set_mapsize failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_set_maxdbs failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

#ifdef HAVE_EBCDIC
	strcpy( path, mdb->mi_dbenv_home );
	__atoe( path );
	dbhome = path;
#else
	dbhome = mdb->mi_dbenv_home;
#endif

	Debug( LDAP_DEBUG_TRACE,
		LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
		"dbenv_open(%s).\n",
		be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0);

	flags = mdb->mi_dbenv_flags;

	if ( slapMode & SLAP_TOOL_QUICK )
		flags |= MDB_NOSYNC|MDB_WRITEMAP;

	if ( slapMode & SLAP_TOOL_READONLY)
		flags |= MDB_RDONLY;

	rc = mdb_env_open( mdb->mi_dbenv, dbhome,
			flags, mdb->mi_dbenv_mode );

	if ( rc ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
			"Restore from backup!\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
	if ( rc ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
			"Restore from backup!\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	/* open (and create) main databases */
	for( i = 0; mdmi_databases[i].bv_val; i++ ) {
		flags = MDB_INTEGERKEY;
		if( i == MDB_ID2ENTRY ) {
			if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
				flags |= MDB_CREATE;
		} else {
			if ( i == MDB_DN2ID )
				flags |= MDB_DUPSORT;
			if ( !(slapMode & SLAP_TOOL_READONLY) )
				flags |= MDB_CREATE;
		}

		rc = mdb_dbi_open( txn,
			mdmi_databases[i].bv_val,
			flags,
			&mdb->mi_dbis[i] );

		if ( rc != 0 ) {
			snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
				"mdb_dbi_open(%s/%s) failed: %s (%d).", 
				be->be_suffix[0].bv_val, 
				mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
				mdb_strerror(rc), rc );
			Debug( LDAP_DEBUG_ANY,
				LDAP_XSTRING(mdb_db_open) ": %s\n",
				cr->msg, 0, 0 );
			goto fail;
		}

		if ( i == MDB_ID2ENTRY )
			mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
		else if ( i == MDB_DN2ID ) {
			MDB_cursor *mc;
			MDB_val key, data;
			ID id;
			mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
			/* check for old dn2id format */
			rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
			/* first record is always ID 0 */
			rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
			if ( rc == 0 ) {
				rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
				if ( rc == 0 ) {
					int len;
					unsigned char *ptr;
					ptr = data.mv_data;
					len = (ptr[0] & 0x7f) << 8 | ptr[1];
					if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
						snprintf( cr->msg, sizeof(cr->msg),
						"database \"%s\": DN index needs upgrade, "
						"run \"slapindex entryDN\".",
						be->be_suffix[0].bv_val );
						Debug( LDAP_DEBUG_ANY,
							LDAP_XSTRING(mdb_db_open) ": %s\n",
							cr->msg, 0, 0 );
						if ( !(slapMode & SLAP_TOOL_READMAIN ))
							rc = LDAP_OTHER;
						mdb->mi_flags |= MDB_NEED_UPGRADE;
					}
				}
			}
			mdb_cursor_close( mc );
			if ( rc == LDAP_OTHER )
				goto fail;
		}
	}

	rc = mdb_ad_read( mdb, txn );
	if ( rc ) {
		mdb_txn_abort( txn );
		goto fail;
	}

	/* slapcat doesn't need indexes. avoid a failure if
	 * a configured index wasn't created yet.
	 */
	if ( !(slapMode & SLAP_TOOL_READONLY) ) {
		rc = mdb_attr_dbs_open( be, txn, cr );
		if ( rc ) {
			mdb_txn_abort( txn );
			goto fail;
		}
	}

	rc = mdb_txn_commit(txn);
	if ( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database %s: "
			"txn_commit failed: %s (%d)\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	/* monitor setup */
	rc = mdb_monitor_db_open( be );
	if ( rc != 0 ) {
		goto fail;
	}

	mdb->mi_flags |= MDB_IS_OPEN;

	return 0;

fail:
	mdb_db_close( be, NULL );
	return rc;
}
Пример #8
0
int main(int argc,char * argv[])
{
	int i = 0, j = 0, rc;
	MDB_env *env;
	MDB_dbi dbi;
	MDB_val key, data;
	MDB_txn *txn;
	MDB_stat mst;
	MDB_cursor *cursor;
	int count;
	int *values;
	char sval[32] = "";
	int env_oflags;
	struct stat db_stat, exe_stat;

	(void) argc;
	(void) argv;
	srand(time(NULL));

	count = (rand()%384) + 64;
	values = (int *)malloc(count*sizeof(int));

	for(i = 0;i<count;i++) {
		values[i] = rand()%1024;
	}

	E(mdb_env_create(&env));
	E(mdb_env_set_maxreaders(env, 1));
	E(mdb_env_set_mapsize(env, 10485760));
	E(mdb_env_set_maxdbs(env, 4));

	E(stat("/proc/self/exe", &exe_stat)?errno:0);
	E(stat(DBPATH "/.", &db_stat)?errno:0);
	env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
	if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
		/* LY: Assume running inside a CI-environment:
		 *  1) don't use FIXEDMAP to avoid EBUSY in case collision,
		 *     which could be inspired by address space randomisation feature.
		 *  2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
		 */
		env_oflags = 0;
	}
	/* LY: especially here we always needs MDB_NOSYNC
	 * for testing mdbx_env_close_ex() and "redo-to-steady" on open. */
	env_oflags |= MDB_NOSYNC;
	E(mdb_env_open(env, DBPATH, env_oflags, 0664));

	E(mdb_txn_begin(env, NULL, 0, &txn));
	if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS)
		E(mdb_drop(txn, dbi, 1));
	E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));

	key.mv_size = sizeof(int);
	key.mv_data = sval;
	data.mv_size = sizeof(sval);
	data.mv_data = sval;

	printf("Adding %d values\n", count);
	for (i=0;i<count;i++) {
		sprintf(sval, "%03x %d foo bar", values[i], values[i]);
		if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
			j++;
	}
	if (j) printf("%d duplicates skipped\n", j);
	E(mdb_txn_commit(txn));
	E(mdb_env_stat(env, &mst));

	printf("check-preset-a\n");
	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
	int present_a = 0;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %p %.*s, data: %p %.*s\n",
			key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
			data.mv_data, (int) data.mv_size, (char *) data.mv_data);
		++present_a;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_a == count - j, "mismatch");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);
	mdb_env_sync(env, 1);

	int deleted = 0;
	key.mv_data = sval;
	for (i = count - 1; i > -1; i -= (rand()%5)) {
		txn=NULL;
		E(mdb_txn_begin(env, NULL, 0, &txn));
		sprintf(sval, "%03x ", values[i]);
		if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
			mdb_txn_abort(txn);
		} else {
			E(mdb_txn_commit(txn));
			deleted++;
		}
	}
	free(values);
	printf("Deleted %d values\n", deleted);

	printf("check-preset-b.cursor-next\n");
	E(mdb_env_stat(env, &mst));
	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
	int present_b = 0;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		++present_b;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_b == present_a - deleted, "mismatch");

	printf("check-preset-b.cursor-prev\n");
	j = 1;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		++j;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_b == j, "mismatch");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);

	mdb_dbi_close(env, dbi);
	/********************* LY: kept DB dirty ****************/
	mdbx_env_close_ex(env, 1);
	E(mdb_env_create(&env));
	E(mdb_env_set_maxdbs(env, 4));
	E(mdb_env_open(env, DBPATH, env_oflags, 0664));

	printf("check-preset-c.cursor-next\n");
	E(mdb_env_stat(env, &mst));
	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_dbi_open(txn, "id1", 0, &dbi));
	E(mdb_cursor_open(txn, dbi, &cursor));
	int present_c = 0;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		++present_c;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	printf("Rolled back %d deletion(s)\n", present_c - (present_a - deleted));
	CHECK(present_c > present_a - deleted, "mismatch");

	printf("check-preset-d.cursor-prev\n");
	j = 1;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		++j;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_c == j, "mismatch");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);

	mdb_dbi_close(env, dbi);
	mdbx_env_close_ex(env, 0);

	return 0;
}
Пример #9
0
int main(int argc,char * argv[])
{
	int i = 0, j = 0, rc;
	MDB_env *env;
	MDB_dbi dbi;
	MDB_val key, data;
	MDB_txn *txn;
	MDB_stat mst;
	MDB_cursor *cursor;
	int count;
	int *values;
	char sval[32] = "";

	srand(time(NULL));

	count = (rand()%384) + 64;
	values = (int *)malloc(count*sizeof(int));

	for(i = 0;i<count;i++) {
		values[i] = rand()%1024;
	}

	E(mdb_env_create(&env));

	E(mdb_env_set_maxreaders(env, 1));
	E(mdb_env_set_mapsize(env, 10485760));
	E(mdb_env_set_maxdbs(env, 4));
	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));

	E(mdb_txn_begin(env, NULL, 0, &txn));
	if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS)
		E(mdb_drop(txn, dbi, 1));
	E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));

	key.mv_size = sizeof(int);
	key.mv_data = sval;
	data.mv_size = sizeof(sval);
	data.mv_data = sval;

	printf("Adding %d values\n", count);
	for (i=0;i<count;i++) {
		sprintf(sval, "%03x %d foo bar", values[i], values[i]);
		if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
			j++;
	}
	if (j) printf("%d duplicates skipped\n", j);
	E(mdb_txn_commit(txn));
	E(mdb_env_stat(env, &mst));

	printf("check-preset-a\n");
	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
	int present_a = 0;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %p %.*s, data: %p %.*s\n",
			key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
			data.mv_data, (int) data.mv_size, (char *) data.mv_data);
		++present_a;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_a == count - j, "mismatch");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);
	mdb_env_sync(env, 1);

	j=0;
	key.mv_data = sval;
	for (i= count - 1; i > -1; i-= (rand()%5)) {
		j++;
		txn=NULL;
		E(mdb_txn_begin(env, NULL, 0, &txn));
		sprintf(sval, "%03x ", values[i]);
		if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
			j--;
			mdb_txn_abort(txn);
		} else {
			E(mdb_txn_commit(txn));
		}
	}
	free(values);
	printf("Deleted %d values\n", j);

	printf("check-preset-b.cursor-next\n");
	E(mdb_env_stat(env, &mst));
	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
	int present_b = 0;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		++present_b;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_b == present_a - j, "mismatch");

	printf("check-preset-b.cursor-prev\n");
	j = 1;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		++j;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_b == j, "mismatch");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);

	mdb_dbi_close(env, dbi);
	/********************* LY: kept DB dirty ****************/
	mdb_env_close_ex(env, 1);
	E(mdb_env_create(&env));
	E(mdb_env_set_maxdbs(env, 4));
	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));

	printf("check-preset-c.cursor-next\n");
	E(mdb_env_stat(env, &mst));
	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_dbi_open(txn, "id1", 0, &dbi));
	E(mdb_cursor_open(txn, dbi, &cursor));
	int present_c = 0;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		++present_c;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_c == present_a, "mismatch");

	printf("check-preset-d.cursor-prev\n");
	j = 1;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		++j;
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	CHECK(present_c == j, "mismatch");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);

	mdb_dbi_close(env, dbi);
	mdb_env_close_ex(env, 0);

	return 0;
}
Пример #10
0
bool LMDBFileIndex::create_env()
{
	int rc;
	if(env==NULL)
	{
		rc = mdb_env_create(&env);
		if(rc)
		{
			Server->Log("LMDB: Failed to create LMDB env ("+(std::string)mdb_strerror(rc)+")", LL_ERROR);
			return false;
		}

		{
			std::auto_ptr<IFile> lmdb_f(Server->openFile("urbackup/fileindex/backup_server_files_index.lmdb", MODE_READ));
			if(lmdb_f.get()!=NULL)
			{
				while(lmdb_f->Size()>static_cast<_i64>(map_size))
				{
					map_size*=2;
				}
			}
		}
		
		rc = mdb_env_set_maxreaders(env, 4094);

		if (rc)
		{
			Server->Log("LMDB: Failed to set max readers (" + (std::string)mdb_strerror(rc) + ")", LL_ERROR);
			return false;
		}

		rc = mdb_env_set_mapsize(env, map_size);

		if(rc)
		{
			Server->Log("LMDB: Failed to set map size ("+(std::string)mdb_strerror(rc)+")", LL_ERROR);
			return false;
		}

		os_create_dir("urbackup/fileindex");

		unsigned int flags = MDB_NOSUBDIR|MDB_NOMETASYNC;
		if(no_sync)
		{
			flags|=MDB_NOSYNC;
		}
		rc = mdb_env_open(env, "urbackup/fileindex/backup_server_files_index.lmdb", flags, 0664);

		if(rc)
		{
			Server->Log("LMDB: Failed to open LMDB database file ("+(std::string)mdb_strerror(rc)+")", LL_ERROR);
			return false;
		}

		MDB_txn* l_txn;
		rc = mdb_txn_begin(env, NULL, flags, &l_txn);

		if (rc)
		{
			Server->Log("LMDB: Failed to open transaction handle for dbi open (" + (std::string)mdb_strerror(rc) + ")", LL_ERROR);
			return false;
		}

		rc = mdb_dbi_open(l_txn, NULL, 0, &dbi);

		if (rc)
		{
			Server->Log("LMDB: Failed to open database (" + (std::string)mdb_strerror(rc) + ")", LL_ERROR);
			return false;
		}

		rc = mdb_txn_commit(l_txn);

		if (rc)
		{
			Server->Log("LMDB: Failed to commit txn for dbi handle (" + (std::string)mdb_strerror(rc) + ")", LL_ERROR);
			return false;
		}

		return true;
	}
	else
	{
		return true;
	}
}
Пример #11
0
int main(int argc,char * argv[])
{
	int i = 0, j = 0, rc;
	MDB_env *env;
	MDB_dbi dbi;
	MDB_val key, data;
	MDB_txn *txn;
	MDB_stat mst;
	MDB_cursor *cursor;
	int count;
	int *values;
	char sval[32] = "";
	int env_oflags;
	struct stat db_stat, exe_stat;

	srand(time(NULL));

	count = (rand()%384) + 64;
	values = (int *)malloc(count*sizeof(int));

	for(i = 0;i<count;i++) {
		values[i] = rand()%1024;
	}

	E(mdb_env_create(&env));
	E(mdb_env_set_maxreaders(env, 1));
	E(mdb_env_set_mapsize(env, 10485760));
	E(mdb_env_set_maxdbs(env, 4));

	E(stat("/proc/self/exe", &exe_stat)?errno:0);
	E(stat(DBPATH "/.", &db_stat)?errno:0);
	env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
	if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
		/* LY: Assume running inside a CI-environment:
		 *  1) don't use FIXEDMAP to avoid EBUSY in case collision,
		 *     which could be inspired by address space randomisation feature.
		 *  2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
		 */
		env_oflags = 0;
	}
	E(mdb_env_open(env, DBPATH, env_oflags, 0664));

	E(mdb_txn_begin(env, NULL, 0, &txn));
	if (mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi) == MDB_SUCCESS)
		E(mdb_drop(txn, dbi, 1));
	E(mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi));

	key.mv_size = sizeof(int);
	key.mv_data = sval;

	printf("Adding %d values\n", count);
	for (i=0;i<count;i++) {
		sprintf(sval, "%03x %d foo bar", values[i], values[i]);
		data.mv_size = sizeof(sval);
		data.mv_data = sval;
		if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
			j++;
	}
	if (j) printf("%d duplicates skipped\n", j);
	E(mdb_txn_commit(txn));
	E(mdb_env_stat(env, &mst));

	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %p %.*s, data: %p %.*s\n",
			key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
			data.mv_data, (int) data.mv_size, (char *) data.mv_data);
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);

	j=0;
	key.mv_data = sval;
	for (i= count - 1; i > -1; i-= (rand()%5)) {
		j++;
		txn=NULL;
		E(mdb_txn_begin(env, NULL, 0, &txn));
		sprintf(sval, "%03x ", values[i]);
		if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
			j--;
			mdb_txn_abort(txn);
		} else {
			E(mdb_txn_commit(txn));
		}
	}
	free(values);
	printf("Deleted %d values\n", j);

	E(mdb_env_stat(env, &mst));
	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
	printf("Cursor next\n");
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	printf("Cursor prev\n");
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);

	mdb_dbi_close(env, dbi);
	mdb_env_close(env);
	return 0;
}
Пример #12
0
/*
 * Initialize MDB db
 * (reference openldap 2.4.31 back-mdb/init.c)
 */
DWORD
VmDirMDBInitializeDB(
    BOOLEAN bMainDB,
    const char *dbHomeDir,
    PVDIR_DB_HANDLE *phHandle
    )
{
    DWORD           dwError = 0;
    unsigned int    envFlags = 0;
    mdb_mode_t      mode = 0;
    uint64_t        db_max_mapsize = BE_MDB_ENV_MAX_MEM_MAPSIZE;
    DWORD           db_max_size_mb = 0;
    DWORD           db_chkpt_interval = 0;
    BOOLEAN         bMdbWalEnable = TRUE;
    PVDIR_MDB_DB pDB = NULL;

    VmDirLog( LDAP_DEBUG_TRACE, "MDBInitializeDB: Begin, DB Home Dir = %s", dbHomeDir );

    //Make a db entry for path. fail if already exist
    dwError = _VmDirMDBInitializeDBEntry(dbHomeDir, &pDB);
    BAIL_ON_VMDIR_ERROR( dwError );

    if (bMainDB)
    {
      pDB->bIsMainDB = TRUE;
    } else
    {
      pDB->bIsMainDB = FALSE;
    }

    dwError = (sizeof(ENTRYID) == sizeof(VDIR_DB_SEQ_T)) ? 0 : ERROR_BACKEND_ERROR;
    BAIL_ON_VMDIR_ERROR( dwError );

    dwError = MDBInitConfig(pDB);
    BAIL_ON_VMDIR_ERROR( dwError );

    /* Create the environment */
    dwError = mdb_env_create ( &pDB->mdbEnv );
    BAIL_ON_VMDIR_ERROR( dwError );

    dwError = mdb_env_set_maxreaders( pDB->mdbEnv, BE_MDB_ENV_MAX_READERS );
    BAIL_ON_VMDIR_ERROR( dwError );

    /* FROM mdb.h
     * The size should be a multiple of the OS page size. The default is
     * 10485760 bytes. The size of the memory map is also the maximum size
     * of the database. The value should be chosen as large as possible,
     * to accommodate future growth of the database.
     *
     * // TODO, this is also the max size of database (per logical mdb db or the total dbs)
     */

     dwError = VmDirGetMaxDbSizeMb(&db_max_size_mb);
     if (dwError != 0)
     {
         VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "Use default max-database-size %llu", BE_MDB_ENV_MAX_MEM_MAPSIZE);
     } else
     {
         db_max_mapsize = (uint64_t)(db_max_size_mb)*1024*1024;
         if (db_max_mapsize < BE_MDB_ENV_MAX_MEM_MAPSIZE)
         {
             db_max_mapsize = BE_MDB_ENV_MAX_MEM_MAPSIZE;
             VMDIR_LOG_WARNING(VMDIR_LOG_MASK_ALL, "RegKey %s value (%u) is too small. Use default max-database-size %llu",
                            VMDIR_REG_KEY_MAXIMUM_DB_SIZE_MB, db_max_size_mb, db_max_mapsize);
         } else
         {
             VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "max-database-size is set to %llu per RegKey %s",
                            db_max_mapsize, VMDIR_REG_KEY_MAXIMUM_DB_SIZE_MB);
         }
     }

     dwError = mdb_env_set_mapsize( pDB->mdbEnv, db_max_mapsize);
     BAIL_ON_VMDIR_ERROR( dwError );

     dwError = mdb_env_set_maxdbs ( pDB->mdbEnv, BE_MDB_ENV_MAX_DBS );
     BAIL_ON_VMDIR_ERROR( dwError );

     dwError = VmDirGetMdbChkptInterval(&db_chkpt_interval);
     if (dwError)
     {
         db_chkpt_interval = VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_DEFAULT;
         dwError = 0;
     }

     VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: %s is set to %d",
        __func__, VMDIR_REG_KEY_MDB_CHKPT_INTERVAL, db_chkpt_interval);

     dwError = mdb_env_set_chkpt_interval(pDB->mdbEnv, db_chkpt_interval);
     BAIL_ON_VMDIR_ERROR( dwError );

     if(bMainDB)
     {
         mdb_set_raft_prepare_commit_func(pDB->mdbEnv, VmDirRaftPrepareCommit);

         mdb_set_raft_post_commit_func(pDB->mdbEnv, VmDirRaftPostCommit);

         mdb_set_raft_commit_fail_func(pDB->mdbEnv, VmDirRaftCommitFail);
     }

#ifdef MDB_NOTLS
     envFlags = MDB_NOTLS; // Required for versions of mdb which have this flag
#endif

     // this is experimental from mdb.h comments
     //envFlags = MDB_FIXEDMAP;        /* use a fixed address for the mmap region */

     //envFlags |= MDB_NOSYNC       need sync for durability
     //envFlags |= MDB_RDONLY       need to open for read and write

    /* Open the environment.  */

#ifndef _WIN32
    mode = S_IRUSR | S_IWUSR;
#else
    mode = GENERIC_READ|GENERIC_WRITE;
#endif

    //MDB WAL is the default mode and can be turned off with reg key MdbEnableWal set to 0
    dwError = VmDirGetMdbWalEnable(&bMdbWalEnable);
    if (dwError)
    {
        bMdbWalEnable = TRUE;
        dwError = 0;
    }

    VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: %s is set to %s",
      __func__, VMDIR_REG_KEY_MDB_ENABLE_WAL, bMdbWalEnable?"True":"False");

    if (bMdbWalEnable)
    {
        envFlags |= MDB_WAL;
    }

    dwError = mdb_env_open ( pDB->mdbEnv, dbHomeDir, envFlags, mode );
//TODO, what if open failed?  how to recover??
    BAIL_ON_VMDIR_ERROR( dwError );

    mdb_set_error_log_func(&VmDirMdbErrorLog);

    /* Open main database. */
    dwError = MDBOpenMainDB(pDB);
    BAIL_ON_VMDIR_ERROR( dwError );

    /* Open sequences */
    dwError = MDBOpenSequence(pDB);
    BAIL_ON_VMDIR_ERROR( dwError );

    /* Open generic */
    dwError = MDBOpenGeneric(pDB);
    BAIL_ON_VMDIR_ERROR( dwError );

    /* Initialize indices */
    dwError = VmDirMDBInitializeIndexDB(pDB);
    BAIL_ON_VMDIR_ERROR( dwError );

    dwError = VmDirInitMdbStateGlobals();
    BAIL_ON_VMDIR_ERROR(dwError);

    VmDirLogDBStats(pDB);

    /* TODO: require per db txn stats for raft groups */
    if (bMainDB)
    {
        dwError = _VmDirWtxnStatsInit();
        BAIL_ON_VMDIR_ERROR( dwError );
    }

    *phHandle = pDB;

    VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: succeeded with dbHomeDir %s", __func__, dbHomeDir); 

cleanup:
    VmDirLog( LDAP_DEBUG_TRACE, "MDBInitializeDB: End" );

    return dwError;

error:

    VmDirLog( LDAP_DEBUG_ANY, "MDBInitializeDB failed with error code: %d, error string: %s", dwError, mdb_strerror(dwError) );

//TODO, should shutdown here or caller will do that?
//gVdirMdbGlobals.mdbEnv = NULL;

    goto cleanup;
}
int main(int argc,char * argv[])
{
	int i = 0, j = 0, rc;
	MDB_env *env;
	MDB_dbi dbi;
	MDB_val key, data;
	MDB_txn *txn;
	MDB_stat mst;
	MDB_cursor *cursor, *cur2;
	MDB_cursor_op op;
	int count;
	int *values;
	char sval[32] = "";

	srand(time(NULL));

	    count = (rand()%384) + 64;
	    values = (int *)malloc(count*sizeof(int));

	    for(i = 0;i<count;i++) {
			values[i] = rand()%1024;
	    }
    
		E(mdb_env_create(&env));
		E(mdb_env_set_maxreaders(env, 1));
		E(mdb_env_set_mapsize(env, 10485760));
		E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664));

		E(mdb_txn_begin(env, NULL, 0, &txn));
		E(mdb_dbi_open(txn, NULL, 0, &dbi));
   
		key.mv_size = sizeof(int);
		key.mv_data = sval;

		printf("Adding %d values\n", count);
	    for (i=0;i<count;i++) {	
			sprintf(sval, "%03x %d foo bar", values[i], values[i]);
			/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
			data.mv_size = sizeof(sval);
			data.mv_data = sval;
			if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
				j++;
				data.mv_size = sizeof(sval);
				data.mv_data = sval;
			}
	    }
		if (j) printf("%d duplicates skipped\n", j);
		E(mdb_txn_commit(txn));
		E(mdb_env_stat(env, &mst));

		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
		E(mdb_cursor_open(txn, dbi, &cursor));
		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
			printf("key: %p %.*s, data: %p %.*s\n",
				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
		}
		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
		mdb_cursor_close(cursor);
		mdb_txn_abort(txn);

		j=0;
		key.mv_data = sval;
	    for (i= count - 1; i > -1; i-= (rand()%5)) {
			j++;
			txn=NULL;
			E(mdb_txn_begin(env, NULL, 0, &txn));
			sprintf(sval, "%03x ", values[i]);
			if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
				j--;
				mdb_txn_abort(txn);
			} else {
				E(mdb_txn_commit(txn));
			}
	    }
	    free(values);
		printf("Deleted %d values\n", j);

		E(mdb_env_stat(env, &mst));
		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
		E(mdb_cursor_open(txn, dbi, &cursor));
		printf("Cursor next\n");
		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
			printf("key: %.*s, data: %.*s\n",
				(int) key.mv_size,  (char *) key.mv_data,
				(int) data.mv_size, (char *) data.mv_data);
		}
		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
		printf("Cursor last\n");
		E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
		printf("Cursor prev\n");
		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
			printf("key: %.*s, data: %.*s\n",
				(int) key.mv_size,  (char *) key.mv_data,
				(int) data.mv_size, (char *) data.mv_data);
		}
		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
		printf("Cursor last/prev\n");
		E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
			printf("key: %.*s, data: %.*s\n",
				(int) key.mv_size,  (char *) key.mv_data,
				(int) data.mv_size, (char *) data.mv_data);
		E(mdb_cursor_get(cursor, &key, &data, MDB_PREV));
			printf("key: %.*s, data: %.*s\n",
				(int) key.mv_size,  (char *) key.mv_data,
				(int) data.mv_size, (char *) data.mv_data);

		mdb_cursor_close(cursor);
		mdb_txn_abort(txn);

		printf("Deleting with cursor\n");
		E(mdb_txn_begin(env, NULL, 0, &txn));
		E(mdb_cursor_open(txn, dbi, &cur2));
		for (i=0; i<50; i++) {
			if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT)))
				break;
			printf("key: %p %.*s, data: %p %.*s\n",
				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
			E(mdb_del(txn, dbi, &key, NULL));
		}

		printf("Restarting cursor in txn\n");
		for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
			if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op)))
				break;
			printf("key: %p %.*s, data: %p %.*s\n",
				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
		}
		mdb_cursor_close(cur2);
		E(mdb_txn_commit(txn));

		printf("Restarting cursor outside txn\n");
		E(mdb_txn_begin(env, NULL, 0, &txn));
		E(mdb_cursor_open(txn, dbi, &cursor));
		for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
			if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op)))
				break;
			printf("key: %p %.*s, data: %p %.*s\n",
				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
		}
		mdb_cursor_close(cursor);
		mdb_txn_abort(txn);

		mdb_dbi_close(env, dbi);
		mdb_env_close(env);

	return 0;
}
Пример #14
0
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
	int rc, i;
	struct mdb_info *mdb = (struct mdb_info *) be->be_private;
	struct stat stat1;
	uint32_t flags;
	char *dbhome;
	MDB_txn *txn;

	if ( be->be_suffix == NULL ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": need suffix.\n",
			1, 0, 0 );
		return -1;
	}

	Debug( LDAP_DEBUG_ARGS,
		LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
		be->be_suffix[0].bv_val, 0, 0 );

	/* Check existence of dbenv_home. Any error means trouble */
	rc = stat( mdb->mi_dbenv_home, &stat1 );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"cannot access database directory \"%s\" (%d).\n",
			be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
		return -1;
	}

	/* mdb is always clean */
	be->be_flags |= SLAP_DBFLAG_CLEAN;

	rc = mdb_env_create( &mdb->mi_dbenv );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_create failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	if ( mdb->mi_readers ) {
		rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
		if( rc != 0 ) {
			Debug( LDAP_DEBUG_ANY,
				LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
				"mdb_env_set_maxreaders failed: %s (%d).\n",
				be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
			goto fail;
		}
	}

	rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_set_mapsize failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_set_maxdbs failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

#ifdef HAVE_EBCDIC
	strcpy( path, mdb->mi_dbenv_home );
	__atoe( path );
	dbhome = path;
#else
	dbhome = mdb->mi_dbenv_home;
#endif

	Debug( LDAP_DEBUG_TRACE,
		LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
		"dbenv_open(%s).\n",
		be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0);

	flags = mdb->mi_dbenv_flags;

	if ( slapMode & SLAP_TOOL_QUICK )
		flags |= MDB_NOSYNC;

	if ( slapMode & SLAP_TOOL_READONLY)
		flags |= MDB_RDONLY;

	rc = mdb_env_open( mdb->mi_dbenv, dbhome,
			flags, mdb->mi_dbenv_mode );

	if ( rc ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. "
			"Restore from backup!\n",
			be->be_suffix[0].bv_val, rc, 0 );
		goto fail;
	}

	rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
	if ( rc ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. "
			"Restore from backup!\n",
			be->be_suffix[0].bv_val, rc, 0 );
		goto fail;
	}

	/* open (and create) main databases */
	for( i = 0; mdmi_databases[i].bv_val; i++ ) {
		flags = MDB_INTEGERKEY;
		if( i == MDB_ID2ENTRY ) {
			if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
				flags |= MDB_CREATE;
		} else {
			if ( i == MDB_DN2ID )
				flags |= MDB_DUPSORT;
			if ( !(slapMode & SLAP_TOOL_READONLY) )
				flags |= MDB_CREATE;
		}

		rc = mdb_open( txn,
			mdmi_databases[i].bv_val,
			flags,
			&mdb->mi_dbis[i] );

		if ( rc != 0 ) {
			snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
				"mdb_open(%s/%s) failed: %s (%d).", 
				be->be_suffix[0].bv_val, 
				mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
				mdb_strerror(rc), rc );
			Debug( LDAP_DEBUG_ANY,
				LDAP_XSTRING(mdb_db_open) ": %s\n",
				cr->msg, 0, 0 );
			goto fail;
		}

		if ( i == MDB_ID2ENTRY )
			mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
		else if ( i == MDB_DN2ID )
			mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );

	}

	rc = mdb_ad_read( mdb, txn );
	if ( rc ) {
		mdb_txn_abort( txn );
		goto fail;
	}

	rc = mdb_attr_dbs_open( be, txn, cr );
	if ( rc ) {
		mdb_txn_abort( txn );
		goto fail;
	}

	rc = mdb_txn_commit(txn);
	if ( rc != 0 ) {
		goto fail;
	}

	/* monitor setup */
	rc = mdb_monitor_db_open( be );
	if ( rc != 0 ) {
		goto fail;
	}

	mdb->mi_flags |= MDB_IS_OPEN;

	return 0;

fail:
	mdb_db_close( be, NULL );
	return rc;
}
Пример #15
0
int main(int argc, char *argv[])
{
	int i, rc;
	MDB_env *env;
	MDB_txn *txn;
	MDB_cursor *mc;
	MDB_dbi dbi;
	char *envname;
	int envflags = 0, putflags = 0;
	int dohdr = 0;

	prog = argv[0];

	if (argc < 2) {
		usage();
	}

	/* -f: load file instead of stdin
	 * -n: use NOSUBDIR flag on env_open
	 * -s: load into named subDB
	 * -N: use NOOVERWRITE on puts
	 * -T: read plaintext
	 * -V: print version and exit
	 */
	while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
		switch(i) {
		case 'V':
			printf("%s\n", MDB_VERSION_STRING);
			exit(0);
			break;
		case 'f':
			if (freopen(optarg, "r", stdin) == NULL) {
				fprintf(stderr, "%s: %s: reopen: %s\n",
					prog, optarg, strerror(errno));
				exit(EXIT_FAILURE);
			}
			break;
		case 'n':
			envflags |= MDB_NOSUBDIR;
			break;
		case 's':
			subname = strdup(optarg);
			break;
		case 'N':
			putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
			break;
		case 'T':
			mode |= NOHDR | PRINT;
			break;
		default:
			usage();
		}
	}

	if (optind != argc - 1)
		usage();

	dbuf.mv_size = 4096;
	dbuf.mv_data = malloc(dbuf.mv_size);

	if (!(mode & NOHDR))
		readhdr();

	envname = argv[optind];
	rc = mdb_env_create(&env);
	if (rc) {
		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
		return EXIT_FAILURE;
	}

	mdb_env_set_maxdbs(env, 2);

	if (info.me_maxreaders)
		mdb_env_set_maxreaders(env, info.me_maxreaders);

	if (info.me_mapsize)
		mdb_env_set_mapsize(env, info.me_mapsize);

	if (info.me_mapaddr)
		envflags |= MDB_FIXEDMAP;

	rc = mdb_env_open(env, envname, envflags, 0664);
	if (rc) {
		fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
		goto env_close;
	}

	kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
	kbuf.mv_data = malloc(kbuf.mv_size);

	while(!Eof) {
		MDB_val key, data;
		int batch = 0;
		flags = 0;

		if (!dohdr) {
			dohdr = 1;
		} else if (!(mode & NOHDR))
			readhdr();
		
		rc = mdb_txn_begin(env, NULL, 0, &txn);
		if (rc) {
			fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
			goto env_close;
		}

		rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
		if (rc) {
			fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
			goto txn_abort;
		}

		rc = mdb_cursor_open(txn, dbi, &mc);
		if (rc) {
			fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
			goto txn_abort;
		}

		while(1) {
			rc = readline(&key, &kbuf);
			if (rc)  /* rc == EOF */
				break;

			rc = readline(&data, &dbuf);
			if (rc) {
				fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
				goto txn_abort;
			}

			rc = mdb_cursor_put(mc, &key, &data, putflags);
			if (rc == MDB_KEYEXIST && putflags)
				continue;
			if (rc) {
				fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
				goto txn_abort;
			}
			batch++;
			if (batch == 100) {
				rc = mdb_txn_commit(txn);
				if (rc) {
					fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
						prog, lineno, mdb_strerror(rc));
					goto env_close;
				}
				rc = mdb_txn_begin(env, NULL, 0, &txn);
				if (rc) {
					fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
					goto env_close;
				}
				rc = mdb_cursor_open(txn, dbi, &mc);
				if (rc) {
					fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
					goto txn_abort;
				}
				batch = 0;
			}
		}
		rc = mdb_txn_commit(txn);
		txn = NULL;
		if (rc) {
			fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
				prog, lineno, mdb_strerror(rc));
			goto env_close;
		}
		mdb_dbi_close(env, dbi);
	}

txn_abort:
	mdb_txn_abort(txn);
env_close:
	mdb_env_close(env);

	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
Пример #16
0
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
	int rc, i;
	struct mdb_info *mdb = (struct mdb_info *) be->be_private;
	struct stat stat1;
	uint32_t flags;
	char *dbhome;
	MDB_txn *txn;

	if ( be->be_suffix == NULL ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": need suffix.\n" );
		return -1;
	}

	Debug( LDAP_DEBUG_ARGS,
		LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
		be->be_suffix[0].bv_val );

	/* Check existence of dbenv_home. Any error means trouble */
	rc = stat( mdb->mi_dbenv_home, &stat1 );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"cannot access database directory \"%s\" (%d).\n",
			be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
		return -1;
	}

	/* mdb is always clean */
	be->be_flags |= SLAP_DBFLAG_CLEAN;

	rc = mdb_env_create( &mdb->mi_dbenv );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_create failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	mdb_env_set_userctx(mdb->mi_dbenv, mdb);

	if ( mdb->mi_readers ) {
		rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
		if( rc != 0 ) {
			Debug( LDAP_DEBUG_ANY,
				LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
				"mdb_env_set_maxreaders failed: %s (%d).\n",
				be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
			goto fail;
		}
	}

	rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_set_mapsize failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_set_maxdbs failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

#ifdef MDBX_LIFORECLAIM
#if MDBX_MODE_ENABLED
	rc = mdbx_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul);
#else
	rc = mdb_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul);
#endif /* MDBX_MODE_ENABLED */
	if( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
			"mdb_env_set_sync_threshold failed: %s (%d).\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

#if MDBX_MODE_ENABLED
	mdbx_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler );
#else
	mdb_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler );
#endif /* MDBX_MODE_ENABLED */

	if ( (slapMode & SLAP_SERVER_MODE) && SLAP_MULTIMASTER(be) &&
			((MDBX_OOM_YIELD & mdb->mi_oom_flags) == 0 || mdb->mi_renew_lag == 0)) {
		snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
			"for properly operation in multi-master mode"
			" 'oom-handler yield' and 'dreamcatcher' are recommended",
			be->be_suffix[0].bv_val );
		Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg );
	}
#endif /* MDBX_LIFORECLAIM */

	dbhome = mdb->mi_dbenv_home;

	Debug( LDAP_DEBUG_TRACE,
		LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
		"dbenv_open(%s).\n",
		be->be_suffix[0].bv_val, mdb->mi_dbenv_home);

	flags = mdb->mi_dbenv_flags;

#ifdef MDBX_PAGEPERTURB
	if (reopenldap_mode_check())
		flags |= MDBX_PAGEPERTURB;
#endif

	if ( slapMode & SLAP_TOOL_QUICK )
		flags |= MDB_NOSYNC|MDB_WRITEMAP;

	if ( slapMode & SLAP_TOOL_READONLY)
		flags |= MDB_RDONLY;

	rc = mdb_env_open( mdb->mi_dbenv, dbhome,
			flags, mdb->mi_dbenv_mode );

	if ( rc ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
			"Restore from backup!\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
	if ( rc ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
			"Restore from backup!\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	/* open (and create) main databases */
	for( i = 0; mdmi_databases[i].bv_val; i++ ) {
		flags = MDB_INTEGERKEY;
		if( i == MDB_ID2ENTRY ) {
			if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
				flags |= MDB_CREATE;
		} else {
			if ( i == MDB_DN2ID )
				flags |= MDB_DUPSORT;
			if ( !(slapMode & SLAP_TOOL_READONLY) )
				flags |= MDB_CREATE;
		}

		rc = mdb_dbi_open( txn,
			mdmi_databases[i].bv_val,
			flags,
			&mdb->mi_dbis[i] );

		if ( rc != 0 ) {
			snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
				"mdb_dbi_open(%s/%s) failed: %s (%d).",
				be->be_suffix[0].bv_val,
				mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
				mdb_strerror(rc), rc );
			Debug( LDAP_DEBUG_ANY,
				LDAP_XSTRING(mdb_db_open) ": %s\n",
				cr->msg );
			goto fail;
		}

		if ( i == MDB_ID2ENTRY )
			mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
		else if ( i == MDB_DN2ID ) {
			MDB_cursor *mc;
			MDB_val key, data;
			mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
			/* check for old dn2id format */
			rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
			/* first record is always ID 0 */
			rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
			if ( rc == 0 ) {
				rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
				if ( rc == 0 ) {
					int len;
					unsigned char *ptr;
					ptr = data.mv_data;
					len = (ptr[0] & 0x7f) << 8 | ptr[1];
					if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
						snprintf( cr->msg, sizeof(cr->msg),
						"database \"%s\": DN index needs upgrade, "
						"run \"slapindex entryDN\".",
						be->be_suffix[0].bv_val );
						Debug( LDAP_DEBUG_ANY,
							LDAP_XSTRING(mdb_db_open) ": %s\n",
							cr->msg );
						if ( !(slapMode & SLAP_TOOL_READMAIN ))
							rc = LDAP_OTHER;
						mdb->mi_flags |= MDB_NEED_UPGRADE;
					}
				}
			}
			mdb_cursor_close( mc );
			if ( rc == LDAP_OTHER )
				goto fail;
		}
	}

	rc = mdb_ad_read( mdb, txn );
	if ( rc ) {
		mdb_txn_abort( txn );
		goto fail;
	}

	/* slapcat doesn't need indexes. avoid a failure if
	 * a configured index wasn't created yet.
	 */
	if ( !(slapMode & SLAP_TOOL_READONLY) ) {
		rc = mdb_attr_dbs_open( be, txn, cr );
		if ( rc ) {
			mdb_txn_abort( txn );
			goto fail;
		}
	}

	rc = mdb_txn_commit(txn);
	if ( rc != 0 ) {
		Debug( LDAP_DEBUG_ANY,
			LDAP_XSTRING(mdb_db_open) ": database %s: "
			"txn_commit failed: %s (%d)\n",
			be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
		goto fail;
	}

	/* monitor setup */
	rc = mdb_monitor_db_open( be );
	if ( rc != 0 ) {
		goto fail;
	}

	mdb->mi_flags |= MDB_IS_OPEN;

	return 0;

fail:
	mdb_db_close( be, NULL );
	return rc;
}
Пример #17
0
/*
 * Open the environment
 */
static int do_open(const sasl_utils_t *utils,
			 sasl_conn_t *conn,
			 int rdwr, MDB_txn **mtxn)
{
    const char *path = SASL_DB_PATH;
    void *cntxt;
    MDB_env *env;
    MDB_txn *txn;
    sasl_getopt_t *getopt;
    size_t mapsize = 0;
    int readers = 0;
    int ret;
    int flags;

    if (!db_env) {

	if (utils->getcallback(conn, SASL_CB_GETOPT,
			       (sasl_callback_ft *)&getopt, &cntxt) == SASL_OK) {
	    const char *p;
	    if (getopt(cntxt, NULL, "sasldb_path", &p, NULL) == SASL_OK
		&& p != NULL && *p != 0) {
		path = p;
	    }
	    if (getopt(cntxt, NULL, "sasldb_maxreaders", &p, NULL) == SASL_OK
		&& p != NULL && *p != 0) {
		readers = atoi(p);
	    }
	    if (getopt(cntxt, NULL, "sasldb_mapsize", &p, NULL) == SASL_OK
		&& p != NULL && *p != 0) {
		mapsize = atoi(p);
		mapsize *= KILO;
	    }
	}

	ret = mdb_env_create(&env);
	if (ret) {
	    utils->log(conn, SASL_LOG_ERR,
		       "unable to create MDB environment: %s",
		       mdb_strerror(ret));
	    utils->seterror(conn, SASL_NOLOG, "Unable to create MDB environment");
	    return SASL_FAIL;
	}

	if (readers) {
	    ret = mdb_env_set_maxreaders(env, readers);
	    if (ret) {
		utils->log(conn, SASL_LOG_ERR,
		       "unable to set MDB maxreaders: %s",
		       mdb_strerror(ret));
		utils->seterror(conn, SASL_NOLOG, "Unable to set MDB maxreaders");
		return SASL_FAIL;
	    }
	}

	if (mapsize) {
	    ret = mdb_env_set_mapsize(env, mapsize);
	    if (ret) {
		utils->log(conn, SASL_LOG_ERR,
		       "unable to set MDB mapsize: %s",
		       mdb_strerror(ret));
		utils->seterror(conn, SASL_NOLOG, "Unable to set MDB mapsize");
		return SASL_FAIL;
	    }
	}

	flags = MDB_NOSUBDIR;
	if (!rdwr) flags |= MDB_RDONLY;
	ret = mdb_env_open(env, path, flags, 0660);
	if (ret) {
	    mdb_env_close(env);
	    if (!rdwr && ret == ENOENT) {
		/* File not found and we are only reading the data.
		   Treat as SASL_NOUSER. */
		return SASL_NOUSER;
	    }
	    utils->log(conn, SASL_LOG_ERR,
		       "unable to open MDB environment %s: %s",
		       path, mdb_strerror(ret));
	    utils->seterror(conn, SASL_NOLOG, "Unable to open MDB environment");
	    return SASL_FAIL;
	}
    } else {
    	env = db_env;
    }

    ret = mdb_txn_begin(env, NULL, rdwr ? 0 : MDB_RDONLY, &txn);
    if (ret) {
    	mdb_env_close(env);
	utils->log(conn, SASL_LOG_ERR,
		   "unable to open MDB transaction: %s",
		   mdb_strerror(ret));
	utils->seterror(conn, SASL_NOLOG, "Unable to open MDB transaction");
	return SASL_FAIL;
    }

    if (!db_dbi) {
	ret = mdb_open(txn, NULL, 0, &db_dbi);
	if (ret) {
	    mdb_txn_abort(txn);
	    mdb_env_close(env);
	    utils->log(conn, SASL_LOG_ERR,
		       "unable to open MDB database: %s",
		       mdb_strerror(ret));
	    utils->seterror(conn, SASL_NOLOG, "Unable to open MDB database");
	    return SASL_FAIL;
	}
    }

    if (!db_env)
	db_env = env;
    *mtxn = txn;

    return SASL_OK;
}
Пример #18
0
bool B_ACCURATE_LMDB::init(JCR *jcr, uint32_t nbfile)
{
   int result;
   MDB_env *env;
   size_t mapsize = 10485760;

   if (!m_db_env) {
      result = mdb_env_create(&env);
      if (result) {
         Jmsg1(jcr, M_FATAL, 0, _("Unable to create MDB environment: %s\n"), mdb_strerror(result));
         return false;
      }

      if ((nbfile * AVG_NR_BYTES_PER_ENTRY) > mapsize) {
         size_t pagesize;

#ifdef HAVE_GETPAGESIZE
         pagesize = getpagesize();
#else
         pagesize = B_PAGE_SIZE;
#endif

         mapsize = (((nbfile * AVG_NR_BYTES_PER_ENTRY) / pagesize) + 1) * pagesize;
      }
      result = mdb_env_set_mapsize(env, mapsize);
      if (result) {
         Jmsg1(jcr, M_FATAL, 0, _("Unable to set MDB mapsize: %s\n"), mdb_strerror(result));
         goto bail_out;
      }

      /*
       * Explicitly set the number of readers to 1.
       */
      result = mdb_env_set_maxreaders(env, 1);
      if (result) {
         Jmsg1(jcr, M_FATAL, 0, _("Unable to set MDB maxreaders: %s\n"), mdb_strerror(result));
         goto bail_out;
      }

      Mmsg(m_lmdb_name, "%s/.accurate_lmdb.%d", me->working_directory, jcr->JobId);
      result = mdb_env_open(env, m_lmdb_name, MDB_NOSUBDIR | MDB_NOLOCK | MDB_NOSYNC, 0600);
      if (result) {
         Jmsg2(jcr, M_FATAL, 0, _("Unable create LDMD database %s: %s\n"), m_lmdb_name, mdb_strerror(result));
         goto bail_out;
      }

      result = mdb_txn_begin(env, NULL, 0, &m_db_rw_txn);
      if (result) {
         Jmsg1(jcr, M_FATAL, 0, _("Unable to start a write transaction: %s\n"), mdb_strerror(result));
         goto bail_out;
      }

      result = mdb_dbi_open(m_db_rw_txn, NULL, MDB_CREATE, &m_db_dbi);
      if (result) {
         Jmsg1(jcr, M_FATAL, 0, _("Unable to open LMDB internal database: %s\n"), mdb_strerror(result));
         mdb_txn_abort(m_db_rw_txn);
         m_db_rw_txn = NULL;
         goto bail_out;
      }

      m_db_env = env;
   }

   if (!m_pay_load) {
      m_pay_load = get_pool_memory(PM_MESSAGE);
   }

   if (!m_lmdb_name) {
      m_pay_load = get_pool_memory(PM_FNAME);
   }

   if (!m_seen_bitmap) {
      m_seen_bitmap = (char *)malloc(nbytes_for_bits(nbfile));
      clear_all_bits(nbfile, m_seen_bitmap);
   }

   return true;

bail_out:
   if (env) {
      mdb_env_close(env);
   }

   return false;
}
Пример #19
0
DBPriv *DBPrivOpenDB(const char *dbpath, dbid id)
{
    DBPriv *db = xcalloc(1, sizeof(DBPriv));
    MDB_txn *txn = NULL;
    int rc;

    rc = pthread_key_create(&db->txn_key, &DestroyTransaction);
    if (rc)
    {
        Log(LOG_LEVEL_ERR, "Could not create transaction key. (pthread_key_create: '%s')",
            GetErrorStrFromCode(rc));
        free(db);
        return NULL;
    }

    rc = mdb_env_create(&db->env);
    if (rc)
    {
        Log(LOG_LEVEL_ERR, "Could not create handle for database %s: %s",
              dbpath, mdb_strerror(rc));
        goto err;
    }
    rc = mdb_env_set_mapsize(db->env, LMDB_MAXSIZE);
    if (rc)
    {
        Log(LOG_LEVEL_ERR, "Could not set mapsize for database %s: %s",
              dbpath, mdb_strerror(rc));
        goto err;
    }
    if (DB_MAX_READERS > 0)
    {
        rc = mdb_env_set_maxreaders(db->env, DB_MAX_READERS);
        if (rc)
        {
            Log(LOG_LEVEL_ERR, "Could not set maxreaders for database %s: %s",
                dbpath, mdb_strerror(rc));
            goto err;
        }
    }

    unsigned int open_flags = MDB_NOSUBDIR;
    if (id == dbid_locks
        || (GetAmPolicyHub() && id == dbid_lastseen))
    {
        open_flags |= MDB_NOSYNC;
    }

#ifdef __hpux
    /*
     * On HP-UX, a unified file cache was not introduced until version 11.31.
     * This means that on 11.23 there are separate file caches for mmap()'ed
     * files and open()'ed files. When these two are mixed, changes made using
     * one mode won't be immediately seen by the other mode, which is an
     * assumption LMDB is relying on. The MDB_WRITEMAP flag causes LMDB to use
     * mmap() only, so that we stay within one file cache.
     */
    open_flags |= MDB_WRITEMAP;
#endif

    rc = mdb_env_open(db->env, dbpath, open_flags, 0644);
    if (rc)
    {
        Log(LOG_LEVEL_ERR, "Could not open database %s: %s",
              dbpath, mdb_strerror(rc));
        goto err;
    }
    if (DB_MAX_READERS > 0)
    {
        int max_readers;
        rc = mdb_env_get_maxreaders(db->env, &max_readers);
        if (rc)
        {
            Log(LOG_LEVEL_ERR, "Could not get maxreaders for database %s: %s",
                dbpath, mdb_strerror(rc));
            goto err;
        }
        if (max_readers < DB_MAX_READERS)
        {
            // LMDB will only reinitialize maxreaders if no database handles are
            // open, including in other processes, which is how we might end up
            // here.
            Log(LOG_LEVEL_VERBOSE, "Failed to set LMDB max reader limit on database '%s', "
                "consider restarting CFEngine",
                dbpath);
        }
    }
    rc = mdb_txn_begin(db->env, NULL, MDB_RDONLY, &txn);
    if (rc)
    {
        Log(LOG_LEVEL_ERR, "Could not open database txn %s: %s",
              dbpath, mdb_strerror(rc));
        goto err;
    }
    rc = mdb_open(txn, NULL, 0, &db->dbi);
    if (rc)
    {
        Log(LOG_LEVEL_ERR, "Could not open database dbi %s: %s",
              dbpath, mdb_strerror(rc));
        mdb_txn_abort(txn);
        goto err;
    }
    rc = mdb_txn_commit(txn);
    if (rc)
    {
        Log(LOG_LEVEL_ERR, "Could not commit database dbi %s: %s",
              dbpath, mdb_strerror(rc));
        goto err;
    }

    return db;

err:
    if (db->env)
    {
        mdb_env_close(db->env);
    }
    pthread_key_delete(db->txn_key);
    free(db);
    if (rc == MDB_INVALID)
    {
        return DB_PRIV_DATABASE_BROKEN;
    }
    return NULL;
}