int main(void)
{
check(mdb_env_create(&env));
check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL));
check(mdb_env_set_maxreaders(env, 126));
check(mdb_env_set_maxdbs(env, 1));
if(! access(DB_PATH, X_OK)) {
system("rm -rf " DB_PATH);
}
check(mkdir(DB_PATH, 0777));
check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644));
new_txn();
check(mdb_dbi_open(txn, NULL, 0, &dbi));
put("a");
put("b");
put("baa");
put("d");
new_txn();
check(mdb_cursor_open(txn, dbi, &c1));
check(mdb_cursor_get(c1, &keyv, &valv, MDB_LAST));
check(mdb_cursor_del(c1, 0));
check(mdb_cursor_del(c1, 0));
new_txn();
}
CAMLprim value caml_mdb_env_set_maxdbs(value env,value m){
CAMLparam2(env,m);
if(mdb_env_set_maxdbs((MDB_env*)env,Int_val(m))){
caml_failwith("error in mdb_env_set_maxdbs");
}
CAMLreturn0;
}
static void db_connect() {
MDB_dbi dbi_session;
MDB_dbi dbi_session_id;
MDB_dbi dbi_event;
MDB_dbi dbi_ip;
LMDB_CHECK(mdb_env_create(&env));
LMDB_CHECK(mdb_env_set_mapsize(env, 300000L * 4096L));
LMDB_CHECK(mdb_env_set_maxdbs(env, 30));
#if defined(MDBX_LIFORECLAIM)
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664));
#else
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664));
#endif
MDB_txn *txn;
// transaction init
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
// transaction commit
LMDB_CHECK(mdb_txn_commit(txn));
printf("Connection open\n");
}
/*
* Class: jmdb_DatabaseWrapper
* Method: envSetMaxDbs
* Signature: (JI)V
*/
JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_envSetMaxDbs(JNIEnv *vm,
jclass clazz, jlong envL, jint dbs) {
MDB_env *envC = (MDB_env*) envL;
int code = mdb_env_set_maxdbs(envC, dbs);
if (code) {
throwDatabaseException(vm, code);
}
}
kvenv_t *kvenv_new_lmdb(const char *dbpath, uint64_t max_dbsize, uint32_t max_dbs)
{
kvenv_lmdb_t *kvenv_lmdb = (kvenv_lmdb_t*)zmalloc(sizeof(kvenv_lmdb_t));
memset(kvenv_lmdb, 0, sizeof(kvenv_lmdb_t));
kvenv_lmdb->kvenv.dbclass = "lmdb";
kvenv_lmdb->kvenv.max_dbsize = max_dbsize;
kvenv_lmdb->kvenv.max_dbs = max_dbs;
int rc = mdb_env_create(&kvenv_lmdb->env);
if ( rc != 0 ) {
zfree(kvenv_lmdb);
error_log("mdb_env_create() failed.");
return NULL;
}
int maxreaders = 256;
if ( max_dbs == 0 ){
max_dbs = 256;
}
if ( max_dbsize == 0 ) {
max_dbsize = 1024L * 1024L * 1024L * 4L;
}
rc = mdb_env_set_mapsize(kvenv_lmdb->env, max_dbsize);
if ( rc != 0 ) {
zfree(kvenv_lmdb);
error_log("mdb_env_set_mapsize() failed.");
return NULL;
}
rc = mdb_env_set_maxreaders(kvenv_lmdb->env, maxreaders);
if ( rc != 0 ) {
zfree(kvenv_lmdb);
error_log("mdb_env_set_maxreaders() failed.");
return NULL;
}
rc = mdb_env_set_maxdbs(kvenv_lmdb->env, max_dbs);
if ( rc != 0 ) {
zfree(kvenv_lmdb);
error_log("mdb_env_set_maxdbs() failed.");
return NULL;
}
/*rc = mdb_env_open(lmdb->env, dbpath, MDB_FIXEDMAP | MDB_NOSYNC, 0640); */
/*rc = mdb_env_open(lmdb->env, dbpath, MDB_MAPASYNC | MDB_WRITEMAP | MDB_NOTLS , 0640); */
/*rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_MAPASYNC | MDB_WRITEMAP, 0640); */
/*rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_NOMETASYNC, 0640); */
rc = mdb_env_open(kvenv_lmdb->env, dbpath, MDB_NOSYNC, 0640);
if ( rc != 0 ) {
zfree(kvenv_lmdb);
error_log("mdb_env_open() failed. dbpath=%s error: %s", dbpath, mdb_strerror(rc));
return NULL;
}
return (kvenv_t*)kvenv_lmdb;
}
static void db_connect() {
LMDB_CHECK(mdb_env_create(&env));
LMDB_CHECK(mdb_env_set_mapsize(env, 3L * 1024L * 1024L * 1024L));
LMDB_CHECK(mdb_env_set_maxdbs(env, 30));
#if defined(MDBX_LIFORECLAIM)
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664));
#else
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664));
#endif
printf("Connection open\n");
}
int LMDBStore::open(const char *path, bool durable)
{
int rc;
rc = mdb_env_create(&env);
rc = mdb_env_set_mapsize(env, size_t(1048576000));
rc = mdb_env_set_maxdbs(env, 1024);
rc = mdb_env_open(env, path, MDB_NOSYNC | MDB_WRITEMAP, 0664);
MDB_txn *txn;
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, NULL, 0, &dbi);
rc = mdb_txn_commit(txn);
return HCAT_SUCCESS;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_cursor *cursor;
char sval[MAX_DATA_ALLOCATE_SIZE], kval[MAX_KEY_ALLOCATE_SIZE];
/* Note: Most error checking omitted for simplicity */
rc = mdb_env_create(&env);
mdb_env_set_mapsize(env, MAX_DB_SIZE);
mdb_env_set_maxdbs(env, (MDB_dbi)10);
rc = mdb_env_open(env, "./demoDB", 0, 0664);
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_dbi_open(txn, "what", 0, &dbi);
// rc = mdb_put(txn, dbi, &key, &data, 0);
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc));
goto leave;
}
fprintf(stderr, "print out old data:\n");
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
rc = mdb_cursor_open(txn, dbi, &cursor);
key.mv_size = sizeof(kval);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
memset(kval, argv[1][0], sizeof(kval));
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET);
fprintf(stderr, "key: %s, data: %s\n",(char *) key.mv_data,(char *) data.mv_data);
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
leave:
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDB_stat mst;
char *envname = argv[1];
char *subname = NULL;
rc = mdb_env_create(&env);
if (argc > 2) {
mdb_env_set_maxdbs(env, 4);
subname = argv[2];
}
rc = mdb_env_open(env, envname, MDB_RDONLY, 0);
if (rc) {
printf("mdb_env_open failed, error %d\n", rc);
exit(1);
}
rc = mdb_txn_begin(env, NULL, 1, &txn);
if (rc) {
printf("mdb_txn_begin failed, error %d\n", rc);
exit(1);
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
printf("mdb_open failed, error %d\n", rc);
exit(1);
}
rc = mdb_stat(txn, dbi, &mst);
printf("Page size: %u\n", mst.ms_psize);
printf("Tree depth: %u\n", mst.ms_depth);
printf("Branch pages: %zu\n", mst.ms_branch_pages);
printf("Leaf pages: %zu\n", mst.ms_leaf_pages);
printf("Overflow pages: %zu\n", mst.ms_overflow_pages);
printf("Entries: %zu\n", mst.ms_entries);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
CDbxMdb::CDbxMdb(const TCHAR *tszFileName, int iMode) :
m_safetyMode(true),
m_bReadOnly((iMode & DBMODE_READONLY) != 0),
m_bShared((iMode & DBMODE_SHARED) != 0),
m_dwMaxContactId(1),
m_lMods(50, ModCompare),
m_lOfs(50, OfsCompare),
m_lResidentSettings(50, stringCompare2)
{
m_tszProfileName = mir_tstrdup(tszFileName);
InitDbInstance(this);
mdb_env_create(&m_pMdbEnv);
mdb_env_set_maxdbs(m_pMdbEnv, 10);
m_codePage = CallService(MS_LANGPACK_GETCODEPAGE, 0, 0);
m_hModHeap = HeapCreate(0, 0, 0);
}
bool Context::Open(bool setSize, size_t dbMegabytes,size_t osPageSize){
size_t mbbytes = 10485760;
if ((dbMegabytes*mbbytes % osPageSize) != 0){
return false;
}
else if (envCreateReturn!=0){
return false;
}
else {
MDB_dbi dbs = 1;
int maxdbsReturn = mdb_env_set_maxdbs(envHandle,dbs);
if (maxdbsReturn !=0){
Error(maxdbsReturn);
return false;
}
else{
int mapsizeReturn=0;
if (setSize==true){
size_t bytes = dbMegabytes*mbbytes;
mapsizeReturn = mdb_env_set_mapsize(envHandle,bytes);
}
if(setSize==true && mapsizeReturn!=0){
Error(mapsizeReturn);
return false;
}
envOpenReturn = mdb_env_open(envHandle,
"/root/mydb",0,0664);
if (envOpenReturn!=0){
Error(envOpenReturn);
mdb_env_close(envHandle);
envHandle=nullptr;
return false;
}
else return true;
}
}
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n" );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
mdb_env_set_userctx(mdb->mi_dbenv, mdb);
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef MDBX_LIFORECLAIM
#if MDBX_MODE_ENABLED
rc = mdbx_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul);
#else
rc = mdb_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul);
#endif /* MDBX_MODE_ENABLED */
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_sync_threshold failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#if MDBX_MODE_ENABLED
mdbx_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler );
#else
mdb_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler );
#endif /* MDBX_MODE_ENABLED */
if ( (slapMode & SLAP_SERVER_MODE) && SLAP_MULTIMASTER(be) &&
((MDBX_OOM_YIELD & mdb->mi_oom_flags) == 0 || mdb->mi_renew_lag == 0)) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"for properly operation in multi-master mode"
" 'oom-handler yield' and 'dreamcatcher' are recommended",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg );
}
#endif /* MDBX_LIFORECLAIM */
dbhome = mdb->mi_dbenv_home;
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home);
flags = mdb->mi_dbenv_flags;
#ifdef MDBX_PAGEPERTURB
if (reopenldap_mode_check())
flags |= MDBX_PAGEPERTURB;
#endif
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC|MDB_WRITEMAP;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_dbi_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_dbi_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID ) {
MDB_cursor *mc;
MDB_val key, data;
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
/* check for old dn2id format */
rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
/* first record is always ID 0 */
rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
if ( rc == 0 ) {
rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
if ( rc == 0 ) {
int len;
unsigned char *ptr;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
snprintf( cr->msg, sizeof(cr->msg),
"database \"%s\": DN index needs upgrade, "
"run \"slapindex entryDN\".",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg );
if ( !(slapMode & SLAP_TOOL_READMAIN ))
rc = LDAP_OTHER;
mdb->mi_flags |= MDB_NEED_UPGRADE;
}
}
}
mdb_cursor_close( mc );
if ( rc == LDAP_OTHER )
goto fail;
}
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
/* slapcat doesn't need indexes. avoid a failure if
* a configured index wasn't created yet.
*/
if ( !(slapMode & SLAP_TOOL_READONLY) ) {
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database %s: "
"txn_commit failed: %s (%d)\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
int env_oflags;
struct stat db_stat, exe_stat;
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_cursor *mc;
MDB_dbi dbi;
char *envname;
int envflags = 0, putflags = 0;
int dohdr = 0;
prog = argv[0];
if (argc < 2) {
usage();
}
/* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 's':
subname = strdup(optarg);
break;
case 'N':
putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
dbuf.mv_size = 4096;
dbuf.mv_data = malloc(dbuf.mv_size);
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
mdb_env_set_maxdbs(env, 2);
if (info.me_maxreaders)
mdb_env_set_maxreaders(env, info.me_maxreaders);
if (info.me_mapsize)
mdb_env_set_mapsize(env, info.me_mapsize);
if (info.me_mapaddr)
envflags |= MDB_FIXEDMAP;
rc = mdb_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
kbuf.mv_data = malloc(kbuf.mv_size);
while(!Eof) {
MDB_val key, data;
int batch = 0;
flags = 0;
if (!dohdr) {
dohdr = 1;
} else if (!(mode & NOHDR))
readhdr();
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while(1) {
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
goto txn_abort;
}
rc = mdb_cursor_put(mc, &key, &data, putflags);
if (rc == MDB_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdb_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
mdb_dbi_close(env, dbi);
}
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
PageDBError
page_db_new(PageDB **db, const char *path) {
PageDB *p = *db = malloc(sizeof(*p));
if (p == 0)
return page_db_error_memory;
p->error = error_new();
if (p->error == 0) {
free(p);
return page_db_error_memory;
}
p->persist = PAGE_DB_DEFAULT_PERSIST;
p->domain_temp = 0;
// create directory if not present yet
const char *error = make_dir(path);
if ((p->path = strdup(path)) == 0) {
error = "could not duplicate path string";
}
if (error != 0) {
page_db_set_error(p, page_db_error_invalid_path, __func__);
page_db_add_error(p, error);
return p->error->code;
}
if (txn_manager_new(&p->txn_manager, 0) != 0) {
page_db_set_error(p, page_db_error_internal, __func__);
page_db_add_error(p, p->txn_manager?
p->txn_manager->error->message:
"NULL");
return p->error->code;
}
// initialize LMDB on the directory
MDB_txn *txn;
MDB_dbi dbi;
int mdb_rc = 0;
if ((mdb_rc = mdb_env_create(&p->txn_manager->env) != 0))
error = "creating environment";
else if ((mdb_rc = mdb_env_set_mapsize(
p->txn_manager->env, PAGE_DB_DEFAULT_SIZE)) != 0)
error = "setting map size";
else if ((mdb_rc = mdb_env_set_maxdbs(p->txn_manager->env, 5)) != 0)
error = "setting number of databases";
else if ((mdb_rc = mdb_env_open(
p->txn_manager->env,
path,
MDB_NOTLS | MDB_NOSYNC, 0664) != 0))
error = "opening environment";
else if ((mdb_rc = txn_manager_begin(p->txn_manager, 0, &txn)) != 0)
error = "starting transaction";
// create all database
else if ((mdb_rc = mdb_dbi_open(txn,
"hash2info",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating hash2info database";
else if ((mdb_rc = mdb_dbi_open(txn,
"hash2idx",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating hash2idx database";
else if ((mdb_rc = mdb_dbi_open(txn,
"links",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating links database";
else if ((mdb_rc = mdb_dbi_open(txn, "info", MDB_CREATE, &dbi)) != 0)
error = "creating info database";
else {
// initialize n_pages inside info database
size_t n_pages = 0;
MDB_val key = {
.mv_size = sizeof(info_n_pages),
.mv_data = info_n_pages
};
MDB_val val = {
.mv_size = sizeof(size_t),
.mv_data = &n_pages
};
switch (mdb_rc = mdb_put(txn, dbi, &key, &val, MDB_NOOVERWRITE)) {
case MDB_KEYEXIST:
case 0:
if (txn_manager_commit(p->txn_manager, txn) != 0)
error = p->txn_manager->error->message;
break; // we good
default:
error = "could not initialize info.n_pages";
}
}
if (error != 0) {
page_db_set_error(p, page_db_error_internal, __func__);
page_db_add_error(p, error);
page_db_add_error(p, mdb_strerror(mdb_rc));
mdb_env_close(p->txn_manager->env);
}
return p->error->code;
}
/** Store a new or updated @ref PageInfo for a crawled page.
*
* @param cur An open cursor to the hash2info database
* @param key The key (hash) to the page
* @param page
* @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter
* will be set with the error (otherwise is set to zero).
* @return 0 if success, -1 if failure.
*/
static int
page_db_add_crawled_page_info(MDB_cursor *cur,
MDB_val *key,
const CrawledPage *page,
PageInfo **page_info,
int *mdb_error) {
MDB_val val;
*page_info = 0;
int mdb_rc = mdb_cursor_get(cur, key, &val, MDB_SET);
int put_flags = 0;
switch (mdb_rc) {
case 0:
if (!(*page_info = page_info_load(&val)))
goto on_error;
if ((page_info_update(*page_info, page) != 0))
goto on_error;
put_flags = MDB_CURRENT;
break;
case MDB_NOTFOUND:
if (!(*page_info = page_info_new_crawled(page)))
goto on_error;
break;
default:
goto on_error;
}
if ((page_info_dump(*page_info, &val) != 0))
goto on_error;
if ((mdb_rc = mdb_cursor_put(cur, key, &val, put_flags)) != 0)
goto on_error;
free(val.mv_data);
val.mv_data = 0;
*mdb_error = 0;
return 0;
on_error:
if (val.mv_data)
free(val.mv_data);
*mdb_error = mdb_rc;
page_info_delete(*page_info);
return -1;
}
/** Store a new or updated @ref PageInfo for an uncrawled link.
*
* @param cur An open cursor to the hash2info database
* @param key The key (hash) to the page
* @param url
* @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter
* will be set with the error (otherwise is set to zero).
* @return 0 if success, -1 if failure.
*/
static int
page_db_add_link_page_info(MDB_cursor *cur,
MDB_val *key,
uint64_t linked_from,
uint64_t depth,
const LinkInfo *link,
PageInfo **page_info,
int *mdb_error) {
MDB_val val;
int mdb_rc = 0;
PageInfo *pi = *page_info =
page_info_new_link(link->url, linked_from, depth, link->score);
if (!pi)
goto on_error;
if ((page_info_dump(pi, &val) != 0))
goto on_error;
if ((mdb_rc = mdb_cursor_put(cur, key, &val, MDB_NOOVERWRITE)) != 0)
goto on_error;
free(val.mv_data);
val.mv_data = 0;
*mdb_error = 0;
return 0;
on_error:
if (val.mv_data)
free(val.mv_data);
*mdb_error = mdb_rc;
page_info_delete(pi);
return -1;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32];
srandom(time(NULL));
count = (random()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = random()%1024;
}
rc = mdb_env_create(&env);
rc = mdb_env_set_mapsize(env, 10485760);
rc = mdb_env_set_maxdbs(env, 4);
rc = mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664);
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, "id1", MDB_CREATE, &dbi);
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
rc = mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE);
if (rc) j++;
}
if (j) printf("%d duplicates skipped\n", j);
rc = mdb_txn_commit(txn);
rc = mdb_env_stat(env, &mst);
rc = mdb_txn_begin(env, NULL, 1, &txn);
rc = mdb_cursor_open(txn, dbi, &cursor);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (random()%5)) {
j++;
txn=NULL;
rc = mdb_txn_begin(env, NULL, 0, &txn);
sprintf(sval, "%03x ", values[i]);
rc = mdb_del(txn, dbi, &key, NULL);
if (rc) {
j--;
mdb_txn_abort(txn);
} else {
rc = mdb_txn_commit(txn);
}
}
free(values);
printf("Deleted %d values\n", j);
rc = mdb_env_stat(env, &mst);
rc = mdb_txn_begin(env, NULL, 1, &txn);
rc = mdb_cursor_open(txn, dbi, &cursor);
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[8];
char kval[sizeof(int)];
memset(sval, 0, sizeof(sval));
count = 510;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = i*5;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
strcpy(kval, "001");
for (i=0;i<count;i++) {
sprintf(sval, "%07x", values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
/* there should be one full page of dups now.
*/
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
/* test all 3 branches of split code:
* 1: new key in lower half
* 2: new key at split point
* 3: new key in upper half
*/
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%07x", values[3]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[255]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[500]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
E(mdb_txn_commit(txn));
/* Try MDB_NEXT_MULTIPLE */
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%3)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%07x", values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
db_t db_init(const size_t size, const char * const path, const int flags, const int mode, int mdb_flags, int ndbi, dbi_t *dbis, size_t padsize){
int fd = -1, err = 0xdeadbeef;
struct stat st;
db_t db;
db = malloc(size < sizeof(*db) ? sizeof(*db) : size);
assert(db);
// always do this
mdb_flags |= MDB_NOSUBDIR;
int r;
// ignore MDB_RDONLY - key off of OS flags
// see docs for open() - O_RDONLY is not a bit!
if((flags & 0x3) == O_RDONLY){
mdb_flags |= MDB_RDONLY;
// disable NOSYNC/NOMETASYNC for readonly, as that burns another file descriptor
mdb_flags &= ~(MDB_NOSYNC|MDB_NOMETASYNC);
}else{
mdb_flags &= ~MDB_RDONLY;
}
// default to 100mb minimum pad
db->padsize = padsize ? padsize : (1<<27);
db->env = NULL;
db->txns = 0;
db->handles = NULL;
db->updated = 0;
r = pthread_mutex_init(&db->mutex, NULL);
FAIL(r, err, errno, fail);
r = pthread_cond_init(&db->cond, NULL);
FAIL(r, err, errno, fail);
// unless MDB_RDONLY is specified, lmdb will automatically create non-existant databases,
// which is not what I want. Try to emulate standard unix open() flags:
fd = open(path, flags, mode);
FAIL(-1 == fd, err, errno, fail);
r = mdb_env_create(&db->env);
FAIL(r, err, r, fail);
r = mdb_env_set_maxdbs(db->env, ndbi);
FAIL(r, err, r, fail);
size_t mapsize;
do{
r = fstat(fd, &st);
FAIL(r, err, errno, fail);
if(!st.st_size &&( flags & O_CREAT))
db->updated = 1;
// pad out such that we have at least 1gb of map overhead
mapsize = (1 + st.st_size / db->padsize) * db->padsize;
r = mdb_env_set_mapsize(db->env, mapsize);
}while(MDB_SUCCESS != r);
close(fd);
fd = -1;
r = mdb_env_open(db->env, path, mdb_flags, mode);
// mdb_env_open can return EAGAIN somehow, but I think it really means:
if(EAGAIN == r)
r = EMFILE;
FAIL(r, err, r, fail);
r = mdb_env_get_fd(db->env, &db->fd);
FAIL(r, err, r, fail);
if(dbis && ndbi){
db->handles = malloc(sizeof(db->handles[0]) * ndbi);
FAIL(!db->handles, err, errno, fail);
// open the indexes - try read-only first
unsigned int txn_flags = MDB_RDONLY;
txn_t txn = db_txn_new(db, NULL, txn_flags);
int i;
for(i = 0; i < ndbi; i++){
int dbflags = dbis[i].flags;
retry:
dbflags = (txn_flags & MDB_RDONLY) ? (dbflags & ~MDB_CREATE) : (dbflags | MDB_CREATE);
r = mdb_dbi_open(txn->txn, dbis[i].name, dbflags, &db->handles[i]);
if(MDB_SUCCESS != r){
if(MDB_NOTFOUND == r && (txn_flags & MDB_RDONLY)){
// we were in read-only and a sub-db was missing
// end txn
txn_commit(txn);
// switch to read-write
txn_flags &= ~MDB_RDONLY;
txn = db_txn_new(db, NULL, txn_flags);
// and pick up where we left off
goto retry;
}else{
txn_abort(txn);
}
FAIL(r, err, r, fail);
}
}
r = txn_commit(txn);
FAIL(r, err, r, fail);
}
return db;
fail:
db_close(db);
if(fd != -1){
if((flags & (O_CREAT|O_EXCL)) == (O_CREAT|O_EXCL))
unlink(path);
close(fd);
}
errno = err;
return NULL;
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n",
1, 0, 0 );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val, 0, 0 );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef HAVE_EBCDIC
strcpy( path, mdb->mi_dbenv_home );
__atoe( path );
dbhome = path;
#else
dbhome = mdb->mi_dbenv_home;
#endif
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0);
flags = mdb->mi_dbenv_flags;
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. "
"Restore from backup!\n",
be->be_suffix[0].bv_val, rc, 0 );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. "
"Restore from backup!\n",
be->be_suffix[0].bv_val, rc, 0 );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID )
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDB_stat mst;
MDB_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2) {
usage(prog);
}
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdb_env_set_maxdbs(env, 4);
}
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (envinfo) {
(void)mdb_env_stat(env, &mst);
(void)mdb_env_info(env, &mei);
printf("Environment Info\n");
printf(" Map address: %p\n", mei.me_mapaddr);
printf(" Map size: %"Z"u\n", mei.me_mapsize);
printf(" Page size: %u\n", mst.ms_psize);
printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize);
printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1);
printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid);
printf(" Max readers: %u\n", mei.me_maxreaders);
printf(" Number of readers used: %u\n", mei.me_numreaders);
}
if (rdrinfo) {
printf("Reader Table Status\n");
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
if (rdrinfo > 1) {
int dead;
mdb_reader_check(env, &dead);
printf(" %d stale readers cleared.\n", dead);
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (freinfo) {
MDB_cursor *cursor;
MDB_val key, data;
size_t pages = 0, *iptr;
printf("Freelist Status\n");
dbi = 0;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
iptr = data.mv_data;
pages += *iptr;
if (freinfo > 1) {
char *bad = "";
size_t pg, prev;
ssize_t i, j, span = 0;
j = *iptr++;
for (i = j, prev = 1; --i >= 0; ) {
pg = iptr[i];
if (pg <= prev)
bad = " [bad sequence]";
prev = pg;
pg += span;
for (; i >= span && iptr[i-span] == pg; span++, pg++) ;
}
printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n",
*(size_t *)key.mv_data, j, span, bad);
if (freinfo > 2) {
for (--j; j >= 0; ) {
pg = iptr[j];
for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ;
printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n",
pg, span);
}
}
}
}
mdb_cursor_close(cursor);
printf(" Free pages: %"Z"u\n", pages);
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDB_cursor *cursor;
MDB_val key;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
char *str;
MDB_dbi db2;
if (memchr(key.mv_data, '\0', key.mv_size))
continue;
str = malloc(key.mv_size+1);
memcpy(str, key.mv_data, key.mv_size);
str[key.mv_size] = '\0';
rc = mdb_open(txn, str, 0, &db2);
if (rc == MDB_SUCCESS)
printf("Status of %s\n", str);
free(str);
if (rc) continue;
rc = mdb_stat(txn, db2, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdb_close(env, db2);
}
mdb_cursor_close(cursor);
}
if (rc == MDB_NOTFOUND)
rc = MDB_SUCCESS;
mdb_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
printf("check-preset-a\n");
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_a = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
++present_a;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_a == count - j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_env_sync(env, 1);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
printf("check-preset-b.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_b = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_b;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == present_a - j, "mismatch");
printf("check-preset-b.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
/********************* LY: kept DB dirty ****************/
mdb_env_close_ex(env, 1);
E(mdb_env_create(&env));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
printf("check-preset-c.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_dbi_open(txn, "id1", 0, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_c = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_c;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_c == present_a, "mismatch");
printf("check-preset-d.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_c == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close_ex(env, 0);
return 0;
}
int main(int argc, char *argv[])
{
int size;
char string[LINE_SIZE+1];
char s_key[LINE_SIZE+1];
char s_value[LINE_SIZE+1];
char* token;
char* save;
int cnt_line;
int rc;
MDB_env* env;
MDB_txn* txn;
MDB_cursor* mc;
MDB_dbi dbi;
MDB_val key, data;
char* envname;
int envflags=0;
char* subname;
char* prog = argv[0];
size_t map_size = (SIZE_MAX / (1024*1024*1024) / 4)*6; // 4giga * 6
struct timeval tv1, tv2;
if(argc != 3) {
fprintf(stderr,"%s <envname> <subname>\n",prog);
exit(1);
}
envflags = MDB_NOSUBDIR | MDB_NOLOCK;
envname = argv[1];
rc = mdb_env_create(&env);
if(rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
mdb_env_set_maxdbs(env, 2);
mdb_env_set_mapsize(env, map_size);
rc = mdb_env_open(env, envname, envflags, 0664);
if(rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if(rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
subname = argv[2];
rc = mdb_open(txn, subname, MDB_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
gettimeofday(&tv1, NULL);
cnt_line = 0;
while(fgets(string, LINE_SIZE, stdin) != NULL) {
size = strlen(string);
if(string[size-1] == '\n'){
string[size-1] = '\0';
--size;
}
if(size > 1 && string[size-1] == '\r'){
string[size-1] = '\0';
--size;
}
if(string[0] == '\0')
continue;
if(cnt_line % 10000 == 0)
fprintf(stderr,"[linecount]\t%d\n",cnt_line);
/*
token = strtok_r(string, "\t", &save);
if(token != NULL) {
strcpy(s_key, token);
token = strtok_r(NULL, "\t", &save);
if(token != NULL) {
strcpy(s_value, token);
} else continue;
} else continue;
*/
token = strtok_r(string, "\t", &save);
if(token != NULL) {
strcpy(s_key, token);
} else continue;
key.mv_data = s_key;
key.mv_size = strlen(s_key) + 1;
rc = mdb_get(txn, dbi, &key, &data);
if(!rc) {
fprintf(stdout, "%s\t%s\n", s_key, (char*)data.mv_data);
}
cnt_line++;
}
gettimeofday(&tv2, NULL);
fprintf(stderr, "<-end > : t2.sec = %d t2.usec = %d\n",(int)tv2.tv_sec,(int)tv2.tv_usec);
fprintf(stderr, "<+time> : sec = %d usec = %d\n",(int)(tv2.tv_sec-tv1.tv_sec),(int)(tv2.tv_usec-tv1.tv_usec));
rc = mdb_txn_commit(txn);
txn = NULL;
if(rc) {
fprintf(stderr, "%s: txn_commit fail: %s\n", prog, mdb_strerror(rc));
goto env_close;
}
mdb_dbi_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
int main(int argc, const char* argv[])
{
g_log = stdout;
db_thread thr;
db_thread threads[RTHREADS];
pthread_t tids[RTHREADS];
priv_data pd;
mdbinf* mdb = &thr.mdb;
int i, rc;
db_connection *cons;
g_pd = &pd;
char commit = 1;
MDB_env *menv = NULL;
char *lmpath = "lmdb";
MDB_txn *txn;
MDB_val key = {1,(void*)"?"}, data = {0,NULL};
MDB_envinfo stat;
sqlite3_initialize();
sqlite3_vfs_register(sqlite3_nullvfs(), 1);
unlink(lmpath);
memset(threads, 0, sizeof(threads));
memset(&thr, 0, sizeof(db_thread));
memset(&pd, 0, sizeof(priv_data));
pd.wmdb = calloc(1,sizeof(mdbinf));
pd.nEnvs = 1;
pd.nReadThreads = RTHREADS;
pd.nWriteThreads = 1;
pd.syncNumbers = calloc(1,sizeof(u64));
pd.actorIndexes = calloc(1,sizeof(atomic_llong));
atomic_init(pd.actorIndexes,0);
g_cons = cons = calloc(NCONS, sizeof(db_connection));
g_tsd_cursync = 0;
g_tsd_conn = NULL;
g_tsd_wmdb = NULL;
g_tsd_thread = &thr;
if (mdb_env_create(&menv) != MDB_SUCCESS)
return -1;
if (mdb_env_set_maxdbs(menv,5) != MDB_SUCCESS)
return -1;
if (mdb_env_set_mapsize(menv,1024*1024*1024) != MDB_SUCCESS)
return -1;
// Syncs are handled from erlang.
if (mdb_env_open(menv, lmpath, MDB_NOSUBDIR|MDB_NOTLS|MDB_NOSYNC, 0664) != MDB_SUCCESS) //MDB_NOSYNC
return -1;
if (mdb_txn_begin(menv, NULL, 0, &txn) != MDB_SUCCESS)
return -1;
if (mdb_dbi_open(txn, "info", MDB_INTEGERKEY | MDB_CREATE, &pd.wmdb[0].infodb) != MDB_SUCCESS)
return -1;
if (mdb_dbi_open(txn, "actors", MDB_CREATE, &pd.wmdb[0].actorsdb) != MDB_SUCCESS)
return -1;
if (mdb_dbi_open(txn, "log", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP,
&pd.wmdb[0].logdb) != MDB_SUCCESS)
return -1;
if (mdb_dbi_open(txn, "pages", MDB_CREATE | MDB_DUPSORT, &pd.wmdb[0].pagesdb) != MDB_SUCCESS)
return -1;
if (mdb_txn_commit(txn) != MDB_SUCCESS)
return -1;
pd.wmdb[0].env = menv;
thr.nEnv = 0;
thr.isreadonly = 0;
thr.mdb.env = menv;
thr.mdb.infodb = pd.wmdb[0].infodb;
thr.mdb.actorsdb = pd.wmdb[0].actorsdb;
thr.mdb.logdb = pd.wmdb[0].logdb;
thr.mdb.pagesdb = pd.wmdb[0].pagesdb;
thr.maxvalsize = mdb_env_get_maxkeysize(mdb->env);
thr.resFrames = alloca((SQLITE_DEFAULT_PAGE_SIZE/thr.maxvalsize + 1)*sizeof(MDB_val));
open_txn(&thr.mdb, MDB_RDONLY);
for (i = 0; i < NCONS; i++)
{
char filename[256];
char commit = 1;
g_tsd_conn = &cons[i];
sprintf(filename, "ac%d.db", i);
pthread_mutex_init(&cons[i].wal.mtx, NULL);
thr.pagesChanged = 0;
rc = sqlite3_open(filename,&(cons[i].db));
if(rc != SQLITE_OK)
{
DBG("Unable to open db");
break;
}
rc = sqlite3_exec(cons[i].db,"PRAGMA synchronous=0;PRAGMA journal_mode=wal;",NULL,NULL,NULL);
if (rc != SQLITE_OK)
{
DBG("unable to open wal");
break;
}
cons[i].wal.inProgressTerm = 1;
cons[i].wal.inProgressEvnum = 1;
rc = sqlite3_exec(cons[i].db,"CREATE TABLE tab (id INTEGER PRIMARY KEY, txt TEXT);"
"insert into tab values (1,'aaaa');",NULL,NULL,NULL);
if (rc != SQLITE_OK)
{
DBG("Cant create table");
break;
}
unlock_write_txn(thr.nEnv, 0, &commit);
mdb_txn_reset(thr.mdb.txn);
rc = mdb_txn_renew(thr.mdb.txn);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorLog);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorPages);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorInfo);
if (rc != MDB_SUCCESS)
break;
}
// mdb_cursor_close(thr.mdb.cursorLog);
// mdb_cursor_close(thr.mdb.cursorPages);
// mdb_cursor_close(thr.mdb.cursorInfo);
// mdb_txn_abort(thr.mdb.txn);
for (i = 0; i < RTHREADS; i++)
{
threads[i].nEnv = 0;
threads[i].isreadonly = 0;
threads[i].mdb.env = menv;
threads[i].mdb.infodb = pd.wmdb[0].infodb;
threads[i].mdb.actorsdb = pd.wmdb[0].actorsdb;
threads[i].mdb.logdb = pd.wmdb[0].logdb;
threads[i].mdb.pagesdb = pd.wmdb[0].pagesdb;
threads[i].maxvalsize = mdb_env_get_maxkeysize(mdb->env);
pthread_create(&tids[i], NULL, perform, (void *)&threads[i]);
}
srand((u32)pthread_self() + time(NULL));
for (i = 0; i < 1000*200; i++)
{
char commit = 1;
int j = rand() % NCONS;
db_connection *con = &g_cons[j];
char str[100];
if (pthread_mutex_trylock(&con->wal.mtx) != 0)
{
i--;
continue;
}
if (i % 1000 == 0)
printf("w %d\n",i);
g_tsd_conn = con;
lock_wtxn(thr.nEnv);
thr.pagesChanged = 0;
if (con->wal.firstCompleteEvnum+10 < con->wal.lastCompleteEvnum)
{
// printf("CHECKPOINT? %llu %llu\n",con->wal.firstCompleteEvnum,con->wal.lastCompleteEvnum);
if (checkpoint(&con->wal, con->wal.lastCompleteEvnum-10) != SQLITE_OK)
{
printf("Checkpoint failed\n");
break;
}
}
con->wal.inProgressTerm = 1;
con->wal.inProgressEvnum = con->wal.lastCompleteEvnum+1;
sprintf(str,"INSERT INTO tab VALUES (%d,'VALUE VALUE13456');", i);
sqlite3_exec(con->db,str,NULL,NULL,NULL);
pthread_mutex_unlock(&con->wal.mtx);
unlock_write_txn(thr.nEnv, 0, &commit);
mdb_txn_reset(thr.mdb.txn);
rc = mdb_txn_renew(thr.mdb.txn);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorLog);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorPages);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorInfo);
if (rc != MDB_SUCCESS)
break;
}
unlock_write_txn(thr.nEnv, 1, &commit);
for (i = 0; i < RTHREADS; i++)
pthread_join(tids[i],NULL);
return 1;
}
/*
* Initialize MDB db
* (reference openldap 2.4.31 back-mdb/init.c)
*/
DWORD
VmDirMDBInitializeDB(
BOOLEAN bMainDB,
const char *dbHomeDir,
PVDIR_DB_HANDLE *phHandle
)
{
DWORD dwError = 0;
unsigned int envFlags = 0;
mdb_mode_t mode = 0;
uint64_t db_max_mapsize = BE_MDB_ENV_MAX_MEM_MAPSIZE;
DWORD db_max_size_mb = 0;
DWORD db_chkpt_interval = 0;
BOOLEAN bMdbWalEnable = TRUE;
PVDIR_MDB_DB pDB = NULL;
VmDirLog( LDAP_DEBUG_TRACE, "MDBInitializeDB: Begin, DB Home Dir = %s", dbHomeDir );
//Make a db entry for path. fail if already exist
dwError = _VmDirMDBInitializeDBEntry(dbHomeDir, &pDB);
BAIL_ON_VMDIR_ERROR( dwError );
if (bMainDB)
{
pDB->bIsMainDB = TRUE;
} else
{
pDB->bIsMainDB = FALSE;
}
dwError = (sizeof(ENTRYID) == sizeof(VDIR_DB_SEQ_T)) ? 0 : ERROR_BACKEND_ERROR;
BAIL_ON_VMDIR_ERROR( dwError );
dwError = MDBInitConfig(pDB);
BAIL_ON_VMDIR_ERROR( dwError );
/* Create the environment */
dwError = mdb_env_create ( &pDB->mdbEnv );
BAIL_ON_VMDIR_ERROR( dwError );
dwError = mdb_env_set_maxreaders( pDB->mdbEnv, BE_MDB_ENV_MAX_READERS );
BAIL_ON_VMDIR_ERROR( dwError );
/* FROM mdb.h
* The size should be a multiple of the OS page size. The default is
* 10485760 bytes. The size of the memory map is also the maximum size
* of the database. The value should be chosen as large as possible,
* to accommodate future growth of the database.
*
* // TODO, this is also the max size of database (per logical mdb db or the total dbs)
*/
dwError = VmDirGetMaxDbSizeMb(&db_max_size_mb);
if (dwError != 0)
{
VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "Use default max-database-size %llu", BE_MDB_ENV_MAX_MEM_MAPSIZE);
} else
{
db_max_mapsize = (uint64_t)(db_max_size_mb)*1024*1024;
if (db_max_mapsize < BE_MDB_ENV_MAX_MEM_MAPSIZE)
{
db_max_mapsize = BE_MDB_ENV_MAX_MEM_MAPSIZE;
VMDIR_LOG_WARNING(VMDIR_LOG_MASK_ALL, "RegKey %s value (%u) is too small. Use default max-database-size %llu",
VMDIR_REG_KEY_MAXIMUM_DB_SIZE_MB, db_max_size_mb, db_max_mapsize);
} else
{
VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "max-database-size is set to %llu per RegKey %s",
db_max_mapsize, VMDIR_REG_KEY_MAXIMUM_DB_SIZE_MB);
}
}
dwError = mdb_env_set_mapsize( pDB->mdbEnv, db_max_mapsize);
BAIL_ON_VMDIR_ERROR( dwError );
dwError = mdb_env_set_maxdbs ( pDB->mdbEnv, BE_MDB_ENV_MAX_DBS );
BAIL_ON_VMDIR_ERROR( dwError );
dwError = VmDirGetMdbChkptInterval(&db_chkpt_interval);
if (dwError)
{
db_chkpt_interval = VMDIR_REG_KEY_MDB_CHKPT_INTERVAL_DEFAULT;
dwError = 0;
}
VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: %s is set to %d",
__func__, VMDIR_REG_KEY_MDB_CHKPT_INTERVAL, db_chkpt_interval);
dwError = mdb_env_set_chkpt_interval(pDB->mdbEnv, db_chkpt_interval);
BAIL_ON_VMDIR_ERROR( dwError );
if(bMainDB)
{
mdb_set_raft_prepare_commit_func(pDB->mdbEnv, VmDirRaftPrepareCommit);
mdb_set_raft_post_commit_func(pDB->mdbEnv, VmDirRaftPostCommit);
mdb_set_raft_commit_fail_func(pDB->mdbEnv, VmDirRaftCommitFail);
}
#ifdef MDB_NOTLS
envFlags = MDB_NOTLS; // Required for versions of mdb which have this flag
#endif
// this is experimental from mdb.h comments
//envFlags = MDB_FIXEDMAP; /* use a fixed address for the mmap region */
//envFlags |= MDB_NOSYNC need sync for durability
//envFlags |= MDB_RDONLY need to open for read and write
/* Open the environment. */
#ifndef _WIN32
mode = S_IRUSR | S_IWUSR;
#else
mode = GENERIC_READ|GENERIC_WRITE;
#endif
//MDB WAL is the default mode and can be turned off with reg key MdbEnableWal set to 0
dwError = VmDirGetMdbWalEnable(&bMdbWalEnable);
if (dwError)
{
bMdbWalEnable = TRUE;
dwError = 0;
}
VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: %s is set to %s",
__func__, VMDIR_REG_KEY_MDB_ENABLE_WAL, bMdbWalEnable?"True":"False");
if (bMdbWalEnable)
{
envFlags |= MDB_WAL;
}
dwError = mdb_env_open ( pDB->mdbEnv, dbHomeDir, envFlags, mode );
//TODO, what if open failed? how to recover??
BAIL_ON_VMDIR_ERROR( dwError );
mdb_set_error_log_func(&VmDirMdbErrorLog);
/* Open main database. */
dwError = MDBOpenMainDB(pDB);
BAIL_ON_VMDIR_ERROR( dwError );
/* Open sequences */
dwError = MDBOpenSequence(pDB);
BAIL_ON_VMDIR_ERROR( dwError );
/* Open generic */
dwError = MDBOpenGeneric(pDB);
BAIL_ON_VMDIR_ERROR( dwError );
/* Initialize indices */
dwError = VmDirMDBInitializeIndexDB(pDB);
BAIL_ON_VMDIR_ERROR( dwError );
dwError = VmDirInitMdbStateGlobals();
BAIL_ON_VMDIR_ERROR(dwError);
VmDirLogDBStats(pDB);
/* TODO: require per db txn stats for raft groups */
if (bMainDB)
{
dwError = _VmDirWtxnStatsInit();
BAIL_ON_VMDIR_ERROR( dwError );
}
*phHandle = pDB;
VMDIR_LOG_INFO(VMDIR_LOG_MASK_ALL, "%s: succeeded with dbHomeDir %s", __func__, dbHomeDir);
cleanup:
VmDirLog( LDAP_DEBUG_TRACE, "MDBInitializeDB: End" );
return dwError;
error:
VmDirLog( LDAP_DEBUG_ANY, "MDBInitializeDB failed with error code: %d, error string: %s", dwError, mdb_strerror(dwError) );
//TODO, should shutdown here or caller will do that?
//gVdirMdbGlobals.mdbEnv = NULL;
goto cleanup;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data, sdata;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
long kval;
char *sval;
srand(time(NULL));
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
sdata.mv_size = mst.ms_psize / 4 - 30;
sdata.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
do {
printf("key: %p %s, data: %p %.*s\n",
key.mv_data, mdb_dkey(&key, dkbuf),
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
} while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0);
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
#if 0
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
#endif
mdb_env_close(env);
return 0;
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n",
1, 0, 0 );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val, 0, 0 );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef HAVE_EBCDIC
strcpy( path, mdb->mi_dbenv_home );
__atoe( path );
dbhome = path;
#else
dbhome = mdb->mi_dbenv_home;
#endif
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0);
flags = mdb->mi_dbenv_flags;
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC|MDB_WRITEMAP;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_dbi_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_dbi_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID ) {
MDB_cursor *mc;
MDB_val key, data;
ID id;
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
/* check for old dn2id format */
rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
/* first record is always ID 0 */
rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
if ( rc == 0 ) {
rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
if ( rc == 0 ) {
int len;
unsigned char *ptr;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
snprintf( cr->msg, sizeof(cr->msg),
"database \"%s\": DN index needs upgrade, "
"run \"slapindex entryDN\".",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
if ( !(slapMode & SLAP_TOOL_READMAIN ))
rc = LDAP_OTHER;
mdb->mi_flags |= MDB_NEED_UPGRADE;
}
}
}
mdb_cursor_close( mc );
if ( rc == LDAP_OTHER )
goto fail;
}
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
/* slapcat doesn't need indexes. avoid a failure if
* a configured index wasn't created yet.
*/
if ( !(slapMode & SLAP_TOOL_READONLY) ) {
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database %s: "
"txn_commit failed: %s (%d)\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi db;
rc = mdb_env_create(&env);
if (rc != 0) {
return rc;
}
rc = mdb_env_set_mapsize(env, 107374182400); //100G
if (rc != 0) {
return rc;
}
rc = mdb_env_set_maxdbs(env, 27);
if (rc != 0) {
return rc;
}
rc = mdb_env_open(env, "/home/pieter/Downloads/thundergraph/src/main/native/testdb", MDB_NOSYNC, 0664);
if (rc != 0) {
return rc;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_open(txn, "testdb", MDB_CREATE | MDB_INTEGERKEY, &db);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_set_compare(txn, db, compareTestDbId);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_txn_commit(txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
int i= 0;
for (i = 0; i < 1000; i++) {
MDB_cursor *cursor;
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
//Create a core vertex
MDB_val key, data;
TestDbId testDbId;
testDbId.testId = i;
testDbId.coreOrPropertyEnum = 0;
testDbId.labelId = -1;
testDbId.propertykeyId = -1;
testDbId.edgeId = -1LL;
key.mv_size = sizeof(TestDbId);
key.mv_data = &testDbId;
data.mv_size = 50;
char *value = malloc(5);
char v[] = "12345";
memcpy(value, v, 5);
data.mv_data = &v;
mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE);
free(value);
//Create a vertex property
MDB_val propertyKey, propertyData;
TestDbId propertyTestDbId;
propertyTestDbId.testId = i;
propertyTestDbId.coreOrPropertyEnum = 1;
propertyTestDbId.labelId = -1;
propertyTestDbId.propertykeyId = -1;
propertyTestDbId.edgeId = -1LL;
propertyKey.mv_size = sizeof(TestDbId);
propertyKey.mv_data = &propertyTestDbId;
data.mv_size = 50;
char *propertyValue = malloc(5);
char propertyV[] = "12345";
memcpy(propertyValue, propertyV, 5);
data.mv_data = &propertyV;
mdb_cursor_put(cursor, &propertyKey, &propertyData, MDB_NOOVERWRITE);
free(propertyValue);
//Create a vertex in edge
MDB_val inEdgeKey, inEdgeData;
TestDbId inEdgeTestDbId;
inEdgeTestDbId.testId = i;
inEdgeTestDbId.coreOrPropertyEnum = 2;
inEdgeTestDbId.labelId = -1;
inEdgeTestDbId.propertykeyId = -1;
inEdgeTestDbId.edgeId = -1LL;
inEdgeKey.mv_size = sizeof(TestDbId);
inEdgeKey.mv_data = &inEdgeTestDbId;
data.mv_size = 50;
char *inEdgeValue = malloc(5);
char inEdgeV[] = "12345";
memcpy(inEdgeValue, inEdgeV, 5);
data.mv_data = &inEdgeV;
mdb_cursor_put(cursor, &inEdgeKey, &inEdgeData, MDB_NOOVERWRITE);
mdb_cursor_close(cursor);
free(inEdgeValue);
}
MDB_cursor *cursor;
mdb_cursor_close(cursor);
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
MDB_val key, data;
for (i = 0; i < 1000; i++) {
TestDbId testDbId;
testDbId.testId = i;
testDbId.coreOrPropertyEnum = 0;
testDbId.labelId = -1;
testDbId.propertykeyId = -1;
testDbId.edgeId = -1LL;
key.mv_size = sizeof(TestDbId);
key.mv_data = &testDbId;
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
printf("mdb_cursor_get index = %i result = %i\n", i, rc);
MDB_val keyDelete, dataDelete;
TestDbId deleteTestDbId;
deleteTestDbId.testId = i;
deleteTestDbId.coreOrPropertyEnum = 0;
deleteTestDbId.labelId = -1;
deleteTestDbId.propertykeyId = -1;
deleteTestDbId.edgeId = -1LL;
keyDelete.mv_size = sizeof(TestDbId);
keyDelete.mv_data = &deleteTestDbId;
rc = mdb_del(txn, db, &keyDelete, &dataDelete);
printf("mdb_del = %i\n", rc);
MDB_val propertyKeyDelete, propertyDataDelete;
TestDbId propertyDeleteTestDbId;
propertyDeleteTestDbId.testId = i;
propertyDeleteTestDbId.coreOrPropertyEnum = 1;
propertyDeleteTestDbId.labelId = -1;
propertyDeleteTestDbId.propertykeyId = -1;
propertyDeleteTestDbId.edgeId = -1LL;
propertyKeyDelete.mv_size = sizeof(TestDbId);
propertyKeyDelete.mv_data = &propertyDeleteTestDbId;
rc = mdb_del(txn, db, &propertyKeyDelete, &propertyDataDelete);
printf("mdb_del = %i\n", rc);
MDB_val inEdgeKeyDelete, inEdgeDataDelete;
TestDbId inEdgeDeleteTestDbId;
inEdgeDeleteTestDbId.testId = i;
inEdgeDeleteTestDbId.coreOrPropertyEnum = 2;
inEdgeDeleteTestDbId.labelId = -1;
inEdgeDeleteTestDbId.propertykeyId = -1;
inEdgeDeleteTestDbId.edgeId = -1LL;
inEdgeKeyDelete.mv_size = sizeof(TestDbId);
inEdgeKeyDelete.mv_data = &inEdgeDeleteTestDbId;
rc = mdb_del(txn, db, &inEdgeKeyDelete, &inEdgeDataDelete);
printf("mdb_del = %i\n", rc);
}
mdb_cursor_close(cursor);
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
TestDbId testDbId = *((TestDbId *) (key.mv_data));
printf("keyId = %llu\n", testDbId.testId);
}
mdb_cursor_close(cursor);
fail:
mdb_close(env, db);
mdb_env_close(env);
printf("closing graph!\n");
return 0;
}
FreqSchedulerError
freq_scheduler_new(FreqScheduler **sch, PageDB *db, const char *path) {
FreqScheduler *p = *sch = malloc(sizeof(*p));
if (p == 0)
return freq_scheduler_error_memory;
p->error = error_new();
if (p->error == 0) {
free(p);
return freq_scheduler_error_memory;
}
p->page_db = db;
p->persist = FREQ_SCHEDULER_DEFAULT_PERSIST;
p->margin = -1.0; // disabled
p->max_n_crawls = 0;
// create directory if not present yet
char *error = 0;
p->path = path? strdup(path): concat(db->path, "freqs", '_');
if (!p->path)
error = "building scheduler path";
else
error = make_dir(p->path);
if (error != 0) {
freq_scheduler_set_error(p, freq_scheduler_error_invalid_path, __func__);
freq_scheduler_add_error(p, error);
return p->error->code;
}
if (txn_manager_new(&p->txn_manager, 0) != 0) {
freq_scheduler_set_error(p, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(p, p->txn_manager?
p->txn_manager->error->message:
"NULL");
return p->error->code;
}
int rc;
// initialize LMDB on the directory
if ((rc = mdb_env_create(&p->txn_manager->env) != 0))
error = "creating environment";
else if ((rc = mdb_env_set_mapsize(p->txn_manager->env,
FREQ_SCHEDULER_DEFAULT_SIZE)) != 0)
error = "setting map size";
else if ((rc = mdb_env_set_maxdbs(p->txn_manager->env, 1)) != 0)
error = "setting number of databases";
else if ((rc = mdb_env_open(
p->txn_manager->env,
p->path,
MDB_NOTLS | MDB_NOSYNC, 0664) != 0))
error = "opening environment";
if (error != 0) {
freq_scheduler_set_error(p, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(p, error);
freq_scheduler_add_error(p, mdb_strerror(rc));
return p->error->code;
}
return p->error->code;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32];
char kval[sizeof(int)];
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = (rand()%384) + 64;
values = (int *)je_malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "/tmp/testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
if (!(i & 0x0f))
sprintf(kval, "%03x", values[i]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
je_free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
/* LY: especially here we always needs MDB_NOSYNC
* for testing mdbx_env_close_ex() and "redo-to-steady" on open. */
env_oflags |= MDB_NOSYNC;
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
printf("check-preset-a\n");
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_a = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
++present_a;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_a == count - j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_env_sync(env, 1);
int deleted = 0;
key.mv_data = sval;
for (i = count - 1; i > -1; i -= (rand()%5)) {
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
deleted++;
}
}
free(values);
printf("Deleted %d values\n", deleted);
printf("check-preset-b.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_b = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_b;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == present_a - deleted, "mismatch");
printf("check-preset-b.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
/********************* LY: kept DB dirty ****************/
mdbx_env_close_ex(env, 1);
E(mdb_env_create(&env));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
printf("check-preset-c.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_dbi_open(txn, "id1", 0, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_c = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_c;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Rolled back %d deletion(s)\n", present_c - (present_a - deleted));
CHECK(present_c > present_a - deleted, "mismatch");
printf("check-preset-d.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_c == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdbx_env_close_ex(env, 0);
return 0;
}
