static int mdb_waitfixup( Operation *op, ww_ctx *ww, MDB_cursor *mci, MDB_cursor *mcd ) { int rc = 0; ww->flag = 0; mdb_txn_renew( ww->txn ); mdb_cursor_renew( ww->txn, mci ); mdb_cursor_renew( ww->txn, mcd ); if ( ww->mcd ) { MDB_val key, data; key.mv_size = sizeof(ID); key.mv_data = &ww->key; data = ww->data; rc = mdb_cursor_get( mcd, &key, &data, MDB_GET_BOTH ); if ( rc == MDB_NOTFOUND ) { data = ww->data; rc = mdb_cursor_get( mcd, &key, &data, MDB_GET_BOTH_RANGE ); /* the loop will skip this node using NEXT_DUP but we want it * sent, so go back one space first */ if ( rc == MDB_SUCCESS ) mdb_cursor_get( mcd, &key, &data, MDB_PREV_DUP ); else rc = LDAP_BUSY; } else if ( rc ) { rc = LDAP_OTHER; } op->o_tmpfree( ww->data.mv_data, op->o_tmpmemctx ); ww->data.mv_data = NULL; } return rc; }
/* * Class: jmdb_DatabaseWrapper * Method: txnRenew * Signature: (J)V */ JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_txnRenew(JNIEnv *vm, jclass clazz, jlong txnL) { int code = mdb_txn_renew((MDB_txn*) txnL); if (code) { throwDatabaseException(vm, code); } }
bool B_ACCURATE_LMDB::send_base_file_list(JCR *jcr) { int result; int32_t LinkFIc; FF_PKT *ff_pkt; MDB_cursor *cursor; MDB_val key, data; bool retval = false; accurate_payload *payload; int stream = STREAM_UNIX_ATTRIBUTES; if (!jcr->accurate || jcr->getJobLevel() != L_FULL) { return true; } /* * Commit any pending write transactions. */ if (m_db_rw_txn) { result = mdb_txn_commit(m_db_rw_txn); if (result != 0) { Jmsg1(jcr, M_FATAL, 0, _("Unable close write transaction: %s\n"), mdb_strerror(result)); goto bail_out; } m_db_rw_txn = NULL; } ff_pkt = init_find_files(); ff_pkt->type = FT_BASE; result = mdb_cursor_open(m_db_ro_txn, m_db_dbi, &cursor); if (result == 0) { while ((result = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { payload = (accurate_payload *)data.mv_data; if (bit_is_set(payload->filenr, m_seen_bitmap)) { Dmsg1(dbglvl, "base file fname=%s\n", key.mv_data); decode_stat(payload->lstat, &ff_pkt->statp, sizeof(struct stat), &LinkFIc); /* decode catalog stat */ ff_pkt->fname = (char *)key.mv_data; encode_and_send_attributes(jcr, ff_pkt, stream); } } mdb_cursor_close(cursor); } else { Jmsg1(jcr, M_FATAL, 0, _("Unable create cursor: %s\n"), mdb_strerror(result)); } mdb_txn_reset(m_db_ro_txn); result = mdb_txn_renew(m_db_ro_txn); if (result != 0) { Jmsg1(jcr, M_FATAL, 0, _("Unable to renew read transaction: %s\n"), mdb_strerror(result)); goto bail_out; } retval = true; bail_out: term_find_files(ff_pkt); return retval; }
static int mdb_waitfixup( Operation *op, ww_ctx *ww, MDB_cursor *mci, MDB_cursor *mcd, IdScopes *isc ) { MDB_val key; int rc = 0; ww->flag = 0; ww->nentries = 0; rc = mdb_txn_renew( ww->txn ); assert(rc == MDB_SUCCESS); rc = mdb_cursor_renew( ww->txn, mci ); assert(rc == MDB_SUCCESS); rc = mdb_cursor_renew( ww->txn, mcd ); assert(rc == MDB_SUCCESS); key.mv_size = sizeof(ID); if ( ww->mcd ) { /* scope-based search using dn2id_walk */ if ( isc->numrdns ) mdb_dn2id_wrestore( op, isc ); if ( ww->data.mv_data ) { MDB_val data; key.mv_data = &ww->key; data = ww->data; rc = mdb_cursor_get( mcd, &key, &data, MDB_GET_BOTH ); if ( rc == MDB_NOTFOUND ) { data = ww->data; rc = mdb_cursor_get( mcd, &key, &data, MDB_GET_BOTH_RANGE ); /* the loop will skip this node using NEXT_DUP but we want it * sent, so go back one space first */ if ( rc == MDB_SUCCESS ) mdb_cursor_get( mcd, &key, &data, MDB_PREV_DUP ); else rc = LDAP_BUSY; } else if ( rc ) { rc = LDAP_OTHER; } op->o_tmpfree( ww->data.mv_data, op->o_tmpmemctx ); ww->data.mv_data = NULL; } ww->flag = 0; } else if ( isc->scopes[0].mid > 1 ) { /* candidate-based search */ int i; for ( i=1; i<isc->scopes[0].mid; i++ ) { if ( !isc->scopes[i].mval.mv_data ) continue; key.mv_data = &isc->scopes[i].mid; rc = mdb_cursor_get( mcd, &key, &isc->scopes[i].mval, MDB_SET_RANGE ); if ( rc != MDB_SUCCESS ) { /* LY: Yea, this is my paranoia */ rc = (rc == MDB_NOTFOUND) ? LDAP_BUSY : LDAP_OTHER; break; } } } return rc; }
accurate_payload *B_ACCURATE_LMDB::lookup_payload(JCR *jcr, char *fname) { int result; int lstat_length; MDB_val key, data; accurate_payload *payload = NULL; key.mv_data = fname; key.mv_size = strlen(fname) + 1; result = mdb_get(m_db_ro_txn, m_db_dbi, &key, &data); switch (result) { case 0: /* * Success. * * We need to make a private copy of the LDMB data as we are not * allowed to change its content and we need to update the lstat * and chksum pointer to point to the actual lstat and chksum that * is stored behind the accurate_payload structure in the LMDB. */ m_pay_load = check_pool_memory_size(m_pay_load, data.mv_size); payload = (accurate_payload *)m_pay_load; memcpy(payload, data.mv_data, data.mv_size); payload->lstat = (char *)payload + sizeof(accurate_payload); lstat_length = strlen(payload->lstat); payload->chksum = (char *)payload->lstat + lstat_length + 1; /* * We keep the transaction as short a possible so after a lookup * and copying the actual data out we reset the read transaction * and do a renew of the read transaction for a new run. */ mdb_txn_reset(m_db_ro_txn); result = mdb_txn_renew(m_db_ro_txn); if (result != 0) { Jmsg1(jcr, M_FATAL, 0, _("Unable to renew read transaction: %s\n"), mdb_strerror(result)); return NULL; } break; case MDB_NOTFOUND: /* * Failed to find the given key. */ break; default: break; } return payload; }
static void *perform(void *arg) { db_thread *thr = (db_thread*)arg; int i,rc; mdbinf* mdb = &thr->mdb; srand((u32)pthread_self()); open_txn(mdb, MDB_RDONLY); thr->resFrames = alloca((SQLITE_DEFAULT_PAGE_SIZE/thr->maxvalsize + 1)*sizeof(MDB_val)); for (i = 0; i < 1000*100; i++) { int j = rand() % NCONS; if (i % 1000 == 0) printf("r %lld %d\n",(i64)pthread_self(),i); if (pthread_mutex_trylock(&g_cons[j].wal.mtx) != 0) continue; g_tsd_conn = &g_cons[j]; rc = sqlite3_exec(g_cons[j].db,"SELECT max(id) FROM tab;",NULL,NULL,NULL); if (rc != SQLITE_OK) { printf("Error select"); break; } pthread_mutex_unlock(&g_cons[j].wal.mtx); mdb_txn_reset(thr->mdb.txn); rc = mdb_txn_renew(thr->mdb.txn); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(mdb->txn, mdb->cursorLog); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(mdb->txn, mdb->cursorPages); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(mdb->txn, mdb->cursorInfo); if (rc != MDB_SUCCESS) break; } mdb_cursor_close(mdb->cursorLog); mdb_cursor_close(mdb->cursorPages); mdb_cursor_close(mdb->cursorInfo); mdb_txn_abort(mdb->txn); return NULL; }
int main(int argc, const char* argv[]) { g_log = stdout; db_thread thr; db_thread threads[RTHREADS]; pthread_t tids[RTHREADS]; priv_data pd; mdbinf* mdb = &thr.mdb; int i, rc; db_connection *cons; g_pd = &pd; char commit = 1; MDB_env *menv = NULL; char *lmpath = "lmdb"; MDB_txn *txn; MDB_val key = {1,(void*)"?"}, data = {0,NULL}; MDB_envinfo stat; sqlite3_initialize(); sqlite3_vfs_register(sqlite3_nullvfs(), 1); unlink(lmpath); memset(threads, 0, sizeof(threads)); memset(&thr, 0, sizeof(db_thread)); memset(&pd, 0, sizeof(priv_data)); pd.wmdb = calloc(1,sizeof(mdbinf)); pd.nEnvs = 1; pd.nReadThreads = RTHREADS; pd.nWriteThreads = 1; pd.syncNumbers = calloc(1,sizeof(u64)); pd.actorIndexes = calloc(1,sizeof(atomic_llong)); atomic_init(pd.actorIndexes,0); g_cons = cons = calloc(NCONS, sizeof(db_connection)); g_tsd_cursync = 0; g_tsd_conn = NULL; g_tsd_wmdb = NULL; g_tsd_thread = &thr; if (mdb_env_create(&menv) != MDB_SUCCESS) return -1; if (mdb_env_set_maxdbs(menv,5) != MDB_SUCCESS) return -1; if (mdb_env_set_mapsize(menv,1024*1024*1024) != MDB_SUCCESS) return -1; // Syncs are handled from erlang. if (mdb_env_open(menv, lmpath, MDB_NOSUBDIR|MDB_NOTLS|MDB_NOSYNC, 0664) != MDB_SUCCESS) //MDB_NOSYNC return -1; if (mdb_txn_begin(menv, NULL, 0, &txn) != MDB_SUCCESS) return -1; if (mdb_dbi_open(txn, "info", MDB_INTEGERKEY | MDB_CREATE, &pd.wmdb[0].infodb) != MDB_SUCCESS) return -1; if (mdb_dbi_open(txn, "actors", MDB_CREATE, &pd.wmdb[0].actorsdb) != MDB_SUCCESS) return -1; if (mdb_dbi_open(txn, "log", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP, &pd.wmdb[0].logdb) != MDB_SUCCESS) return -1; if (mdb_dbi_open(txn, "pages", MDB_CREATE | MDB_DUPSORT, &pd.wmdb[0].pagesdb) != MDB_SUCCESS) return -1; if (mdb_txn_commit(txn) != MDB_SUCCESS) return -1; pd.wmdb[0].env = menv; thr.nEnv = 0; thr.isreadonly = 0; thr.mdb.env = menv; thr.mdb.infodb = pd.wmdb[0].infodb; thr.mdb.actorsdb = pd.wmdb[0].actorsdb; thr.mdb.logdb = pd.wmdb[0].logdb; thr.mdb.pagesdb = pd.wmdb[0].pagesdb; thr.maxvalsize = mdb_env_get_maxkeysize(mdb->env); thr.resFrames = alloca((SQLITE_DEFAULT_PAGE_SIZE/thr.maxvalsize + 1)*sizeof(MDB_val)); open_txn(&thr.mdb, MDB_RDONLY); for (i = 0; i < NCONS; i++) { char filename[256]; char commit = 1; g_tsd_conn = &cons[i]; sprintf(filename, "ac%d.db", i); pthread_mutex_init(&cons[i].wal.mtx, NULL); thr.pagesChanged = 0; rc = sqlite3_open(filename,&(cons[i].db)); if(rc != SQLITE_OK) { DBG("Unable to open db"); break; } rc = sqlite3_exec(cons[i].db,"PRAGMA synchronous=0;PRAGMA journal_mode=wal;",NULL,NULL,NULL); if (rc != SQLITE_OK) { DBG("unable to open wal"); break; } cons[i].wal.inProgressTerm = 1; cons[i].wal.inProgressEvnum = 1; rc = sqlite3_exec(cons[i].db,"CREATE TABLE tab (id INTEGER PRIMARY KEY, txt TEXT);" "insert into tab values (1,'aaaa');",NULL,NULL,NULL); if (rc != SQLITE_OK) { DBG("Cant create table"); break; } unlock_write_txn(thr.nEnv, 0, &commit); mdb_txn_reset(thr.mdb.txn); rc = mdb_txn_renew(thr.mdb.txn); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorLog); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorPages); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorInfo); if (rc != MDB_SUCCESS) break; } // mdb_cursor_close(thr.mdb.cursorLog); // mdb_cursor_close(thr.mdb.cursorPages); // mdb_cursor_close(thr.mdb.cursorInfo); // mdb_txn_abort(thr.mdb.txn); for (i = 0; i < RTHREADS; i++) { threads[i].nEnv = 0; threads[i].isreadonly = 0; threads[i].mdb.env = menv; threads[i].mdb.infodb = pd.wmdb[0].infodb; threads[i].mdb.actorsdb = pd.wmdb[0].actorsdb; threads[i].mdb.logdb = pd.wmdb[0].logdb; threads[i].mdb.pagesdb = pd.wmdb[0].pagesdb; threads[i].maxvalsize = mdb_env_get_maxkeysize(mdb->env); pthread_create(&tids[i], NULL, perform, (void *)&threads[i]); } srand((u32)pthread_self() + time(NULL)); for (i = 0; i < 1000*200; i++) { char commit = 1; int j = rand() % NCONS; db_connection *con = &g_cons[j]; char str[100]; if (pthread_mutex_trylock(&con->wal.mtx) != 0) { i--; continue; } if (i % 1000 == 0) printf("w %d\n",i); g_tsd_conn = con; lock_wtxn(thr.nEnv); thr.pagesChanged = 0; if (con->wal.firstCompleteEvnum+10 < con->wal.lastCompleteEvnum) { // printf("CHECKPOINT? %llu %llu\n",con->wal.firstCompleteEvnum,con->wal.lastCompleteEvnum); if (checkpoint(&con->wal, con->wal.lastCompleteEvnum-10) != SQLITE_OK) { printf("Checkpoint failed\n"); break; } } con->wal.inProgressTerm = 1; con->wal.inProgressEvnum = con->wal.lastCompleteEvnum+1; sprintf(str,"INSERT INTO tab VALUES (%d,'VALUE VALUE13456');", i); sqlite3_exec(con->db,str,NULL,NULL,NULL); pthread_mutex_unlock(&con->wal.mtx); unlock_write_txn(thr.nEnv, 0, &commit); mdb_txn_reset(thr.mdb.txn); rc = mdb_txn_renew(thr.mdb.txn); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorLog); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorPages); if (rc != MDB_SUCCESS) break; rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorInfo); if (rc != MDB_SUCCESS) break; } unlock_write_txn(thr.nEnv, 1, &commit); for (i = 0; i < RTHREADS; i++) pthread_join(tids[i],NULL); return 1; }