static int
mdb_waitfixup( Operation *op, ww_ctx *ww, MDB_cursor *mci, MDB_cursor *mcd )
{
int rc = 0;
ww->flag = 0;
mdb_txn_renew( ww->txn );
mdb_cursor_renew( ww->txn, mci );
mdb_cursor_renew( ww->txn, mcd );
if ( ww->mcd ) {
MDB_val key, data;
key.mv_size = sizeof(ID);
key.mv_data = &ww->key;
data = ww->data;
rc = mdb_cursor_get( mcd, &key, &data, MDB_GET_BOTH );
if ( rc == MDB_NOTFOUND ) {
data = ww->data;
rc = mdb_cursor_get( mcd, &key, &data, MDB_GET_BOTH_RANGE );
/* the loop will skip this node using NEXT_DUP but we want it
* sent, so go back one space first
*/
if ( rc == MDB_SUCCESS )
mdb_cursor_get( mcd, &key, &data, MDB_PREV_DUP );
else
rc = LDAP_BUSY;
} else if ( rc ) {
rc = LDAP_OTHER;
}
op->o_tmpfree( ww->data.mv_data, op->o_tmpmemctx );
ww->data.mv_data = NULL;
}
return rc;
}
/*
* Class: jmdb_DatabaseWrapper
* Method: txnRenew
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_txnRenew(JNIEnv *vm,
jclass clazz, jlong txnL) {
int code = mdb_txn_renew((MDB_txn*) txnL);
if (code) {
throwDatabaseException(vm, code);
}
}
bool B_ACCURATE_LMDB::send_base_file_list(JCR *jcr)
{
int result;
int32_t LinkFIc;
FF_PKT *ff_pkt;
MDB_cursor *cursor;
MDB_val key, data;
bool retval = false;
accurate_payload *payload;
int stream = STREAM_UNIX_ATTRIBUTES;
if (!jcr->accurate || jcr->getJobLevel() != L_FULL) {
return true;
}
/*
* Commit any pending write transactions.
*/
if (m_db_rw_txn) {
result = mdb_txn_commit(m_db_rw_txn);
if (result != 0) {
Jmsg1(jcr, M_FATAL, 0, _("Unable close write transaction: %s\n"), mdb_strerror(result));
goto bail_out;
}
m_db_rw_txn = NULL;
}
ff_pkt = init_find_files();
ff_pkt->type = FT_BASE;
result = mdb_cursor_open(m_db_ro_txn, m_db_dbi, &cursor);
if (result == 0) {
while ((result = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
payload = (accurate_payload *)data.mv_data;
if (bit_is_set(payload->filenr, m_seen_bitmap)) {
Dmsg1(dbglvl, "base file fname=%s\n", key.mv_data);
decode_stat(payload->lstat, &ff_pkt->statp, sizeof(struct stat), &LinkFIc); /* decode catalog stat */
ff_pkt->fname = (char *)key.mv_data;
encode_and_send_attributes(jcr, ff_pkt, stream);
}
}
mdb_cursor_close(cursor);
} else {
Jmsg1(jcr, M_FATAL, 0, _("Unable create cursor: %s\n"), mdb_strerror(result));
}
mdb_txn_reset(m_db_ro_txn);
result = mdb_txn_renew(m_db_ro_txn);
if (result != 0) {
Jmsg1(jcr, M_FATAL, 0, _("Unable to renew read transaction: %s\n"), mdb_strerror(result));
goto bail_out;
}
retval = true;
bail_out:
term_find_files(ff_pkt);
return retval;
}
static int
mdb_waitfixup( Operation *op, ww_ctx *ww, MDB_cursor *mci, MDB_cursor *mcd, IdScopes *isc )
{
MDB_val key;
int rc = 0;
ww->flag = 0;
ww->nentries = 0;
rc = mdb_txn_renew( ww->txn );
assert(rc == MDB_SUCCESS);
rc = mdb_cursor_renew( ww->txn, mci );
assert(rc == MDB_SUCCESS);
rc = mdb_cursor_renew( ww->txn, mcd );
assert(rc == MDB_SUCCESS);
key.mv_size = sizeof(ID);
if ( ww->mcd ) { /* scope-based search using dn2id_walk */
if ( isc->numrdns )
mdb_dn2id_wrestore( op, isc );
if ( ww->data.mv_data ) {
MDB_val data;
key.mv_data = &ww->key;
data = ww->data;
rc = mdb_cursor_get( mcd, &key, &data, MDB_GET_BOTH );
if ( rc == MDB_NOTFOUND ) {
data = ww->data;
rc = mdb_cursor_get( mcd, &key, &data, MDB_GET_BOTH_RANGE );
/* the loop will skip this node using NEXT_DUP but we want it
* sent, so go back one space first
*/
if ( rc == MDB_SUCCESS )
mdb_cursor_get( mcd, &key, &data, MDB_PREV_DUP );
else
rc = LDAP_BUSY;
} else if ( rc ) {
rc = LDAP_OTHER;
}
op->o_tmpfree( ww->data.mv_data, op->o_tmpmemctx );
ww->data.mv_data = NULL;
}
ww->flag = 0;
} else if ( isc->scopes[0].mid > 1 ) { /* candidate-based search */
int i;
for ( i=1; i<isc->scopes[0].mid; i++ ) {
if ( !isc->scopes[i].mval.mv_data )
continue;
key.mv_data = &isc->scopes[i].mid;
rc = mdb_cursor_get( mcd, &key, &isc->scopes[i].mval, MDB_SET_RANGE );
if ( rc != MDB_SUCCESS ) {
/* LY: Yea, this is my paranoia */
rc = (rc == MDB_NOTFOUND) ? LDAP_BUSY : LDAP_OTHER;
break;
}
}
}
return rc;
}
accurate_payload *B_ACCURATE_LMDB::lookup_payload(JCR *jcr, char *fname)
{
int result;
int lstat_length;
MDB_val key, data;
accurate_payload *payload = NULL;
key.mv_data = fname;
key.mv_size = strlen(fname) + 1;
result = mdb_get(m_db_ro_txn, m_db_dbi, &key, &data);
switch (result) {
case 0:
/*
* Success.
*
* We need to make a private copy of the LDMB data as we are not
* allowed to change its content and we need to update the lstat
* and chksum pointer to point to the actual lstat and chksum that
* is stored behind the accurate_payload structure in the LMDB.
*/
m_pay_load = check_pool_memory_size(m_pay_load, data.mv_size);
payload = (accurate_payload *)m_pay_load;
memcpy(payload, data.mv_data, data.mv_size);
payload->lstat = (char *)payload + sizeof(accurate_payload);
lstat_length = strlen(payload->lstat);
payload->chksum = (char *)payload->lstat + lstat_length + 1;
/*
* We keep the transaction as short a possible so after a lookup
* and copying the actual data out we reset the read transaction
* and do a renew of the read transaction for a new run.
*/
mdb_txn_reset(m_db_ro_txn);
result = mdb_txn_renew(m_db_ro_txn);
if (result != 0) {
Jmsg1(jcr, M_FATAL, 0, _("Unable to renew read transaction: %s\n"), mdb_strerror(result));
return NULL;
}
break;
case MDB_NOTFOUND:
/*
* Failed to find the given key.
*/
break;
default:
break;
}
return payload;
}
static void *perform(void *arg)
{
db_thread *thr = (db_thread*)arg;
int i,rc;
mdbinf* mdb = &thr->mdb;
srand((u32)pthread_self());
open_txn(mdb, MDB_RDONLY);
thr->resFrames = alloca((SQLITE_DEFAULT_PAGE_SIZE/thr->maxvalsize + 1)*sizeof(MDB_val));
for (i = 0; i < 1000*100; i++)
{
int j = rand() % NCONS;
if (i % 1000 == 0)
printf("r %lld %d\n",(i64)pthread_self(),i);
if (pthread_mutex_trylock(&g_cons[j].wal.mtx) != 0)
continue;
g_tsd_conn = &g_cons[j];
rc = sqlite3_exec(g_cons[j].db,"SELECT max(id) FROM tab;",NULL,NULL,NULL);
if (rc != SQLITE_OK)
{
printf("Error select");
break;
}
pthread_mutex_unlock(&g_cons[j].wal.mtx);
mdb_txn_reset(thr->mdb.txn);
rc = mdb_txn_renew(thr->mdb.txn);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(mdb->txn, mdb->cursorLog);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(mdb->txn, mdb->cursorPages);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(mdb->txn, mdb->cursorInfo);
if (rc != MDB_SUCCESS)
break;
}
mdb_cursor_close(mdb->cursorLog);
mdb_cursor_close(mdb->cursorPages);
mdb_cursor_close(mdb->cursorInfo);
mdb_txn_abort(mdb->txn);
return NULL;
}
int main(int argc, const char* argv[])
{
g_log = stdout;
db_thread thr;
db_thread threads[RTHREADS];
pthread_t tids[RTHREADS];
priv_data pd;
mdbinf* mdb = &thr.mdb;
int i, rc;
db_connection *cons;
g_pd = &pd;
char commit = 1;
MDB_env *menv = NULL;
char *lmpath = "lmdb";
MDB_txn *txn;
MDB_val key = {1,(void*)"?"}, data = {0,NULL};
MDB_envinfo stat;
sqlite3_initialize();
sqlite3_vfs_register(sqlite3_nullvfs(), 1);
unlink(lmpath);
memset(threads, 0, sizeof(threads));
memset(&thr, 0, sizeof(db_thread));
memset(&pd, 0, sizeof(priv_data));
pd.wmdb = calloc(1,sizeof(mdbinf));
pd.nEnvs = 1;
pd.nReadThreads = RTHREADS;
pd.nWriteThreads = 1;
pd.syncNumbers = calloc(1,sizeof(u64));
pd.actorIndexes = calloc(1,sizeof(atomic_llong));
atomic_init(pd.actorIndexes,0);
g_cons = cons = calloc(NCONS, sizeof(db_connection));
g_tsd_cursync = 0;
g_tsd_conn = NULL;
g_tsd_wmdb = NULL;
g_tsd_thread = &thr;
if (mdb_env_create(&menv) != MDB_SUCCESS)
return -1;
if (mdb_env_set_maxdbs(menv,5) != MDB_SUCCESS)
return -1;
if (mdb_env_set_mapsize(menv,1024*1024*1024) != MDB_SUCCESS)
return -1;
// Syncs are handled from erlang.
if (mdb_env_open(menv, lmpath, MDB_NOSUBDIR|MDB_NOTLS|MDB_NOSYNC, 0664) != MDB_SUCCESS) //MDB_NOSYNC
return -1;
if (mdb_txn_begin(menv, NULL, 0, &txn) != MDB_SUCCESS)
return -1;
if (mdb_dbi_open(txn, "info", MDB_INTEGERKEY | MDB_CREATE, &pd.wmdb[0].infodb) != MDB_SUCCESS)
return -1;
if (mdb_dbi_open(txn, "actors", MDB_CREATE, &pd.wmdb[0].actorsdb) != MDB_SUCCESS)
return -1;
if (mdb_dbi_open(txn, "log", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP,
&pd.wmdb[0].logdb) != MDB_SUCCESS)
return -1;
if (mdb_dbi_open(txn, "pages", MDB_CREATE | MDB_DUPSORT, &pd.wmdb[0].pagesdb) != MDB_SUCCESS)
return -1;
if (mdb_txn_commit(txn) != MDB_SUCCESS)
return -1;
pd.wmdb[0].env = menv;
thr.nEnv = 0;
thr.isreadonly = 0;
thr.mdb.env = menv;
thr.mdb.infodb = pd.wmdb[0].infodb;
thr.mdb.actorsdb = pd.wmdb[0].actorsdb;
thr.mdb.logdb = pd.wmdb[0].logdb;
thr.mdb.pagesdb = pd.wmdb[0].pagesdb;
thr.maxvalsize = mdb_env_get_maxkeysize(mdb->env);
thr.resFrames = alloca((SQLITE_DEFAULT_PAGE_SIZE/thr.maxvalsize + 1)*sizeof(MDB_val));
open_txn(&thr.mdb, MDB_RDONLY);
for (i = 0; i < NCONS; i++)
{
char filename[256];
char commit = 1;
g_tsd_conn = &cons[i];
sprintf(filename, "ac%d.db", i);
pthread_mutex_init(&cons[i].wal.mtx, NULL);
thr.pagesChanged = 0;
rc = sqlite3_open(filename,&(cons[i].db));
if(rc != SQLITE_OK)
{
DBG("Unable to open db");
break;
}
rc = sqlite3_exec(cons[i].db,"PRAGMA synchronous=0;PRAGMA journal_mode=wal;",NULL,NULL,NULL);
if (rc != SQLITE_OK)
{
DBG("unable to open wal");
break;
}
cons[i].wal.inProgressTerm = 1;
cons[i].wal.inProgressEvnum = 1;
rc = sqlite3_exec(cons[i].db,"CREATE TABLE tab (id INTEGER PRIMARY KEY, txt TEXT);"
"insert into tab values (1,'aaaa');",NULL,NULL,NULL);
if (rc != SQLITE_OK)
{
DBG("Cant create table");
break;
}
unlock_write_txn(thr.nEnv, 0, &commit);
mdb_txn_reset(thr.mdb.txn);
rc = mdb_txn_renew(thr.mdb.txn);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorLog);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorPages);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorInfo);
if (rc != MDB_SUCCESS)
break;
}
// mdb_cursor_close(thr.mdb.cursorLog);
// mdb_cursor_close(thr.mdb.cursorPages);
// mdb_cursor_close(thr.mdb.cursorInfo);
// mdb_txn_abort(thr.mdb.txn);
for (i = 0; i < RTHREADS; i++)
{
threads[i].nEnv = 0;
threads[i].isreadonly = 0;
threads[i].mdb.env = menv;
threads[i].mdb.infodb = pd.wmdb[0].infodb;
threads[i].mdb.actorsdb = pd.wmdb[0].actorsdb;
threads[i].mdb.logdb = pd.wmdb[0].logdb;
threads[i].mdb.pagesdb = pd.wmdb[0].pagesdb;
threads[i].maxvalsize = mdb_env_get_maxkeysize(mdb->env);
pthread_create(&tids[i], NULL, perform, (void *)&threads[i]);
}
srand((u32)pthread_self() + time(NULL));
for (i = 0; i < 1000*200; i++)
{
char commit = 1;
int j = rand() % NCONS;
db_connection *con = &g_cons[j];
char str[100];
if (pthread_mutex_trylock(&con->wal.mtx) != 0)
{
i--;
continue;
}
if (i % 1000 == 0)
printf("w %d\n",i);
g_tsd_conn = con;
lock_wtxn(thr.nEnv);
thr.pagesChanged = 0;
if (con->wal.firstCompleteEvnum+10 < con->wal.lastCompleteEvnum)
{
// printf("CHECKPOINT? %llu %llu\n",con->wal.firstCompleteEvnum,con->wal.lastCompleteEvnum);
if (checkpoint(&con->wal, con->wal.lastCompleteEvnum-10) != SQLITE_OK)
{
printf("Checkpoint failed\n");
break;
}
}
con->wal.inProgressTerm = 1;
con->wal.inProgressEvnum = con->wal.lastCompleteEvnum+1;
sprintf(str,"INSERT INTO tab VALUES (%d,'VALUE VALUE13456');", i);
sqlite3_exec(con->db,str,NULL,NULL,NULL);
pthread_mutex_unlock(&con->wal.mtx);
unlock_write_txn(thr.nEnv, 0, &commit);
mdb_txn_reset(thr.mdb.txn);
rc = mdb_txn_renew(thr.mdb.txn);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorLog);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorPages);
if (rc != MDB_SUCCESS)
break;
rc = mdb_cursor_renew(thr.mdb.txn, mdb->cursorInfo);
if (rc != MDB_SUCCESS)
break;
}
unlock_write_txn(thr.nEnv, 1, &commit);
for (i = 0; i < RTHREADS; i++)
pthread_join(tids[i],NULL);
return 1;
}
