int cursor_get(cursor_t cursor, MDB_val *key, MDB_val *data, MDB_cursor_op op){
return cursor->prev ? mdb_cursor_get(cursor->cursor, key, data, op) : MDB_BAD_TXN;
}
static int findframe(db_thread *thr, Wal *pWal, Pgno pgno, u32 *piRead, u64 limitTerm,
u64 limitEvnum, u64 *outTerm, u64 *outEvnum)
{
MDB_val key, data;
int rc;
size_t ndupl = 0;
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
mdbinf *mdb;
if (thr->pagesChanged)
{
#if ATOMIC
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
}
else
mdb = &thr->mdb;
track_time(7,thr);
DBG("FIND FRAME pgno=%u, index=%llu, limitterm=%llu, limitevnum=%llu",
pgno,pWal->index,limitTerm,limitEvnum);
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>,
// <<Evterm:64,Evnum:64,Counter,CompressedPage/binary>>}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
// u32 pgno2 = *(u32*)(key.mv_data+sizeof(u64));
// DBG("RUN %d",pgno2);
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY);
if (rc == MDB_SUCCESS)
{
mdb_cursor_count(mdb->cursorPages,&ndupl);
if (ndupl == 0)
{
*piRead = 0;
return SQLITE_OK;
}
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP);
if (rc == MDB_SUCCESS)
{
while (1)
{
char frag1 = *(char*)(((char*)data.mv_data)+sizeof(u64)*2);
int frag = frag1;
u64 term, evnum;
memcpy(&term, data.mv_data, sizeof(u64));
memcpy(&evnum, ((u8*)data.mv_data) + sizeof(u64), sizeof(u64));
if (term > limitTerm || evnum > limitEvnum)
{
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP);
if (rc == MDB_SUCCESS)
continue;
else
{
DBG("Cant move to prev dup, term=%llu, evnum=%llu,"
" limitterm=%llu, limitevnum=%llu",term,evnum,limitTerm,limitEvnum);
*piRead = 0;
break;
}
}
if (outTerm != NULL)
*outTerm = term;
if (outEvnum != NULL)
*outEvnum = evnum;
DBG("Found page size=%ld, frags=%d",data.mv_size,(int)frag);
thr->nResFrames = frag;
thr->resFrames[frag--] = data;
while (frag >= 0)
{
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP);
frag = *(((u8*)data.mv_data)+sizeof(u64)*2);
// DBG("SUCCESS? %d frag=%d, size=%ld",pgno,frag,data.mv_size);
thr->resFrames[frag--] = data;
}
*piRead = 1;
break;
}
}
else
{
DBG("Find page no last dup");
*piRead = 0;
}
}
else if (rc == MDB_NOTFOUND)
{
DBG("Frame not found!");
*piRead = 0;
}
else
{
DBG("ERROR findframe: %d",rc);
*piRead = 0;
}
return SQLITE_OK;
}
// Delete all pages up to limitEvterm and limitEvnum
static int checkpoint(Wal *pWal, u64 limitEvnum)
{
MDB_val logKey, logVal;
u8 logKeyBuf[sizeof(u64)*3];
u64 evnum,evterm,aindex;
mdbinf* mdb;
// db_thread* const thr = enif_tsd_get(g_tsd_thread);
#if ATOMIC
db_thread *thr = g_tsd_thread;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
#endif
int logop, mrc = MDB_SUCCESS;
// u8 somethingDeleted = 0;
int allPagesDiff = 0;
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
if (!mdb)
return SQLITE_ERROR;
// if (pWal->inProgressTerm == 0)
// return SQLITE_OK;
DBG("checkpoint actor=%llu, fct=%llu, fcev=%llu, limitEvnum=%llu",pWal->index,
pWal->firstCompleteTerm,pWal->firstCompleteEvnum,limitEvnum);
while (pWal->firstCompleteEvnum < limitEvnum)
{
logKey.mv_data = logKeyBuf;
logKey.mv_size = sizeof(logKeyBuf);
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &pWal->firstCompleteTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &pWal->firstCompleteEvnum,sizeof(u64));
if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS)
{
printf("Key not found in log for checkpoint %llu %llu\n",
pWal->firstCompleteTerm, pWal->firstCompleteEvnum);
return SQLITE_OK;
}
DBG("checkpoint evnum=%llu",pWal->firstCompleteEvnum);
// For every page here
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
// Delete from
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>}
logop = MDB_FIRST_DUP;
while ((mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS)
{
u32 pgno;
size_t ndupl;
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
MDB_val pgKey = {0,NULL}, pgVal = {0,NULL};
u64 pgnoLimitEvnum;
logop = MDB_NEXT_DUP;
memcpy(&pgno, logVal.mv_data,sizeof(u32));
DBG("checkpoint pgno=%u",pgno);
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32));
pgKey.mv_data = pagesKeyBuf;
pgKey.mv_size = sizeof(pagesKeyBuf);
if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS)
{
continue;
}
mdb_cursor_count(mdb->cursorPages,&ndupl);
if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_LAST_DUP) == MDB_SUCCESS)
memcpy(&pgnoLimitEvnum, (u8*)pgVal.mv_data+sizeof(u64),sizeof(u64));
else
continue;
pgnoLimitEvnum = pgnoLimitEvnum < limitEvnum ? pgnoLimitEvnum : limitEvnum;
if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_FIRST_DUP) != MDB_SUCCESS)
continue;
do
{
u8 frag;
MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL};
mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT);
frag = *((u8*)pgDelVal.mv_data+sizeof(u64)*2);
memcpy(&evterm, pgDelVal.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)pgDelVal.mv_data+sizeof(u64),sizeof(u64));
DBG("limit limitevnum %lld, curnum %lld, dupl %zu, frag=%d",
limitEvnum, evnum, ndupl,(int)frag);
if (evnum < pgnoLimitEvnum)
{
mrc = mdb_cursor_del(mdb->cursorPages,0);
if (mrc != MDB_SUCCESS)
{
DBG("Unable to delete page on cursor!.");
break;
}
// else
// {
// DBG("Deleted page!");
// // somethingDeleted = 1;
// }
if (frag == 0)
allPagesDiff++;
}
ndupl--;
if (!ndupl)
break;
mrc = mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_NEXT_DUP);
} while (mrc == MDB_SUCCESS);
}
mrc = mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA);
if (mrc != MDB_SUCCESS)
{
DBG("Can not delete log");
break;
}
// move forward
if ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_NEXT_NODUP)) != MDB_SUCCESS)
{
DBG("Unable to move to next log %d",mrc);
break;
}
// read next key data
memcpy(&aindex, logKey.mv_data, sizeof(u64));
memcpy(&evterm, (u8*)logKey.mv_data + sizeof(u64), sizeof(u64));
memcpy(&evnum, (u8*)logKey.mv_data + sizeof(u64)*2, sizeof(u64));
if (aindex != pWal->index)
{
DBG("Reached another actor");
break;
}
pWal->firstCompleteTerm = evterm;
pWal->firstCompleteEvnum = evnum;
pWal->allPages -= allPagesDiff;
allPagesDiff = 0;
DBG("Checkpint fce now=%lld",(u64)evnum);
}
// no dirty pages, but will write info
if (sqlite3WalFrames(pWal, SQLITE_DEFAULT_PAGE_SIZE, NULL, pWal->mxPage, 1, 0) == SQLITE_OK)
return SQLITE_OK;
else
return SQLITE_ERROR;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
long kval;
char *sval;
srand(time(NULL));
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
data.mv_size = mst.ms_psize / 4 - 30;
data.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
do {
printf("key: %p %s, data: %p %.*s\n",
key.mv_data, mdb_dkey(&key, dkbuf),
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
} while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0);
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
#if 0
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
#endif
mdb_env_close(env);
return 0;
}
static int storeinfo(Wal *pWal, u64 currentTerm, u8 votedForSize, u8 *votedFor)
{
MDB_val key = {0,NULL}, data = {0,NULL};
int rc;
#if ATOMIC
db_thread *thr = g_tsd_thread;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
#endif
mdbinf* mdb;
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
if (!mdb)
return SQLITE_ERROR;
key.mv_size = sizeof(u64);
key.mv_data = &pWal->index;
if (votedFor == NULL)
{
rc = mdb_cursor_get(mdb->cursorInfo,&key,&data,MDB_SET_KEY);
if (rc == MDB_SUCCESS && data.mv_size >= (1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1))
{
memcpy(¤tTerm, (u8*)data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2, sizeof(u64));
votedForSize = (u8)((u8*)data.mv_data)[1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)];
//votedFor = data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1;
votedFor = alloca(votedForSize);
memcpy(votedFor,(u8*)data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1, votedForSize);
// DBG("Voted for %.*s",(int)votedForSize,(char*)votedFor));
}
}
key.mv_size = sizeof(u64);
key.mv_data = &pWal->index;
data.mv_data = NULL;
data.mv_size = 1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1+votedForSize;
rc = mdb_cursor_put(mdb->cursorInfo,&key,&data,MDB_RESERVE);
if (rc == MDB_SUCCESS)
{
u8 *infoBuf = data.mv_data;
infoBuf[0] = 1;
memcpy(infoBuf+1, &pWal->firstCompleteTerm,sizeof(u64));
memcpy(infoBuf+1+sizeof(u64), &pWal->firstCompleteEvnum,sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*2, &pWal->lastCompleteTerm, sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*3, &pWal->lastCompleteEvnum,sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*4, &pWal->inProgressTerm, sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*5, &pWal->inProgressEvnum, sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*6, &pWal->mxPage, sizeof(u32));
memcpy(infoBuf+1+sizeof(u64)*6+sizeof(u32), &pWal->allPages,sizeof(u32));
memcpy(infoBuf+1+sizeof(u64)*6+sizeof(u32)*2, ¤tTerm, sizeof(u64));
infoBuf[1+sizeof(u64)*7+sizeof(u32)*2] = votedForSize;
memcpy(infoBuf+2+sizeof(u64)*7+sizeof(u32)*2, votedFor, votedForSize);
thr->forceCommit = 1;
return SQLITE_OK;
}
return SQLITE_ERROR;
}
/**
* Insert key=file_binary_hash, value=source_id. Return bool, source_id.
* True if new.
*/
bool insert(const std::string& file_binary_hash,
hashdb::lmdb_changes_t& changes, uint64_t& source_id) {
// require valid file_binary_hash
if (file_binary_hash.size() == 0) {
std::cerr << "Usage error: the file_binary_hash value provided to insert is empty.\n";
return false;
}
MUTEX_LOCK(&M);
// maybe grow the DB
lmdb_helper::maybe_grow(env);
// get context
hashdb::lmdb_context_t context(env, true, false); // writable, no duplicates
context.open();
// set key
context.key.mv_size = file_binary_hash.size();
context.key.mv_data =
static_cast<void*>(const_cast<char*>(file_binary_hash.c_str()));
// see if source_id exists yet
int rc = mdb_cursor_get(context.cursor, &context.key, &context.data,
MDB_SET_KEY);
if (rc == 0) {
#ifdef DEBUG_LMDB_SOURCE_ID_MANAGER_HPP
print_mdb_val("source_id_manager insert has key", context.key);
print_mdb_val("source_id_manager insert has data", context.data);
#endif
// source ID found
const uint8_t* p = static_cast<uint8_t*>(context.data.mv_data);
p = lmdb_helper::decode_uint64_t(p, source_id);
// read must align to data record
if (p != static_cast<uint8_t*>(context.data.mv_data) +
context.data.mv_size) {
std::cerr << "data decode error in LMDB source ID store\n";
assert(0);
}
++changes.source_id_already_present;
context.close();
MUTEX_UNLOCK(&M);
return false;
} else if (rc == MDB_NOTFOUND) {
// generate new source ID as DB size + 1
source_id = size() + 1;
uint8_t data[10];
uint8_t* p = data;
p = lmdb_helper::encode_uint64_t(source_id, p);
context.data.mv_size = p - data;
context.data.mv_data = data;
#ifdef DEBUG_LMDB_SOURCE_ID_MANAGER_HPP
print_mdb_val("source_id_manager insert new key", context.key);
print_mdb_val("source_id_manager insert new data", context.data);
#endif
rc = mdb_put(context.txn, context.dbi,
&context.key, &context.data, MDB_NODUPDATA);
// write must work
if (rc != 0) {
std::cerr << "LMDB error: " << mdb_strerror(rc) << "\n";
assert(0);
}
// source ID created
++changes.source_id_inserted;
context.close();
MUTEX_UNLOCK(&M);
return true;
} else {
// invalid rc
std::cerr << "LMDB error: " << mdb_strerror(rc) << "\n";
assert(0);
source_id = 0;
return false; // for mingw
}
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
/* LY: especially here we always needs MDB_NOSYNC
* for testing mdbx_env_close_ex() and "redo-to-steady" on open. */
env_oflags |= MDB_NOSYNC;
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
printf("check-preset-a\n");
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_a = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
++present_a;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_a == count - j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_env_sync(env, 1);
int deleted = 0;
key.mv_data = sval;
for (i = count - 1; i > -1; i -= (rand()%5)) {
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
deleted++;
}
}
free(values);
printf("Deleted %d values\n", deleted);
printf("check-preset-b.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_b = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_b;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == present_a - deleted, "mismatch");
printf("check-preset-b.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
/********************* LY: kept DB dirty ****************/
mdbx_env_close_ex(env, 1);
E(mdb_env_create(&env));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
printf("check-preset-c.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_dbi_open(txn, "id1", 0, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_c = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_c;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Rolled back %d deletion(s)\n", present_c - (present_a - deleted));
CHECK(present_c > present_a - deleted, "mismatch");
printf("check-preset-d.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_c == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdbx_env_close_ex(env, 0);
return 0;
}
DWORD
VmDirMDBIndexIterate(
PVDIR_BACKEND_INDEX_ITERATOR pIterator,
PSTR* ppszVal,
ENTRYID* pEId
)
{
DWORD dwError = 0;
PVDIR_MDB_INDEX_ITERATOR pMdbIterator = NULL;
PVDIR_DB_DBC pCursor = NULL;
VDIR_DB_DBT key = {0};
VDIR_DB_DBT value = {0};
PSTR pszVal = NULL;
ENTRYID eId = 0;
if (!pIterator || !ppszVal || !pEId)
{
dwError = ERROR_INVALID_PARAMETER;
BAIL_ON_VMDIR_ERROR(dwError);
}
pMdbIterator = (PVDIR_MDB_INDEX_ITERATOR)pIterator->pIterator;
pCursor = pMdbIterator->pCursor;
if (pIterator->bHasNext)
{
*ppszVal = pMdbIterator->pszVal;
*pEId = pMdbIterator->eId;
dwError = mdb_cursor_get(pCursor, &key, &value, MDB_NEXT);
if (dwError == 0 && *(char *)(key.mv_data) == BE_INDEX_KEY_TYPE_FWD)
{
dwError = VmDirAllocateMemory(key.mv_size, (PVOID*)&pszVal);
BAIL_ON_VMDIR_ERROR(dwError);
dwError = VmDirCopyMemory(
pszVal, key.mv_size, (char*)key.mv_data + 1, key.mv_size - 1);
BAIL_ON_VMDIR_ERROR(dwError);
MDBDBTToEntryId(&value, &eId);
pIterator->bHasNext = TRUE;
}
else
{
pIterator->bHasNext = FALSE;
dwError = dwError == MDB_NOTFOUND ? 0 : dwError;
pMdbIterator->bAbort = dwError ? TRUE : FALSE;
}
BAIL_ON_VMDIR_ERROR(dwError);
pMdbIterator->pszVal = pszVal;
pMdbIterator->eId = eId;
}
cleanup:
return dwError;
error:
VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL,
"%s failed, error (%d)", __FUNCTION__, dwError );
VMDIR_SAFE_FREE_MEMORY(pszVal);
goto cleanup;
}
DWORD
VmDirMDBIndexIteratorInit(
PVDIR_INDEX_CFG pIndexCfg,
PSTR pszInitVal,
PVDIR_BACKEND_INDEX_ITERATOR* ppIterator
)
{
DWORD dwError = 0;
VDIR_DB mdbDBi = 0;
PSTR pszInitKey = NULL;
VDIR_DB_DBT key = {0};
VDIR_DB_DBT value = {0};
PSTR pszVal = NULL;
ENTRYID eId = 0;
PVDIR_DB_TXN pTxn = NULL;
PVDIR_DB_DBC pCursor = NULL;
PVDIR_BACKEND_INDEX_ITERATOR pIterator = NULL;
PVDIR_MDB_INDEX_ITERATOR pMdbIterator = NULL;
if (!pIndexCfg || !ppIterator)
{
dwError = ERROR_INVALID_PARAMETER;
BAIL_ON_VMDIR_ERROR(dwError);
}
dwError = VmDirAllocateMemory(
sizeof(VDIR_BACKEND_INDEX_ITERATOR),
(PVOID*)&pIterator);
BAIL_ON_VMDIR_ERROR(dwError);
dwError = VmDirAllocateMemory(
sizeof(VDIR_MDB_INDEX_ITERATOR),
(PVOID*)&pMdbIterator);
BAIL_ON_VMDIR_ERROR(dwError);
pIterator->pIterator = (PVOID)pMdbIterator;
dwError = VmDirMDBIndexGetDBi(pIndexCfg, &mdbDBi);
BAIL_ON_VMDIR_ERROR(dwError);
dwError = mdb_txn_begin(gVdirMdbGlobals.mdbEnv, NULL, MDB_RDONLY, &pTxn);
BAIL_ON_VMDIR_ERROR(dwError);
pMdbIterator->pTxn = pTxn;
dwError = mdb_cursor_open(pTxn, mdbDBi, &pCursor);
BAIL_ON_VMDIR_ERROR(dwError);
pMdbIterator->pCursor = pCursor;
if (IsNullOrEmptyString(pszInitVal))
{
dwError = mdb_cursor_get(pCursor, &key, &value, MDB_FIRST);
}
else
{
dwError = VmDirAllocateStringPrintf(
&pszInitKey, "%c%s", BE_INDEX_KEY_TYPE_FWD, pszInitVal);
BAIL_ON_VMDIR_ERROR(dwError);
key.mv_data = pszInitKey;
key.mv_size = VmDirStringLenA(pszInitKey);
dwError = mdb_cursor_get(pCursor, &key, &value, MDB_SET_RANGE);
}
if (dwError == 0 && *(char *)(key.mv_data) == BE_INDEX_KEY_TYPE_FWD)
{
dwError = VmDirAllocateMemory(key.mv_size, (PVOID*)&pszVal);
BAIL_ON_VMDIR_ERROR(dwError);
dwError = VmDirCopyMemory(
pszVal, key.mv_size, (char*)key.mv_data + 1, key.mv_size - 1);
BAIL_ON_VMDIR_ERROR(dwError);
MDBDBTToEntryId(&value, &eId);
pIterator->bHasNext = TRUE;
}
else
{
pIterator->bHasNext = FALSE;
dwError = dwError == MDB_NOTFOUND ? 0 : dwError;
pMdbIterator->bAbort = dwError ? TRUE : FALSE;
}
BAIL_ON_VMDIR_ERROR(dwError);
pMdbIterator->pszVal = pszVal;
pMdbIterator->eId = eId;
*ppIterator = pIterator;
cleanup:
VMDIR_SAFE_FREE_MEMORY(pszInitKey);
return dwError;
error:
VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL,
"%s failed, error (%d)", __FUNCTION__, dwError );
VmDirMDBIndexIteratorFree(pIterator);
VMDIR_SAFE_FREE_MEMORY(pszVal);
goto cleanup;
}
int
mdb_filter_candidates(
Operation *op,
MDB_txn *rtxn,
Filter *f,
ID *ids,
ID *tmp,
ID *stack )
{
int rc = 0;
#ifdef LDAP_COMP_MATCH
AttributeAliasing *aa;
#endif
Debug( LDAP_DEBUG_FILTER, "=> mdb_filter_candidates\n" );
if ( f->f_choice & SLAPD_FILTER_UNDEFINED ) {
MDB_IDL_ZERO( ids );
goto out;
}
switch ( f->f_choice ) {
case SLAPD_FILTER_COMPUTED:
switch( f->f_result ) {
case SLAPD_COMPARE_UNDEFINED:
/* This technically is not the same as FALSE, but it
* certainly will produce no matches.
*/
/* FALL THRU */
case LDAP_COMPARE_FALSE:
MDB_IDL_ZERO( ids );
break;
case LDAP_COMPARE_TRUE:
MDB_IDL_ALL( ids );
break;
case LDAP_SUCCESS:
/* this is a pre-computed scope, leave it alone */
break;
}
break;
case LDAP_FILTER_PRESENT:
Debug( LDAP_DEBUG_FILTER, "\tPRESENT\n" );
rc = presence_candidates( op, rtxn, f->f_desc, ids );
break;
case LDAP_FILTER_EQUALITY:
Debug( LDAP_DEBUG_FILTER, "\tEQUALITY\n" );
#ifdef LDAP_COMP_MATCH
if ( is_aliased_attribute && ( aa = is_aliased_attribute ( f->f_ava->aa_desc ) ) ) {
rc = ava_comp_candidates ( op, rtxn, f->f_ava, aa, ids, tmp, stack );
}
else
#endif
{
rc = equality_candidates( op, rtxn, f->f_ava, ids, tmp );
}
break;
case LDAP_FILTER_APPROX:
Debug( LDAP_DEBUG_FILTER, "\tAPPROX\n" );
rc = approx_candidates( op, rtxn, f->f_ava, ids, tmp );
break;
case LDAP_FILTER_SUBSTRINGS:
Debug( LDAP_DEBUG_FILTER, "\tSUBSTRINGS\n" );
rc = substring_candidates( op, rtxn, f->f_sub, ids, tmp );
break;
case LDAP_FILTER_GE:
/* if no GE index, use pres */
Debug( LDAP_DEBUG_FILTER, "\tGE\n" );
if( f->f_ava->aa_desc->ad_type->sat_ordering &&
( f->f_ava->aa_desc->ad_type->sat_ordering->smr_usage & SLAP_MR_ORDERED_INDEX ) )
rc = inequality_candidates( op, rtxn, f->f_ava, ids, tmp, LDAP_FILTER_GE );
else
rc = presence_candidates( op, rtxn, f->f_ava->aa_desc, ids );
break;
case LDAP_FILTER_LE:
/* if no LE index, use pres */
Debug( LDAP_DEBUG_FILTER, "\tLE\n" );
if( f->f_ava->aa_desc->ad_type->sat_ordering &&
( f->f_ava->aa_desc->ad_type->sat_ordering->smr_usage & SLAP_MR_ORDERED_INDEX ) )
rc = inequality_candidates( op, rtxn, f->f_ava, ids, tmp, LDAP_FILTER_LE );
else
rc = presence_candidates( op, rtxn, f->f_ava->aa_desc, ids );
break;
case LDAP_FILTER_NOT:
/* no indexing to support NOT filters */
Debug( LDAP_DEBUG_FILTER, "\tNOT\n" );
MDB_IDL_ALL( ids );
break;
case LDAP_FILTER_AND:
Debug( LDAP_DEBUG_FILTER, "\tAND\n" );
rc = list_candidates( op, rtxn,
f->f_and, LDAP_FILTER_AND, ids, tmp, stack );
break;
case LDAP_FILTER_OR:
Debug( LDAP_DEBUG_FILTER, "\tOR\n" );
rc = list_candidates( op, rtxn,
f->f_or, LDAP_FILTER_OR, ids, tmp, stack );
break;
case LDAP_FILTER_EXT:
Debug( LDAP_DEBUG_FILTER, "\tEXT\n" );
rc = ext_candidates( op, rtxn, f->f_mra, ids, tmp, stack );
break;
default:
Debug( LDAP_DEBUG_FILTER, "\tUNKNOWN %lu\n",
(unsigned long) f->f_choice );
/* Must not return NULL, otherwise extended filters break */
MDB_IDL_ALL( ids );
}
if ( ids[2] == NOID && MDB_IDL_IS_RANGE( ids )) {
struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
ID last;
if ( mdb->mi_nextid ) {
last = mdb->mi_nextid;
} else {
MDB_cursor *mc;
MDB_val key;
last = 0;
rc = mdb_cursor_open( rtxn, mdb->mi_id2entry, &mc );
if ( !rc ) {
rc = mdb_cursor_get( mc, &key, NULL, MDB_LAST );
if ( !rc )
memcpy( &last, key.mv_data, sizeof( last ));
mdb_cursor_close( mc );
}
}
if ( last ) {
ids[2] = last;
} else {
MDB_IDL_ZERO( ids );
}
}
out:
Debug( LDAP_DEBUG_FILTER,
"<= mdb_filter_candidates: id=%ld first=%ld last=%ld\n",
(long) ids[0],
(long) MDB_IDL_FIRST( ids ),
(long) MDB_IDL_LAST( ids ) );
return rc;
}
FreqSchedulerError
freq_scheduler_cursor_write(FreqScheduler *sch,
MDB_cursor *cursor,
uint64_t hash,
float freq) {
if (freq <= 0)
return 0;
ScheduleKey sk = {
.score = 0,
.hash = hash
};
MDB_val key = {
.mv_size = sizeof(sk),
.mv_data = &sk,
};
MDB_val val = {
.mv_size = sizeof(float),
.mv_data = &freq,
};
int mdb_rc;
if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) {
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, "adding page to schedule");
freq_scheduler_add_error(sch, mdb_strerror(mdb_rc));
}
return sch->error->code;
}
FreqSchedulerError
freq_scheduler_load_simple(FreqScheduler *sch,
float freq_default,
float freq_scale) {
char *error1 = 0;
char *error2 = 0;
MDB_cursor *cursor = 0;
HashInfoStream *st;
if (hashinfo_stream_new(&st, sch->page_db) != 0) {
error1 = "creating stream";
error2 = st? sch->page_db->error->message: "NULL";
goto on_error;
}
if (freq_scheduler_cursor_open(sch, &cursor) != 0)
goto on_error;
StreamState ss;
uint64_t hash;
PageInfo *pi;
while ((ss = hashinfo_stream_next(st, &hash, &pi)) == stream_state_next) {
if ((pi->n_crawls > 0) &&
((sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls)) &&
!page_info_is_seed(pi)){
float freq = freq_default;
if (freq_scale > 0) {
float rate = page_info_rate(pi);
if (rate > 0) {
freq = freq_scale * rate;
}
}
if (freq_scheduler_cursor_write(sch, cursor, hash, freq) != 0)
goto on_error;
}
page_info_delete(pi);
}
if (ss != stream_state_end) {
error1 = "incorrect stream state";
error2 = 0;
hashinfo_stream_delete(st);
goto on_error;
}
hashinfo_stream_delete(st);
if (freq_scheduler_cursor_commit(sch, cursor) != 0)
goto on_error;
return sch->error->code;
on_error:
freq_scheduler_cursor_abort(sch, cursor);
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, error1);
freq_scheduler_add_error(sch, error2);
return sch->error->code;
}
FreqSchedulerError
freq_scheduler_load_mmap(FreqScheduler *sch, MMapArray *freqs) {
char *error1 = 0;
char *error2 = 0;
MDB_cursor *cursor = 0;
if (txn_manager_expand(
sch->txn_manager,
2*freqs->n_elements*freqs->element_size) != 0) {
error1 = "resizing database";
error2 = sch->txn_manager->error->message;
goto on_error;
}
if (freq_scheduler_cursor_open(sch, &cursor) != 0)
goto on_error;
for (size_t i=0; i<freqs->n_elements; ++i) {
PageFreq *f = mmap_array_idx(freqs, i);
ScheduleKey sk = {
.score = 1.0/f->freq,
.hash = f->hash
};
MDB_val key = {
.mv_size = sizeof(sk),
.mv_data = &sk,
};
MDB_val val = {
.mv_size = sizeof(float),
.mv_data = &f->freq,
};
int mdb_rc;
if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) {
error1 = "adding page to schedule";
error2 = mdb_strerror(mdb_rc);
goto on_error;
}
}
if (freq_scheduler_cursor_commit(sch, cursor) != 0)
goto on_error;
return sch->error->code;
on_error:
freq_scheduler_cursor_abort(sch, cursor);
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, error1);
freq_scheduler_add_error(sch, error2);
return sch->error->code;
}
FreqSchedulerError
freq_scheduler_request(FreqScheduler *sch,
size_t max_requests,
PageRequest **request) {
char *error1 = 0;
char *error2 = 0;
MDB_cursor *cursor = 0;
if (freq_scheduler_cursor_open(sch, &cursor) != 0)
goto on_error;
PageRequest *req = *request = page_request_new(max_requests);
if (!req) {
error1 = "allocating memory";
goto on_error;
}
int interrupt_requests = 0;
while ((req->n_urls < max_requests) && !interrupt_requests) {
MDB_val key;
MDB_val val;
ScheduleKey sk;
float freq;
int mdb_rc;
int crawl = 0;
switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, MDB_FIRST)) {
case 0:
// copy data before deleting cursor
sk = *(ScheduleKey*)key.mv_data;
freq = *(float*)val.mv_data;
PageInfo *pi = 0;
if (page_db_get_info(sch->page_db, sk.hash, &pi) != 0) {
error1 = "retrieving PageInfo from PageDB";
error2 = sch->page_db->error->message;
goto on_error;
}
if (pi) {
if (sch->margin >= 0) {
double elapsed = difftime(time(0), 0) - pi->last_crawl;
if (elapsed < 1.0/(freq*(1.0 + sch->margin)))
interrupt_requests = 1;
}
crawl = (sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls);
}
if (!interrupt_requests) {
if ((mdb_rc = mdb_cursor_del(cursor, 0)) != 0) {
error1 = "deleting head of schedule";
error2 = mdb_strerror(mdb_rc);
goto on_error;
}
if (crawl) {
if (page_request_add_url(req, pi->url) != 0) {
error1 = "adding url to request";
goto on_error;
}
sk.score += 1.0/freq;
val.mv_data = &freq;
key.mv_data = &sk;
if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) {
error1 = "moving element inside schedule";
error2 = mdb_strerror(mdb_rc);
goto on_error;
}
}
}
page_info_delete(pi);
break;
case MDB_NOTFOUND: // no more pages left
interrupt_requests = 1;
break;
default:
error1 = "getting head of schedule";
error2 = mdb_strerror(mdb_rc);
goto on_error;
}
}
if (freq_scheduler_cursor_commit(sch, cursor) != 0)
goto on_error;
return sch->error->code;
on_error:
freq_scheduler_cursor_abort(sch, cursor);
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, error1);
freq_scheduler_add_error(sch, error2);
return sch->error->code;
}
FreqSchedulerError
freq_scheduler_add(FreqScheduler *sch, const CrawledPage *page) {
if (page_db_add(sch->page_db, page, 0) != 0) {
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, "adding crawled page");
freq_scheduler_add_error(sch, sch->page_db->error->message);
}
return sch->error->code;
}
void
freq_scheduler_delete(FreqScheduler *sch) {
mdb_env_close(sch->txn_manager->env);
(void)txn_manager_delete(sch->txn_manager);
if (!sch->persist) {
char *data = build_path(sch->path, "data.mdb");
char *lock = build_path(sch->path, "lock.mdb");
remove(data);
remove(lock);
free(data);
free(lock);
remove(sch->path);
}
free(sch->path);
error_delete(sch->error);
free(sch);
}
FreqSchedulerError
freq_scheduler_dump(FreqScheduler *sch, FILE *output) {
MDB_cursor *cursor;
if (freq_scheduler_cursor_open(sch, &cursor) != 0)
return sch->error->code;
int end = 0;
MDB_cursor_op cursor_op = MDB_FIRST;
do {
int mdb_rc;
MDB_val key;
MDB_val val;
ScheduleKey *key_data;
float *val_data;
switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, cursor_op)) {
case 0:
key_data = (ScheduleKey*)key.mv_data;
val_data = (float*)val.mv_data;
fprintf(output, "%.2e %016"PRIx64" %.2e\n",
key_data->score, key_data->hash, *val_data);
break;
case MDB_NOTFOUND:
end = 1;
break;
default:
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, "iterating over database");
freq_scheduler_add_error(sch, mdb_strerror(mdb_rc));
end = 1;
break;
}
cursor_op = MDB_NEXT;
} while (!end);
freq_scheduler_cursor_abort(sch, cursor);
return sch->error->code;
}
void LMDBIterator::SeekToFirst()
{
int rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_FIRST);
CHECK_OP(rc);
m_valid = rc == 0;
}
h2o_start_response(req, &gen->super);
int e;
e = mdb_txn_begin(sv->db_env, NULL, MDB_RDONLY, &gen->txn);
if (0 != e)
mdb_fatal(e);
e = mdb_cursor_open(gen->txn, sv->docs, &gen->curs);
if (0 != e)
mdb_fatal(e);
MDB_val k = { .mv_size = key->len, .mv_data = key->s }, v;
/* Get documents where the key has a prefix which matches */
e = mdb_cursor_get(gen->curs, &k, &v, MDB_SET_RANGE);
switch (e)
{
case 0:
__get_keys_send(gen, e, &k, req);
break;
case MDB_BAD_VALSIZE:
e = mdb_cursor_get(gen->curs, &k, &v, MDB_FIRST);
__get_keys_send(gen, e, &k, req);
break;
default:
mdb_fatal(e);
}
return 0;
fail:
/* reindex entries on the fly */
static void *
mdb_online_index( void *ctx, void *arg )
{
struct re_s *rtask = arg;
BackendDB *be = rtask->arg;
struct mdb_info *mdb = be->be_private;
Connection conn = {0};
OperationBuffer opbuf;
Operation *op;
MDB_cursor *curs;
MDB_val key, data;
MDB_txn *txn;
ID id;
Entry *e;
int rc, getnext = 1;
int i;
connection_fake_init( &conn, &opbuf, ctx );
op = &opbuf.ob_op;
op->o_bd = be;
id = 1;
key.mv_size = sizeof(ID);
while ( 1 ) {
if ( slapd_shutdown )
break;
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
if ( rc )
break;
rc = mdb_cursor_open( txn, mdb->mi_id2entry, &curs );
if ( rc ) {
mdb_txn_abort( txn );
break;
}
if ( getnext ) {
getnext = 0;
key.mv_data = &id;
rc = mdb_cursor_get( curs, &key, &data, MDB_SET_RANGE );
if ( rc ) {
mdb_txn_abort( txn );
if ( rc == MDB_NOTFOUND )
rc = 0;
break;
}
memcpy( &id, key.mv_data, sizeof( id ));
}
rc = mdb_id2entry( op, curs, id, &e );
mdb_cursor_close( curs );
if ( rc ) {
mdb_txn_abort( txn );
if ( rc == MDB_NOTFOUND ) {
id++;
getnext = 1;
continue;
}
break;
}
rc = mdb_index_entry( op, txn, MDB_INDEX_UPDATE_OP, e );
mdb_entry_return( op, e );
if ( rc == 0 ) {
rc = mdb_txn_commit( txn );
txn = NULL;
} else {
mdb_txn_abort( txn );
txn = NULL;
}
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_online_index) ": database %s: "
"txn_commit failed: %s (%d)\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
break;
}
id++;
getnext = 1;
}
for ( i = 0; i < mdb->mi_nattrs; i++ ) {
if ( mdb->mi_attrs[ i ]->ai_indexmask & MDB_INDEX_DELETING
|| mdb->mi_attrs[ i ]->ai_newmask == 0 )
{
continue;
}
mdb->mi_attrs[ i ]->ai_indexmask = mdb->mi_attrs[ i ]->ai_newmask;
mdb->mi_attrs[ i ]->ai_newmask = 0;
}
ldap_pvt_thread_mutex_lock( &slapd_rq.rq_mutex );
ldap_pvt_runqueue_stoptask( &slapd_rq, rtask );
mdb->mi_index_task = NULL;
ldap_pvt_runqueue_remove( &slapd_rq, rtask );
ldap_pvt_thread_mutex_unlock( &slapd_rq.rq_mutex );
return NULL;
}
void DataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
vector<Blob<Dtype>*>* top) {
// Initialize DB
switch (this->layer_param_.data_param().backend()) {
case DataParameter_DB_LEVELDB:
{
leveldb::DB* db_temp;
leveldb::Options options = GetLevelDBOptions();
options.create_if_missing = false;
LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source();
leveldb::Status status = leveldb::DB::Open(
options, this->layer_param_.data_param().source(), &db_temp);
CHECK(status.ok()) << "Failed to open leveldb "
<< this->layer_param_.data_param().source() << std::endl
<< status.ToString();
db_.reset(db_temp);
iter_.reset(db_->NewIterator(leveldb::ReadOptions()));
iter_->SeekToFirst();
}
break;
case DataParameter_DB_LMDB:
CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed";
CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB
CHECK_EQ(mdb_env_open(mdb_env_,
this->layer_param_.data_param().source().c_str(),
MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed";
CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS)
<< "mdb_txn_begin failed";
CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS)
<< "mdb_open failed";
CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS)
<< "mdb_cursor_open failed";
LOG(INFO) << "Opening lmdb " << this->layer_param_.data_param().source();
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST),
MDB_SUCCESS) << "mdb_cursor_get failed";
break;
default:
LOG(FATAL) << "Unknown database backend";
}
// Check if we would need to randomly skip a few data points
if (this->layer_param_.data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
while (skip-- > 0) {
switch (this->layer_param_.data_param().backend()) {
case DataParameter_DB_LEVELDB:
iter_->Next();
if (!iter_->Valid()) {
iter_->SeekToFirst();
}
break;
case DataParameter_DB_LMDB:
if (mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT)
!= MDB_SUCCESS) {
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_,
MDB_FIRST), MDB_SUCCESS);
}
break;
default:
LOG(FATAL) << "Unknown database backend";
}
}
}
// Read a data point, and use it to initialize the top blob.
Datum datum;
switch (this->layer_param_.data_param().backend()) {
case DataParameter_DB_LEVELDB:
datum.ParseFromString(iter_->value().ToString());
break;
case DataParameter_DB_LMDB:
datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
break;
default:
LOG(FATAL) << "Unknown database backend";
}
// image
int crop_size = this->layer_param_.transform_param().crop_size();
if (crop_size > 0) {
(*top)[0]->Reshape(this->layer_param_.data_param().batch_size(),
datum.channels(), crop_size, crop_size);
this->prefetch_data_.Reshape(this->layer_param_.data_param().batch_size(),
datum.channels(), crop_size, crop_size);
} else {
(*top)[0]->Reshape(
this->layer_param_.data_param().batch_size(), datum.channels(),
datum.height(), datum.width());
this->prefetch_data_.Reshape(this->layer_param_.data_param().batch_size(),
datum.channels(), datum.height(), datum.width());
}
LOG(INFO) << "output data size: " << (*top)[0]->num() << ","
<< (*top)[0]->channels() << "," << (*top)[0]->height() << ","
<< (*top)[0]->width();
// label
if (this->output_labels_) {
// liu
(*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 4, 1, 1);
this->prefetch_label_.Reshape(this->layer_param_.data_param().batch_size(),
4, 1, 1);
/*
(*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1);
this->prefetch_label_.Reshape(this->layer_param_.data_param().batch_size(),
1, 1, 1);
*/
}
// datum size
this->datum_channels_ = datum.channels();
this->datum_height_ = datum.height();
this->datum_width_ = datum.width();
this->datum_size_ = datum.channels() * datum.height() * datum.width();
}
static int cache_iter_next(struct iter *it)
{
return mdb_cursor_get(it->cur, &it->key, &it->val, MDB_NEXT_DUP);
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n",
1, 0, 0 );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val, 0, 0 );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef HAVE_EBCDIC
strcpy( path, mdb->mi_dbenv_home );
__atoe( path );
dbhome = path;
#else
dbhome = mdb->mi_dbenv_home;
#endif
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0);
flags = mdb->mi_dbenv_flags;
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC|MDB_WRITEMAP;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. "
"Restore from backup!\n",
be->be_suffix[0].bv_val, rc, 0 );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. "
"Restore from backup!\n",
be->be_suffix[0].bv_val, rc, 0 );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_dbi_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_dbi_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID ) {
MDB_cursor *mc;
MDB_val key, data;
ID id;
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
/* check for old dn2id format */
rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
/* first record is always ID 0 */
rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
if ( rc == 0 ) {
rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
if ( rc == 0 ) {
int len;
unsigned char *ptr;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
snprintf( cr->msg, sizeof(cr->msg),
"database \"%s\": DN index needs upgrade, "
"run \"slapindex entryDN\".",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
if ( !(slapMode & SLAP_TOOL_READMAIN ))
rc = LDAP_OTHER;
mdb->mi_flags |= MDB_NEED_UPGRADE;
}
}
}
mdb_cursor_close( mc );
if ( rc == LDAP_OTHER )
goto fail;
}
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database %s: "
"txn_commit failed: %s (%d)\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
srandom(time(NULL));
count = (random()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = random()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (random()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
printf("check-preset-a\n");
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_a = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
++present_a;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_a == count - j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_env_sync(env, 1);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
printf("check-preset-b.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_b = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_b;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == present_a - j, "mismatch");
printf("check-preset-b.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
/********************* LY: kept DB dirty ****************/
mdb_env_close_ex(env, 1);
E(mdb_env_create(&env));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
printf("check-preset-c.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_dbi_open(txn, "id1", 0, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_c = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_c;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_c == present_a, "mismatch");
printf("check-preset-d.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_c == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close_ex(env, 0);
return 0;
}
void LMDBIterator::SeekToLast()
{
int rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_LAST);
m_valid = rc == 0;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[8];
char kval[sizeof(int)];
memset(sval, 0, sizeof(sval));
count = 510;
values = (int *)je_malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = i*5;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "/tmp/testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
strcpy(kval, "001");
for (i=0;i<count;i++) {
sprintf(sval, "%07x", values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
/* there should be one full page of dups now.
*/
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
/* test all 3 branches of split code:
* 1: new key in lower half
* 2: new key at split point
* 3: new key in upper half
*/
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%07x", values[3]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[255]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[500]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
E(mdb_txn_commit(txn));
/* Try MDB_NEXT_MULTIPLE */
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%3)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%07x", values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
je_free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
void LMDBIterator::Next()
{
int rc;
rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_NEXT);
m_valid = rc == 0;
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDB_stat mst;
MDB_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2) {
usage(prog);
}
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -v: use previous snapshot
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'v':
envflags |= MDB_PREVSNAPSHOT;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdb_env_set_maxdbs(env, 4);
}
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (envinfo) {
(void)mdb_env_stat(env, &mst);
(void)mdb_env_info(env, &mei);
printf("Environment Info\n");
printf(" Map address: %p\n", mei.me_mapaddr);
printf(" Map size: %"Y"u\n", mei.me_mapsize);
printf(" Page size: %u\n", mst.ms_psize);
printf(" Max pages: %"Y"u\n", mei.me_mapsize / mst.ms_psize);
printf(" Number of pages used: %"Y"u\n", mei.me_last_pgno+1);
printf(" Last transaction ID: %"Y"u\n", mei.me_last_txnid);
printf(" Max readers: %u\n", mei.me_maxreaders);
printf(" Number of readers used: %u\n", mei.me_numreaders);
}
if (rdrinfo) {
printf("Reader Table Status\n");
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
if (rdrinfo > 1) {
int dead;
mdb_reader_check(env, &dead);
printf(" %d stale readers cleared.\n", dead);
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (freinfo) {
MDB_cursor *cursor;
MDB_val key, data;
size_t pages = 0, *iptr;
printf("Freelist Status\n");
dbi = 0;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
iptr = data.mv_data;
pages += *iptr;
if (freinfo > 1) {
char *bad = "";
size_t pg, prev;
ssize_t i, j, span = 0;
j = *iptr++;
for (i = j, prev = 1; --i >= 0; ) {
pg = iptr[i];
if (pg <= prev)
bad = " [bad sequence]";
prev = pg;
pg += span;
for (; i >= span && iptr[i-span] == pg; span++, pg++) ;
}
printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n",
*(size_t *)key.mv_data, j, span, bad);
if (freinfo > 2) {
for (--j; j >= 0; ) {
pg = iptr[j];
for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ;
printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n",
pg, span);
}
}
}
}
mdb_cursor_close(cursor);
printf(" Free pages: %"Z"u\n", pages);
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDB_cursor *cursor;
MDB_val key;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
char *str;
MDB_dbi db2;
if (memchr(key.mv_data, '\0', key.mv_size))
continue;
str = malloc(key.mv_size+1);
memcpy(str, key.mv_data, key.mv_size);
str[key.mv_size] = '\0';
rc = mdb_open(txn, str, 0, &db2);
if (rc == MDB_SUCCESS)
printf("Status of %s\n", str);
free(str);
if (rc) continue;
rc = mdb_stat(txn, db2, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdb_close(env, db2);
}
mdb_cursor_close(cursor);
}
if (rc == MDB_NOTFOUND)
rc = MDB_SUCCESS;
mdb_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
void LMDBIterator::Prev()
{
int rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_PREV);
m_valid = rc == 0;
}
/* Write a frame or frames to the log. */
int sqlite3WalFrames(Wal *pWal, int szPage, PgHdr *pList, Pgno nTruncate, int isCommit, int sync_flags)
{
PgHdr *p;
MDB_val key, data;
int rc;
mdbinf* mdb;
MDB_txn* txn;
#if ATOMIC
db_thread *thr = g_tsd_thread;
db_connection* pCon = g_tsd_conn;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
db_connection* pCon = enif_tsd_get(g_tsd_conn);
#endif
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
txn = mdb->txn;
if (!mdb)
return SQLITE_ERROR;
key.mv_size = sizeof(u64);
key.mv_data = (void*)&pWal->index;
// Term/evnum must always be increasing
if ((pWal->inProgressTerm > 0 && pWal->inProgressTerm < pWal->lastCompleteTerm) ||
(pWal->inProgressEvnum > 0 && pWal->inProgressEvnum < pWal->lastCompleteEvnum))
return SQLITE_ERROR;
track_time(2,thr);
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Fragment,CompressedPage/binary>>}
for(p=pList; p; p=p->pDirty)
{
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
u8 pagesBuf[PAGE_BUFF_SIZE];
int full_size = 0;
int page_size = LZ4_compress_default((char*)p->pData,(char*)pagesBuf+sizeof(u64)*2+1,szPage,sizeof(pagesBuf));
char fragment_index = 0;
int skipped = 0;
track_time(3,thr);
DBG("Insert frame, actor=%lld, pgno=%u, "
"term=%lld, evnum=%lld, commit=%d, truncate=%d, compressedsize=%d",
pWal->index,p->pgno,pWal->inProgressTerm,pWal->inProgressEvnum,
isCommit,nTruncate,page_size);
if (pCon->doReplicate)
{
u8 hdr[sizeof(u64)*2+sizeof(u32)*2];
put8byte(hdr, pWal->inProgressTerm);
put8byte(hdr+sizeof(u64), pWal->inProgressEvnum);
put4byte(hdr+sizeof(u64)*2, p->pgno);
if (p->pDirty)
put4byte(hdr+sizeof(u64)*2+sizeof(u32), 0);
else
put4byte(hdr+sizeof(u64)*2+sizeof(u32), nTruncate);
#ifndef _TESTAPP_
wal_page_hook(thr,pagesBuf+sizeof(u64)*2+1, page_size, hdr, sizeof(hdr));
#endif
}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
// Check if there are pages with the same or higher evnum/evterm. If there are, delete them.
// This can happen if sqlite flushed some page to disk before commiting, because there were
// so many pages that they could not be held in memory. Or it could happen if pages need to be
// overwritten because there was a write that did not pass raft consensus.
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY);
if (rc == MDB_SUCCESS)
{
size_t ndupl;
mdb_cursor_count(mdb->cursorPages,&ndupl);
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP);
if (rc == MDB_SUCCESS)
{
MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL};
u64 evnum, evterm;
u8 frag = *((u8*)data.mv_data+sizeof(u64)*2);
memcpy(&evterm, data.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64));
while ((evterm > pWal->inProgressTerm || evnum >= pWal->inProgressEvnum))
//(pWal->inProgressTerm + pWal->inProgressEvnum) > 0)
{
DBG("Deleting pages higher or equal to current. "
"Evterm=%llu, evnum=%llu, curterm=%llu, curevn=%llu, dupl=%ld",
evterm,evnum,pWal->inProgressTerm,pWal->inProgressEvnum,ndupl);
if (pgDelKey.mv_data != NULL)
{
if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS)
{
DBG("Unable to cleanup page from pagedb %d",rc);
break;
}
pgDelKey.mv_data = NULL;
}
mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT);
// if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS)
// {
// DBG("Cant delete!");
// break;
// }
if (frag == 0)
pWal->allPages--;
ndupl--;
if (!ndupl)
break;
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP);
if (rc != MDB_SUCCESS)
break;
memcpy(&evterm, data.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64));
frag = *((u8*)data.mv_data+sizeof(u64)*2);
}
if (pgDelKey.mv_data != NULL)
{
if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS)
{
DBG("Unable to cleanup page from pagedb %d",rc);
break;
}
pgDelKey.mv_data = NULL;
}
}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
}
track_time(4,thr);
memcpy(pagesBuf, &pWal->inProgressTerm, sizeof(u64));
memcpy(pagesBuf + sizeof(u64), &pWal->inProgressEvnum, sizeof(u64));
full_size = page_size + sizeof(u64)*2 + 1;
if (full_size < thr->maxvalsize)
fragment_index = 0;
else
{
full_size = page_size;
skipped = thr->maxvalsize - sizeof(u64)*2 - 1;
full_size -= skipped;
while(full_size > 0)
{
full_size -= (thr->maxvalsize - sizeof(u64)*2 - 1);
fragment_index++;
}
full_size = page_size + sizeof(u64)*2 +1;
}
pagesBuf[sizeof(u64)*2] = fragment_index;
data.mv_size = fragment_index == 0 ? full_size : thr->maxvalsize;
data.mv_data = pagesBuf;
// fragment_index == 0 ? MDB_APPENDDUP : 0
if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS)
{
// printf("Cursor put failed to pages %d",rc);
DBG("ERROR: cursor put failed=%d, datasize=%d",rc,full_size);
return SQLITE_ERROR;
}
fragment_index--;
skipped = data.mv_size;
while (fragment_index >= 0)
{
DBG("Insert fragment %d",(int)fragment_index);
if (fragment_index == 0)
data.mv_size = full_size - skipped + sizeof(u64)*2 + 1;
else
data.mv_size = thr->maxvalsize;
data.mv_data = pagesBuf + skipped - (sizeof(u64)*2+1);
memcpy(pagesBuf + skipped - (sizeof(u64)*2+1), &pWal->inProgressTerm, sizeof(u64));
memcpy(pagesBuf + skipped - (sizeof(u64)+1), &pWal->inProgressEvnum, sizeof(u64));
pagesBuf[skipped-1] = fragment_index;
if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS)
{
DBG("ERROR: cursor secondary put failed: err=%d, datasize=%d, skipped=%d, frag=%d",
rc,full_size, skipped, (int)fragment_index);
return SQLITE_ERROR;
}
skipped += data.mv_size - sizeof(u64)*2 - 1;
fragment_index--;
}
thr->pagesChanged++;
}
// printf("");
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
if (pWal->inProgressTerm > 0)
{
for(p=pList; p; p=p->pDirty)
{
u8 logKeyBuf[sizeof(u64)*3];
DBG("Inserting to log");
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum, sizeof(u64));
key.mv_size = sizeof(logKeyBuf);
key.mv_data = logKeyBuf;
data.mv_size = sizeof(u32);
data.mv_data = &p->pgno;
if (mdb_cursor_put(mdb->cursorLog,&key,&data,0) != MDB_SUCCESS)
{
// printf("Cursor put failed to log");
DBG("ERROR: cursor put to log failed: %d",rc);
return SQLITE_ERROR;
}
pWal->allPages++;
}
}
else
{
DBG("Skipping log");
for(p=pList; p; p=p->pDirty)
pWal->allPages++;
}
/** - Info DB: {<<ActorIndex:64>>, <<V,FirstCompleteTerm:64,FirstCompleteEvnum:64,
LastCompleteTerm:64,LastCompleteEvnum:64,
InprogressTerm:64,InProgressEvnum:64>>} */
{
if (isCommit)
{
DBG("Commit actor=%llu fct=%llu, fcev=%llu, lct=%llu, lcev=%llu, int=%llu, inev=%llu",
pWal->index,
pWal->firstCompleteTerm, pWal->firstCompleteEvnum, pWal->lastCompleteTerm,
pWal->lastCompleteEvnum, pWal->inProgressTerm,pWal->inProgressEvnum);
#ifndef _TESTAPP_
enif_mutex_lock(pWal->mtx);
#endif
pWal->lastCompleteTerm = pWal->inProgressTerm > 0 ?
pWal->inProgressTerm : pWal->lastCompleteTerm;
pWal->lastCompleteEvnum = pWal->inProgressEvnum > 0 ?
pWal->inProgressEvnum : pWal->lastCompleteEvnum;
if (pWal->firstCompleteTerm == 0)
{
pWal->firstCompleteTerm = pWal->inProgressTerm;
pWal->firstCompleteEvnum = pWal->inProgressEvnum;
}
pWal->inProgressTerm = pWal->inProgressEvnum = 0;
pWal->mxPage = pWal->mxPage > nTruncate ? pWal->mxPage : nTruncate;
// pWal->changed = 0;
thr->forceCommit = 1;
pCon->dirty = 0;
#ifndef _TESTAPP_
enif_mutex_unlock(pWal->mtx);
#endif
DBG("cur mxpage=%u",pWal->mxPage);
}
else
{
// pWal->changed = 1;
pCon->dirty = 1;
}
thr->pagesChanged++;
rc = storeinfo(pWal,0,0,NULL);
if (rc != SQLITE_OK)
return rc;
track_time(5,thr);
}
return SQLITE_OK;
}
int lmdbgo_mdb_cursor_get2(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, MDB_val *key, MDB_val *val, MDB_cursor_op op) {
LMDBGO_SET_VAL(key, kn, kdata);
LMDBGO_SET_VAL(val, vn, vdata);
return mdb_cursor_get(cur, key, val, op);
}
// return number of bytes written
static int wal_iterate(Wal *pWal, iterate_resource *iter, u8 *buf, int bufsize, u8 *hdr, u32 *done)
{
// db_thread* const thr = enif_tsd_get(g_tsd_thread);
#if ATOMIC
db_thread *thr = g_tsd_thread;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
#endif
mdbinf* const mdb = &thr->mdb;
u32 mxPage;
u64 readSafeEvnum, readSafeTerm;
#ifndef _TESTAPP_
enif_mutex_lock(pWal->mtx);
#endif
readSafeEvnum = pWal->lastCompleteEvnum;
readSafeTerm = pWal->lastCompleteTerm;
mxPage = pWal->mxPage;
#ifndef _TESTAPP_
enif_mutex_unlock(pWal->mtx);
#endif
if (!iter->started)
{
if (iter->evnum + iter->evterm == 0)
{
// If any writes come after iterator started, we must ignore those pages.
iter->evnum = readSafeEvnum;
iter->evterm = readSafeTerm;
iter->pgnoPos = 1;
iter->entiredb = 1;
iter->mxPage = mxPage;
if (pWal->mxPage == 0)
{
DBG("ERROR: Iterate on empty DB %llu",pWal->lastCompleteEvnum);
*done = 1;
return 0;
}
}
else
{
// set mxPage to highest pgno we find.
iter->pgnoPos = iter->mxPage = 0;
DBG("Iterate rsterm=%llu rsevnum=%llu",readSafeTerm,readSafeEvnum);
}
iter->started = 1;
}
// send entire db (without history)
if (iter->entiredb)
{
u32 iRead = 0;
findframe(thr, pWal, iter->pgnoPos, &iRead, iter->evterm, iter->evnum, NULL, NULL);
if (!iRead)
{
DBG("Iterate did not find page");
*done = iter->mxPage;
return 0;
}
DBG("Iter pos=%u, mx=%u, safemx=%u",iter->pgnoPos, iter->mxPage, mxPage);
if (iter->pgnoPos == iter->mxPage)
*done = iter->mxPage;
put8byte(hdr, iter->evterm);
put8byte(hdr+sizeof(u64), iter->evnum);
put4byte(hdr+sizeof(u64)*2, iter->pgnoPos);
put4byte(hdr+sizeof(u64)*2+sizeof(u32), *done);
iter->pgnoPos++;
return fillbuff(thr, pWal, iter, buf, bufsize);
}
else
{
MDB_val logKey, logVal;
int logop;
u8 logKeyBuf[sizeof(u64)*3];
int rc;
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &iter->evterm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &iter->evnum, sizeof(u64));
logKey.mv_data = logKeyBuf;
logKey.mv_size = sizeof(logKeyBuf);
DBG("iterate looking for, matchterm=%llu matchevnum=%llu",iter->evterm,iter->evnum);
if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS)
{
// Evterm/evnum combination not found. Check if evnum is there.
// If so return evterm. It will mean a node is in conflict.
DBG("Key not found in log");
if (readSafeEvnum == iter->evnum)
{
iter->evterm = readSafeTerm;
iter->termMismatch = 1;
}
else
{
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &readSafeTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &readSafeEvnum,sizeof(u64));
if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS)
{
DBG("Key not found in log for undo");
*done = 1;
return 0;
}
while (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_PREV_NODUP) == MDB_SUCCESS)
{
u64 aindex, term, evnum;
mdb_cursor_get(mdb->cursorLog,&logKey, &logVal, MDB_GET_CURRENT);
memcpy(&aindex, logKey.mv_data, sizeof(u64));
memcpy(&term, (u8*)logKey.mv_data+sizeof(u64), sizeof(u64));
memcpy(&evnum, (u8*)logKey.mv_data+sizeof(u64)*2,sizeof(u64));
DBG("Iterate on term=%llu, evnum=%llu, looking for=%llu",term,evnum,iter->evnum);
if (aindex != pWal->index)
break;
if (iter->evnum == evnum)
{
iter->evterm = term;
iter->termMismatch = 1;
break;
}
}
}
*done = 1;
return 0;
}
// We start iterate from next evnum not current. Input evterm/evnum is match_index and match_term.
// It needs next.
if (iter->started == 1 &&
(rc = mdb_cursor_get(mdb->cursorLog,&logKey, &logVal, MDB_NEXT_NODUP)) != MDB_SUCCESS)
{
*done = 1;
return 0;
}
else
{
u64 aindex;
rc = mdb_cursor_get(mdb->cursorLog,&logKey, &logVal, MDB_GET_CURRENT);
if (rc != MDB_SUCCESS)
{
*done = 1;
return 0;
}
memcpy(&aindex, (u8*)logKey.mv_data, sizeof(u64));
memcpy(&iter->evterm, (u8*)logKey.mv_data+sizeof(u64), sizeof(u64));
memcpy(&iter->evnum, (u8*)logKey.mv_data+sizeof(u64)*2,sizeof(u64));
if (aindex != pWal->index)
{
*done = 1;
return 0;
}
// To keep from moving iter->evterm/iter->evnum forward more than once.
iter->started = 2;
}
logop = MDB_FIRST_DUP;
while ((rc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS)
{
u64 evnum,evterm;
u32 pgno;
u32 iRead;
logop = MDB_NEXT_DUP;
mdb_cursor_get(mdb->cursorLog,&logKey, &logVal, MDB_GET_CURRENT);
memcpy(&pgno,logVal.mv_data,sizeof(u32));
DBG("iterate at pgno=%u, pgnopos=%u",pgno,iter->pgnoPos);
if (pgno <= iter->pgnoPos)
continue;
findframe(thr, pWal, pgno, &iRead, iter->evterm, iter->evnum, &evterm, &evnum);
if (iRead == 0)
{
DBG("ERROR: Did not find frame for pgno=%u, evterm=%llu, evnum=%llu",
pgno, iter->evterm, iter->evnum);
*done = 1;
return 0;
}
if (evterm != iter->evterm || evnum != iter->evnum)
{
DBG("ERROR: Evterm/evnum does not match,looking for: evterm=%llu, evnum=%llu, "
"got: evterm=%llu, evnum=%llu", iter->evterm, iter->evnum, evterm, evnum);
*done = 1;
return 0;
}
iter->pgnoPos = pgno;
if ((rc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS)
*done = 0;
else
*done = iter->pgnoPos;
DBG( "logcursor get next %d, done=%u",rc,*done);
put8byte(hdr, iter->evterm);
put8byte(hdr+sizeof(u64), iter->evnum);
put4byte(hdr+sizeof(u64)*2, iter->pgnoPos);
put4byte(hdr+sizeof(u64)*2+sizeof(u32), *done);
return fillbuff(thr, pWal, iter, buf, bufsize);
}
*done = 1;
return 0;
}
}
void DataLayer<Dtype>::InternalThreadEntry() {
Datum datum;
CHECK(this->prefetch_data_.count());
Dtype* top_data = this->prefetch_data_.mutable_cpu_data();
Dtype* top_label = NULL; // suppress warnings about uninitialized variables
if (this->output_labels_) {
top_label = this->prefetch_label_.mutable_cpu_data();
}
const int batch_size = this->layer_param_.data_param().batch_size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
switch (this->layer_param_.data_param().backend()) {
case DataParameter_DB_LEVELDB:
CHECK(iter_);
CHECK(iter_->Valid());
datum.ParseFromString(iter_->value().ToString());
break;
case DataParameter_DB_LMDB:
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
&mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS);
datum.ParseFromArray(mdb_value_.mv_data,
mdb_value_.mv_size);
break;
default:
LOG(FATAL) << "Unknown database backend";
}
// Apply data transformations (mirror, scale, crop...)
this->data_transformer_.Transform(item_id, datum, this->mean_, top_data);
if (this->output_labels_) {
// liu
// top_label[item_id] = datum.label();
// LOG(ERROR) << "label size " << datum.label_size() << " " << datum.label(0) \
<< " " << datum.label(1) << " " << datum.label(2) << " " << datum.label(3);
for(int label_i=0; label_i < datum.label_size(); label_i++){
top_label[item_id * datum.label_size() + label_i] = datum.label(label_i);
}
}
// go to the next iter
switch (this->layer_param_.data_param().backend()) {
case DataParameter_DB_LEVELDB:
iter_->Next();
if (!iter_->Valid()) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
iter_->SeekToFirst();
}
break;
case DataParameter_DB_LMDB:
if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
&mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
&mdb_value_, MDB_FIRST), MDB_SUCCESS);
}
break;
default:
LOG(FATAL) << "Unknown database backend";
}
}
}
static int doundo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, u8 delPages)
{
MDB_val logKey, logVal;
MDB_val pgKey, pgVal;
u8 logKeyBuf[sizeof(u64)*3];
int logop, pgop, rc, mrc;
mdbinf *mdb;
// db_thread* const thr = enif_tsd_get(g_tsd_thread);
#if ATOMIC
db_thread *thr = g_tsd_thread;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
#endif
rc = SQLITE_OK;
if (pWal->inProgressTerm == 0)
return SQLITE_OK;
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
if (!mdb)
return SQLITE_ERROR;
logKey.mv_data = logKeyBuf;
logKey.mv_size = sizeof(logKeyBuf);
DBG("Undo");
// For every page here
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
// Delete from
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>}
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum,sizeof(u64));
if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS)
{
DBG("Key not found in log for undo, index=%llu, term=%llu, evnum=%llu",
pWal->index, pWal->inProgressTerm, pWal->inProgressEvnum);
return SQLITE_OK;
}
logop = MDB_FIRST_DUP;
while ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS)
{
u32 pgno;
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
u64 term,evnum;
memcpy(&pgno,logVal.mv_data,sizeof(u32));
if (delPages)
{
size_t ndupl;
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32));
pgKey.mv_data = pagesKeyBuf;
pgKey.mv_size = sizeof(pagesKeyBuf);
DBG("UNDO pgno=%d",pgno);
pgop = MDB_FIRST_DUP;
if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS)
{
DBG("Key not found in log for undo");
continue;
}
mdb_cursor_count(mdb->cursorPages,&ndupl);
while (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,pgop) == MDB_SUCCESS)
{
u8 frag = *((u8*)pgVal.mv_data+sizeof(u64)*2);
memcpy(&term, pgVal.mv_data, sizeof(u64));
memcpy(&evnum,(u8*)pgVal.mv_data+sizeof(u64),sizeof(u64));
DBG("progress term %lld, progress evnum %lld, curterm %lld, curnum %lld",
pWal->inProgressTerm, pWal->inProgressEvnum, term, evnum);
if (term >= pWal->inProgressTerm && evnum >= pWal->inProgressEvnum)
{
if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS)
{
DBG("Can not delete undo");
rc = SQLITE_ERROR;
break;
}
if (frag == 0)
pWal->allPages--;
ndupl--;
if (!ndupl)
break;
}
pgop = MDB_NEXT_DUP;
}
pWal->inProgressTerm = pWal->inProgressEvnum = 0;
storeinfo(pWal,0,0,NULL);
thr->pagesChanged++;
}
if (xUndo)
rc = xUndo(pUndoCtx, pgno);
logop = MDB_NEXT_DUP;
}
// if (mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA) != MDB_SUCCESS)
// {
// DBG("Unable to cleanup key from logdb"));
// }
DBG("Undo done!");
return rc;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
int env_oflags;
struct stat db_stat, exe_stat;
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}
