int cursor_get(cursor_t cursor, MDB_val *key, MDB_val *data, MDB_cursor_op op){ return cursor->prev ? mdb_cursor_get(cursor->cursor, key, data, op) : MDB_BAD_TXN; }
static int findframe(db_thread *thr, Wal *pWal, Pgno pgno, u32 *piRead, u64 limitTerm, u64 limitEvnum, u64 *outTerm, u64 *outEvnum) { MDB_val key, data; int rc; size_t ndupl = 0; u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; mdbinf *mdb; if (thr->pagesChanged) { #if ATOMIC mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); #endif } else mdb = &thr->mdb; track_time(7,thr); DBG("FIND FRAME pgno=%u, index=%llu, limitterm=%llu, limitevnum=%llu", pgno,pWal->index,limitTerm,limitEvnum); // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, // <<Evterm:64,Evnum:64,Counter,CompressedPage/binary>>} memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32)); key.mv_size = sizeof(pagesKeyBuf); key.mv_data = pagesKeyBuf; // u32 pgno2 = *(u32*)(key.mv_data+sizeof(u64)); // DBG("RUN %d",pgno2); rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY); if (rc == MDB_SUCCESS) { mdb_cursor_count(mdb->cursorPages,&ndupl); if (ndupl == 0) { *piRead = 0; return SQLITE_OK; } rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP); if (rc == MDB_SUCCESS) { while (1) { char frag1 = *(char*)(((char*)data.mv_data)+sizeof(u64)*2); int frag = frag1; u64 term, evnum; memcpy(&term, data.mv_data, sizeof(u64)); memcpy(&evnum, ((u8*)data.mv_data) + sizeof(u64), sizeof(u64)); if (term > limitTerm || evnum > limitEvnum) { rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP); if (rc == MDB_SUCCESS) continue; else { DBG("Cant move to prev dup, term=%llu, evnum=%llu," " limitterm=%llu, limitevnum=%llu",term,evnum,limitTerm,limitEvnum); *piRead = 0; break; } } if (outTerm != NULL) *outTerm = term; if (outEvnum != NULL) *outEvnum = evnum; DBG("Found page size=%ld, frags=%d",data.mv_size,(int)frag); thr->nResFrames = frag; thr->resFrames[frag--] = data; while (frag >= 0) { rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP); frag = *(((u8*)data.mv_data)+sizeof(u64)*2); // DBG("SUCCESS? %d frag=%d, size=%ld",pgno,frag,data.mv_size); thr->resFrames[frag--] = data; } *piRead = 1; break; } } else { DBG("Find page no last dup"); *piRead = 0; } } else if (rc == MDB_NOTFOUND) { DBG("Frame not found!"); *piRead = 0; } else { DBG("ERROR findframe: %d",rc); *piRead = 0; } return SQLITE_OK; }
// Delete all pages up to limitEvterm and limitEvnum static int checkpoint(Wal *pWal, u64 limitEvnum) { MDB_val logKey, logVal; u8 logKeyBuf[sizeof(u64)*3]; u64 evnum,evterm,aindex; mdbinf* mdb; // db_thread* const thr = enif_tsd_get(g_tsd_thread); #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif int logop, mrc = MDB_SUCCESS; // u8 somethingDeleted = 0; int allPagesDiff = 0; #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif if (!mdb) return SQLITE_ERROR; // if (pWal->inProgressTerm == 0) // return SQLITE_OK; DBG("checkpoint actor=%llu, fct=%llu, fcev=%llu, limitEvnum=%llu",pWal->index, pWal->firstCompleteTerm,pWal->firstCompleteEvnum,limitEvnum); while (pWal->firstCompleteEvnum < limitEvnum) { logKey.mv_data = logKeyBuf; logKey.mv_size = sizeof(logKeyBuf); memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->firstCompleteTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->firstCompleteEvnum,sizeof(u64)); if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS) { printf("Key not found in log for checkpoint %llu %llu\n", pWal->firstCompleteTerm, pWal->firstCompleteEvnum); return SQLITE_OK; } DBG("checkpoint evnum=%llu",pWal->firstCompleteEvnum); // For every page here // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} // Delete from // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>} logop = MDB_FIRST_DUP; while ((mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS) { u32 pgno; size_t ndupl; u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; MDB_val pgKey = {0,NULL}, pgVal = {0,NULL}; u64 pgnoLimitEvnum; logop = MDB_NEXT_DUP; memcpy(&pgno, logVal.mv_data,sizeof(u32)); DBG("checkpoint pgno=%u",pgno); memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32)); pgKey.mv_data = pagesKeyBuf; pgKey.mv_size = sizeof(pagesKeyBuf); if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS) { continue; } mdb_cursor_count(mdb->cursorPages,&ndupl); if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_LAST_DUP) == MDB_SUCCESS) memcpy(&pgnoLimitEvnum, (u8*)pgVal.mv_data+sizeof(u64),sizeof(u64)); else continue; pgnoLimitEvnum = pgnoLimitEvnum < limitEvnum ? pgnoLimitEvnum : limitEvnum; if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_FIRST_DUP) != MDB_SUCCESS) continue; do { u8 frag; MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL}; mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT); frag = *((u8*)pgDelVal.mv_data+sizeof(u64)*2); memcpy(&evterm, pgDelVal.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)pgDelVal.mv_data+sizeof(u64),sizeof(u64)); DBG("limit limitevnum %lld, curnum %lld, dupl %zu, frag=%d", limitEvnum, evnum, ndupl,(int)frag); if (evnum < pgnoLimitEvnum) { mrc = mdb_cursor_del(mdb->cursorPages,0); if (mrc != MDB_SUCCESS) { DBG("Unable to delete page on cursor!."); break; } // else // { // DBG("Deleted page!"); // // somethingDeleted = 1; // } if (frag == 0) allPagesDiff++; } ndupl--; if (!ndupl) break; mrc = mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_NEXT_DUP); } while (mrc == MDB_SUCCESS); } mrc = mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA); if (mrc != MDB_SUCCESS) { DBG("Can not delete log"); break; } // move forward if ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_NEXT_NODUP)) != MDB_SUCCESS) { DBG("Unable to move to next log %d",mrc); break; } // read next key data memcpy(&aindex, logKey.mv_data, sizeof(u64)); memcpy(&evterm, (u8*)logKey.mv_data + sizeof(u64), sizeof(u64)); memcpy(&evnum, (u8*)logKey.mv_data + sizeof(u64)*2, sizeof(u64)); if (aindex != pWal->index) { DBG("Reached another actor"); break; } pWal->firstCompleteTerm = evterm; pWal->firstCompleteEvnum = evnum; pWal->allPages -= allPagesDiff; allPagesDiff = 0; DBG("Checkpint fce now=%lld",(u64)evnum); } // no dirty pages, but will write info if (sqlite3WalFrames(pWal, SQLITE_DEFAULT_PAGE_SIZE, NULL, pWal->mxPage, 1, 0) == SQLITE_OK) return SQLITE_OK; else return SQLITE_ERROR; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; long kval; char *sval; srand(time(NULL)); E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi)); E(mdb_cursor_open(txn, dbi, &cursor)); E(mdb_stat(txn, dbi, &mst)); sval = calloc(1, mst.ms_psize / 4); key.mv_size = sizeof(long); key.mv_data = &kval; data.mv_size = mst.ms_psize / 4 - 30; data.mv_data = sval; printf("Adding 12 values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5; sprintf(sval, "%08x", kval); (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } printf("Adding 12 more values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5+4; sprintf(sval, "%08x", kval); (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } printf("Adding 12 more values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5+1; sprintf(sval, "%08x", kval); (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST)); do { printf("key: %p %s, data: %p %.*s\n", key.mv_data, mdb_dkey(&key, dkbuf), data.mv_data, (int) data.mv_size, (char *) data.mv_data); } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_commit(txn); #if 0 j=0; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(kval, "%03x", values[i & ~0x0f]); sprintf(sval, "%03x %d foo bar", values[i], values[i]); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_close(env, dbi); mdb_txn_abort(txn); #endif mdb_env_close(env); return 0; }
static int storeinfo(Wal *pWal, u64 currentTerm, u8 votedForSize, u8 *votedFor) { MDB_val key = {0,NULL}, data = {0,NULL}; int rc; #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif mdbinf* mdb; #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif if (!mdb) return SQLITE_ERROR; key.mv_size = sizeof(u64); key.mv_data = &pWal->index; if (votedFor == NULL) { rc = mdb_cursor_get(mdb->cursorInfo,&key,&data,MDB_SET_KEY); if (rc == MDB_SUCCESS && data.mv_size >= (1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1)) { memcpy(¤tTerm, (u8*)data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2, sizeof(u64)); votedForSize = (u8)((u8*)data.mv_data)[1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)]; //votedFor = data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1; votedFor = alloca(votedForSize); memcpy(votedFor,(u8*)data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1, votedForSize); // DBG("Voted for %.*s",(int)votedForSize,(char*)votedFor)); } } key.mv_size = sizeof(u64); key.mv_data = &pWal->index; data.mv_data = NULL; data.mv_size = 1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1+votedForSize; rc = mdb_cursor_put(mdb->cursorInfo,&key,&data,MDB_RESERVE); if (rc == MDB_SUCCESS) { u8 *infoBuf = data.mv_data; infoBuf[0] = 1; memcpy(infoBuf+1, &pWal->firstCompleteTerm,sizeof(u64)); memcpy(infoBuf+1+sizeof(u64), &pWal->firstCompleteEvnum,sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*2, &pWal->lastCompleteTerm, sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*3, &pWal->lastCompleteEvnum,sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*4, &pWal->inProgressTerm, sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*5, &pWal->inProgressEvnum, sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*6, &pWal->mxPage, sizeof(u32)); memcpy(infoBuf+1+sizeof(u64)*6+sizeof(u32), &pWal->allPages,sizeof(u32)); memcpy(infoBuf+1+sizeof(u64)*6+sizeof(u32)*2, ¤tTerm, sizeof(u64)); infoBuf[1+sizeof(u64)*7+sizeof(u32)*2] = votedForSize; memcpy(infoBuf+2+sizeof(u64)*7+sizeof(u32)*2, votedFor, votedForSize); thr->forceCommit = 1; return SQLITE_OK; } return SQLITE_ERROR; }
/** * Insert key=file_binary_hash, value=source_id. Return bool, source_id. * True if new. */ bool insert(const std::string& file_binary_hash, hashdb::lmdb_changes_t& changes, uint64_t& source_id) { // require valid file_binary_hash if (file_binary_hash.size() == 0) { std::cerr << "Usage error: the file_binary_hash value provided to insert is empty.\n"; return false; } MUTEX_LOCK(&M); // maybe grow the DB lmdb_helper::maybe_grow(env); // get context hashdb::lmdb_context_t context(env, true, false); // writable, no duplicates context.open(); // set key context.key.mv_size = file_binary_hash.size(); context.key.mv_data = static_cast<void*>(const_cast<char*>(file_binary_hash.c_str())); // see if source_id exists yet int rc = mdb_cursor_get(context.cursor, &context.key, &context.data, MDB_SET_KEY); if (rc == 0) { #ifdef DEBUG_LMDB_SOURCE_ID_MANAGER_HPP print_mdb_val("source_id_manager insert has key", context.key); print_mdb_val("source_id_manager insert has data", context.data); #endif // source ID found const uint8_t* p = static_cast<uint8_t*>(context.data.mv_data); p = lmdb_helper::decode_uint64_t(p, source_id); // read must align to data record if (p != static_cast<uint8_t*>(context.data.mv_data) + context.data.mv_size) { std::cerr << "data decode error in LMDB source ID store\n"; assert(0); } ++changes.source_id_already_present; context.close(); MUTEX_UNLOCK(&M); return false; } else if (rc == MDB_NOTFOUND) { // generate new source ID as DB size + 1 source_id = size() + 1; uint8_t data[10]; uint8_t* p = data; p = lmdb_helper::encode_uint64_t(source_id, p); context.data.mv_size = p - data; context.data.mv_data = data; #ifdef DEBUG_LMDB_SOURCE_ID_MANAGER_HPP print_mdb_val("source_id_manager insert new key", context.key); print_mdb_val("source_id_manager insert new data", context.data); #endif rc = mdb_put(context.txn, context.dbi, &context.key, &context.data, MDB_NODUPDATA); // write must work if (rc != 0) { std::cerr << "LMDB error: " << mdb_strerror(rc) << "\n"; assert(0); } // source ID created ++changes.source_id_inserted; context.close(); MUTEX_UNLOCK(&M); return true; } else { // invalid rc std::cerr << "LMDB error: " << mdb_strerror(rc) << "\n"; assert(0); source_id = 0; return false; // for mingw } }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32] = ""; int env_oflags; struct stat db_stat, exe_stat; (void) argc; (void) argv; srand(time(NULL)); count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_maxreaders(env, 1)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(stat("/proc/self/exe", &exe_stat)?errno:0); E(stat(DBPATH "/.", &db_stat)?errno:0); env_oflags = MDB_FIXEDMAP | MDB_NOSYNC; if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { /* LY: Assume running inside a CI-environment: * 1) don't use FIXEDMAP to avoid EBUSY in case collision, * which could be inspired by address space randomisation feature. * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage. */ env_oflags = 0; } /* LY: especially here we always needs MDB_NOSYNC * for testing mdbx_env_close_ex() and "redo-to-steady" on open. */ env_oflags |= MDB_NOSYNC; E(mdb_env_open(env, DBPATH, env_oflags, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS) E(mdb_drop(txn, dbi, 1)); E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); printf("check-preset-a\n"); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_a = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_a; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_a == count - j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_env_sync(env, 1); int deleted = 0; key.mv_data = sval; for (i = count - 1; i > -1; i -= (rand()%5)) { txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); deleted++; } } free(values); printf("Deleted %d values\n", deleted); printf("check-preset-b.cursor-next\n"); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_b = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_b; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_b == present_a - deleted, "mismatch"); printf("check-preset-b.cursor-prev\n"); j = 1; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++j; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_b == j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); /********************* LY: kept DB dirty ****************/ mdbx_env_close_ex(env, 1); E(mdb_env_create(&env)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, DBPATH, env_oflags, 0664)); printf("check-preset-c.cursor-next\n"); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_dbi_open(txn, "id1", 0, &dbi)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_c = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_c; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Rolled back %d deletion(s)\n", present_c - (present_a - deleted)); CHECK(present_c > present_a - deleted, "mismatch"); printf("check-preset-d.cursor-prev\n"); j = 1; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++j; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_c == j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); mdbx_env_close_ex(env, 0); return 0; }
DWORD VmDirMDBIndexIterate( PVDIR_BACKEND_INDEX_ITERATOR pIterator, PSTR* ppszVal, ENTRYID* pEId ) { DWORD dwError = 0; PVDIR_MDB_INDEX_ITERATOR pMdbIterator = NULL; PVDIR_DB_DBC pCursor = NULL; VDIR_DB_DBT key = {0}; VDIR_DB_DBT value = {0}; PSTR pszVal = NULL; ENTRYID eId = 0; if (!pIterator || !ppszVal || !pEId) { dwError = ERROR_INVALID_PARAMETER; BAIL_ON_VMDIR_ERROR(dwError); } pMdbIterator = (PVDIR_MDB_INDEX_ITERATOR)pIterator->pIterator; pCursor = pMdbIterator->pCursor; if (pIterator->bHasNext) { *ppszVal = pMdbIterator->pszVal; *pEId = pMdbIterator->eId; dwError = mdb_cursor_get(pCursor, &key, &value, MDB_NEXT); if (dwError == 0 && *(char *)(key.mv_data) == BE_INDEX_KEY_TYPE_FWD) { dwError = VmDirAllocateMemory(key.mv_size, (PVOID*)&pszVal); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirCopyMemory( pszVal, key.mv_size, (char*)key.mv_data + 1, key.mv_size - 1); BAIL_ON_VMDIR_ERROR(dwError); MDBDBTToEntryId(&value, &eId); pIterator->bHasNext = TRUE; } else { pIterator->bHasNext = FALSE; dwError = dwError == MDB_NOTFOUND ? 0 : dwError; pMdbIterator->bAbort = dwError ? TRUE : FALSE; } BAIL_ON_VMDIR_ERROR(dwError); pMdbIterator->pszVal = pszVal; pMdbIterator->eId = eId; } cleanup: return dwError; error: VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s failed, error (%d)", __FUNCTION__, dwError ); VMDIR_SAFE_FREE_MEMORY(pszVal); goto cleanup; }
DWORD VmDirMDBIndexIteratorInit( PVDIR_INDEX_CFG pIndexCfg, PSTR pszInitVal, PVDIR_BACKEND_INDEX_ITERATOR* ppIterator ) { DWORD dwError = 0; VDIR_DB mdbDBi = 0; PSTR pszInitKey = NULL; VDIR_DB_DBT key = {0}; VDIR_DB_DBT value = {0}; PSTR pszVal = NULL; ENTRYID eId = 0; PVDIR_DB_TXN pTxn = NULL; PVDIR_DB_DBC pCursor = NULL; PVDIR_BACKEND_INDEX_ITERATOR pIterator = NULL; PVDIR_MDB_INDEX_ITERATOR pMdbIterator = NULL; if (!pIndexCfg || !ppIterator) { dwError = ERROR_INVALID_PARAMETER; BAIL_ON_VMDIR_ERROR(dwError); } dwError = VmDirAllocateMemory( sizeof(VDIR_BACKEND_INDEX_ITERATOR), (PVOID*)&pIterator); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirAllocateMemory( sizeof(VDIR_MDB_INDEX_ITERATOR), (PVOID*)&pMdbIterator); BAIL_ON_VMDIR_ERROR(dwError); pIterator->pIterator = (PVOID)pMdbIterator; dwError = VmDirMDBIndexGetDBi(pIndexCfg, &mdbDBi); BAIL_ON_VMDIR_ERROR(dwError); dwError = mdb_txn_begin(gVdirMdbGlobals.mdbEnv, NULL, MDB_RDONLY, &pTxn); BAIL_ON_VMDIR_ERROR(dwError); pMdbIterator->pTxn = pTxn; dwError = mdb_cursor_open(pTxn, mdbDBi, &pCursor); BAIL_ON_VMDIR_ERROR(dwError); pMdbIterator->pCursor = pCursor; if (IsNullOrEmptyString(pszInitVal)) { dwError = mdb_cursor_get(pCursor, &key, &value, MDB_FIRST); } else { dwError = VmDirAllocateStringPrintf( &pszInitKey, "%c%s", BE_INDEX_KEY_TYPE_FWD, pszInitVal); BAIL_ON_VMDIR_ERROR(dwError); key.mv_data = pszInitKey; key.mv_size = VmDirStringLenA(pszInitKey); dwError = mdb_cursor_get(pCursor, &key, &value, MDB_SET_RANGE); } if (dwError == 0 && *(char *)(key.mv_data) == BE_INDEX_KEY_TYPE_FWD) { dwError = VmDirAllocateMemory(key.mv_size, (PVOID*)&pszVal); BAIL_ON_VMDIR_ERROR(dwError); dwError = VmDirCopyMemory( pszVal, key.mv_size, (char*)key.mv_data + 1, key.mv_size - 1); BAIL_ON_VMDIR_ERROR(dwError); MDBDBTToEntryId(&value, &eId); pIterator->bHasNext = TRUE; } else { pIterator->bHasNext = FALSE; dwError = dwError == MDB_NOTFOUND ? 0 : dwError; pMdbIterator->bAbort = dwError ? TRUE : FALSE; } BAIL_ON_VMDIR_ERROR(dwError); pMdbIterator->pszVal = pszVal; pMdbIterator->eId = eId; *ppIterator = pIterator; cleanup: VMDIR_SAFE_FREE_MEMORY(pszInitKey); return dwError; error: VMDIR_LOG_ERROR( VMDIR_LOG_MASK_ALL, "%s failed, error (%d)", __FUNCTION__, dwError ); VmDirMDBIndexIteratorFree(pIterator); VMDIR_SAFE_FREE_MEMORY(pszVal); goto cleanup; }
int mdb_filter_candidates( Operation *op, MDB_txn *rtxn, Filter *f, ID *ids, ID *tmp, ID *stack ) { int rc = 0; #ifdef LDAP_COMP_MATCH AttributeAliasing *aa; #endif Debug( LDAP_DEBUG_FILTER, "=> mdb_filter_candidates\n" ); if ( f->f_choice & SLAPD_FILTER_UNDEFINED ) { MDB_IDL_ZERO( ids ); goto out; } switch ( f->f_choice ) { case SLAPD_FILTER_COMPUTED: switch( f->f_result ) { case SLAPD_COMPARE_UNDEFINED: /* This technically is not the same as FALSE, but it * certainly will produce no matches. */ /* FALL THRU */ case LDAP_COMPARE_FALSE: MDB_IDL_ZERO( ids ); break; case LDAP_COMPARE_TRUE: MDB_IDL_ALL( ids ); break; case LDAP_SUCCESS: /* this is a pre-computed scope, leave it alone */ break; } break; case LDAP_FILTER_PRESENT: Debug( LDAP_DEBUG_FILTER, "\tPRESENT\n" ); rc = presence_candidates( op, rtxn, f->f_desc, ids ); break; case LDAP_FILTER_EQUALITY: Debug( LDAP_DEBUG_FILTER, "\tEQUALITY\n" ); #ifdef LDAP_COMP_MATCH if ( is_aliased_attribute && ( aa = is_aliased_attribute ( f->f_ava->aa_desc ) ) ) { rc = ava_comp_candidates ( op, rtxn, f->f_ava, aa, ids, tmp, stack ); } else #endif { rc = equality_candidates( op, rtxn, f->f_ava, ids, tmp ); } break; case LDAP_FILTER_APPROX: Debug( LDAP_DEBUG_FILTER, "\tAPPROX\n" ); rc = approx_candidates( op, rtxn, f->f_ava, ids, tmp ); break; case LDAP_FILTER_SUBSTRINGS: Debug( LDAP_DEBUG_FILTER, "\tSUBSTRINGS\n" ); rc = substring_candidates( op, rtxn, f->f_sub, ids, tmp ); break; case LDAP_FILTER_GE: /* if no GE index, use pres */ Debug( LDAP_DEBUG_FILTER, "\tGE\n" ); if( f->f_ava->aa_desc->ad_type->sat_ordering && ( f->f_ava->aa_desc->ad_type->sat_ordering->smr_usage & SLAP_MR_ORDERED_INDEX ) ) rc = inequality_candidates( op, rtxn, f->f_ava, ids, tmp, LDAP_FILTER_GE ); else rc = presence_candidates( op, rtxn, f->f_ava->aa_desc, ids ); break; case LDAP_FILTER_LE: /* if no LE index, use pres */ Debug( LDAP_DEBUG_FILTER, "\tLE\n" ); if( f->f_ava->aa_desc->ad_type->sat_ordering && ( f->f_ava->aa_desc->ad_type->sat_ordering->smr_usage & SLAP_MR_ORDERED_INDEX ) ) rc = inequality_candidates( op, rtxn, f->f_ava, ids, tmp, LDAP_FILTER_LE ); else rc = presence_candidates( op, rtxn, f->f_ava->aa_desc, ids ); break; case LDAP_FILTER_NOT: /* no indexing to support NOT filters */ Debug( LDAP_DEBUG_FILTER, "\tNOT\n" ); MDB_IDL_ALL( ids ); break; case LDAP_FILTER_AND: Debug( LDAP_DEBUG_FILTER, "\tAND\n" ); rc = list_candidates( op, rtxn, f->f_and, LDAP_FILTER_AND, ids, tmp, stack ); break; case LDAP_FILTER_OR: Debug( LDAP_DEBUG_FILTER, "\tOR\n" ); rc = list_candidates( op, rtxn, f->f_or, LDAP_FILTER_OR, ids, tmp, stack ); break; case LDAP_FILTER_EXT: Debug( LDAP_DEBUG_FILTER, "\tEXT\n" ); rc = ext_candidates( op, rtxn, f->f_mra, ids, tmp, stack ); break; default: Debug( LDAP_DEBUG_FILTER, "\tUNKNOWN %lu\n", (unsigned long) f->f_choice ); /* Must not return NULL, otherwise extended filters break */ MDB_IDL_ALL( ids ); } if ( ids[2] == NOID && MDB_IDL_IS_RANGE( ids )) { struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; ID last; if ( mdb->mi_nextid ) { last = mdb->mi_nextid; } else { MDB_cursor *mc; MDB_val key; last = 0; rc = mdb_cursor_open( rtxn, mdb->mi_id2entry, &mc ); if ( !rc ) { rc = mdb_cursor_get( mc, &key, NULL, MDB_LAST ); if ( !rc ) memcpy( &last, key.mv_data, sizeof( last )); mdb_cursor_close( mc ); } } if ( last ) { ids[2] = last; } else { MDB_IDL_ZERO( ids ); } } out: Debug( LDAP_DEBUG_FILTER, "<= mdb_filter_candidates: id=%ld first=%ld last=%ld\n", (long) ids[0], (long) MDB_IDL_FIRST( ids ), (long) MDB_IDL_LAST( ids ) ); return rc; }
FreqSchedulerError freq_scheduler_cursor_write(FreqScheduler *sch, MDB_cursor *cursor, uint64_t hash, float freq) { if (freq <= 0) return 0; ScheduleKey sk = { .score = 0, .hash = hash }; MDB_val key = { .mv_size = sizeof(sk), .mv_data = &sk, }; MDB_val val = { .mv_size = sizeof(float), .mv_data = &freq, }; int mdb_rc; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "adding page to schedule"); freq_scheduler_add_error(sch, mdb_strerror(mdb_rc)); } return sch->error->code; } FreqSchedulerError freq_scheduler_load_simple(FreqScheduler *sch, float freq_default, float freq_scale) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; HashInfoStream *st; if (hashinfo_stream_new(&st, sch->page_db) != 0) { error1 = "creating stream"; error2 = st? sch->page_db->error->message: "NULL"; goto on_error; } if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; StreamState ss; uint64_t hash; PageInfo *pi; while ((ss = hashinfo_stream_next(st, &hash, &pi)) == stream_state_next) { if ((pi->n_crawls > 0) && ((sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls)) && !page_info_is_seed(pi)){ float freq = freq_default; if (freq_scale > 0) { float rate = page_info_rate(pi); if (rate > 0) { freq = freq_scale * rate; } } if (freq_scheduler_cursor_write(sch, cursor, hash, freq) != 0) goto on_error; } page_info_delete(pi); } if (ss != stream_state_end) { error1 = "incorrect stream state"; error2 = 0; hashinfo_stream_delete(st); goto on_error; } hashinfo_stream_delete(st); if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_load_mmap(FreqScheduler *sch, MMapArray *freqs) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; if (txn_manager_expand( sch->txn_manager, 2*freqs->n_elements*freqs->element_size) != 0) { error1 = "resizing database"; error2 = sch->txn_manager->error->message; goto on_error; } if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; for (size_t i=0; i<freqs->n_elements; ++i) { PageFreq *f = mmap_array_idx(freqs, i); ScheduleKey sk = { .score = 1.0/f->freq, .hash = f->hash }; MDB_val key = { .mv_size = sizeof(sk), .mv_data = &sk, }; MDB_val val = { .mv_size = sizeof(float), .mv_data = &f->freq, }; int mdb_rc; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { error1 = "adding page to schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_request(FreqScheduler *sch, size_t max_requests, PageRequest **request) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; PageRequest *req = *request = page_request_new(max_requests); if (!req) { error1 = "allocating memory"; goto on_error; } int interrupt_requests = 0; while ((req->n_urls < max_requests) && !interrupt_requests) { MDB_val key; MDB_val val; ScheduleKey sk; float freq; int mdb_rc; int crawl = 0; switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, MDB_FIRST)) { case 0: // copy data before deleting cursor sk = *(ScheduleKey*)key.mv_data; freq = *(float*)val.mv_data; PageInfo *pi = 0; if (page_db_get_info(sch->page_db, sk.hash, &pi) != 0) { error1 = "retrieving PageInfo from PageDB"; error2 = sch->page_db->error->message; goto on_error; } if (pi) { if (sch->margin >= 0) { double elapsed = difftime(time(0), 0) - pi->last_crawl; if (elapsed < 1.0/(freq*(1.0 + sch->margin))) interrupt_requests = 1; } crawl = (sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls); } if (!interrupt_requests) { if ((mdb_rc = mdb_cursor_del(cursor, 0)) != 0) { error1 = "deleting head of schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } if (crawl) { if (page_request_add_url(req, pi->url) != 0) { error1 = "adding url to request"; goto on_error; } sk.score += 1.0/freq; val.mv_data = &freq; key.mv_data = &sk; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { error1 = "moving element inside schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } } page_info_delete(pi); break; case MDB_NOTFOUND: // no more pages left interrupt_requests = 1; break; default: error1 = "getting head of schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_add(FreqScheduler *sch, const CrawledPage *page) { if (page_db_add(sch->page_db, page, 0) != 0) { freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "adding crawled page"); freq_scheduler_add_error(sch, sch->page_db->error->message); } return sch->error->code; } void freq_scheduler_delete(FreqScheduler *sch) { mdb_env_close(sch->txn_manager->env); (void)txn_manager_delete(sch->txn_manager); if (!sch->persist) { char *data = build_path(sch->path, "data.mdb"); char *lock = build_path(sch->path, "lock.mdb"); remove(data); remove(lock); free(data); free(lock); remove(sch->path); } free(sch->path); error_delete(sch->error); free(sch); } FreqSchedulerError freq_scheduler_dump(FreqScheduler *sch, FILE *output) { MDB_cursor *cursor; if (freq_scheduler_cursor_open(sch, &cursor) != 0) return sch->error->code; int end = 0; MDB_cursor_op cursor_op = MDB_FIRST; do { int mdb_rc; MDB_val key; MDB_val val; ScheduleKey *key_data; float *val_data; switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, cursor_op)) { case 0: key_data = (ScheduleKey*)key.mv_data; val_data = (float*)val.mv_data; fprintf(output, "%.2e %016"PRIx64" %.2e\n", key_data->score, key_data->hash, *val_data); break; case MDB_NOTFOUND: end = 1; break; default: freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "iterating over database"); freq_scheduler_add_error(sch, mdb_strerror(mdb_rc)); end = 1; break; } cursor_op = MDB_NEXT; } while (!end); freq_scheduler_cursor_abort(sch, cursor); return sch->error->code; }
void LMDBIterator::SeekToFirst() { int rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_FIRST); CHECK_OP(rc); m_valid = rc == 0; }
h2o_start_response(req, &gen->super); int e; e = mdb_txn_begin(sv->db_env, NULL, MDB_RDONLY, &gen->txn); if (0 != e) mdb_fatal(e); e = mdb_cursor_open(gen->txn, sv->docs, &gen->curs); if (0 != e) mdb_fatal(e); MDB_val k = { .mv_size = key->len, .mv_data = key->s }, v; /* Get documents where the key has a prefix which matches */ e = mdb_cursor_get(gen->curs, &k, &v, MDB_SET_RANGE); switch (e) { case 0: __get_keys_send(gen, e, &k, req); break; case MDB_BAD_VALSIZE: e = mdb_cursor_get(gen->curs, &k, &v, MDB_FIRST); __get_keys_send(gen, e, &k, req); break; default: mdb_fatal(e); } return 0; fail:
/* reindex entries on the fly */ static void * mdb_online_index( void *ctx, void *arg ) { struct re_s *rtask = arg; BackendDB *be = rtask->arg; struct mdb_info *mdb = be->be_private; Connection conn = {0}; OperationBuffer opbuf; Operation *op; MDB_cursor *curs; MDB_val key, data; MDB_txn *txn; ID id; Entry *e; int rc, getnext = 1; int i; connection_fake_init( &conn, &opbuf, ctx ); op = &opbuf.ob_op; op->o_bd = be; id = 1; key.mv_size = sizeof(ID); while ( 1 ) { if ( slapd_shutdown ) break; rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn ); if ( rc ) break; rc = mdb_cursor_open( txn, mdb->mi_id2entry, &curs ); if ( rc ) { mdb_txn_abort( txn ); break; } if ( getnext ) { getnext = 0; key.mv_data = &id; rc = mdb_cursor_get( curs, &key, &data, MDB_SET_RANGE ); if ( rc ) { mdb_txn_abort( txn ); if ( rc == MDB_NOTFOUND ) rc = 0; break; } memcpy( &id, key.mv_data, sizeof( id )); } rc = mdb_id2entry( op, curs, id, &e ); mdb_cursor_close( curs ); if ( rc ) { mdb_txn_abort( txn ); if ( rc == MDB_NOTFOUND ) { id++; getnext = 1; continue; } break; } rc = mdb_index_entry( op, txn, MDB_INDEX_UPDATE_OP, e ); mdb_entry_return( op, e ); if ( rc == 0 ) { rc = mdb_txn_commit( txn ); txn = NULL; } else { mdb_txn_abort( txn ); txn = NULL; } if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_online_index) ": database %s: " "txn_commit failed: %s (%d)\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); break; } id++; getnext = 1; } for ( i = 0; i < mdb->mi_nattrs; i++ ) { if ( mdb->mi_attrs[ i ]->ai_indexmask & MDB_INDEX_DELETING || mdb->mi_attrs[ i ]->ai_newmask == 0 ) { continue; } mdb->mi_attrs[ i ]->ai_indexmask = mdb->mi_attrs[ i ]->ai_newmask; mdb->mi_attrs[ i ]->ai_newmask = 0; } ldap_pvt_thread_mutex_lock( &slapd_rq.rq_mutex ); ldap_pvt_runqueue_stoptask( &slapd_rq, rtask ); mdb->mi_index_task = NULL; ldap_pvt_runqueue_remove( &slapd_rq, rtask ); ldap_pvt_thread_mutex_unlock( &slapd_rq.rq_mutex ); return NULL; }
void DataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom, vector<Blob<Dtype>*>* top) { // Initialize DB switch (this->layer_param_.data_param().backend()) { case DataParameter_DB_LEVELDB: { leveldb::DB* db_temp; leveldb::Options options = GetLevelDBOptions(); options.create_if_missing = false; LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source(); leveldb::Status status = leveldb::DB::Open( options, this->layer_param_.data_param().source(), &db_temp); CHECK(status.ok()) << "Failed to open leveldb " << this->layer_param_.data_param().source() << std::endl << status.ToString(); db_.reset(db_temp); iter_.reset(db_->NewIterator(leveldb::ReadOptions())); iter_->SeekToFirst(); } break; case DataParameter_DB_LMDB: CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed"; CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB CHECK_EQ(mdb_env_open(mdb_env_, this->layer_param_.data_param().source().c_str(), MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed"; CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS) << "mdb_txn_begin failed"; CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS) << "mdb_open failed"; CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS) << "mdb_cursor_open failed"; LOG(INFO) << "Opening lmdb " << this->layer_param_.data_param().source(); CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST), MDB_SUCCESS) << "mdb_cursor_get failed"; break; default: LOG(FATAL) << "Unknown database backend"; } // Check if we would need to randomly skip a few data points if (this->layer_param_.data_param().rand_skip()) { unsigned int skip = caffe_rng_rand() % this->layer_param_.data_param().rand_skip(); LOG(INFO) << "Skipping first " << skip << " data points."; while (skip-- > 0) { switch (this->layer_param_.data_param().backend()) { case DataParameter_DB_LEVELDB: iter_->Next(); if (!iter_->Valid()) { iter_->SeekToFirst(); } break; case DataParameter_DB_LMDB: if (mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST), MDB_SUCCESS); } break; default: LOG(FATAL) << "Unknown database backend"; } } } // Read a data point, and use it to initialize the top blob. Datum datum; switch (this->layer_param_.data_param().backend()) { case DataParameter_DB_LEVELDB: datum.ParseFromString(iter_->value().ToString()); break; case DataParameter_DB_LMDB: datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); break; default: LOG(FATAL) << "Unknown database backend"; } // image int crop_size = this->layer_param_.transform_param().crop_size(); if (crop_size > 0) { (*top)[0]->Reshape(this->layer_param_.data_param().batch_size(), datum.channels(), crop_size, crop_size); this->prefetch_data_.Reshape(this->layer_param_.data_param().batch_size(), datum.channels(), crop_size, crop_size); } else { (*top)[0]->Reshape( this->layer_param_.data_param().batch_size(), datum.channels(), datum.height(), datum.width()); this->prefetch_data_.Reshape(this->layer_param_.data_param().batch_size(), datum.channels(), datum.height(), datum.width()); } LOG(INFO) << "output data size: " << (*top)[0]->num() << "," << (*top)[0]->channels() << "," << (*top)[0]->height() << "," << (*top)[0]->width(); // label if (this->output_labels_) { // liu (*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 4, 1, 1); this->prefetch_label_.Reshape(this->layer_param_.data_param().batch_size(), 4, 1, 1); /* (*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1); this->prefetch_label_.Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1); */ } // datum size this->datum_channels_ = datum.channels(); this->datum_height_ = datum.height(); this->datum_width_ = datum.width(); this->datum_size_ = datum.channels() * datum.height() * datum.width(); }
static int cache_iter_next(struct iter *it) { return mdb_cursor_get(it->cur, &it->key, &it->val, MDB_NEXT_DUP); }
static int mdb_db_open( BackendDB *be, ConfigReply *cr ) { int rc, i; struct mdb_info *mdb = (struct mdb_info *) be->be_private; struct stat stat1; uint32_t flags; char *dbhome; MDB_txn *txn; if ( be->be_suffix == NULL ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": need suffix.\n", 1, 0, 0 ); return -1; } Debug( LDAP_DEBUG_ARGS, LDAP_XSTRING(mdb_db_open) ": \"%s\"\n", be->be_suffix[0].bv_val, 0, 0 ); /* Check existence of dbenv_home. Any error means trouble */ rc = stat( mdb->mi_dbenv_home, &stat1 ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "cannot access database directory \"%s\" (%d).\n", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno ); return -1; } /* mdb is always clean */ be->be_flags |= SLAP_DBFLAG_CLEAN; rc = mdb_env_create( &mdb->mi_dbenv ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_create failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } if ( mdb->mi_readers ) { rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_maxreaders failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } } rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_mapsize failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES ); if( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "mdb_env_set_maxdbs failed: %s (%d).\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } #ifdef HAVE_EBCDIC strcpy( path, mdb->mi_dbenv_home ); __atoe( path ); dbhome = path; #else dbhome = mdb->mi_dbenv_home; #endif Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_db_open) ": database \"%s\": " "dbenv_open(%s).\n", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0); flags = mdb->mi_dbenv_flags; if ( slapMode & SLAP_TOOL_QUICK ) flags |= MDB_NOSYNC|MDB_WRITEMAP; if ( slapMode & SLAP_TOOL_READONLY) flags |= MDB_RDONLY; rc = mdb_env_open( mdb->mi_dbenv, dbhome, flags, mdb->mi_dbenv_mode ); if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. " "Restore from backup!\n", be->be_suffix[0].bv_val, rc, 0 ); goto fail; } rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn ); if ( rc ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. " "Restore from backup!\n", be->be_suffix[0].bv_val, rc, 0 ); goto fail; } /* open (and create) main databases */ for( i = 0; mdmi_databases[i].bv_val; i++ ) { flags = MDB_INTEGERKEY; if( i == MDB_ID2ENTRY ) { if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) )) flags |= MDB_CREATE; } else { if ( i == MDB_DN2ID ) flags |= MDB_DUPSORT; if ( !(slapMode & SLAP_TOOL_READONLY) ) flags |= MDB_CREATE; } rc = mdb_dbi_open( txn, mdmi_databases[i].bv_val, flags, &mdb->mi_dbis[i] ); if ( rc != 0 ) { snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": " "mdb_dbi_open(%s/%s) failed: %s (%d).", be->be_suffix[0].bv_val, mdb->mi_dbenv_home, mdmi_databases[i].bv_val, mdb_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg, 0, 0 ); goto fail; } if ( i == MDB_ID2ENTRY ) mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare ); else if ( i == MDB_DN2ID ) { MDB_cursor *mc; MDB_val key, data; ID id; mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare ); /* check for old dn2id format */ rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc ); /* first record is always ID 0 */ rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST ); if ( rc == 0 ) { rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT ); if ( rc == 0 ) { int len; unsigned char *ptr; ptr = data.mv_data; len = (ptr[0] & 0x7f) << 8 | ptr[1]; if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) { snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": DN index needs upgrade, " "run \"slapindex entryDN\".", be->be_suffix[0].bv_val ); Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg, 0, 0 ); if ( !(slapMode & SLAP_TOOL_READMAIN )) rc = LDAP_OTHER; mdb->mi_flags |= MDB_NEED_UPGRADE; } } } mdb_cursor_close( mc ); if ( rc == LDAP_OTHER ) goto fail; } } rc = mdb_ad_read( mdb, txn ); if ( rc ) { mdb_txn_abort( txn ); goto fail; } rc = mdb_attr_dbs_open( be, txn, cr ); if ( rc ) { mdb_txn_abort( txn ); goto fail; } rc = mdb_txn_commit(txn); if ( rc != 0 ) { Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": database %s: " "txn_commit failed: %s (%d)\n", be->be_suffix[0].bv_val, mdb_strerror(rc), rc ); goto fail; } /* monitor setup */ rc = mdb_monitor_db_open( be ); if ( rc != 0 ) { goto fail; } mdb->mi_flags |= MDB_IS_OPEN; return 0; fail: mdb_db_close( be, NULL ); return rc; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32] = ""; srandom(time(NULL)); count = (random()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = random()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_open(txn, "id1", MDB_CREATE, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; key.mv_data = sval; for (i= count - 1; i > -1; i-= (random()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_close(env, dbi); mdb_txn_abort(txn); mdb_env_close(env); return 0; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32] = ""; srand(time(NULL)); count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_maxreaders(env, 1)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS) E(mdb_drop(txn, dbi, 1)); E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); printf("check-preset-a\n"); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_a = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_a; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_a == count - j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_env_sync(env, 1); j=0; key.mv_data = sval; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); printf("check-preset-b.cursor-next\n"); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_b = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_b; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_b == present_a - j, "mismatch"); printf("check-preset-b.cursor-prev\n"); j = 1; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++j; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_b == j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); /********************* LY: kept DB dirty ****************/ mdb_env_close_ex(env, 1); E(mdb_env_create(&env)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); printf("check-preset-c.cursor-next\n"); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_dbi_open(txn, "id1", 0, &dbi)); E(mdb_cursor_open(txn, dbi, &cursor)); int present_c = 0; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++present_c; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_c == present_a, "mismatch"); printf("check-preset-d.cursor-prev\n"); j = 1; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); ++j; } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); CHECK(present_c == j, "mismatch"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); mdb_env_close_ex(env, 0); return 0; }
void LMDBIterator::SeekToLast() { int rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_LAST); m_valid = rc == 0; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[8]; char kval[sizeof(int)]; memset(sval, 0, sizeof(sval)); count = 510; values = (int *)je_malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = i*5; } E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "/tmp/testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi)); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); strcpy(kval, "001"); for (i=0;i<count;i++) { sprintf(sval, "%07x", values[i]); if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); /* there should be one full page of dups now. */ E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); /* test all 3 branches of split code: * 1: new key in lower half * 2: new key at split point * 3: new key in upper half */ key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; sprintf(sval, "%07x", values[3]+1); E(mdb_txn_begin(env, NULL, 0, &txn)); (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); mdb_txn_abort(txn); sprintf(sval, "%07x", values[255]+1); E(mdb_txn_begin(env, NULL, 0, &txn)); (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); mdb_txn_abort(txn); sprintf(sval, "%07x", values[500]+1); E(mdb_txn_begin(env, NULL, 0, &txn)); (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); E(mdb_txn_commit(txn)); /* Try MDB_NEXT_MULTIPLE */ E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; for (i= count - 1; i > -1; i-= (rand()%3)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%07x", values[i]); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } je_free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_close(env, dbi); mdb_txn_abort(txn); mdb_env_close(env); return 0; }
void LMDBIterator::Next() { int rc; rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_NEXT); m_valid = rc == 0; }
int main(int argc, char *argv[]) { int i, rc; MDB_env *env; MDB_txn *txn; MDB_dbi dbi; MDB_stat mst; MDB_envinfo mei; char *prog = argv[0]; char *envname; char *subname = NULL; int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0; if (argc < 2) { usage(prog); } /* -a: print stat of main DB and all subDBs * -s: print stat of only the named subDB * -e: print env info * -f: print freelist info * -r: print reader info * -n: use NOSUBDIR flag on env_open * -v: use previous snapshot * -V: print version and exit * (default) print stat of only the main DB */ while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) { switch(i) { case 'V': printf("%s\n", MDB_VERSION_STRING); exit(0); break; case 'a': if (subname) usage(prog); alldbs++; break; case 'e': envinfo++; break; case 'f': freinfo++; break; case 'n': envflags |= MDB_NOSUBDIR; break; case 'v': envflags |= MDB_PREVSNAPSHOT; break; case 'r': rdrinfo++; break; case 's': if (alldbs) usage(prog); subname = optarg; break; default: usage(prog); } } if (optind != argc - 1) usage(prog); envname = argv[optind]; rc = mdb_env_create(&env); if (rc) { fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); return EXIT_FAILURE; } if (alldbs || subname) { mdb_env_set_maxdbs(env, 4); } rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); if (rc) { fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } if (envinfo) { (void)mdb_env_stat(env, &mst); (void)mdb_env_info(env, &mei); printf("Environment Info\n"); printf(" Map address: %p\n", mei.me_mapaddr); printf(" Map size: %"Y"u\n", mei.me_mapsize); printf(" Page size: %u\n", mst.ms_psize); printf(" Max pages: %"Y"u\n", mei.me_mapsize / mst.ms_psize); printf(" Number of pages used: %"Y"u\n", mei.me_last_pgno+1); printf(" Last transaction ID: %"Y"u\n", mei.me_last_txnid); printf(" Max readers: %u\n", mei.me_maxreaders); printf(" Number of readers used: %u\n", mei.me_numreaders); } if (rdrinfo) { printf("Reader Table Status\n"); rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); if (rdrinfo > 1) { int dead; mdb_reader_check(env, &dead); printf(" %d stale readers cleared.\n", dead); rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); } if (!(subname || alldbs || freinfo)) goto env_close; } rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (rc) { fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } if (freinfo) { MDB_cursor *cursor; MDB_val key, data; size_t pages = 0, *iptr; printf("Freelist Status\n"); dbi = 0; rc = mdb_cursor_open(txn, dbi, &cursor); if (rc) { fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } rc = mdb_stat(txn, dbi, &mst); if (rc) { fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } prstat(&mst); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { iptr = data.mv_data; pages += *iptr; if (freinfo > 1) { char *bad = ""; size_t pg, prev; ssize_t i, j, span = 0; j = *iptr++; for (i = j, prev = 1; --i >= 0; ) { pg = iptr[i]; if (pg <= prev) bad = " [bad sequence]"; prev = pg; pg += span; for (; i >= span && iptr[i-span] == pg; span++, pg++) ; } printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n", *(size_t *)key.mv_data, j, span, bad); if (freinfo > 2) { for (--j; j >= 0; ) { pg = iptr[j]; for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ; printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n", pg, span); } } } } mdb_cursor_close(cursor); printf(" Free pages: %"Z"u\n", pages); } rc = mdb_open(txn, subname, 0, &dbi); if (rc) { fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } rc = mdb_stat(txn, dbi, &mst); if (rc) { fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } printf("Status of %s\n", subname ? subname : "Main DB"); prstat(&mst); if (alldbs) { MDB_cursor *cursor; MDB_val key; rc = mdb_cursor_open(txn, dbi, &cursor); if (rc) { fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { char *str; MDB_dbi db2; if (memchr(key.mv_data, '\0', key.mv_size)) continue; str = malloc(key.mv_size+1); memcpy(str, key.mv_data, key.mv_size); str[key.mv_size] = '\0'; rc = mdb_open(txn, str, 0, &db2); if (rc == MDB_SUCCESS) printf("Status of %s\n", str); free(str); if (rc) continue; rc = mdb_stat(txn, db2, &mst); if (rc) { fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } prstat(&mst); mdb_close(env, db2); } mdb_cursor_close(cursor); } if (rc == MDB_NOTFOUND) rc = MDB_SUCCESS; mdb_close(env, dbi); txn_abort: mdb_txn_abort(txn); env_close: mdb_env_close(env); return rc ? EXIT_FAILURE : EXIT_SUCCESS; }
void LMDBIterator::Prev() { int rc = mdb_cursor_get(m_cursor, &m_key, &m_value, MDB_PREV); m_valid = rc == 0; }
/* Write a frame or frames to the log. */ int sqlite3WalFrames(Wal *pWal, int szPage, PgHdr *pList, Pgno nTruncate, int isCommit, int sync_flags) { PgHdr *p; MDB_val key, data; int rc; mdbinf* mdb; MDB_txn* txn; #if ATOMIC db_thread *thr = g_tsd_thread; db_connection* pCon = g_tsd_conn; #else db_thread* thr = enif_tsd_get(g_tsd_thread); db_connection* pCon = enif_tsd_get(g_tsd_conn); #endif #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif txn = mdb->txn; if (!mdb) return SQLITE_ERROR; key.mv_size = sizeof(u64); key.mv_data = (void*)&pWal->index; // Term/evnum must always be increasing if ((pWal->inProgressTerm > 0 && pWal->inProgressTerm < pWal->lastCompleteTerm) || (pWal->inProgressEvnum > 0 && pWal->inProgressEvnum < pWal->lastCompleteEvnum)) return SQLITE_ERROR; track_time(2,thr); // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Fragment,CompressedPage/binary>>} for(p=pList; p; p=p->pDirty) { u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; u8 pagesBuf[PAGE_BUFF_SIZE]; int full_size = 0; int page_size = LZ4_compress_default((char*)p->pData,(char*)pagesBuf+sizeof(u64)*2+1,szPage,sizeof(pagesBuf)); char fragment_index = 0; int skipped = 0; track_time(3,thr); DBG("Insert frame, actor=%lld, pgno=%u, " "term=%lld, evnum=%lld, commit=%d, truncate=%d, compressedsize=%d", pWal->index,p->pgno,pWal->inProgressTerm,pWal->inProgressEvnum, isCommit,nTruncate,page_size); if (pCon->doReplicate) { u8 hdr[sizeof(u64)*2+sizeof(u32)*2]; put8byte(hdr, pWal->inProgressTerm); put8byte(hdr+sizeof(u64), pWal->inProgressEvnum); put4byte(hdr+sizeof(u64)*2, p->pgno); if (p->pDirty) put4byte(hdr+sizeof(u64)*2+sizeof(u32), 0); else put4byte(hdr+sizeof(u64)*2+sizeof(u32), nTruncate); #ifndef _TESTAPP_ wal_page_hook(thr,pagesBuf+sizeof(u64)*2+1, page_size, hdr, sizeof(hdr)); #endif } memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32)); key.mv_size = sizeof(pagesKeyBuf); key.mv_data = pagesKeyBuf; // Check if there are pages with the same or higher evnum/evterm. If there are, delete them. // This can happen if sqlite flushed some page to disk before commiting, because there were // so many pages that they could not be held in memory. Or it could happen if pages need to be // overwritten because there was a write that did not pass raft consensus. rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY); if (rc == MDB_SUCCESS) { size_t ndupl; mdb_cursor_count(mdb->cursorPages,&ndupl); rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP); if (rc == MDB_SUCCESS) { MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL}; u64 evnum, evterm; u8 frag = *((u8*)data.mv_data+sizeof(u64)*2); memcpy(&evterm, data.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64)); while ((evterm > pWal->inProgressTerm || evnum >= pWal->inProgressEvnum)) //(pWal->inProgressTerm + pWal->inProgressEvnum) > 0) { DBG("Deleting pages higher or equal to current. " "Evterm=%llu, evnum=%llu, curterm=%llu, curevn=%llu, dupl=%ld", evterm,evnum,pWal->inProgressTerm,pWal->inProgressEvnum,ndupl); if (pgDelKey.mv_data != NULL) { if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS) { DBG("Unable to cleanup page from pagedb %d",rc); break; } pgDelKey.mv_data = NULL; } mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT); // if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS) // { // DBG("Cant delete!"); // break; // } if (frag == 0) pWal->allPages--; ndupl--; if (!ndupl) break; rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP); if (rc != MDB_SUCCESS) break; memcpy(&evterm, data.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64)); frag = *((u8*)data.mv_data+sizeof(u64)*2); } if (pgDelKey.mv_data != NULL) { if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS) { DBG("Unable to cleanup page from pagedb %d",rc); break; } pgDelKey.mv_data = NULL; } } memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32)); key.mv_size = sizeof(pagesKeyBuf); key.mv_data = pagesKeyBuf; } track_time(4,thr); memcpy(pagesBuf, &pWal->inProgressTerm, sizeof(u64)); memcpy(pagesBuf + sizeof(u64), &pWal->inProgressEvnum, sizeof(u64)); full_size = page_size + sizeof(u64)*2 + 1; if (full_size < thr->maxvalsize) fragment_index = 0; else { full_size = page_size; skipped = thr->maxvalsize - sizeof(u64)*2 - 1; full_size -= skipped; while(full_size > 0) { full_size -= (thr->maxvalsize - sizeof(u64)*2 - 1); fragment_index++; } full_size = page_size + sizeof(u64)*2 +1; } pagesBuf[sizeof(u64)*2] = fragment_index; data.mv_size = fragment_index == 0 ? full_size : thr->maxvalsize; data.mv_data = pagesBuf; // fragment_index == 0 ? MDB_APPENDDUP : 0 if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS) { // printf("Cursor put failed to pages %d",rc); DBG("ERROR: cursor put failed=%d, datasize=%d",rc,full_size); return SQLITE_ERROR; } fragment_index--; skipped = data.mv_size; while (fragment_index >= 0) { DBG("Insert fragment %d",(int)fragment_index); if (fragment_index == 0) data.mv_size = full_size - skipped + sizeof(u64)*2 + 1; else data.mv_size = thr->maxvalsize; data.mv_data = pagesBuf + skipped - (sizeof(u64)*2+1); memcpy(pagesBuf + skipped - (sizeof(u64)*2+1), &pWal->inProgressTerm, sizeof(u64)); memcpy(pagesBuf + skipped - (sizeof(u64)+1), &pWal->inProgressEvnum, sizeof(u64)); pagesBuf[skipped-1] = fragment_index; if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS) { DBG("ERROR: cursor secondary put failed: err=%d, datasize=%d, skipped=%d, frag=%d", rc,full_size, skipped, (int)fragment_index); return SQLITE_ERROR; } skipped += data.mv_size - sizeof(u64)*2 - 1; fragment_index--; } thr->pagesChanged++; } // printf(""); // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} if (pWal->inProgressTerm > 0) { for(p=pList; p; p=p->pDirty) { u8 logKeyBuf[sizeof(u64)*3]; DBG("Inserting to log"); memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum, sizeof(u64)); key.mv_size = sizeof(logKeyBuf); key.mv_data = logKeyBuf; data.mv_size = sizeof(u32); data.mv_data = &p->pgno; if (mdb_cursor_put(mdb->cursorLog,&key,&data,0) != MDB_SUCCESS) { // printf("Cursor put failed to log"); DBG("ERROR: cursor put to log failed: %d",rc); return SQLITE_ERROR; } pWal->allPages++; } } else { DBG("Skipping log"); for(p=pList; p; p=p->pDirty) pWal->allPages++; } /** - Info DB: {<<ActorIndex:64>>, <<V,FirstCompleteTerm:64,FirstCompleteEvnum:64, LastCompleteTerm:64,LastCompleteEvnum:64, InprogressTerm:64,InProgressEvnum:64>>} */ { if (isCommit) { DBG("Commit actor=%llu fct=%llu, fcev=%llu, lct=%llu, lcev=%llu, int=%llu, inev=%llu", pWal->index, pWal->firstCompleteTerm, pWal->firstCompleteEvnum, pWal->lastCompleteTerm, pWal->lastCompleteEvnum, pWal->inProgressTerm,pWal->inProgressEvnum); #ifndef _TESTAPP_ enif_mutex_lock(pWal->mtx); #endif pWal->lastCompleteTerm = pWal->inProgressTerm > 0 ? pWal->inProgressTerm : pWal->lastCompleteTerm; pWal->lastCompleteEvnum = pWal->inProgressEvnum > 0 ? pWal->inProgressEvnum : pWal->lastCompleteEvnum; if (pWal->firstCompleteTerm == 0) { pWal->firstCompleteTerm = pWal->inProgressTerm; pWal->firstCompleteEvnum = pWal->inProgressEvnum; } pWal->inProgressTerm = pWal->inProgressEvnum = 0; pWal->mxPage = pWal->mxPage > nTruncate ? pWal->mxPage : nTruncate; // pWal->changed = 0; thr->forceCommit = 1; pCon->dirty = 0; #ifndef _TESTAPP_ enif_mutex_unlock(pWal->mtx); #endif DBG("cur mxpage=%u",pWal->mxPage); } else { // pWal->changed = 1; pCon->dirty = 1; } thr->pagesChanged++; rc = storeinfo(pWal,0,0,NULL); if (rc != SQLITE_OK) return rc; track_time(5,thr); } return SQLITE_OK; }
int lmdbgo_mdb_cursor_get2(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, MDB_val *key, MDB_val *val, MDB_cursor_op op) { LMDBGO_SET_VAL(key, kn, kdata); LMDBGO_SET_VAL(val, vn, vdata); return mdb_cursor_get(cur, key, val, op); }
// return number of bytes written static int wal_iterate(Wal *pWal, iterate_resource *iter, u8 *buf, int bufsize, u8 *hdr, u32 *done) { // db_thread* const thr = enif_tsd_get(g_tsd_thread); #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif mdbinf* const mdb = &thr->mdb; u32 mxPage; u64 readSafeEvnum, readSafeTerm; #ifndef _TESTAPP_ enif_mutex_lock(pWal->mtx); #endif readSafeEvnum = pWal->lastCompleteEvnum; readSafeTerm = pWal->lastCompleteTerm; mxPage = pWal->mxPage; #ifndef _TESTAPP_ enif_mutex_unlock(pWal->mtx); #endif if (!iter->started) { if (iter->evnum + iter->evterm == 0) { // If any writes come after iterator started, we must ignore those pages. iter->evnum = readSafeEvnum; iter->evterm = readSafeTerm; iter->pgnoPos = 1; iter->entiredb = 1; iter->mxPage = mxPage; if (pWal->mxPage == 0) { DBG("ERROR: Iterate on empty DB %llu",pWal->lastCompleteEvnum); *done = 1; return 0; } } else { // set mxPage to highest pgno we find. iter->pgnoPos = iter->mxPage = 0; DBG("Iterate rsterm=%llu rsevnum=%llu",readSafeTerm,readSafeEvnum); } iter->started = 1; } // send entire db (without history) if (iter->entiredb) { u32 iRead = 0; findframe(thr, pWal, iter->pgnoPos, &iRead, iter->evterm, iter->evnum, NULL, NULL); if (!iRead) { DBG("Iterate did not find page"); *done = iter->mxPage; return 0; } DBG("Iter pos=%u, mx=%u, safemx=%u",iter->pgnoPos, iter->mxPage, mxPage); if (iter->pgnoPos == iter->mxPage) *done = iter->mxPage; put8byte(hdr, iter->evterm); put8byte(hdr+sizeof(u64), iter->evnum); put4byte(hdr+sizeof(u64)*2, iter->pgnoPos); put4byte(hdr+sizeof(u64)*2+sizeof(u32), *done); iter->pgnoPos++; return fillbuff(thr, pWal, iter, buf, bufsize); } else { MDB_val logKey, logVal; int logop; u8 logKeyBuf[sizeof(u64)*3]; int rc; // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &iter->evterm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &iter->evnum, sizeof(u64)); logKey.mv_data = logKeyBuf; logKey.mv_size = sizeof(logKeyBuf); DBG("iterate looking for, matchterm=%llu matchevnum=%llu",iter->evterm,iter->evnum); if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS) { // Evterm/evnum combination not found. Check if evnum is there. // If so return evterm. It will mean a node is in conflict. DBG("Key not found in log"); if (readSafeEvnum == iter->evnum) { iter->evterm = readSafeTerm; iter->termMismatch = 1; } else { memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &readSafeTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &readSafeEvnum,sizeof(u64)); if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS) { DBG("Key not found in log for undo"); *done = 1; return 0; } while (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_PREV_NODUP) == MDB_SUCCESS) { u64 aindex, term, evnum; mdb_cursor_get(mdb->cursorLog,&logKey, &logVal, MDB_GET_CURRENT); memcpy(&aindex, logKey.mv_data, sizeof(u64)); memcpy(&term, (u8*)logKey.mv_data+sizeof(u64), sizeof(u64)); memcpy(&evnum, (u8*)logKey.mv_data+sizeof(u64)*2,sizeof(u64)); DBG("Iterate on term=%llu, evnum=%llu, looking for=%llu",term,evnum,iter->evnum); if (aindex != pWal->index) break; if (iter->evnum == evnum) { iter->evterm = term; iter->termMismatch = 1; break; } } } *done = 1; return 0; } // We start iterate from next evnum not current. Input evterm/evnum is match_index and match_term. // It needs next. if (iter->started == 1 && (rc = mdb_cursor_get(mdb->cursorLog,&logKey, &logVal, MDB_NEXT_NODUP)) != MDB_SUCCESS) { *done = 1; return 0; } else { u64 aindex; rc = mdb_cursor_get(mdb->cursorLog,&logKey, &logVal, MDB_GET_CURRENT); if (rc != MDB_SUCCESS) { *done = 1; return 0; } memcpy(&aindex, (u8*)logKey.mv_data, sizeof(u64)); memcpy(&iter->evterm, (u8*)logKey.mv_data+sizeof(u64), sizeof(u64)); memcpy(&iter->evnum, (u8*)logKey.mv_data+sizeof(u64)*2,sizeof(u64)); if (aindex != pWal->index) { *done = 1; return 0; } // To keep from moving iter->evterm/iter->evnum forward more than once. iter->started = 2; } logop = MDB_FIRST_DUP; while ((rc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS) { u64 evnum,evterm; u32 pgno; u32 iRead; logop = MDB_NEXT_DUP; mdb_cursor_get(mdb->cursorLog,&logKey, &logVal, MDB_GET_CURRENT); memcpy(&pgno,logVal.mv_data,sizeof(u32)); DBG("iterate at pgno=%u, pgnopos=%u",pgno,iter->pgnoPos); if (pgno <= iter->pgnoPos) continue; findframe(thr, pWal, pgno, &iRead, iter->evterm, iter->evnum, &evterm, &evnum); if (iRead == 0) { DBG("ERROR: Did not find frame for pgno=%u, evterm=%llu, evnum=%llu", pgno, iter->evterm, iter->evnum); *done = 1; return 0; } if (evterm != iter->evterm || evnum != iter->evnum) { DBG("ERROR: Evterm/evnum does not match,looking for: evterm=%llu, evnum=%llu, " "got: evterm=%llu, evnum=%llu", iter->evterm, iter->evnum, evterm, evnum); *done = 1; return 0; } iter->pgnoPos = pgno; if ((rc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS) *done = 0; else *done = iter->pgnoPos; DBG( "logcursor get next %d, done=%u",rc,*done); put8byte(hdr, iter->evterm); put8byte(hdr+sizeof(u64), iter->evnum); put4byte(hdr+sizeof(u64)*2, iter->pgnoPos); put4byte(hdr+sizeof(u64)*2+sizeof(u32), *done); return fillbuff(thr, pWal, iter, buf, bufsize); } *done = 1; return 0; } }
void DataLayer<Dtype>::InternalThreadEntry() { Datum datum; CHECK(this->prefetch_data_.count()); Dtype* top_data = this->prefetch_data_.mutable_cpu_data(); Dtype* top_label = NULL; // suppress warnings about uninitialized variables if (this->output_labels_) { top_label = this->prefetch_label_.mutable_cpu_data(); } const int batch_size = this->layer_param_.data_param().batch_size(); for (int item_id = 0; item_id < batch_size; ++item_id) { // get a blob switch (this->layer_param_.data_param().backend()) { case DataParameter_DB_LEVELDB: CHECK(iter_); CHECK(iter_->Valid()); datum.ParseFromString(iter_->value().ToString()); break; case DataParameter_DB_LMDB: CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS); datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); break; default: LOG(FATAL) << "Unknown database backend"; } // Apply data transformations (mirror, scale, crop...) this->data_transformer_.Transform(item_id, datum, this->mean_, top_data); if (this->output_labels_) { // liu // top_label[item_id] = datum.label(); // LOG(ERROR) << "label size " << datum.label_size() << " " << datum.label(0) \ << " " << datum.label(1) << " " << datum.label(2) << " " << datum.label(3); for(int label_i=0; label_i < datum.label_size(); label_i++){ top_label[item_id * datum.label_size() + label_i] = datum.label(label_i); } } // go to the next iter switch (this->layer_param_.data_param().backend()) { case DataParameter_DB_LEVELDB: iter_->Next(); if (!iter_->Valid()) { // We have reached the end. Restart from the first. DLOG(INFO) << "Restarting data prefetching from start."; iter_->SeekToFirst(); } break; case DataParameter_DB_LMDB: if (mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { // We have reached the end. Restart from the first. DLOG(INFO) << "Restarting data prefetching from start."; CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST), MDB_SUCCESS); } break; default: LOG(FATAL) << "Unknown database backend"; } } }
static int doundo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, u8 delPages) { MDB_val logKey, logVal; MDB_val pgKey, pgVal; u8 logKeyBuf[sizeof(u64)*3]; int logop, pgop, rc, mrc; mdbinf *mdb; // db_thread* const thr = enif_tsd_get(g_tsd_thread); #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif rc = SQLITE_OK; if (pWal->inProgressTerm == 0) return SQLITE_OK; #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif if (!mdb) return SQLITE_ERROR; logKey.mv_data = logKeyBuf; logKey.mv_size = sizeof(logKeyBuf); DBG("Undo"); // For every page here // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} // Delete from // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>} memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum,sizeof(u64)); if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS) { DBG("Key not found in log for undo, index=%llu, term=%llu, evnum=%llu", pWal->index, pWal->inProgressTerm, pWal->inProgressEvnum); return SQLITE_OK; } logop = MDB_FIRST_DUP; while ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS) { u32 pgno; u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; u64 term,evnum; memcpy(&pgno,logVal.mv_data,sizeof(u32)); if (delPages) { size_t ndupl; memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32)); pgKey.mv_data = pagesKeyBuf; pgKey.mv_size = sizeof(pagesKeyBuf); DBG("UNDO pgno=%d",pgno); pgop = MDB_FIRST_DUP; if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS) { DBG("Key not found in log for undo"); continue; } mdb_cursor_count(mdb->cursorPages,&ndupl); while (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,pgop) == MDB_SUCCESS) { u8 frag = *((u8*)pgVal.mv_data+sizeof(u64)*2); memcpy(&term, pgVal.mv_data, sizeof(u64)); memcpy(&evnum,(u8*)pgVal.mv_data+sizeof(u64),sizeof(u64)); DBG("progress term %lld, progress evnum %lld, curterm %lld, curnum %lld", pWal->inProgressTerm, pWal->inProgressEvnum, term, evnum); if (term >= pWal->inProgressTerm && evnum >= pWal->inProgressEvnum) { if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS) { DBG("Can not delete undo"); rc = SQLITE_ERROR; break; } if (frag == 0) pWal->allPages--; ndupl--; if (!ndupl) break; } pgop = MDB_NEXT_DUP; } pWal->inProgressTerm = pWal->inProgressEvnum = 0; storeinfo(pWal,0,0,NULL); thr->pagesChanged++; } if (xUndo) rc = xUndo(pUndoCtx, pgno); logop = MDB_NEXT_DUP; } // if (mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA) != MDB_SUCCESS) // { // DBG("Unable to cleanup key from logdb")); // } DBG("Undo done!"); return rc; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32] = ""; int env_oflags; struct stat db_stat, exe_stat; srand(time(NULL)); count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_maxreaders(env, 1)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(stat("/proc/self/exe", &exe_stat)?errno:0); E(stat(DBPATH "/.", &db_stat)?errno:0); env_oflags = MDB_FIXEDMAP | MDB_NOSYNC; if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { /* LY: Assume running inside a CI-environment: * 1) don't use FIXEDMAP to avoid EBUSY in case collision, * which could be inspired by address space randomisation feature. * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage. */ env_oflags = 0; } E(mdb_env_open(env, DBPATH, env_oflags, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); if (mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi) == MDB_SUCCESS) E(mdb_drop(txn, dbi, 1)); E(mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi)); key.mv_size = sizeof(int); key.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { sprintf(sval, "%03x %d foo bar", values[i], values[i]); data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) j++; } if (j) printf("%d duplicates skipped\n", j); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; key.mv_data = sval; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(sval, "%03x ", values[i]); if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); mdb_env_close(env); return 0; }