int main(void) { check(mdb_env_create(&env)); check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL)); check(mdb_env_set_maxreaders(env, 126)); check(mdb_env_set_maxdbs(env, 1)); if(! access(DB_PATH, X_OK)) { system("rm -rf " DB_PATH); } check(mkdir(DB_PATH, 0777)); check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644)); new_txn(); check(mdb_dbi_open(txn, NULL, 0, &dbi)); put("a"); put("b"); put("baa"); put("d"); new_txn(); check(mdb_cursor_open(txn, dbi, &c1)); check(mdb_cursor_get(c1, &keyv, &valv, MDB_LAST)); check(mdb_cursor_del(c1, 0)); check(mdb_cursor_del(c1, 0)); new_txn(); }
CAMLprim value caml_mdb_cursor_del(value cursor,value flags){ CAMLparam2(cursor,flags); if(mdb_cursor_del((MDB_cursor*)cursor, Int_val(flags))){ caml_failwith("error in mdb_cursor_del"); } CAMLreturn0; }
int deleteEntryFromIntIndex(GLMDB_env *glmdb_env, MDB_txn *mdbTxn, MDB_cursor *vertexCursor, MDB_cursor *intIndexCursor, long long elementId, int propertyKeyId) { int rc; //Get the old value first MDB_val oldData; MDB_val key, data; rc = getVertexProperty(vertexCursor, elementId, propertyKeyId, &oldData); if (rc == 0) { //Delete the current indexed element IntIndexKeyStruct *intIndexKeyStruct = malloc(sizeof(IntIndexKeyStruct)); intIndexKeyStruct->propertyKeyId = propertyKeyId; intIndexKeyStruct->elementId = elementId; intIndexKeyStruct->value = *((int *) oldData.mv_data); key.mv_size = sizeof(IntIndexKeyStruct); key.mv_data = intIndexKeyStruct; rc = mdb_cursor_get(intIndexCursor, &key, &data, MDB_SET_KEY); if (rc != 0) { free(intIndexKeyStruct); goto fail; } free(intIndexKeyStruct); rc = mdb_cursor_del(intIndexCursor, 0); } fail: return rc; }
/* * Class: jmdb_DatabaseWrapper * Method: cursorDel * Signature: (JI)V */ JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_cursorDel(JNIEnv *vm, jclass clazz, jlong cursorL, jint flags) { MDB_cursor *cursorC = (MDB_cursor*) cursorL; int code = mdb_cursor_del(cursorC, flags); if (code) { throwDatabaseException(vm, code); } }
static int remove_entry(MDB_cursor *cur) { int ret = mdb_cursor_del(cur, MDB_NODUPDATA); if (ret != 0) { return KNOT_ERROR; } return KNOT_EOK; }
bool DBPrivAdvanceCursor(DBCursorPriv *cursor, void **key, int *key_size, void **value, int *value_size) { MDB_val mkey, data; int rc; bool retval = false; if (cursor->curkv) { free(cursor->curkv); cursor->curkv = NULL; } if ((rc = mdb_cursor_get(cursor->mc, &mkey, &data, MDB_NEXT)) == MDB_SUCCESS) { // Align second buffer to 64-bit boundary, to avoid alignment errors on // certain platforms. size_t keybuf_size = mkey.mv_size; if (keybuf_size & 0x7) { keybuf_size += 8 - (keybuf_size % 8); } cursor->curkv = xmalloc(keybuf_size + data.mv_size); memcpy(cursor->curkv, mkey.mv_data, mkey.mv_size); *key = cursor->curkv; *key_size = mkey.mv_size; *value_size = data.mv_size; memcpy((char *)cursor->curkv+keybuf_size, data.mv_data, data.mv_size); *value = (char *)cursor->curkv + keybuf_size; retval = true; } else if (rc != MDB_NOTFOUND) { Log(LOG_LEVEL_ERR, "Could not advance cursor: %s", mdb_strerror(rc)); } if (cursor->pending_delete) { int r2; /* Position on key to delete */ r2 = mdb_cursor_get(cursor->mc, &cursor->delkey, NULL, MDB_SET); if (r2 == MDB_SUCCESS) { r2 = mdb_cursor_del(cursor->mc, 0); } /* Reposition the cursor if it was valid before */ if (rc == MDB_SUCCESS) { mkey.mv_data = *key; rc = mdb_cursor_get(cursor->mc, &mkey, NULL, MDB_SET); } cursor->pending_delete = false; } return retval; }
int mail_cache_db_clean_up(struct mail_cache_db * cache_db, chash * exist) { int r; MDB_env *env; MDB_txn *txn; MDB_dbi dbi; MDB_cursor *cursor; MDB_val mdb_key; MDB_val mdb_val; env = cache_db->internal_database; r = mdb_txn_begin(env, NULL, 0, &txn); if (r != 0) return -1; r = mdb_dbi_open(txn, NULL, 0, &dbi); if (r != 0) goto error; r = mdb_cursor_open(txn, dbi, &cursor); if (r != 0) goto error; r = mdb_cursor_get(cursor, &mdb_key, &mdb_val, MDB_FIRST); if (r != 0) goto cursor_error; while (r == 0) { chashdatum hash_key; chashdatum hash_data; hash_key.data = mdb_key.mv_data; hash_key.len = (unsigned int) mdb_key.mv_size; r = chash_get(exist, &hash_key, &hash_data); if (r < 0) { r = mdb_cursor_del(cursor, 0); if (r != 0) goto cursor_error; } r = mdb_cursor_get(cursor, &mdb_key, &mdb_val, MDB_NEXT); } mdb_txn_commit(txn); return 0; cursor_error: mdb_cursor_close(cursor); error: mdb_txn_abort(txn); return -1; }
static int lmdb_storage_trim(void* handle, iid_t iid) { struct lmdb_storage* s = handle; int result; iid_t min = 0; MDB_cursor* cursor = NULL; MDB_val key, data; if (iid == 0) return 0; lmdb_storage_put_trim_instance(handle, iid); if ((result = mdb_cursor_open(s->txn, s->dbi, &cursor)) != 0) { paxos_log_error("Could not create cursor. %s", mdb_strerror(result)); goto cleanup_exit; } key.mv_data = &min; key.mv_size = sizeof(iid_t); do { if ((result = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { assert(key.mv_size = sizeof(iid_t)); min = *(iid_t*)key.mv_data; } else { goto cleanup_exit; } if (min != 0 && min <= iid) { if (mdb_cursor_del(cursor, 0) != 0) { paxos_log_error("mdb_cursor_del failed. %s", mdb_strerror(result)); goto cleanup_exit; } } } while (min <= iid); cleanup_exit: if (cursor) { mdb_cursor_close(cursor); } return 0; }
int removeIntIndex(MDB_cursor *indexCursor, long long elementId, int propertyKeyId, int propertyValue) { int rc; MDB_val key, data; IntIndexKeyStruct *intIndexKeyStruct = malloc(sizeof(IntIndexKeyStruct)); intIndexKeyStruct->propertyKeyId = (int) propertyKeyId; intIndexKeyStruct->elementId = (long long) elementId; intIndexKeyStruct->value = propertyValue; key.mv_size = sizeof(IntIndexKeyStruct); key.mv_data = intIndexKeyStruct; data.mv_size = sizeof(long long); data.mv_data = &elementId; rc = mdb_cursor_get(indexCursor, &key, &data, MDB_SET_KEY); if (rc == 0) { rc = mdb_cursor_del(indexCursor, 0); } free(intIndexKeyStruct); return rc; }
int deleteIntIndex(GLMDB_env * glmdb_env, MDB_txn * mdbTxn, int propertyKeyId, unsigned char vertex) { int rc; MDB_cursor *cursor; if (vertex) { rc = mdb_cursor_open(mdbTxn, glmdb_env->vertexIntIndexDb, &cursor); } else { rc = mdb_cursor_open(mdbTxn, glmdb_env->edgeIntIndexDb, &cursor); } if (rc == 0) { MDB_val key, data; IntIndexKeyStruct *intIndexKeyStruct = malloc(sizeof(IntIndexKeyStruct)); intIndexKeyStruct->propertyKeyId = (int) propertyKeyId; intIndexKeyStruct->elementId = -1LL; intIndexKeyStruct->value = -1; key.mv_size = sizeof(IntIndexKeyStruct); key.mv_data = intIndexKeyStruct; while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_RANGE) == 0)) { IntIndexKeyStruct *intIndexKeyStructToDelete = (IntIndexKeyStruct *) (key.mv_data); if (intIndexKeyStructToDelete->propertyKeyId == propertyKeyId) { rc = mdb_cursor_del(cursor, 0); if (rc != 0) { break; } else { //reset the key intIndexKeyStruct->propertyKeyId = (int) propertyKeyId; intIndexKeyStruct->elementId = -1LL; intIndexKeyStruct->value = -1; } } else { break; } } free(intIndexKeyStruct); } return rc; }
void DBPrivCloseCursor(DBCursorPriv *cursor) { DBTxn *txn; int rc = GetWriteTransaction(cursor->db, &txn); CF_ASSERT(rc == MDB_SUCCESS, "Could not get write transaction"); CF_ASSERT(txn->cursor_open, "Transaction not open"); txn->cursor_open = false; if (cursor->curkv) { free(cursor->curkv); } if (cursor->pending_delete) { mdb_cursor_del(cursor->mc, 0); } mdb_cursor_close(cursor->mc); free(cursor); }
/* mc must have been set by mdb_dn2id */ int mdb_dn2id_delete( Operation *op, MDB_cursor *mc, ID id, ID nsubs ) { ID nid; char *ptr; int rc; Debug( LDAP_DEBUG_TRACE, "=> mdb_dn2id_delete 0x%lx\n", id ); /* Delete our ID from the parent's list */ rc = mdb_cursor_del( mc, 0 ); /* Delete our ID from the tree. With sorted duplicates, this * will leave any child nodes still hanging around. This is OK * for modrdn, which will add our info back in later. */ if ( rc == 0 ) { MDB_val key, data; if ( nsubs ) { mdb_cursor_get( mc, &key, NULL, MDB_GET_CURRENT ); memcpy( &nid, key.mv_data, sizeof( ID )); } key.mv_size = sizeof(ID); key.mv_data = &id; rc = mdb_cursor_get( mc, &key, &data, MDB_SET ); if ( rc == 0 ) rc = mdb_cursor_del( mc, 0 ); } /* Delete our subtree count from all superiors */ if ( rc == 0 && nsubs && nid ) { MDB_val key, data; ID subs; key.mv_data = &nid; key.mv_size = sizeof( ID ); do { rc = mdb_cursor_get( mc, &key, &data, MDB_SET ); if ( !rc ) { char *p2; diskNode *d; int rlen; ptr = (char *)data.mv_data + data.mv_size - sizeof( ID ); memcpy( &nid, ptr, sizeof( ID )); /* Get parent's node under grandparent */ d = data.mv_data; rlen = ( d->nrdnlen[0] << 8 ) | d->nrdnlen[1]; p2 = op->o_tmpalloc( rlen + 2, op->o_tmpmemctx ); memcpy( p2, data.mv_data, rlen+2 ); *p2 ^= 0x80; data.mv_data = p2; rc = mdb_cursor_get( mc, &key, &data, MDB_GET_BOTH ); op->o_tmpfree( p2, op->o_tmpmemctx ); if ( !rc ) { /* Get parent's subtree count */ ptr = (char *)data.mv_data + data.mv_size - sizeof( ID ); memcpy( &subs, ptr, sizeof( ID )); subs -= nsubs; p2 = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx ); memcpy( p2, data.mv_data, data.mv_size - sizeof( ID )); memcpy( p2+data.mv_size - sizeof( ID ), &subs, sizeof( ID )); data.mv_data = p2; rc = mdb_cursor_put( mc, &key, &data, MDB_CURRENT ); op->o_tmpfree( p2, op->o_tmpmemctx ); } } if ( rc ) break; } while ( nid ); } Debug( LDAP_DEBUG_TRACE, "<= mdb_dn2id_delete 0x%lx: %d\n", id, rc ); return rc; }
static int doundo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, u8 delPages) { MDB_val logKey, logVal; MDB_val pgKey, pgVal; u8 logKeyBuf[sizeof(u64)*3]; int logop, pgop, rc, mrc; mdbinf *mdb; // db_thread* const thr = enif_tsd_get(g_tsd_thread); #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif rc = SQLITE_OK; if (pWal->inProgressTerm == 0) return SQLITE_OK; #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif if (!mdb) return SQLITE_ERROR; logKey.mv_data = logKeyBuf; logKey.mv_size = sizeof(logKeyBuf); DBG("Undo"); // For every page here // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} // Delete from // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>} memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum,sizeof(u64)); if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS) { DBG("Key not found in log for undo, index=%llu, term=%llu, evnum=%llu", pWal->index, pWal->inProgressTerm, pWal->inProgressEvnum); return SQLITE_OK; } logop = MDB_FIRST_DUP; while ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS) { u32 pgno; u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; u64 term,evnum; memcpy(&pgno,logVal.mv_data,sizeof(u32)); if (delPages) { size_t ndupl; memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32)); pgKey.mv_data = pagesKeyBuf; pgKey.mv_size = sizeof(pagesKeyBuf); DBG("UNDO pgno=%d",pgno); pgop = MDB_FIRST_DUP; if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS) { DBG("Key not found in log for undo"); continue; } mdb_cursor_count(mdb->cursorPages,&ndupl); while (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,pgop) == MDB_SUCCESS) { u8 frag = *((u8*)pgVal.mv_data+sizeof(u64)*2); memcpy(&term, pgVal.mv_data, sizeof(u64)); memcpy(&evnum,(u8*)pgVal.mv_data+sizeof(u64),sizeof(u64)); DBG("progress term %lld, progress evnum %lld, curterm %lld, curnum %lld", pWal->inProgressTerm, pWal->inProgressEvnum, term, evnum); if (term >= pWal->inProgressTerm && evnum >= pWal->inProgressEvnum) { if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS) { DBG("Can not delete undo"); rc = SQLITE_ERROR; break; } if (frag == 0) pWal->allPages--; ndupl--; if (!ndupl) break; } pgop = MDB_NEXT_DUP; } pWal->inProgressTerm = pWal->inProgressEvnum = 0; storeinfo(pWal,0,0,NULL); thr->pagesChanged++; } if (xUndo) rc = xUndo(pUndoCtx, pgno); logop = MDB_NEXT_DUP; } // if (mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA) != MDB_SUCCESS) // { // DBG("Unable to cleanup key from logdb")); // } DBG("Undo done!"); return rc; }
// Delete all pages up to limitEvterm and limitEvnum static int checkpoint(Wal *pWal, u64 limitEvnum) { MDB_val logKey, logVal; u8 logKeyBuf[sizeof(u64)*3]; u64 evnum,evterm,aindex; mdbinf* mdb; // db_thread* const thr = enif_tsd_get(g_tsd_thread); #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif int logop, mrc = MDB_SUCCESS; // u8 somethingDeleted = 0; int allPagesDiff = 0; #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif if (!mdb) return SQLITE_ERROR; // if (pWal->inProgressTerm == 0) // return SQLITE_OK; DBG("checkpoint actor=%llu, fct=%llu, fcev=%llu, limitEvnum=%llu",pWal->index, pWal->firstCompleteTerm,pWal->firstCompleteEvnum,limitEvnum); while (pWal->firstCompleteEvnum < limitEvnum) { logKey.mv_data = logKeyBuf; logKey.mv_size = sizeof(logKeyBuf); memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->firstCompleteTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->firstCompleteEvnum,sizeof(u64)); if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS) { printf("Key not found in log for checkpoint %llu %llu\n", pWal->firstCompleteTerm, pWal->firstCompleteEvnum); return SQLITE_OK; } DBG("checkpoint evnum=%llu",pWal->firstCompleteEvnum); // For every page here // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} // Delete from // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>} logop = MDB_FIRST_DUP; while ((mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS) { u32 pgno; size_t ndupl; u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; MDB_val pgKey = {0,NULL}, pgVal = {0,NULL}; u64 pgnoLimitEvnum; logop = MDB_NEXT_DUP; memcpy(&pgno, logVal.mv_data,sizeof(u32)); DBG("checkpoint pgno=%u",pgno); memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32)); pgKey.mv_data = pagesKeyBuf; pgKey.mv_size = sizeof(pagesKeyBuf); if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS) { continue; } mdb_cursor_count(mdb->cursorPages,&ndupl); if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_LAST_DUP) == MDB_SUCCESS) memcpy(&pgnoLimitEvnum, (u8*)pgVal.mv_data+sizeof(u64),sizeof(u64)); else continue; pgnoLimitEvnum = pgnoLimitEvnum < limitEvnum ? pgnoLimitEvnum : limitEvnum; if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_FIRST_DUP) != MDB_SUCCESS) continue; do { u8 frag; MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL}; mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT); frag = *((u8*)pgDelVal.mv_data+sizeof(u64)*2); memcpy(&evterm, pgDelVal.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)pgDelVal.mv_data+sizeof(u64),sizeof(u64)); DBG("limit limitevnum %lld, curnum %lld, dupl %zu, frag=%d", limitEvnum, evnum, ndupl,(int)frag); if (evnum < pgnoLimitEvnum) { mrc = mdb_cursor_del(mdb->cursorPages,0); if (mrc != MDB_SUCCESS) { DBG("Unable to delete page on cursor!."); break; } // else // { // DBG("Deleted page!"); // // somethingDeleted = 1; // } if (frag == 0) allPagesDiff++; } ndupl--; if (!ndupl) break; mrc = mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_NEXT_DUP); } while (mrc == MDB_SUCCESS); } mrc = mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA); if (mrc != MDB_SUCCESS) { DBG("Can not delete log"); break; } // move forward if ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_NEXT_NODUP)) != MDB_SUCCESS) { DBG("Unable to move to next log %d",mrc); break; } // read next key data memcpy(&aindex, logKey.mv_data, sizeof(u64)); memcpy(&evterm, (u8*)logKey.mv_data + sizeof(u64), sizeof(u64)); memcpy(&evnum, (u8*)logKey.mv_data + sizeof(u64)*2, sizeof(u64)); if (aindex != pWal->index) { DBG("Reached another actor"); break; } pWal->firstCompleteTerm = evterm; pWal->firstCompleteEvnum = evnum; pWal->allPages -= allPagesDiff; allPagesDiff = 0; DBG("Checkpint fce now=%lld",(u64)evnum); } // no dirty pages, but will write info if (sqlite3WalFrames(pWal, SQLITE_DEFAULT_PAGE_SIZE, NULL, pWal->mxPage, 1, 0) == SQLITE_OK) return SQLITE_OK; else return SQLITE_ERROR; }
static int mdb_dn2id_upgrade( BackendDB *be ) { struct mdb_info *mi = (struct mdb_info *) be->be_private; MDB_txn *mt; MDB_cursor *mc = NULL; MDB_val key, data; char *ptr; int rc, writes=0, depth=0; int enable_meter = 0; ID id = 0, *num, count = 0; rec *stack; lutil_meter_t meter; if (!(mi->mi_flags & MDB_NEED_UPGRADE)) { Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n", be->be_suffix[0].bv_val, 0, 0 ); return 0; } { MDB_stat st; mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st); if (!st.ms_entries) { /* Empty DB, nothing to upgrade? */ return 0; } if (isatty(2)) enable_meter = !lutil_meter_open(&meter, &lutil_meter_text_display, &lutil_meter_linear_estimator, st.ms_entries); } num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec))); stack = (rec *)(num + STACKSIZ); rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } key.mv_size = sizeof(ID); /* post-order depth-first update */ for(;;) { size_t dkids; unsigned char *ptr; /* visit */ key.mv_data = &id; stack[depth].id = id; rc = mdb_cursor_get(mc, &key, &data, MDB_SET); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } num[depth] = 1; rc = mdb_cursor_count(mc, &dkids); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } if (dkids > 1) { rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); down: ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID); memcpy(&id, ptr, sizeof(ID)); depth++; memcpy(stack[depth].rdn, data.mv_data, data.mv_size); stack[depth].len = data.mv_size; continue; } /* pop: write updated count, advance to next node */ pop: /* update superior counts */ if (depth) num[depth-1] += num[depth]; key.mv_data = &id; id = stack[depth-1].id; data.mv_data = stack[depth].rdn; data.mv_size = stack[depth].len; rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } data.mv_data = stack[depth].rdn; ptr = (unsigned char *)data.mv_data + data.mv_size; memcpy(ptr, &num[depth], sizeof(ID)); data.mv_size += sizeof(ID); rc = mdb_cursor_del(mc, 0); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_put(mc, &key, &data, 0); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } count++; #if 1 if (enable_meter) lutil_meter_update(&meter, count, 0); #else { int len; ptr = data.mv_data; len = (ptr[0] & 0x7f) << 8 | ptr[1]; printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2); } #endif writes++; if (writes == 1000) { mdb_cursor_close(mc); rc = mdb_txn_commit(mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } writes = 0; } depth--; rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); if (rc == 0) goto down; rc = 0; if (depth) goto pop; else break; } leave: mdb_cursor_close(mc); if (mt) { int r2; r2 = mdb_txn_commit(mt); if (r2) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n", mdb_strerror(r2), r2, 0 ); if (!rc) rc = r2; } } ch_free(num); if (enable_meter) { lutil_meter_update(&meter, count, 1); lutil_meter_close(&meter); } return rc; }
static int mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic ) { mdb_tool_idl_cache_entry *ice; MDB_val key, data[2]; int i, rc; ID id, nid; /* Freshly allocated, ignore it */ if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) { return 0; } key.mv_data = ic->kstr.bv_val; key.mv_size = ic->kstr.bv_len; if ( ic->count > MDB_IDL_DB_SIZE ) { while ( ic->flags & WAS_FOUND ) { rc = mdb_cursor_get( mc, &key, data, MDB_SET ); if ( rc ) { /* FIXME: find out why this happens */ ic->flags = 0; break; } if ( ic->flags & WAS_RANGE ) { /* Skip lo */ rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); /* Get hi */ rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); /* Store range hi */ data[0].mv_data = &ic->last; rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT ); } else { /* Delete old data, replace with range */ ic->first = *(ID *)data[0].mv_data; mdb_cursor_del( mc, MDB_NODUPDATA ); } break; } if ( !(ic->flags & WAS_RANGE)) { /* range, didn't exist before */ nid = 0; data[0].mv_size = sizeof(ID); data[0].mv_data = &nid; rc = mdb_cursor_put( mc, &key, data, 0 ); if ( rc == 0 ) { data[0].mv_data = &ic->first; rc = mdb_cursor_put( mc, &key, data, 0 ); if ( rc == 0 ) { data[0].mv_data = &ic->last; rc = mdb_cursor_put( mc, &key, data, 0 ); } } if ( rc ) { rc = -1; } } } else { /* Normal write */ int n; data[0].mv_size = sizeof(ID); rc = 0; i = ic->offset; for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) { int end; if ( ice->next ) { end = IDBLOCK; } else { end = ic->count & (IDBLOCK-1); if ( !end ) end = IDBLOCK; } data[1].mv_size = end - i; data[0].mv_data = &ice->ids[i]; i = 0; rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE ); if ( rc ) { if ( rc == MDB_KEYEXIST ) { rc = 0; continue; } rc = -1; break; } } if ( ic->head ) { ic->tail->next = ai->ai_flist; ai->ai_flist = ic->head; } } ic->head = ai->ai_clist; ai->ai_clist = ic; return rc; }
FreqSchedulerError freq_scheduler_cursor_write(FreqScheduler *sch, MDB_cursor *cursor, uint64_t hash, float freq) { if (freq <= 0) return 0; ScheduleKey sk = { .score = 0, .hash = hash }; MDB_val key = { .mv_size = sizeof(sk), .mv_data = &sk, }; MDB_val val = { .mv_size = sizeof(float), .mv_data = &freq, }; int mdb_rc; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "adding page to schedule"); freq_scheduler_add_error(sch, mdb_strerror(mdb_rc)); } return sch->error->code; } FreqSchedulerError freq_scheduler_load_simple(FreqScheduler *sch, float freq_default, float freq_scale) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; HashInfoStream *st; if (hashinfo_stream_new(&st, sch->page_db) != 0) { error1 = "creating stream"; error2 = st? sch->page_db->error->message: "NULL"; goto on_error; } if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; StreamState ss; uint64_t hash; PageInfo *pi; while ((ss = hashinfo_stream_next(st, &hash, &pi)) == stream_state_next) { if ((pi->n_crawls > 0) && ((sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls)) && !page_info_is_seed(pi)){ float freq = freq_default; if (freq_scale > 0) { float rate = page_info_rate(pi); if (rate > 0) { freq = freq_scale * rate; } } if (freq_scheduler_cursor_write(sch, cursor, hash, freq) != 0) goto on_error; } page_info_delete(pi); } if (ss != stream_state_end) { error1 = "incorrect stream state"; error2 = 0; hashinfo_stream_delete(st); goto on_error; } hashinfo_stream_delete(st); if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_load_mmap(FreqScheduler *sch, MMapArray *freqs) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; if (txn_manager_expand( sch->txn_manager, 2*freqs->n_elements*freqs->element_size) != 0) { error1 = "resizing database"; error2 = sch->txn_manager->error->message; goto on_error; } if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; for (size_t i=0; i<freqs->n_elements; ++i) { PageFreq *f = mmap_array_idx(freqs, i); ScheduleKey sk = { .score = 1.0/f->freq, .hash = f->hash }; MDB_val key = { .mv_size = sizeof(sk), .mv_data = &sk, }; MDB_val val = { .mv_size = sizeof(float), .mv_data = &f->freq, }; int mdb_rc; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { error1 = "adding page to schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_request(FreqScheduler *sch, size_t max_requests, PageRequest **request) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; PageRequest *req = *request = page_request_new(max_requests); if (!req) { error1 = "allocating memory"; goto on_error; } int interrupt_requests = 0; while ((req->n_urls < max_requests) && !interrupt_requests) { MDB_val key; MDB_val val; ScheduleKey sk; float freq; int mdb_rc; int crawl = 0; switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, MDB_FIRST)) { case 0: // copy data before deleting cursor sk = *(ScheduleKey*)key.mv_data; freq = *(float*)val.mv_data; PageInfo *pi = 0; if (page_db_get_info(sch->page_db, sk.hash, &pi) != 0) { error1 = "retrieving PageInfo from PageDB"; error2 = sch->page_db->error->message; goto on_error; } if (pi) { if (sch->margin >= 0) { double elapsed = difftime(time(0), 0) - pi->last_crawl; if (elapsed < 1.0/(freq*(1.0 + sch->margin))) interrupt_requests = 1; } crawl = (sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls); } if (!interrupt_requests) { if ((mdb_rc = mdb_cursor_del(cursor, 0)) != 0) { error1 = "deleting head of schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } if (crawl) { if (page_request_add_url(req, pi->url) != 0) { error1 = "adding url to request"; goto on_error; } sk.score += 1.0/freq; val.mv_data = &freq; key.mv_data = &sk; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { error1 = "moving element inside schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } } page_info_delete(pi); break; case MDB_NOTFOUND: // no more pages left interrupt_requests = 1; break; default: error1 = "getting head of schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_add(FreqScheduler *sch, const CrawledPage *page) { if (page_db_add(sch->page_db, page, 0) != 0) { freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "adding crawled page"); freq_scheduler_add_error(sch, sch->page_db->error->message); } return sch->error->code; } void freq_scheduler_delete(FreqScheduler *sch) { mdb_env_close(sch->txn_manager->env); (void)txn_manager_delete(sch->txn_manager); if (!sch->persist) { char *data = build_path(sch->path, "data.mdb"); char *lock = build_path(sch->path, "lock.mdb"); remove(data); remove(lock); free(data); free(lock); remove(sch->path); } free(sch->path); error_delete(sch->error); free(sch); } FreqSchedulerError freq_scheduler_dump(FreqScheduler *sch, FILE *output) { MDB_cursor *cursor; if (freq_scheduler_cursor_open(sch, &cursor) != 0) return sch->error->code; int end = 0; MDB_cursor_op cursor_op = MDB_FIRST; do { int mdb_rc; MDB_val key; MDB_val val; ScheduleKey *key_data; float *val_data; switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, cursor_op)) { case 0: key_data = (ScheduleKey*)key.mv_data; val_data = (float*)val.mv_data; fprintf(output, "%.2e %016"PRIx64" %.2e\n", key_data->score, key_data->hash, *val_data); break; case MDB_NOTFOUND: end = 1; break; default: freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "iterating over database"); freq_scheduler_add_error(sch, mdb_strerror(mdb_rc)); end = 1; break; } cursor_op = MDB_NEXT; } while (!end); freq_scheduler_cursor_abort(sch, cursor); return sch->error->code; }
int cursor_del(cursor_t cursor, unsigned int flags){ int ret = cursor->prev ? mdb_cursor_del(cursor->cursor, flags) : MDB_BAD_TXN; if(MDB_SUCCESS == ret) cursor->txn->updated = 1; return ret; }
int mdb_idl_delete_keys( BackendDB *be, MDB_cursor *cursor, struct berval *keys, ID id ) { int rc = 0, k; MDB_val key, data; ID lo, hi, tmp, *i; char *err; #ifndef MISALIGNED_OK int kbuf[2]; #endif { char buf[16]; Debug( LDAP_DEBUG_ARGS, "mdb_idl_delete_keys: %lx %s\n", (long) id, mdb_show_key( buf, keys->bv_val, keys->bv_len ), 0 ); } assert( id != NOID ); #ifndef MISALIGNED_OK if (keys[0].bv_len & ALIGNER) kbuf[1] = 0; #endif for ( k=0; keys[k].bv_val; k++) { /* Fetch the first data item for this key, to see if it * exists and if it's a range. */ #ifndef MISALIGNED_OK if (keys[k].bv_len & ALIGNER) { key.mv_size = sizeof(kbuf); key.mv_data = kbuf; memcpy(key.mv_data, keys[k].bv_val, keys[k].bv_len); } else #endif { key.mv_size = keys[k].bv_len; key.mv_data = keys[k].bv_val; } rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); err = "c_get"; if ( rc == 0 ) { memcpy( &tmp, data.mv_data, sizeof(ID) ); i = data.mv_data; if ( tmp != 0 ) { /* Not a range, just delete it */ data.mv_data = &id; rc = mdb_cursor_get( cursor, &key, &data, MDB_GET_BOTH ); if ( rc != 0 ) { err = "c_get id"; goto fail; } rc = mdb_cursor_del( cursor, 0 ); if ( rc != 0 ) { err = "c_del id"; goto fail; } } else { /* It's a range, see if we need to rewrite * the boundaries */ lo = i[1]; hi = i[2]; if ( id == lo || id == hi ) { ID lo2 = lo, hi2 = hi; if ( id == lo ) { lo2++; } else if ( id == hi ) { hi2--; } if ( lo2 >= hi2 ) { /* The range has collapsed... */ rc = mdb_cursor_del( cursor, MDB_NODUPDATA ); if ( rc != 0 ) { err = "c_del dup"; goto fail; } } else { /* position on lo */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); if ( id == lo ) data.mv_data = &lo2; else { /* position on hi */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); data.mv_data = &hi2; } /* Replace the current lo/hi */ data.mv_size = sizeof(ID); rc = mdb_cursor_put( cursor, &key, &data, MDB_CURRENT ); if ( rc != 0 ) { err = "c_put lo/hi"; goto fail; } } } } } else { /* initial c_get failed, nothing was done */ fail: if ( rc == MDB_NOTFOUND ) rc = 0; if ( rc ) { Debug( LDAP_DEBUG_ANY, "=> mdb_idl_delete_key: " "%s failed: %s (%d)\n", err, mdb_strerror(rc), rc ); break; } } } return rc; }
int mdb_idl_insert_keys( BackendDB *be, MDB_cursor *cursor, struct berval *keys, ID id ) { struct mdb_info *mdb = be->be_private; MDB_val key, data; ID lo, hi, *i; char *err; int rc = 0, k; unsigned int flag = MDB_NODUPDATA; #ifndef MISALIGNED_OK int kbuf[2]; #endif { char buf[16]; Debug( LDAP_DEBUG_ARGS, "mdb_idl_insert_keys: %lx %s\n", (long) id, mdb_show_key( buf, keys->bv_val, keys->bv_len ), 0 ); } assert( id != NOID ); #ifndef MISALIGNED_OK if (keys[0].bv_len & ALIGNER) kbuf[1] = 0; #endif for ( k=0; keys[k].bv_val; k++ ) { /* Fetch the first data item for this key, to see if it * exists and if it's a range. */ #ifndef MISALIGNED_OK if (keys[k].bv_len & ALIGNER) { key.mv_size = sizeof(kbuf); key.mv_data = kbuf; memcpy(key.mv_data, keys[k].bv_val, keys[k].bv_len); } else #endif { key.mv_size = keys[k].bv_len; key.mv_data = keys[k].bv_val; } rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); err = "c_get"; if ( rc == 0 ) { i = data.mv_data; memcpy(&lo, data.mv_data, sizeof(ID)); if ( lo != 0 ) { /* not a range, count the number of items */ size_t count; rc = mdb_cursor_count( cursor, &count ); if ( rc != 0 ) { err = "c_count"; goto fail; } if ( count >= MDB_IDL_DB_MAX ) { /* No room, convert to a range */ lo = *i; rc = mdb_cursor_get( cursor, &key, &data, MDB_LAST_DUP ); if ( rc != 0 && rc != MDB_NOTFOUND ) { err = "c_get last_dup"; goto fail; } i = data.mv_data; hi = *i; /* Update hi/lo if needed */ if ( id < lo ) { lo = id; } else if ( id > hi ) { hi = id; } /* delete the old key */ rc = mdb_cursor_del( cursor, MDB_NODUPDATA ); if ( rc != 0 ) { err = "c_del dups"; goto fail; } /* Store the range */ data.mv_size = sizeof(ID); data.mv_data = &id; id = 0; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put range"; goto fail; } id = lo; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put lo"; goto fail; } id = hi; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put hi"; goto fail; } } else { /* There's room, just store it */ if (id == mdb->mi_nextid) flag |= MDB_APPENDDUP; goto put1; } } else { /* It's a range, see if we need to rewrite * the boundaries */ lo = i[1]; hi = i[2]; if ( id < lo || id > hi ) { /* position on lo */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); if ( rc != 0 ) { err = "c_get lo"; goto fail; } if ( id > hi ) { /* position on hi */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); if ( rc != 0 ) { err = "c_get hi"; goto fail; } } data.mv_size = sizeof(ID); data.mv_data = &id; /* Replace the current lo/hi */ rc = mdb_cursor_put( cursor, &key, &data, MDB_CURRENT ); if ( rc != 0 ) { err = "c_put lo/hi"; goto fail; } } } } else if ( rc == MDB_NOTFOUND ) { flag &= ~MDB_APPENDDUP; put1: data.mv_data = &id; data.mv_size = sizeof(ID); rc = mdb_cursor_put( cursor, &key, &data, flag ); /* Don't worry if it's already there */ if ( rc == MDB_KEYEXIST ) rc = 0; if ( rc ) { err = "c_put id"; goto fail; } } else { /* initial c_get failed, nothing was done */ fail: Debug( LDAP_DEBUG_ANY, "=> mdb_idl_insert_keys: " "%s failed: %s (%d)\n", err, mdb_strerror(rc), rc ); break; } } return rc; }