int mdb_dn2id_children( Operation *op, MDB_txn *txn, Entry *e ) { struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; MDB_dbi dbi = mdb->mi_dn2id; MDB_val key, data; MDB_cursor *cursor; int rc; ID id; key.mv_size = sizeof(ID); key.mv_data = &id; id = e->e_id; rc = mdb_cursor_open( txn, dbi, &cursor ); if ( rc ) return rc; rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); if ( rc == 0 ) { size_t dkids; rc = mdb_cursor_count( cursor, &dkids ); if ( rc == 0 ) { if ( dkids < 2 ) rc = MDB_NOTFOUND; } } mdb_cursor_close( cursor ); return rc; }
CAMLprim value caml_mdb_cursor_count(value cursor){ CAMLparam1(cursor); size_t count; if(mdb_cursor_count((MDB_cursor*)cursor, &count)){ caml_failwith("error in mdb_cursor_count"); } CAMLreturn(Val_int(count)); }
// erase hash, return count erased // note: hashdb does not use erase. size_t erase(const std::string& binary_hash) { // get context lmdb_context_t context(env, true, true); context.open(); // set key lmdb_helper::point_to_string(binary_hash, context.key); // truncate key if truncation is used and binary_hash is longer if (hash_truncation != 0 && context.key.mv_size > hash_truncation) { context.key.mv_size = hash_truncation; } // set the cursor to this key int rc = mdb_cursor_get(context.cursor, &context.key, NULL, MDB_SET_RANGE); size_t key_count; if (rc == 0) { // DB has key so look up the key count rc = mdb_cursor_count(context.cursor, &key_count); if (rc != 0) { std::cerr << "LMDB erase count error: " << mdb_strerror(rc) << "\n"; assert(0); } } else if (rc == MDB_NOTFOUND) { // DB does not have key key_count = 0; } else { std::cerr << "LMDB erase cursor get error: " << mdb_strerror(rc) << "\n"; assert(0); } if (key_count > 0) { // delete rc = mdb_del(context.txn, context.dbi, &context.key, NULL); if (rc != 0) { std::cerr << "LMDB erase delete error: " << mdb_strerror(rc) << "\n"; assert(0); } } // close context context.close(); return key_count; }
// count of entries with this hash value size_t find_count(const std::string& binary_hash) const { // get context lmdb_context_t context(env, false, true); context.open(); // set key lmdb_helper::point_to_string(binary_hash, context.key); // truncate key if truncation is used and binary_hash is longer if (hash_truncation != 0 && context.key.mv_size > hash_truncation) { context.key.mv_size = hash_truncation; } // set the cursor to this key int rc = mdb_cursor_get(context.cursor, &context.key, &context.data, MDB_SET_KEY); size_t key_count = 0; if (rc == 0) { rc = mdb_cursor_count(context.cursor, &key_count); if (rc != 0) { std::cerr << "LMDB count error: " << mdb_strerror(rc) << "\n"; assert(0); } } else if (rc == MDB_NOTFOUND) { // fine, key count is zero } else { std::cerr << "LMDB get error: " << mdb_strerror(rc) << "\n"; assert(0); } // close context context.close(); return key_count; }
int mdb_search( Operation *op, SlapReply *rs ) { struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; ID id, cursor, nsubs, ncand, cscope; ID lastid = NOID; ID candidates[MDB_IDL_UM_SIZE]; ID iscopes[MDB_IDL_DB_SIZE]; ID2 *scopes; void *stack; Entry *e = NULL, *base = NULL; Entry *matched = NULL; AttributeName *attrs; slap_mask_t mask; time_t stoptime; int manageDSAit; int tentries = 0; IdScopes isc; MDB_cursor *mci, *mcd; ww_ctx wwctx; slap_callback cb = { 0 }; mdb_op_info opinfo = {{{0}}}, *moi = &opinfo; MDB_txn *ltid = NULL; Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_search) "\n", 0, 0, 0); attrs = op->oq_search.rs_attrs; manageDSAit = get_manageDSAit( op ); rs->sr_err = mdb_opinfo_get( op, mdb, 1, &moi ); switch(rs->sr_err) { case 0: break; default: send_ldap_error( op, rs, LDAP_OTHER, "internal error" ); return rs->sr_err; } ltid = moi->moi_txn; rs->sr_err = mdb_cursor_open( ltid, mdb->mi_id2entry, &mci ); if ( rs->sr_err ) { send_ldap_error( op, rs, LDAP_OTHER, "internal error" ); return rs->sr_err; } rs->sr_err = mdb_cursor_open( ltid, mdb->mi_dn2id, &mcd ); if ( rs->sr_err ) { mdb_cursor_close( mci ); send_ldap_error( op, rs, LDAP_OTHER, "internal error" ); return rs->sr_err; } scopes = scope_chunk_get( op ); stack = search_stack( op ); isc.mt = ltid; isc.mc = mcd; isc.scopes = scopes; isc.oscope = op->ors_scope; isc.sctmp = stack; if ( op->ors_deref & LDAP_DEREF_FINDING ) { MDB_IDL_ZERO(candidates); } dn2entry_retry: /* get entry with reader lock */ rs->sr_err = mdb_dn2entry( op, ltid, mcd, &op->o_req_ndn, &e, &nsubs, 1 ); switch(rs->sr_err) { case MDB_NOTFOUND: matched = e; e = NULL; break; case 0: break; case LDAP_BUSY: send_ldap_error( op, rs, LDAP_BUSY, "ldap server busy" ); goto done; default: send_ldap_error( op, rs, LDAP_OTHER, "internal error" ); goto done; } if ( op->ors_deref & LDAP_DEREF_FINDING ) { if ( matched && is_entry_alias( matched )) { struct berval stub; stub.bv_val = op->o_req_ndn.bv_val; stub.bv_len = op->o_req_ndn.bv_len - matched->e_nname.bv_len - 1; e = deref_base( op, rs, matched, &matched, ltid, candidates, NULL ); if ( e ) { build_new_dn( &op->o_req_ndn, &e->e_nname, &stub, op->o_tmpmemctx ); mdb_entry_return(op, e); matched = NULL; goto dn2entry_retry; } } else if ( e && is_entry_alias( e )) { e = deref_base( op, rs, e, &matched, ltid, candidates, NULL ); } } if ( e == NULL ) { struct berval matched_dn = BER_BVNULL; if ( matched != NULL ) { BerVarray erefs = NULL; /* return referral only if "disclose" * is granted on the object */ if ( ! access_allowed( op, matched, slap_schema.si_ad_entry, NULL, ACL_DISCLOSE, NULL ) ) { rs->sr_err = LDAP_NO_SUCH_OBJECT; } else { ber_dupbv( &matched_dn, &matched->e_name ); erefs = is_entry_referral( matched ) ? get_entry_referrals( op, matched ) : NULL; if ( rs->sr_err == MDB_NOTFOUND ) rs->sr_err = LDAP_REFERRAL; rs->sr_matched = matched_dn.bv_val; } mdb_entry_return(op, matched); matched = NULL; if ( erefs ) { rs->sr_ref = referral_rewrite( erefs, &matched_dn, &op->o_req_dn, op->oq_search.rs_scope ); ber_bvarray_free( erefs ); } } else { rs->sr_ref = referral_rewrite( default_referral, NULL, &op->o_req_dn, op->oq_search.rs_scope ); rs->sr_err = rs->sr_ref != NULL ? LDAP_REFERRAL : LDAP_NO_SUCH_OBJECT; } send_ldap_result( op, rs ); if ( rs->sr_ref ) { ber_bvarray_free( rs->sr_ref ); rs->sr_ref = NULL; } if ( !BER_BVISNULL( &matched_dn ) ) { ber_memfree( matched_dn.bv_val ); rs->sr_matched = NULL; } goto done; } /* NOTE: __NEW__ "search" access is required * on searchBase object */ if ( ! access_allowed_mask( op, e, slap_schema.si_ad_entry, NULL, ACL_SEARCH, NULL, &mask ) ) { if ( !ACL_GRANT( mask, ACL_DISCLOSE ) ) { rs->sr_err = LDAP_NO_SUCH_OBJECT; } else { rs->sr_err = LDAP_INSUFFICIENT_ACCESS; } mdb_entry_return( op,e); send_ldap_result( op, rs ); goto done; } if ( !manageDSAit && is_entry_referral( e ) ) { /* entry is a referral */ struct berval matched_dn = BER_BVNULL; BerVarray erefs = NULL; ber_dupbv( &matched_dn, &e->e_name ); erefs = get_entry_referrals( op, e ); rs->sr_err = LDAP_REFERRAL; mdb_entry_return( op, e ); e = NULL; if ( erefs ) { rs->sr_ref = referral_rewrite( erefs, &matched_dn, &op->o_req_dn, op->oq_search.rs_scope ); ber_bvarray_free( erefs ); if ( !rs->sr_ref ) { rs->sr_text = "bad_referral object"; } } Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_search) ": entry is referral\n", 0, 0, 0 ); rs->sr_matched = matched_dn.bv_val; send_ldap_result( op, rs ); ber_bvarray_free( rs->sr_ref ); rs->sr_ref = NULL; ber_memfree( matched_dn.bv_val ); rs->sr_matched = NULL; goto done; } if ( get_assert( op ) && ( test_filter( op, e, get_assertion( op )) != LDAP_COMPARE_TRUE )) { rs->sr_err = LDAP_ASSERTION_FAILED; mdb_entry_return( op,e); send_ldap_result( op, rs ); goto done; } /* compute it anyway; root does not use it */ stoptime = op->o_time + op->ors_tlimit; base = e; e = NULL; /* select candidates */ if ( op->oq_search.rs_scope == LDAP_SCOPE_BASE ) { rs->sr_err = base_candidate( op->o_bd, base, candidates ); scopes[0].mid = 0; ncand = 1; } else { if ( op->ors_scope == LDAP_SCOPE_ONELEVEL ) { size_t nkids; MDB_val key, data; key.mv_data = &base->e_id; key.mv_size = sizeof( ID ); mdb_cursor_get( mcd, &key, &data, MDB_SET ); mdb_cursor_count( mcd, &nkids ); nsubs = nkids - 1; } else if ( !base->e_id ) { /* we don't maintain nsubs for entryID 0. * just grab entry count from id2entry stat */ MDB_stat ms; mdb_stat( ltid, mdb->mi_id2entry, &ms ); nsubs = ms.ms_entries; } MDB_IDL_ZERO( candidates ); scopes[0].mid = 1; scopes[1].mid = base->e_id; scopes[1].mval.mv_data = NULL; rs->sr_err = search_candidates( op, rs, base, &isc, mci, candidates, stack ); ncand = MDB_IDL_N( candidates ); if ( !base->e_id || ncand == NOID ) { /* grab entry count from id2entry stat */ MDB_stat ms; mdb_stat( ltid, mdb->mi_id2entry, &ms ); if ( !base->e_id ) nsubs = ms.ms_entries; if ( ncand == NOID ) ncand = ms.ms_entries; } } /* start cursor at beginning of candidates. */ cursor = 0; if ( candidates[0] == 0 ) { Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_search) ": no candidates\n", 0, 0, 0 ); goto nochange; } /* if not root and candidates exceed to-be-checked entries, abort */ if ( op->ors_limit /* isroot == FALSE */ && op->ors_limit->lms_s_unchecked != -1 && ncand > (unsigned) op->ors_limit->lms_s_unchecked ) { rs->sr_err = LDAP_ADMINLIMIT_EXCEEDED; send_ldap_result( op, rs ); rs->sr_err = LDAP_SUCCESS; goto done; } if ( op->ors_limit == NULL /* isroot == TRUE */ || !op->ors_limit->lms_s_pr_hide ) { tentries = ncand; } wwctx.flag = 0; /* If we're running in our own read txn */ if ( moi == &opinfo ) { cb.sc_writewait = mdb_writewait; cb.sc_private = &wwctx; wwctx.txn = ltid; wwctx.mcd = NULL; cb.sc_next = op->o_callback; op->o_callback = &cb; } if ( get_pagedresults( op ) > SLAP_CONTROL_IGNORED ) { PagedResultsState *ps = op->o_pagedresults_state; /* deferred cookie parsing */ rs->sr_err = parse_paged_cookie( op, rs ); if ( rs->sr_err != LDAP_SUCCESS ) { send_ldap_result( op, rs ); goto done; } cursor = (ID) ps->ps_cookie; if ( cursor && ps->ps_size == 0 ) { rs->sr_err = LDAP_SUCCESS; rs->sr_text = "search abandoned by pagedResult size=0"; send_ldap_result( op, rs ); goto done; } id = mdb_idl_first( candidates, &cursor ); if ( id == NOID ) { Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_search) ": no paged results candidates\n", 0, 0, 0 ); send_paged_response( op, rs, &lastid, 0 ); rs->sr_err = LDAP_OTHER; goto done; } if ( id == (ID)ps->ps_cookie ) id = mdb_idl_next( candidates, &cursor ); nsubs = ncand; /* always bypass scope'd search */ goto loop_begin; } if ( nsubs < ncand ) { int rc; /* Do scope-based search */ /* if any alias scopes were set, save them */ if (scopes[0].mid > 1) { cursor = 1; for (cscope = 1; cscope <= scopes[0].mid; cscope++) { /* Ignore the original base */ if (scopes[cscope].mid == base->e_id) continue; iscopes[cursor++] = scopes[cscope].mid; } iscopes[0] = scopes[0].mid - 1; } else { iscopes[0] = 0; } wwctx.mcd = mcd; isc.id = base->e_id; isc.numrdns = 0; rc = mdb_dn2id_walk( op, &isc ); if ( rc ) id = NOID; else id = isc.id; cscope = 0; } else { id = mdb_idl_first( candidates, &cursor ); } while (id != NOID) { int scopeok; MDB_val edata; loop_begin: /* check for abandon */ if ( op->o_abandon ) { rs->sr_err = SLAPD_ABANDON; send_ldap_result( op, rs ); goto done; } /* mostly needed by internal searches, * e.g. related to syncrepl, for whom * abandon does not get set... */ if ( slapd_shutdown ) { rs->sr_err = LDAP_UNAVAILABLE; send_ldap_disconnect( op, rs ); goto done; } /* check time limit */ if ( op->ors_tlimit != SLAP_NO_LIMIT && slap_get_time() > stoptime ) { rs->sr_err = LDAP_TIMELIMIT_EXCEEDED; rs->sr_ref = rs->sr_v2ref; send_ldap_result( op, rs ); rs->sr_err = LDAP_SUCCESS; goto done; } if ( nsubs < ncand ) { unsigned i; /* Is this entry in the candidate list? */ scopeok = 0; if (MDB_IDL_IS_RANGE( candidates )) { if ( id >= MDB_IDL_RANGE_FIRST( candidates ) && id <= MDB_IDL_RANGE_LAST( candidates )) scopeok = 1; } else { i = mdb_idl_search( candidates, id ); if ( candidates[i] == id ) scopeok = 1; } if ( scopeok ) goto scopeok; goto loop_continue; } /* Does this candidate actually satisfy the search scope? */ scopeok = 0; isc.numrdns = 0; switch( op->ors_scope ) { case LDAP_SCOPE_BASE: /* This is always true, yes? */ if ( id == base->e_id ) scopeok = 1; break; #ifdef LDAP_SCOPE_CHILDREN case LDAP_SCOPE_CHILDREN: if ( id == base->e_id ) break; /* Fall-thru */ #endif case LDAP_SCOPE_SUBTREE: if ( id == base->e_id ) { scopeok = 1; break; } /* Fall-thru */ case LDAP_SCOPE_ONELEVEL: isc.id = id; isc.nscope = 0; rs->sr_err = mdb_idscopes( op, &isc ); if ( rs->sr_err == MDB_SUCCESS ) { if ( isc.nscope ) scopeok = 1; } else { if ( rs->sr_err == MDB_NOTFOUND ) goto notfound; } break; } /* Not in scope, ignore it */ if ( !scopeok ) { Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_search) ": %ld scope not okay\n", (long) id, 0, 0 ); goto loop_continue; } scopeok: if ( id == base->e_id ) { e = base; } else { /* get the entry */ rs->sr_err = mdb_id2edata( op, mci, id, &edata ); if ( rs->sr_err == MDB_NOTFOUND ) { notfound: if( nsubs < ncand ) goto loop_continue; if( !MDB_IDL_IS_RANGE(candidates) ) { /* only complain for non-range IDLs */ Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_search) ": candidate %ld not found\n", (long) id, 0, 0 ); } else { /* get the next ID from the DB */ rs->sr_err = mdb_get_nextid( mci, &cursor ); if ( rs->sr_err == MDB_NOTFOUND ) { break; } if ( rs->sr_err ) { rs->sr_err = LDAP_OTHER; rs->sr_text = "internal error in get_nextid"; send_ldap_result( op, rs ); goto done; } cursor--; } goto loop_continue; } else if ( rs->sr_err ) { rs->sr_err = LDAP_OTHER; rs->sr_text = "internal error in mdb_id2edata"; send_ldap_result( op, rs ); goto done; } rs->sr_err = mdb_entry_decode( op, ltid, &edata, &e ); if ( rs->sr_err ) { rs->sr_err = LDAP_OTHER; rs->sr_text = "internal error in mdb_entry_decode"; send_ldap_result( op, rs ); goto done; } e->e_id = id; e->e_name.bv_val = NULL; e->e_nname.bv_val = NULL; } if ( is_entry_subentry( e ) ) { if( op->oq_search.rs_scope != LDAP_SCOPE_BASE ) { if(!get_subentries_visibility( op )) { /* only subentries are visible */ goto loop_continue; } } else if ( get_subentries( op ) && !get_subentries_visibility( op )) { /* only subentries are visible */ goto loop_continue; } } else if ( get_subentries_visibility( op )) { /* only subentries are visible */ goto loop_continue; } /* aliases were already dereferenced in candidate list */ if ( op->ors_deref & LDAP_DEREF_SEARCHING ) { /* but if the search base is an alias, and we didn't * deref it when finding, return it. */ if ( is_entry_alias(e) && ((op->ors_deref & LDAP_DEREF_FINDING) || e != base )) { goto loop_continue; } } if ( !manageDSAit && is_entry_glue( e )) { goto loop_continue; } if (e != base) { struct berval pdn, pndn; char *d, *n; int i; /* child of base, just append RDNs to base->e_name */ if ( nsubs < ncand || isc.scopes[isc.nscope].mid == base->e_id ) { pdn = base->e_name; pndn = base->e_nname; } else { mdb_id2name( op, ltid, &isc.mc, scopes[isc.nscope].mid, &pdn, &pndn ); } e->e_name.bv_len = pdn.bv_len; e->e_nname.bv_len = pndn.bv_len; for (i=0; i<isc.numrdns; i++) { e->e_name.bv_len += isc.rdns[i].bv_len + 1; e->e_nname.bv_len += isc.nrdns[i].bv_len + 1; } e->e_name.bv_val = op->o_tmpalloc(e->e_name.bv_len + 1, op->o_tmpmemctx); e->e_nname.bv_val = op->o_tmpalloc(e->e_nname.bv_len + 1, op->o_tmpmemctx); d = e->e_name.bv_val; n = e->e_nname.bv_val; if (nsubs < ncand) { /* RDNs are in top-down order */ for (i=isc.numrdns-1; i>=0; i--) { memcpy(d, isc.rdns[i].bv_val, isc.rdns[i].bv_len); d += isc.rdns[i].bv_len; *d++ = ','; memcpy(n, isc.nrdns[i].bv_val, isc.nrdns[i].bv_len); n += isc.nrdns[i].bv_len; *n++ = ','; } } else { /* RDNs are in bottom-up order */ for (i=0; i<isc.numrdns; i++) { memcpy(d, isc.rdns[i].bv_val, isc.rdns[i].bv_len); d += isc.rdns[i].bv_len; *d++ = ','; memcpy(n, isc.nrdns[i].bv_val, isc.nrdns[i].bv_len); n += isc.nrdns[i].bv_len; *n++ = ','; } } if (pdn.bv_len) { memcpy(d, pdn.bv_val, pdn.bv_len+1); memcpy(n, pndn.bv_val, pndn.bv_len+1); } else { *--d = '\0'; *--n = '\0'; e->e_name.bv_len--; e->e_nname.bv_len--; } if (pndn.bv_val != base->e_nname.bv_val) { op->o_tmpfree(pndn.bv_val, op->o_tmpmemctx); op->o_tmpfree(pdn.bv_val, op->o_tmpmemctx); } } /* * if it's a referral, add it to the list of referrals. only do * this for non-base searches, and don't check the filter * explicitly here since it's only a candidate anyway. */ if ( !manageDSAit && op->oq_search.rs_scope != LDAP_SCOPE_BASE && is_entry_referral( e ) ) { BerVarray erefs = get_entry_referrals( op, e ); rs->sr_ref = referral_rewrite( erefs, &e->e_name, NULL, op->oq_search.rs_scope == LDAP_SCOPE_ONELEVEL ? LDAP_SCOPE_BASE : LDAP_SCOPE_SUBTREE ); rs->sr_entry = e; rs->sr_flags = 0; send_search_reference( op, rs ); if (e != base) mdb_entry_return( op, e ); rs->sr_entry = NULL; e = NULL; ber_bvarray_free( rs->sr_ref ); ber_bvarray_free( erefs ); rs->sr_ref = NULL; if ( wwctx.flag ) { rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd ); if ( rs->sr_err ) { send_ldap_result( op, rs ); goto done; } } goto loop_continue; } /* if it matches the filter and scope, send it */ rs->sr_err = test_filter( op, e, op->oq_search.rs_filter ); if ( rs->sr_err == LDAP_COMPARE_TRUE ) { /* check size limit */ if ( get_pagedresults(op) > SLAP_CONTROL_IGNORED ) { if ( rs->sr_nentries >= ((PagedResultsState *)op->o_pagedresults_state)->ps_size ) { mdb_entry_return( op, e ); e = NULL; send_paged_response( op, rs, &lastid, tentries ); goto done; } lastid = id; } if (e) { /* safe default */ rs->sr_attrs = op->oq_search.rs_attrs; rs->sr_operational_attrs = NULL; rs->sr_ctrls = NULL; rs->sr_entry = e; RS_ASSERT( e->e_private != NULL ); rs->sr_flags = 0; rs->sr_err = LDAP_SUCCESS; rs->sr_err = send_search_entry( op, rs ); rs->sr_attrs = NULL; rs->sr_entry = NULL; if (e != base) mdb_entry_return( op, e ); e = NULL; switch ( rs->sr_err ) { case LDAP_SUCCESS: /* entry sent ok */ break; default: /* entry not sent */ break; case LDAP_BUSY: send_ldap_result( op, rs ); goto done; case LDAP_UNAVAILABLE: case LDAP_SIZELIMIT_EXCEEDED: if ( rs->sr_err == LDAP_SIZELIMIT_EXCEEDED ) { rs->sr_ref = rs->sr_v2ref; send_ldap_result( op, rs ); rs->sr_err = LDAP_SUCCESS; } else { rs->sr_err = LDAP_OTHER; } goto done; } if ( wwctx.flag ) { rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd ); if ( rs->sr_err ) { send_ldap_result( op, rs ); goto done; } } } } else { Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(mdb_search) ": %ld does not match filter\n", (long) id, 0, 0 ); } loop_continue: if( e != NULL ) { if ( e != base ) mdb_entry_return( op, e ); RS_ASSERT( rs->sr_entry == NULL ); e = NULL; rs->sr_entry = NULL; } if ( nsubs < ncand ) { int rc = mdb_dn2id_walk( op, &isc ); if (rc) { id = NOID; /* We got to the end of a subtree. If there are any * alias scopes left, search them too. */ while (iscopes[0] && cscope < iscopes[0]) { cscope++; isc.id = iscopes[cscope]; if ( base ) mdb_entry_return( op, base ); rs->sr_err = mdb_id2entry(op, mci, isc.id, &base); if ( !rs->sr_err ) { mdb_id2name( op, ltid, &isc.mc, isc.id, &base->e_name, &base->e_nname ); isc.numrdns = 0; if (isc.oscope == LDAP_SCOPE_ONELEVEL) isc.oscope = LDAP_SCOPE_BASE; rc = mdb_dn2id_walk( op, &isc ); if ( !rc ) { id = isc.id; break; } } } } else id = isc.id; } else { id = mdb_idl_next( candidates, &cursor ); } } nochange: rs->sr_ctrls = NULL; rs->sr_ref = rs->sr_v2ref; rs->sr_err = (rs->sr_v2ref == NULL) ? LDAP_SUCCESS : LDAP_REFERRAL; rs->sr_rspoid = NULL; if ( get_pagedresults(op) > SLAP_CONTROL_IGNORED ) { send_paged_response( op, rs, NULL, 0 ); } else { send_ldap_result( op, rs ); } rs->sr_err = LDAP_SUCCESS; done: if ( cb.sc_private ) { /* remove our writewait callback */ slap_callback **scp = &op->o_callback; while ( *scp ) { if ( *scp == &cb ) { *scp = cb.sc_next; cb.sc_private = NULL; break; } } } mdb_cursor_close( mcd ); mdb_cursor_close( mci ); if ( moi == &opinfo ) { mdb_txn_reset( moi->moi_txn ); LDAP_SLIST_REMOVE( &op->o_extra, &moi->moi_oe, OpExtra, oe_next ); } else { moi->moi_ref--; } if( rs->sr_v2ref ) { ber_bvarray_free( rs->sr_v2ref ); rs->sr_v2ref = NULL; } if (base) mdb_entry_return( op, base ); scope_chunk_ret( op, scopes ); return rs->sr_err; }
static int doundo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, u8 delPages) { MDB_val logKey, logVal; MDB_val pgKey, pgVal; u8 logKeyBuf[sizeof(u64)*3]; int logop, pgop, rc, mrc; mdbinf *mdb; // db_thread* const thr = enif_tsd_get(g_tsd_thread); #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif rc = SQLITE_OK; if (pWal->inProgressTerm == 0) return SQLITE_OK; #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif if (!mdb) return SQLITE_ERROR; logKey.mv_data = logKeyBuf; logKey.mv_size = sizeof(logKeyBuf); DBG("Undo"); // For every page here // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} // Delete from // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>} memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum,sizeof(u64)); if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS) { DBG("Key not found in log for undo, index=%llu, term=%llu, evnum=%llu", pWal->index, pWal->inProgressTerm, pWal->inProgressEvnum); return SQLITE_OK; } logop = MDB_FIRST_DUP; while ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS) { u32 pgno; u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; u64 term,evnum; memcpy(&pgno,logVal.mv_data,sizeof(u32)); if (delPages) { size_t ndupl; memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32)); pgKey.mv_data = pagesKeyBuf; pgKey.mv_size = sizeof(pagesKeyBuf); DBG("UNDO pgno=%d",pgno); pgop = MDB_FIRST_DUP; if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS) { DBG("Key not found in log for undo"); continue; } mdb_cursor_count(mdb->cursorPages,&ndupl); while (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,pgop) == MDB_SUCCESS) { u8 frag = *((u8*)pgVal.mv_data+sizeof(u64)*2); memcpy(&term, pgVal.mv_data, sizeof(u64)); memcpy(&evnum,(u8*)pgVal.mv_data+sizeof(u64),sizeof(u64)); DBG("progress term %lld, progress evnum %lld, curterm %lld, curnum %lld", pWal->inProgressTerm, pWal->inProgressEvnum, term, evnum); if (term >= pWal->inProgressTerm && evnum >= pWal->inProgressEvnum) { if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS) { DBG("Can not delete undo"); rc = SQLITE_ERROR; break; } if (frag == 0) pWal->allPages--; ndupl--; if (!ndupl) break; } pgop = MDB_NEXT_DUP; } pWal->inProgressTerm = pWal->inProgressEvnum = 0; storeinfo(pWal,0,0,NULL); thr->pagesChanged++; } if (xUndo) rc = xUndo(pUndoCtx, pgno); logop = MDB_NEXT_DUP; } // if (mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA) != MDB_SUCCESS) // { // DBG("Unable to cleanup key from logdb")); // } DBG("Undo done!"); return rc; }
// Delete all pages up to limitEvterm and limitEvnum static int checkpoint(Wal *pWal, u64 limitEvnum) { MDB_val logKey, logVal; u8 logKeyBuf[sizeof(u64)*3]; u64 evnum,evterm,aindex; mdbinf* mdb; // db_thread* const thr = enif_tsd_get(g_tsd_thread); #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif int logop, mrc = MDB_SUCCESS; // u8 somethingDeleted = 0; int allPagesDiff = 0; #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif if (!mdb) return SQLITE_ERROR; // if (pWal->inProgressTerm == 0) // return SQLITE_OK; DBG("checkpoint actor=%llu, fct=%llu, fcev=%llu, limitEvnum=%llu",pWal->index, pWal->firstCompleteTerm,pWal->firstCompleteEvnum,limitEvnum); while (pWal->firstCompleteEvnum < limitEvnum) { logKey.mv_data = logKeyBuf; logKey.mv_size = sizeof(logKeyBuf); memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->firstCompleteTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->firstCompleteEvnum,sizeof(u64)); if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS) { printf("Key not found in log for checkpoint %llu %llu\n", pWal->firstCompleteTerm, pWal->firstCompleteEvnum); return SQLITE_OK; } DBG("checkpoint evnum=%llu",pWal->firstCompleteEvnum); // For every page here // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} // Delete from // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>} logop = MDB_FIRST_DUP; while ((mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS) { u32 pgno; size_t ndupl; u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; MDB_val pgKey = {0,NULL}, pgVal = {0,NULL}; u64 pgnoLimitEvnum; logop = MDB_NEXT_DUP; memcpy(&pgno, logVal.mv_data,sizeof(u32)); DBG("checkpoint pgno=%u",pgno); memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32)); pgKey.mv_data = pagesKeyBuf; pgKey.mv_size = sizeof(pagesKeyBuf); if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS) { continue; } mdb_cursor_count(mdb->cursorPages,&ndupl); if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_LAST_DUP) == MDB_SUCCESS) memcpy(&pgnoLimitEvnum, (u8*)pgVal.mv_data+sizeof(u64),sizeof(u64)); else continue; pgnoLimitEvnum = pgnoLimitEvnum < limitEvnum ? pgnoLimitEvnum : limitEvnum; if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_FIRST_DUP) != MDB_SUCCESS) continue; do { u8 frag; MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL}; mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT); frag = *((u8*)pgDelVal.mv_data+sizeof(u64)*2); memcpy(&evterm, pgDelVal.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)pgDelVal.mv_data+sizeof(u64),sizeof(u64)); DBG("limit limitevnum %lld, curnum %lld, dupl %zu, frag=%d", limitEvnum, evnum, ndupl,(int)frag); if (evnum < pgnoLimitEvnum) { mrc = mdb_cursor_del(mdb->cursorPages,0); if (mrc != MDB_SUCCESS) { DBG("Unable to delete page on cursor!."); break; } // else // { // DBG("Deleted page!"); // // somethingDeleted = 1; // } if (frag == 0) allPagesDiff++; } ndupl--; if (!ndupl) break; mrc = mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_NEXT_DUP); } while (mrc == MDB_SUCCESS); } mrc = mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA); if (mrc != MDB_SUCCESS) { DBG("Can not delete log"); break; } // move forward if ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_NEXT_NODUP)) != MDB_SUCCESS) { DBG("Unable to move to next log %d",mrc); break; } // read next key data memcpy(&aindex, logKey.mv_data, sizeof(u64)); memcpy(&evterm, (u8*)logKey.mv_data + sizeof(u64), sizeof(u64)); memcpy(&evnum, (u8*)logKey.mv_data + sizeof(u64)*2, sizeof(u64)); if (aindex != pWal->index) { DBG("Reached another actor"); break; } pWal->firstCompleteTerm = evterm; pWal->firstCompleteEvnum = evnum; pWal->allPages -= allPagesDiff; allPagesDiff = 0; DBG("Checkpint fce now=%lld",(u64)evnum); } // no dirty pages, but will write info if (sqlite3WalFrames(pWal, SQLITE_DEFAULT_PAGE_SIZE, NULL, pWal->mxPage, 1, 0) == SQLITE_OK) return SQLITE_OK; else return SQLITE_ERROR; }
static int findframe(db_thread *thr, Wal *pWal, Pgno pgno, u32 *piRead, u64 limitTerm, u64 limitEvnum, u64 *outTerm, u64 *outEvnum) { MDB_val key, data; int rc; size_t ndupl = 0; u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; mdbinf *mdb; if (thr->pagesChanged) { #if ATOMIC mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); #endif } else mdb = &thr->mdb; track_time(7,thr); DBG("FIND FRAME pgno=%u, index=%llu, limitterm=%llu, limitevnum=%llu", pgno,pWal->index,limitTerm,limitEvnum); // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, // <<Evterm:64,Evnum:64,Counter,CompressedPage/binary>>} memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32)); key.mv_size = sizeof(pagesKeyBuf); key.mv_data = pagesKeyBuf; // u32 pgno2 = *(u32*)(key.mv_data+sizeof(u64)); // DBG("RUN %d",pgno2); rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY); if (rc == MDB_SUCCESS) { mdb_cursor_count(mdb->cursorPages,&ndupl); if (ndupl == 0) { *piRead = 0; return SQLITE_OK; } rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP); if (rc == MDB_SUCCESS) { while (1) { char frag1 = *(char*)(((char*)data.mv_data)+sizeof(u64)*2); int frag = frag1; u64 term, evnum; memcpy(&term, data.mv_data, sizeof(u64)); memcpy(&evnum, ((u8*)data.mv_data) + sizeof(u64), sizeof(u64)); if (term > limitTerm || evnum > limitEvnum) { rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP); if (rc == MDB_SUCCESS) continue; else { DBG("Cant move to prev dup, term=%llu, evnum=%llu," " limitterm=%llu, limitevnum=%llu",term,evnum,limitTerm,limitEvnum); *piRead = 0; break; } } if (outTerm != NULL) *outTerm = term; if (outEvnum != NULL) *outEvnum = evnum; DBG("Found page size=%ld, frags=%d",data.mv_size,(int)frag); thr->nResFrames = frag; thr->resFrames[frag--] = data; while (frag >= 0) { rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP); frag = *(((u8*)data.mv_data)+sizeof(u64)*2); // DBG("SUCCESS? %d frag=%d, size=%ld",pgno,frag,data.mv_size); thr->resFrames[frag--] = data; } *piRead = 1; break; } } else { DBG("Find page no last dup"); *piRead = 0; } } else if (rc == MDB_NOTFOUND) { DBG("Frame not found!"); *piRead = 0; } else { DBG("ERROR findframe: %d",rc); *piRead = 0; } return SQLITE_OK; }
/* Write a frame or frames to the log. */ int sqlite3WalFrames(Wal *pWal, int szPage, PgHdr *pList, Pgno nTruncate, int isCommit, int sync_flags) { PgHdr *p; MDB_val key, data; int rc; mdbinf* mdb; MDB_txn* txn; #if ATOMIC db_thread *thr = g_tsd_thread; db_connection* pCon = g_tsd_conn; #else db_thread* thr = enif_tsd_get(g_tsd_thread); db_connection* pCon = enif_tsd_get(g_tsd_conn); #endif #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif txn = mdb->txn; if (!mdb) return SQLITE_ERROR; key.mv_size = sizeof(u64); key.mv_data = (void*)&pWal->index; // Term/evnum must always be increasing if ((pWal->inProgressTerm > 0 && pWal->inProgressTerm < pWal->lastCompleteTerm) || (pWal->inProgressEvnum > 0 && pWal->inProgressEvnum < pWal->lastCompleteEvnum)) return SQLITE_ERROR; track_time(2,thr); // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Fragment,CompressedPage/binary>>} for(p=pList; p; p=p->pDirty) { u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; u8 pagesBuf[PAGE_BUFF_SIZE]; int full_size = 0; int page_size = LZ4_compress_default((char*)p->pData,(char*)pagesBuf+sizeof(u64)*2+1,szPage,sizeof(pagesBuf)); char fragment_index = 0; int skipped = 0; track_time(3,thr); DBG("Insert frame, actor=%lld, pgno=%u, " "term=%lld, evnum=%lld, commit=%d, truncate=%d, compressedsize=%d", pWal->index,p->pgno,pWal->inProgressTerm,pWal->inProgressEvnum, isCommit,nTruncate,page_size); if (pCon->doReplicate) { u8 hdr[sizeof(u64)*2+sizeof(u32)*2]; put8byte(hdr, pWal->inProgressTerm); put8byte(hdr+sizeof(u64), pWal->inProgressEvnum); put4byte(hdr+sizeof(u64)*2, p->pgno); if (p->pDirty) put4byte(hdr+sizeof(u64)*2+sizeof(u32), 0); else put4byte(hdr+sizeof(u64)*2+sizeof(u32), nTruncate); #ifndef _TESTAPP_ wal_page_hook(thr,pagesBuf+sizeof(u64)*2+1, page_size, hdr, sizeof(hdr)); #endif } memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32)); key.mv_size = sizeof(pagesKeyBuf); key.mv_data = pagesKeyBuf; // Check if there are pages with the same or higher evnum/evterm. If there are, delete them. // This can happen if sqlite flushed some page to disk before commiting, because there were // so many pages that they could not be held in memory. Or it could happen if pages need to be // overwritten because there was a write that did not pass raft consensus. rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY); if (rc == MDB_SUCCESS) { size_t ndupl; mdb_cursor_count(mdb->cursorPages,&ndupl); rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP); if (rc == MDB_SUCCESS) { MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL}; u64 evnum, evterm; u8 frag = *((u8*)data.mv_data+sizeof(u64)*2); memcpy(&evterm, data.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64)); while ((evterm > pWal->inProgressTerm || evnum >= pWal->inProgressEvnum)) //(pWal->inProgressTerm + pWal->inProgressEvnum) > 0) { DBG("Deleting pages higher or equal to current. " "Evterm=%llu, evnum=%llu, curterm=%llu, curevn=%llu, dupl=%ld", evterm,evnum,pWal->inProgressTerm,pWal->inProgressEvnum,ndupl); if (pgDelKey.mv_data != NULL) { if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS) { DBG("Unable to cleanup page from pagedb %d",rc); break; } pgDelKey.mv_data = NULL; } mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT); // if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS) // { // DBG("Cant delete!"); // break; // } if (frag == 0) pWal->allPages--; ndupl--; if (!ndupl) break; rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP); if (rc != MDB_SUCCESS) break; memcpy(&evterm, data.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64)); frag = *((u8*)data.mv_data+sizeof(u64)*2); } if (pgDelKey.mv_data != NULL) { if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS) { DBG("Unable to cleanup page from pagedb %d",rc); break; } pgDelKey.mv_data = NULL; } } memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32)); key.mv_size = sizeof(pagesKeyBuf); key.mv_data = pagesKeyBuf; } track_time(4,thr); memcpy(pagesBuf, &pWal->inProgressTerm, sizeof(u64)); memcpy(pagesBuf + sizeof(u64), &pWal->inProgressEvnum, sizeof(u64)); full_size = page_size + sizeof(u64)*2 + 1; if (full_size < thr->maxvalsize) fragment_index = 0; else { full_size = page_size; skipped = thr->maxvalsize - sizeof(u64)*2 - 1; full_size -= skipped; while(full_size > 0) { full_size -= (thr->maxvalsize - sizeof(u64)*2 - 1); fragment_index++; } full_size = page_size + sizeof(u64)*2 +1; } pagesBuf[sizeof(u64)*2] = fragment_index; data.mv_size = fragment_index == 0 ? full_size : thr->maxvalsize; data.mv_data = pagesBuf; // fragment_index == 0 ? MDB_APPENDDUP : 0 if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS) { // printf("Cursor put failed to pages %d",rc); DBG("ERROR: cursor put failed=%d, datasize=%d",rc,full_size); return SQLITE_ERROR; } fragment_index--; skipped = data.mv_size; while (fragment_index >= 0) { DBG("Insert fragment %d",(int)fragment_index); if (fragment_index == 0) data.mv_size = full_size - skipped + sizeof(u64)*2 + 1; else data.mv_size = thr->maxvalsize; data.mv_data = pagesBuf + skipped - (sizeof(u64)*2+1); memcpy(pagesBuf + skipped - (sizeof(u64)*2+1), &pWal->inProgressTerm, sizeof(u64)); memcpy(pagesBuf + skipped - (sizeof(u64)+1), &pWal->inProgressEvnum, sizeof(u64)); pagesBuf[skipped-1] = fragment_index; if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS) { DBG("ERROR: cursor secondary put failed: err=%d, datasize=%d, skipped=%d, frag=%d", rc,full_size, skipped, (int)fragment_index); return SQLITE_ERROR; } skipped += data.mv_size - sizeof(u64)*2 - 1; fragment_index--; } thr->pagesChanged++; } // printf(""); // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} if (pWal->inProgressTerm > 0) { for(p=pList; p; p=p->pDirty) { u8 logKeyBuf[sizeof(u64)*3]; DBG("Inserting to log"); memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum, sizeof(u64)); key.mv_size = sizeof(logKeyBuf); key.mv_data = logKeyBuf; data.mv_size = sizeof(u32); data.mv_data = &p->pgno; if (mdb_cursor_put(mdb->cursorLog,&key,&data,0) != MDB_SUCCESS) { // printf("Cursor put failed to log"); DBG("ERROR: cursor put to log failed: %d",rc); return SQLITE_ERROR; } pWal->allPages++; } } else { DBG("Skipping log"); for(p=pList; p; p=p->pDirty) pWal->allPages++; } /** - Info DB: {<<ActorIndex:64>>, <<V,FirstCompleteTerm:64,FirstCompleteEvnum:64, LastCompleteTerm:64,LastCompleteEvnum:64, InprogressTerm:64,InProgressEvnum:64>>} */ { if (isCommit) { DBG("Commit actor=%llu fct=%llu, fcev=%llu, lct=%llu, lcev=%llu, int=%llu, inev=%llu", pWal->index, pWal->firstCompleteTerm, pWal->firstCompleteEvnum, pWal->lastCompleteTerm, pWal->lastCompleteEvnum, pWal->inProgressTerm,pWal->inProgressEvnum); #ifndef _TESTAPP_ enif_mutex_lock(pWal->mtx); #endif pWal->lastCompleteTerm = pWal->inProgressTerm > 0 ? pWal->inProgressTerm : pWal->lastCompleteTerm; pWal->lastCompleteEvnum = pWal->inProgressEvnum > 0 ? pWal->inProgressEvnum : pWal->lastCompleteEvnum; if (pWal->firstCompleteTerm == 0) { pWal->firstCompleteTerm = pWal->inProgressTerm; pWal->firstCompleteEvnum = pWal->inProgressEvnum; } pWal->inProgressTerm = pWal->inProgressEvnum = 0; pWal->mxPage = pWal->mxPage > nTruncate ? pWal->mxPage : nTruncate; // pWal->changed = 0; thr->forceCommit = 1; pCon->dirty = 0; #ifndef _TESTAPP_ enif_mutex_unlock(pWal->mtx); #endif DBG("cur mxpage=%u",pWal->mxPage); } else { // pWal->changed = 1; pCon->dirty = 1; } thr->pagesChanged++; rc = storeinfo(pWal,0,0,NULL); if (rc != SQLITE_OK) return rc; track_time(5,thr); } return SQLITE_OK; }
int cursor_count(cursor_t cursor, size_t *count){ return cursor->prev ? mdb_cursor_count(cursor->cursor, count) : MDB_BAD_TXN; }
int mdb_idl_insert_keys( BackendDB *be, MDB_cursor *cursor, struct berval *keys, ID id ) { struct mdb_info *mdb = be->be_private; MDB_val key, data; ID lo, hi, *i; char *err; int rc = 0, k; unsigned int flag = MDB_NODUPDATA; #ifndef MISALIGNED_OK int kbuf[2]; #endif { char buf[16]; Debug( LDAP_DEBUG_ARGS, "mdb_idl_insert_keys: %lx %s\n", (long) id, mdb_show_key( buf, keys->bv_val, keys->bv_len ), 0 ); } assert( id != NOID ); #ifndef MISALIGNED_OK if (keys[0].bv_len & ALIGNER) kbuf[1] = 0; #endif for ( k=0; keys[k].bv_val; k++ ) { /* Fetch the first data item for this key, to see if it * exists and if it's a range. */ #ifndef MISALIGNED_OK if (keys[k].bv_len & ALIGNER) { key.mv_size = sizeof(kbuf); key.mv_data = kbuf; memcpy(key.mv_data, keys[k].bv_val, keys[k].bv_len); } else #endif { key.mv_size = keys[k].bv_len; key.mv_data = keys[k].bv_val; } rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); err = "c_get"; if ( rc == 0 ) { i = data.mv_data; memcpy(&lo, data.mv_data, sizeof(ID)); if ( lo != 0 ) { /* not a range, count the number of items */ size_t count; rc = mdb_cursor_count( cursor, &count ); if ( rc != 0 ) { err = "c_count"; goto fail; } if ( count >= MDB_IDL_DB_MAX ) { /* No room, convert to a range */ lo = *i; rc = mdb_cursor_get( cursor, &key, &data, MDB_LAST_DUP ); if ( rc != 0 && rc != MDB_NOTFOUND ) { err = "c_get last_dup"; goto fail; } i = data.mv_data; hi = *i; /* Update hi/lo if needed */ if ( id < lo ) { lo = id; } else if ( id > hi ) { hi = id; } /* delete the old key */ rc = mdb_cursor_del( cursor, MDB_NODUPDATA ); if ( rc != 0 ) { err = "c_del dups"; goto fail; } /* Store the range */ data.mv_size = sizeof(ID); data.mv_data = &id; id = 0; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put range"; goto fail; } id = lo; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put lo"; goto fail; } id = hi; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put hi"; goto fail; } } else { /* There's room, just store it */ if (id == mdb->mi_nextid) flag |= MDB_APPENDDUP; goto put1; } } else { /* It's a range, see if we need to rewrite * the boundaries */ lo = i[1]; hi = i[2]; if ( id < lo || id > hi ) { /* position on lo */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); if ( rc != 0 ) { err = "c_get lo"; goto fail; } if ( id > hi ) { /* position on hi */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); if ( rc != 0 ) { err = "c_get hi"; goto fail; } } data.mv_size = sizeof(ID); data.mv_data = &id; /* Replace the current lo/hi */ rc = mdb_cursor_put( cursor, &key, &data, MDB_CURRENT ); if ( rc != 0 ) { err = "c_put lo/hi"; goto fail; } } } } else if ( rc == MDB_NOTFOUND ) { flag &= ~MDB_APPENDDUP; put1: data.mv_data = &id; data.mv_size = sizeof(ID); rc = mdb_cursor_put( cursor, &key, &data, flag ); /* Don't worry if it's already there */ if ( rc == MDB_KEYEXIST ) rc = 0; if ( rc ) { err = "c_put id"; goto fail; } } else { /* initial c_get failed, nothing was done */ fail: Debug( LDAP_DEBUG_ANY, "=> mdb_idl_insert_keys: " "%s failed: %s (%d)\n", err, mdb_strerror(rc), rc ); break; } } return rc; }
int mdb_tool_idl_add( MDB_cursor *mc, struct berval *keys, ID id ) { MDB_dbi dbi; mdb_tool_idl_cache *ic, itmp; mdb_tool_idl_cache_entry *ice; int i, rc, lcount; AttrInfo *ai = (AttrInfo *)mc; mc = ai->ai_cursor; dbi = ai->ai_dbi; for (i=0; keys[i].bv_val; i++) { itmp.kstr = keys[i]; ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp ); /* No entry yet, create one */ if ( !ic ) { MDB_val key, data; ID nid; int rc; if ( ai->ai_clist ) { ic = ai->ai_clist; ai->ai_clist = ic->head; } else { ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 ); } ic->kstr.bv_len = itmp.kstr.bv_len; ic->kstr.bv_val = (char *)(ic+1); memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len ); ic->head = ic->tail = NULL; ic->last = 0; ic->count = 0; ic->offset = 0; ic->flags = 0; tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp, avl_dup_error ); /* load existing key count here */ key.mv_size = keys[i].bv_len; key.mv_data = keys[i].bv_val; rc = mdb_cursor_get( mc, &key, &data, MDB_SET ); if ( rc == 0 ) { ic->flags |= WAS_FOUND; nid = *(ID *)data.mv_data; if ( nid == 0 ) { ic->count = MDB_IDL_DB_SIZE+1; ic->flags |= WAS_RANGE; } else { size_t count; mdb_cursor_count( mc, &count ); ic->count = count; ic->first = nid; ic->offset = count & (IDBLOCK-1); } } } /* are we a range already? */ if ( ic->count > MDB_IDL_DB_SIZE ) { ic->last = id; continue; /* Are we at the limit, and converting to a range? */ } else if ( ic->count == MDB_IDL_DB_SIZE ) { if ( ic->head ) { ic->tail->next = ai->ai_flist; ai->ai_flist = ic->head; } ic->head = ic->tail = NULL; ic->last = id; ic->count++; continue; } /* No free block, create that too */ lcount = ic->count & (IDBLOCK-1); if ( !ic->tail || lcount == 0) { if ( ai->ai_flist ) { ice = ai->ai_flist; ai->ai_flist = ice->next; } else { ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry )); } ice->next = NULL; if ( !ic->head ) { ic->head = ice; } else { ic->tail->next = ice; } ic->tail = ice; if ( lcount ) ice->ids[lcount-1] = 0; if ( !ic->count ) ic->first = id; } ice = ic->tail; if (!lcount || ice->ids[lcount-1] != id) ice->ids[lcount] = id; ic->count++; } return 0; }
static int mdb_dn2id_upgrade( BackendDB *be ) { struct mdb_info *mi = (struct mdb_info *) be->be_private; MDB_txn *mt; MDB_cursor *mc = NULL; MDB_val key, data; char *ptr; int rc, writes=0, depth=0; int enable_meter = 0; ID id = 0, *num, count = 0; rec *stack; lutil_meter_t meter; if (!(mi->mi_flags & MDB_NEED_UPGRADE)) { Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n", be->be_suffix[0].bv_val, 0, 0 ); return 0; } { MDB_stat st; mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st); if (!st.ms_entries) { /* Empty DB, nothing to upgrade? */ return 0; } if (isatty(2)) enable_meter = !lutil_meter_open(&meter, &lutil_meter_text_display, &lutil_meter_linear_estimator, st.ms_entries); } num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec))); stack = (rec *)(num + STACKSIZ); rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } key.mv_size = sizeof(ID); /* post-order depth-first update */ for(;;) { size_t dkids; unsigned char *ptr; /* visit */ key.mv_data = &id; stack[depth].id = id; rc = mdb_cursor_get(mc, &key, &data, MDB_SET); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } num[depth] = 1; rc = mdb_cursor_count(mc, &dkids); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } if (dkids > 1) { rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); down: ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID); memcpy(&id, ptr, sizeof(ID)); depth++; memcpy(stack[depth].rdn, data.mv_data, data.mv_size); stack[depth].len = data.mv_size; continue; } /* pop: write updated count, advance to next node */ pop: /* update superior counts */ if (depth) num[depth-1] += num[depth]; key.mv_data = &id; id = stack[depth-1].id; data.mv_data = stack[depth].rdn; data.mv_size = stack[depth].len; rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } data.mv_data = stack[depth].rdn; ptr = (unsigned char *)data.mv_data + data.mv_size; memcpy(ptr, &num[depth], sizeof(ID)); data.mv_size += sizeof(ID); rc = mdb_cursor_del(mc, 0); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_put(mc, &key, &data, 0); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } count++; #if 1 if (enable_meter) lutil_meter_update(&meter, count, 0); #else { int len; ptr = data.mv_data; len = (ptr[0] & 0x7f) << 8 | ptr[1]; printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2); } #endif writes++; if (writes == 1000) { mdb_cursor_close(mc); rc = mdb_txn_commit(mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } writes = 0; } depth--; rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); if (rc == 0) goto down; rc = 0; if (depth) goto pop; else break; } leave: mdb_cursor_close(mc); if (mt) { int r2; r2 = mdb_txn_commit(mt); if (r2) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n", mdb_strerror(r2), r2, 0 ); if (!rc) rc = r2; } } ch_free(num); if (enable_meter) { lutil_meter_update(&meter, count, 1); lutil_meter_close(&meter); } return rc; }