int
mdb_dn2id_children(
Operation *op,
MDB_txn *txn,
Entry *e )
{
struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
MDB_dbi dbi = mdb->mi_dn2id;
MDB_val key, data;
MDB_cursor *cursor;
int rc;
ID id;
key.mv_size = sizeof(ID);
key.mv_data = &id;
id = e->e_id;
rc = mdb_cursor_open( txn, dbi, &cursor );
if ( rc ) return rc;
rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
if ( rc == 0 ) {
size_t dkids;
rc = mdb_cursor_count( cursor, &dkids );
if ( rc == 0 ) {
if ( dkids < 2 ) rc = MDB_NOTFOUND;
}
}
mdb_cursor_close( cursor );
return rc;
}
CAMLprim value caml_mdb_cursor_count(value cursor){
CAMLparam1(cursor);
size_t count;
if(mdb_cursor_count((MDB_cursor*)cursor, &count)){
caml_failwith("error in mdb_cursor_count");
}
CAMLreturn(Val_int(count));
}
// erase hash, return count erased
// note: hashdb does not use erase.
size_t erase(const std::string& binary_hash) {
// get context
lmdb_context_t context(env, true, true);
context.open();
// set key
lmdb_helper::point_to_string(binary_hash, context.key);
// truncate key if truncation is used and binary_hash is longer
if (hash_truncation != 0 && context.key.mv_size > hash_truncation) {
context.key.mv_size = hash_truncation;
}
// set the cursor to this key
int rc = mdb_cursor_get(context.cursor, &context.key,
NULL, MDB_SET_RANGE);
size_t key_count;
if (rc == 0) {
// DB has key so look up the key count
rc = mdb_cursor_count(context.cursor, &key_count);
if (rc != 0) {
std::cerr << "LMDB erase count error: " << mdb_strerror(rc) << "\n";
assert(0);
}
} else if (rc == MDB_NOTFOUND) {
// DB does not have key
key_count = 0;
} else {
std::cerr << "LMDB erase cursor get error: " << mdb_strerror(rc) << "\n";
assert(0);
}
if (key_count > 0) {
// delete
rc = mdb_del(context.txn, context.dbi,
&context.key, NULL);
if (rc != 0) {
std::cerr << "LMDB erase delete error: " << mdb_strerror(rc) << "\n";
assert(0);
}
}
// close context
context.close();
return key_count;
}
// count of entries with this hash value
size_t find_count(const std::string& binary_hash) const {
// get context
lmdb_context_t context(env, false, true);
context.open();
// set key
lmdb_helper::point_to_string(binary_hash, context.key);
// truncate key if truncation is used and binary_hash is longer
if (hash_truncation != 0 && context.key.mv_size > hash_truncation) {
context.key.mv_size = hash_truncation;
}
// set the cursor to this key
int rc = mdb_cursor_get(context.cursor, &context.key, &context.data,
MDB_SET_KEY);
size_t key_count = 0;
if (rc == 0) {
rc = mdb_cursor_count(context.cursor, &key_count);
if (rc != 0) {
std::cerr << "LMDB count error: " << mdb_strerror(rc) << "\n";
assert(0);
}
} else if (rc == MDB_NOTFOUND) {
// fine, key count is zero
} else {
std::cerr << "LMDB get error: " << mdb_strerror(rc) << "\n";
assert(0);
}
// close context
context.close();
return key_count;
}
int
mdb_search( Operation *op, SlapReply *rs )
{
struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
ID id, cursor, nsubs, ncand, cscope;
ID lastid = NOID;
ID candidates[MDB_IDL_UM_SIZE];
ID iscopes[MDB_IDL_DB_SIZE];
ID2 *scopes;
void *stack;
Entry *e = NULL, *base = NULL;
Entry *matched = NULL;
AttributeName *attrs;
slap_mask_t mask;
time_t stoptime;
int manageDSAit;
int tentries = 0;
IdScopes isc;
MDB_cursor *mci, *mcd;
ww_ctx wwctx;
slap_callback cb = { 0 };
mdb_op_info opinfo = {{{0}}}, *moi = &opinfo;
MDB_txn *ltid = NULL;
Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(mdb_search) "\n", 0, 0, 0);
attrs = op->oq_search.rs_attrs;
manageDSAit = get_manageDSAit( op );
rs->sr_err = mdb_opinfo_get( op, mdb, 1, &moi );
switch(rs->sr_err) {
case 0:
break;
default:
send_ldap_error( op, rs, LDAP_OTHER, "internal error" );
return rs->sr_err;
}
ltid = moi->moi_txn;
rs->sr_err = mdb_cursor_open( ltid, mdb->mi_id2entry, &mci );
if ( rs->sr_err ) {
send_ldap_error( op, rs, LDAP_OTHER, "internal error" );
return rs->sr_err;
}
rs->sr_err = mdb_cursor_open( ltid, mdb->mi_dn2id, &mcd );
if ( rs->sr_err ) {
mdb_cursor_close( mci );
send_ldap_error( op, rs, LDAP_OTHER, "internal error" );
return rs->sr_err;
}
scopes = scope_chunk_get( op );
stack = search_stack( op );
isc.mt = ltid;
isc.mc = mcd;
isc.scopes = scopes;
isc.oscope = op->ors_scope;
isc.sctmp = stack;
if ( op->ors_deref & LDAP_DEREF_FINDING ) {
MDB_IDL_ZERO(candidates);
}
dn2entry_retry:
/* get entry with reader lock */
rs->sr_err = mdb_dn2entry( op, ltid, mcd, &op->o_req_ndn, &e, &nsubs, 1 );
switch(rs->sr_err) {
case MDB_NOTFOUND:
matched = e;
e = NULL;
break;
case 0:
break;
case LDAP_BUSY:
send_ldap_error( op, rs, LDAP_BUSY, "ldap server busy" );
goto done;
default:
send_ldap_error( op, rs, LDAP_OTHER, "internal error" );
goto done;
}
if ( op->ors_deref & LDAP_DEREF_FINDING ) {
if ( matched && is_entry_alias( matched )) {
struct berval stub;
stub.bv_val = op->o_req_ndn.bv_val;
stub.bv_len = op->o_req_ndn.bv_len - matched->e_nname.bv_len - 1;
e = deref_base( op, rs, matched, &matched, ltid,
candidates, NULL );
if ( e ) {
build_new_dn( &op->o_req_ndn, &e->e_nname, &stub,
op->o_tmpmemctx );
mdb_entry_return(op, e);
matched = NULL;
goto dn2entry_retry;
}
} else if ( e && is_entry_alias( e )) {
e = deref_base( op, rs, e, &matched, ltid,
candidates, NULL );
}
}
if ( e == NULL ) {
struct berval matched_dn = BER_BVNULL;
if ( matched != NULL ) {
BerVarray erefs = NULL;
/* return referral only if "disclose"
* is granted on the object */
if ( ! access_allowed( op, matched,
slap_schema.si_ad_entry,
NULL, ACL_DISCLOSE, NULL ) )
{
rs->sr_err = LDAP_NO_SUCH_OBJECT;
} else {
ber_dupbv( &matched_dn, &matched->e_name );
erefs = is_entry_referral( matched )
? get_entry_referrals( op, matched )
: NULL;
if ( rs->sr_err == MDB_NOTFOUND )
rs->sr_err = LDAP_REFERRAL;
rs->sr_matched = matched_dn.bv_val;
}
mdb_entry_return(op, matched);
matched = NULL;
if ( erefs ) {
rs->sr_ref = referral_rewrite( erefs, &matched_dn,
&op->o_req_dn, op->oq_search.rs_scope );
ber_bvarray_free( erefs );
}
} else {
rs->sr_ref = referral_rewrite( default_referral,
NULL, &op->o_req_dn, op->oq_search.rs_scope );
rs->sr_err = rs->sr_ref != NULL ? LDAP_REFERRAL : LDAP_NO_SUCH_OBJECT;
}
send_ldap_result( op, rs );
if ( rs->sr_ref ) {
ber_bvarray_free( rs->sr_ref );
rs->sr_ref = NULL;
}
if ( !BER_BVISNULL( &matched_dn ) ) {
ber_memfree( matched_dn.bv_val );
rs->sr_matched = NULL;
}
goto done;
}
/* NOTE: __NEW__ "search" access is required
* on searchBase object */
if ( ! access_allowed_mask( op, e, slap_schema.si_ad_entry,
NULL, ACL_SEARCH, NULL, &mask ) )
{
if ( !ACL_GRANT( mask, ACL_DISCLOSE ) ) {
rs->sr_err = LDAP_NO_SUCH_OBJECT;
} else {
rs->sr_err = LDAP_INSUFFICIENT_ACCESS;
}
mdb_entry_return( op,e);
send_ldap_result( op, rs );
goto done;
}
if ( !manageDSAit && is_entry_referral( e ) ) {
/* entry is a referral */
struct berval matched_dn = BER_BVNULL;
BerVarray erefs = NULL;
ber_dupbv( &matched_dn, &e->e_name );
erefs = get_entry_referrals( op, e );
rs->sr_err = LDAP_REFERRAL;
mdb_entry_return( op, e );
e = NULL;
if ( erefs ) {
rs->sr_ref = referral_rewrite( erefs, &matched_dn,
&op->o_req_dn, op->oq_search.rs_scope );
ber_bvarray_free( erefs );
if ( !rs->sr_ref ) {
rs->sr_text = "bad_referral object";
}
}
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_search) ": entry is referral\n",
0, 0, 0 );
rs->sr_matched = matched_dn.bv_val;
send_ldap_result( op, rs );
ber_bvarray_free( rs->sr_ref );
rs->sr_ref = NULL;
ber_memfree( matched_dn.bv_val );
rs->sr_matched = NULL;
goto done;
}
if ( get_assert( op ) &&
( test_filter( op, e, get_assertion( op )) != LDAP_COMPARE_TRUE ))
{
rs->sr_err = LDAP_ASSERTION_FAILED;
mdb_entry_return( op,e);
send_ldap_result( op, rs );
goto done;
}
/* compute it anyway; root does not use it */
stoptime = op->o_time + op->ors_tlimit;
base = e;
e = NULL;
/* select candidates */
if ( op->oq_search.rs_scope == LDAP_SCOPE_BASE ) {
rs->sr_err = base_candidate( op->o_bd, base, candidates );
scopes[0].mid = 0;
ncand = 1;
} else {
if ( op->ors_scope == LDAP_SCOPE_ONELEVEL ) {
size_t nkids;
MDB_val key, data;
key.mv_data = &base->e_id;
key.mv_size = sizeof( ID );
mdb_cursor_get( mcd, &key, &data, MDB_SET );
mdb_cursor_count( mcd, &nkids );
nsubs = nkids - 1;
} else if ( !base->e_id ) {
/* we don't maintain nsubs for entryID 0.
* just grab entry count from id2entry stat
*/
MDB_stat ms;
mdb_stat( ltid, mdb->mi_id2entry, &ms );
nsubs = ms.ms_entries;
}
MDB_IDL_ZERO( candidates );
scopes[0].mid = 1;
scopes[1].mid = base->e_id;
scopes[1].mval.mv_data = NULL;
rs->sr_err = search_candidates( op, rs, base,
&isc, mci, candidates, stack );
ncand = MDB_IDL_N( candidates );
if ( !base->e_id || ncand == NOID ) {
/* grab entry count from id2entry stat
*/
MDB_stat ms;
mdb_stat( ltid, mdb->mi_id2entry, &ms );
if ( !base->e_id )
nsubs = ms.ms_entries;
if ( ncand == NOID )
ncand = ms.ms_entries;
}
}
/* start cursor at beginning of candidates.
*/
cursor = 0;
if ( candidates[0] == 0 ) {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_search) ": no candidates\n",
0, 0, 0 );
goto nochange;
}
/* if not root and candidates exceed to-be-checked entries, abort */
if ( op->ors_limit /* isroot == FALSE */ &&
op->ors_limit->lms_s_unchecked != -1 &&
ncand > (unsigned) op->ors_limit->lms_s_unchecked )
{
rs->sr_err = LDAP_ADMINLIMIT_EXCEEDED;
send_ldap_result( op, rs );
rs->sr_err = LDAP_SUCCESS;
goto done;
}
if ( op->ors_limit == NULL /* isroot == TRUE */ ||
!op->ors_limit->lms_s_pr_hide )
{
tentries = ncand;
}
wwctx.flag = 0;
/* If we're running in our own read txn */
if ( moi == &opinfo ) {
cb.sc_writewait = mdb_writewait;
cb.sc_private = &wwctx;
wwctx.txn = ltid;
wwctx.mcd = NULL;
cb.sc_next = op->o_callback;
op->o_callback = &cb;
}
if ( get_pagedresults( op ) > SLAP_CONTROL_IGNORED ) {
PagedResultsState *ps = op->o_pagedresults_state;
/* deferred cookie parsing */
rs->sr_err = parse_paged_cookie( op, rs );
if ( rs->sr_err != LDAP_SUCCESS ) {
send_ldap_result( op, rs );
goto done;
}
cursor = (ID) ps->ps_cookie;
if ( cursor && ps->ps_size == 0 ) {
rs->sr_err = LDAP_SUCCESS;
rs->sr_text = "search abandoned by pagedResult size=0";
send_ldap_result( op, rs );
goto done;
}
id = mdb_idl_first( candidates, &cursor );
if ( id == NOID ) {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_search)
": no paged results candidates\n",
0, 0, 0 );
send_paged_response( op, rs, &lastid, 0 );
rs->sr_err = LDAP_OTHER;
goto done;
}
if ( id == (ID)ps->ps_cookie )
id = mdb_idl_next( candidates, &cursor );
nsubs = ncand; /* always bypass scope'd search */
goto loop_begin;
}
if ( nsubs < ncand ) {
int rc;
/* Do scope-based search */
/* if any alias scopes were set, save them */
if (scopes[0].mid > 1) {
cursor = 1;
for (cscope = 1; cscope <= scopes[0].mid; cscope++) {
/* Ignore the original base */
if (scopes[cscope].mid == base->e_id)
continue;
iscopes[cursor++] = scopes[cscope].mid;
}
iscopes[0] = scopes[0].mid - 1;
} else {
iscopes[0] = 0;
}
wwctx.mcd = mcd;
isc.id = base->e_id;
isc.numrdns = 0;
rc = mdb_dn2id_walk( op, &isc );
if ( rc )
id = NOID;
else
id = isc.id;
cscope = 0;
} else {
id = mdb_idl_first( candidates, &cursor );
}
while (id != NOID)
{
int scopeok;
MDB_val edata;
loop_begin:
/* check for abandon */
if ( op->o_abandon ) {
rs->sr_err = SLAPD_ABANDON;
send_ldap_result( op, rs );
goto done;
}
/* mostly needed by internal searches,
* e.g. related to syncrepl, for whom
* abandon does not get set... */
if ( slapd_shutdown ) {
rs->sr_err = LDAP_UNAVAILABLE;
send_ldap_disconnect( op, rs );
goto done;
}
/* check time limit */
if ( op->ors_tlimit != SLAP_NO_LIMIT
&& slap_get_time() > stoptime )
{
rs->sr_err = LDAP_TIMELIMIT_EXCEEDED;
rs->sr_ref = rs->sr_v2ref;
send_ldap_result( op, rs );
rs->sr_err = LDAP_SUCCESS;
goto done;
}
if ( nsubs < ncand ) {
unsigned i;
/* Is this entry in the candidate list? */
scopeok = 0;
if (MDB_IDL_IS_RANGE( candidates )) {
if ( id >= MDB_IDL_RANGE_FIRST( candidates ) &&
id <= MDB_IDL_RANGE_LAST( candidates ))
scopeok = 1;
} else {
i = mdb_idl_search( candidates, id );
if ( candidates[i] == id )
scopeok = 1;
}
if ( scopeok )
goto scopeok;
goto loop_continue;
}
/* Does this candidate actually satisfy the search scope?
*/
scopeok = 0;
isc.numrdns = 0;
switch( op->ors_scope ) {
case LDAP_SCOPE_BASE:
/* This is always true, yes? */
if ( id == base->e_id ) scopeok = 1;
break;
#ifdef LDAP_SCOPE_CHILDREN
case LDAP_SCOPE_CHILDREN:
if ( id == base->e_id ) break;
/* Fall-thru */
#endif
case LDAP_SCOPE_SUBTREE:
if ( id == base->e_id ) {
scopeok = 1;
break;
}
/* Fall-thru */
case LDAP_SCOPE_ONELEVEL:
isc.id = id;
isc.nscope = 0;
rs->sr_err = mdb_idscopes( op, &isc );
if ( rs->sr_err == MDB_SUCCESS ) {
if ( isc.nscope )
scopeok = 1;
} else {
if ( rs->sr_err == MDB_NOTFOUND )
goto notfound;
}
break;
}
/* Not in scope, ignore it */
if ( !scopeok )
{
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_search)
": %ld scope not okay\n",
(long) id, 0, 0 );
goto loop_continue;
}
scopeok:
if ( id == base->e_id ) {
e = base;
} else {
/* get the entry */
rs->sr_err = mdb_id2edata( op, mci, id, &edata );
if ( rs->sr_err == MDB_NOTFOUND ) {
notfound:
if( nsubs < ncand )
goto loop_continue;
if( !MDB_IDL_IS_RANGE(candidates) ) {
/* only complain for non-range IDLs */
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_search)
": candidate %ld not found\n",
(long) id, 0, 0 );
} else {
/* get the next ID from the DB */
rs->sr_err = mdb_get_nextid( mci, &cursor );
if ( rs->sr_err == MDB_NOTFOUND ) {
break;
}
if ( rs->sr_err ) {
rs->sr_err = LDAP_OTHER;
rs->sr_text = "internal error in get_nextid";
send_ldap_result( op, rs );
goto done;
}
cursor--;
}
goto loop_continue;
} else if ( rs->sr_err ) {
rs->sr_err = LDAP_OTHER;
rs->sr_text = "internal error in mdb_id2edata";
send_ldap_result( op, rs );
goto done;
}
rs->sr_err = mdb_entry_decode( op, ltid, &edata, &e );
if ( rs->sr_err ) {
rs->sr_err = LDAP_OTHER;
rs->sr_text = "internal error in mdb_entry_decode";
send_ldap_result( op, rs );
goto done;
}
e->e_id = id;
e->e_name.bv_val = NULL;
e->e_nname.bv_val = NULL;
}
if ( is_entry_subentry( e ) ) {
if( op->oq_search.rs_scope != LDAP_SCOPE_BASE ) {
if(!get_subentries_visibility( op )) {
/* only subentries are visible */
goto loop_continue;
}
} else if ( get_subentries( op ) &&
!get_subentries_visibility( op ))
{
/* only subentries are visible */
goto loop_continue;
}
} else if ( get_subentries_visibility( op )) {
/* only subentries are visible */
goto loop_continue;
}
/* aliases were already dereferenced in candidate list */
if ( op->ors_deref & LDAP_DEREF_SEARCHING ) {
/* but if the search base is an alias, and we didn't
* deref it when finding, return it.
*/
if ( is_entry_alias(e) &&
((op->ors_deref & LDAP_DEREF_FINDING) || e != base ))
{
goto loop_continue;
}
}
if ( !manageDSAit && is_entry_glue( e )) {
goto loop_continue;
}
if (e != base) {
struct berval pdn, pndn;
char *d, *n;
int i;
/* child of base, just append RDNs to base->e_name */
if ( nsubs < ncand || isc.scopes[isc.nscope].mid == base->e_id ) {
pdn = base->e_name;
pndn = base->e_nname;
} else {
mdb_id2name( op, ltid, &isc.mc, scopes[isc.nscope].mid, &pdn, &pndn );
}
e->e_name.bv_len = pdn.bv_len;
e->e_nname.bv_len = pndn.bv_len;
for (i=0; i<isc.numrdns; i++) {
e->e_name.bv_len += isc.rdns[i].bv_len + 1;
e->e_nname.bv_len += isc.nrdns[i].bv_len + 1;
}
e->e_name.bv_val = op->o_tmpalloc(e->e_name.bv_len + 1, op->o_tmpmemctx);
e->e_nname.bv_val = op->o_tmpalloc(e->e_nname.bv_len + 1, op->o_tmpmemctx);
d = e->e_name.bv_val;
n = e->e_nname.bv_val;
if (nsubs < ncand) {
/* RDNs are in top-down order */
for (i=isc.numrdns-1; i>=0; i--) {
memcpy(d, isc.rdns[i].bv_val, isc.rdns[i].bv_len);
d += isc.rdns[i].bv_len;
*d++ = ',';
memcpy(n, isc.nrdns[i].bv_val, isc.nrdns[i].bv_len);
n += isc.nrdns[i].bv_len;
*n++ = ',';
}
} else {
/* RDNs are in bottom-up order */
for (i=0; i<isc.numrdns; i++) {
memcpy(d, isc.rdns[i].bv_val, isc.rdns[i].bv_len);
d += isc.rdns[i].bv_len;
*d++ = ',';
memcpy(n, isc.nrdns[i].bv_val, isc.nrdns[i].bv_len);
n += isc.nrdns[i].bv_len;
*n++ = ',';
}
}
if (pdn.bv_len) {
memcpy(d, pdn.bv_val, pdn.bv_len+1);
memcpy(n, pndn.bv_val, pndn.bv_len+1);
} else {
*--d = '\0';
*--n = '\0';
e->e_name.bv_len--;
e->e_nname.bv_len--;
}
if (pndn.bv_val != base->e_nname.bv_val) {
op->o_tmpfree(pndn.bv_val, op->o_tmpmemctx);
op->o_tmpfree(pdn.bv_val, op->o_tmpmemctx);
}
}
/*
* if it's a referral, add it to the list of referrals. only do
* this for non-base searches, and don't check the filter
* explicitly here since it's only a candidate anyway.
*/
if ( !manageDSAit && op->oq_search.rs_scope != LDAP_SCOPE_BASE
&& is_entry_referral( e ) )
{
BerVarray erefs = get_entry_referrals( op, e );
rs->sr_ref = referral_rewrite( erefs, &e->e_name, NULL,
op->oq_search.rs_scope == LDAP_SCOPE_ONELEVEL
? LDAP_SCOPE_BASE : LDAP_SCOPE_SUBTREE );
rs->sr_entry = e;
rs->sr_flags = 0;
send_search_reference( op, rs );
if (e != base)
mdb_entry_return( op, e );
rs->sr_entry = NULL;
e = NULL;
ber_bvarray_free( rs->sr_ref );
ber_bvarray_free( erefs );
rs->sr_ref = NULL;
if ( wwctx.flag ) {
rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd );
if ( rs->sr_err ) {
send_ldap_result( op, rs );
goto done;
}
}
goto loop_continue;
}
/* if it matches the filter and scope, send it */
rs->sr_err = test_filter( op, e, op->oq_search.rs_filter );
if ( rs->sr_err == LDAP_COMPARE_TRUE ) {
/* check size limit */
if ( get_pagedresults(op) > SLAP_CONTROL_IGNORED ) {
if ( rs->sr_nentries >= ((PagedResultsState *)op->o_pagedresults_state)->ps_size ) {
mdb_entry_return( op, e );
e = NULL;
send_paged_response( op, rs, &lastid, tentries );
goto done;
}
lastid = id;
}
if (e) {
/* safe default */
rs->sr_attrs = op->oq_search.rs_attrs;
rs->sr_operational_attrs = NULL;
rs->sr_ctrls = NULL;
rs->sr_entry = e;
RS_ASSERT( e->e_private != NULL );
rs->sr_flags = 0;
rs->sr_err = LDAP_SUCCESS;
rs->sr_err = send_search_entry( op, rs );
rs->sr_attrs = NULL;
rs->sr_entry = NULL;
if (e != base)
mdb_entry_return( op, e );
e = NULL;
switch ( rs->sr_err ) {
case LDAP_SUCCESS: /* entry sent ok */
break;
default: /* entry not sent */
break;
case LDAP_BUSY:
send_ldap_result( op, rs );
goto done;
case LDAP_UNAVAILABLE:
case LDAP_SIZELIMIT_EXCEEDED:
if ( rs->sr_err == LDAP_SIZELIMIT_EXCEEDED ) {
rs->sr_ref = rs->sr_v2ref;
send_ldap_result( op, rs );
rs->sr_err = LDAP_SUCCESS;
} else {
rs->sr_err = LDAP_OTHER;
}
goto done;
}
if ( wwctx.flag ) {
rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd );
if ( rs->sr_err ) {
send_ldap_result( op, rs );
goto done;
}
}
}
} else {
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_search)
": %ld does not match filter\n",
(long) id, 0, 0 );
}
loop_continue:
if( e != NULL ) {
if ( e != base )
mdb_entry_return( op, e );
RS_ASSERT( rs->sr_entry == NULL );
e = NULL;
rs->sr_entry = NULL;
}
if ( nsubs < ncand ) {
int rc = mdb_dn2id_walk( op, &isc );
if (rc) {
id = NOID;
/* We got to the end of a subtree. If there are any
* alias scopes left, search them too.
*/
while (iscopes[0] && cscope < iscopes[0]) {
cscope++;
isc.id = iscopes[cscope];
if ( base )
mdb_entry_return( op, base );
rs->sr_err = mdb_id2entry(op, mci, isc.id, &base);
if ( !rs->sr_err ) {
mdb_id2name( op, ltid, &isc.mc, isc.id, &base->e_name, &base->e_nname );
isc.numrdns = 0;
if (isc.oscope == LDAP_SCOPE_ONELEVEL)
isc.oscope = LDAP_SCOPE_BASE;
rc = mdb_dn2id_walk( op, &isc );
if ( !rc ) {
id = isc.id;
break;
}
}
}
} else
id = isc.id;
} else {
id = mdb_idl_next( candidates, &cursor );
}
}
nochange:
rs->sr_ctrls = NULL;
rs->sr_ref = rs->sr_v2ref;
rs->sr_err = (rs->sr_v2ref == NULL) ? LDAP_SUCCESS : LDAP_REFERRAL;
rs->sr_rspoid = NULL;
if ( get_pagedresults(op) > SLAP_CONTROL_IGNORED ) {
send_paged_response( op, rs, NULL, 0 );
} else {
send_ldap_result( op, rs );
}
rs->sr_err = LDAP_SUCCESS;
done:
if ( cb.sc_private ) {
/* remove our writewait callback */
slap_callback **scp = &op->o_callback;
while ( *scp ) {
if ( *scp == &cb ) {
*scp = cb.sc_next;
cb.sc_private = NULL;
break;
}
}
}
mdb_cursor_close( mcd );
mdb_cursor_close( mci );
if ( moi == &opinfo ) {
mdb_txn_reset( moi->moi_txn );
LDAP_SLIST_REMOVE( &op->o_extra, &moi->moi_oe, OpExtra, oe_next );
} else {
moi->moi_ref--;
}
if( rs->sr_v2ref ) {
ber_bvarray_free( rs->sr_v2ref );
rs->sr_v2ref = NULL;
}
if (base)
mdb_entry_return( op, base );
scope_chunk_ret( op, scopes );
return rs->sr_err;
}
static int doundo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, u8 delPages)
{
MDB_val logKey, logVal;
MDB_val pgKey, pgVal;
u8 logKeyBuf[sizeof(u64)*3];
int logop, pgop, rc, mrc;
mdbinf *mdb;
// db_thread* const thr = enif_tsd_get(g_tsd_thread);
#if ATOMIC
db_thread *thr = g_tsd_thread;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
#endif
rc = SQLITE_OK;
if (pWal->inProgressTerm == 0)
return SQLITE_OK;
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
if (!mdb)
return SQLITE_ERROR;
logKey.mv_data = logKeyBuf;
logKey.mv_size = sizeof(logKeyBuf);
DBG("Undo");
// For every page here
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
// Delete from
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>}
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum,sizeof(u64));
if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS)
{
DBG("Key not found in log for undo, index=%llu, term=%llu, evnum=%llu",
pWal->index, pWal->inProgressTerm, pWal->inProgressEvnum);
return SQLITE_OK;
}
logop = MDB_FIRST_DUP;
while ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS)
{
u32 pgno;
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
u64 term,evnum;
memcpy(&pgno,logVal.mv_data,sizeof(u32));
if (delPages)
{
size_t ndupl;
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32));
pgKey.mv_data = pagesKeyBuf;
pgKey.mv_size = sizeof(pagesKeyBuf);
DBG("UNDO pgno=%d",pgno);
pgop = MDB_FIRST_DUP;
if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS)
{
DBG("Key not found in log for undo");
continue;
}
mdb_cursor_count(mdb->cursorPages,&ndupl);
while (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,pgop) == MDB_SUCCESS)
{
u8 frag = *((u8*)pgVal.mv_data+sizeof(u64)*2);
memcpy(&term, pgVal.mv_data, sizeof(u64));
memcpy(&evnum,(u8*)pgVal.mv_data+sizeof(u64),sizeof(u64));
DBG("progress term %lld, progress evnum %lld, curterm %lld, curnum %lld",
pWal->inProgressTerm, pWal->inProgressEvnum, term, evnum);
if (term >= pWal->inProgressTerm && evnum >= pWal->inProgressEvnum)
{
if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS)
{
DBG("Can not delete undo");
rc = SQLITE_ERROR;
break;
}
if (frag == 0)
pWal->allPages--;
ndupl--;
if (!ndupl)
break;
}
pgop = MDB_NEXT_DUP;
}
pWal->inProgressTerm = pWal->inProgressEvnum = 0;
storeinfo(pWal,0,0,NULL);
thr->pagesChanged++;
}
if (xUndo)
rc = xUndo(pUndoCtx, pgno);
logop = MDB_NEXT_DUP;
}
// if (mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA) != MDB_SUCCESS)
// {
// DBG("Unable to cleanup key from logdb"));
// }
DBG("Undo done!");
return rc;
}
// Delete all pages up to limitEvterm and limitEvnum
static int checkpoint(Wal *pWal, u64 limitEvnum)
{
MDB_val logKey, logVal;
u8 logKeyBuf[sizeof(u64)*3];
u64 evnum,evterm,aindex;
mdbinf* mdb;
// db_thread* const thr = enif_tsd_get(g_tsd_thread);
#if ATOMIC
db_thread *thr = g_tsd_thread;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
#endif
int logop, mrc = MDB_SUCCESS;
// u8 somethingDeleted = 0;
int allPagesDiff = 0;
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
if (!mdb)
return SQLITE_ERROR;
// if (pWal->inProgressTerm == 0)
// return SQLITE_OK;
DBG("checkpoint actor=%llu, fct=%llu, fcev=%llu, limitEvnum=%llu",pWal->index,
pWal->firstCompleteTerm,pWal->firstCompleteEvnum,limitEvnum);
while (pWal->firstCompleteEvnum < limitEvnum)
{
logKey.mv_data = logKeyBuf;
logKey.mv_size = sizeof(logKeyBuf);
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &pWal->firstCompleteTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &pWal->firstCompleteEvnum,sizeof(u64));
if (mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_SET) != MDB_SUCCESS)
{
printf("Key not found in log for checkpoint %llu %llu\n",
pWal->firstCompleteTerm, pWal->firstCompleteEvnum);
return SQLITE_OK;
}
DBG("checkpoint evnum=%llu",pWal->firstCompleteEvnum);
// For every page here
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
// Delete from
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Count,CompressedPage/binary>>}
logop = MDB_FIRST_DUP;
while ((mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,logop)) == MDB_SUCCESS)
{
u32 pgno;
size_t ndupl;
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
MDB_val pgKey = {0,NULL}, pgVal = {0,NULL};
u64 pgnoLimitEvnum;
logop = MDB_NEXT_DUP;
memcpy(&pgno, logVal.mv_data,sizeof(u32));
DBG("checkpoint pgno=%u",pgno);
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32));
pgKey.mv_data = pagesKeyBuf;
pgKey.mv_size = sizeof(pagesKeyBuf);
if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_SET) != MDB_SUCCESS)
{
continue;
}
mdb_cursor_count(mdb->cursorPages,&ndupl);
if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_LAST_DUP) == MDB_SUCCESS)
memcpy(&pgnoLimitEvnum, (u8*)pgVal.mv_data+sizeof(u64),sizeof(u64));
else
continue;
pgnoLimitEvnum = pgnoLimitEvnum < limitEvnum ? pgnoLimitEvnum : limitEvnum;
if (mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_FIRST_DUP) != MDB_SUCCESS)
continue;
do
{
u8 frag;
MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL};
mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT);
frag = *((u8*)pgDelVal.mv_data+sizeof(u64)*2);
memcpy(&evterm, pgDelVal.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)pgDelVal.mv_data+sizeof(u64),sizeof(u64));
DBG("limit limitevnum %lld, curnum %lld, dupl %zu, frag=%d",
limitEvnum, evnum, ndupl,(int)frag);
if (evnum < pgnoLimitEvnum)
{
mrc = mdb_cursor_del(mdb->cursorPages,0);
if (mrc != MDB_SUCCESS)
{
DBG("Unable to delete page on cursor!.");
break;
}
// else
// {
// DBG("Deleted page!");
// // somethingDeleted = 1;
// }
if (frag == 0)
allPagesDiff++;
}
ndupl--;
if (!ndupl)
break;
mrc = mdb_cursor_get(mdb->cursorPages,&pgKey,&pgVal,MDB_NEXT_DUP);
} while (mrc == MDB_SUCCESS);
}
mrc = mdb_cursor_del(mdb->cursorLog,MDB_NODUPDATA);
if (mrc != MDB_SUCCESS)
{
DBG("Can not delete log");
break;
}
// move forward
if ((mrc = mdb_cursor_get(mdb->cursorLog,&logKey,&logVal,MDB_NEXT_NODUP)) != MDB_SUCCESS)
{
DBG("Unable to move to next log %d",mrc);
break;
}
// read next key data
memcpy(&aindex, logKey.mv_data, sizeof(u64));
memcpy(&evterm, (u8*)logKey.mv_data + sizeof(u64), sizeof(u64));
memcpy(&evnum, (u8*)logKey.mv_data + sizeof(u64)*2, sizeof(u64));
if (aindex != pWal->index)
{
DBG("Reached another actor");
break;
}
pWal->firstCompleteTerm = evterm;
pWal->firstCompleteEvnum = evnum;
pWal->allPages -= allPagesDiff;
allPagesDiff = 0;
DBG("Checkpint fce now=%lld",(u64)evnum);
}
// no dirty pages, but will write info
if (sqlite3WalFrames(pWal, SQLITE_DEFAULT_PAGE_SIZE, NULL, pWal->mxPage, 1, 0) == SQLITE_OK)
return SQLITE_OK;
else
return SQLITE_ERROR;
}
static int findframe(db_thread *thr, Wal *pWal, Pgno pgno, u32 *piRead, u64 limitTerm,
u64 limitEvnum, u64 *outTerm, u64 *outEvnum)
{
MDB_val key, data;
int rc;
size_t ndupl = 0;
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
mdbinf *mdb;
if (thr->pagesChanged)
{
#if ATOMIC
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
}
else
mdb = &thr->mdb;
track_time(7,thr);
DBG("FIND FRAME pgno=%u, index=%llu, limitterm=%llu, limitevnum=%llu",
pgno,pWal->index,limitTerm,limitEvnum);
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>,
// <<Evterm:64,Evnum:64,Counter,CompressedPage/binary>>}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
// u32 pgno2 = *(u32*)(key.mv_data+sizeof(u64));
// DBG("RUN %d",pgno2);
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY);
if (rc == MDB_SUCCESS)
{
mdb_cursor_count(mdb->cursorPages,&ndupl);
if (ndupl == 0)
{
*piRead = 0;
return SQLITE_OK;
}
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP);
if (rc == MDB_SUCCESS)
{
while (1)
{
char frag1 = *(char*)(((char*)data.mv_data)+sizeof(u64)*2);
int frag = frag1;
u64 term, evnum;
memcpy(&term, data.mv_data, sizeof(u64));
memcpy(&evnum, ((u8*)data.mv_data) + sizeof(u64), sizeof(u64));
if (term > limitTerm || evnum > limitEvnum)
{
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP);
if (rc == MDB_SUCCESS)
continue;
else
{
DBG("Cant move to prev dup, term=%llu, evnum=%llu,"
" limitterm=%llu, limitevnum=%llu",term,evnum,limitTerm,limitEvnum);
*piRead = 0;
break;
}
}
if (outTerm != NULL)
*outTerm = term;
if (outEvnum != NULL)
*outEvnum = evnum;
DBG("Found page size=%ld, frags=%d",data.mv_size,(int)frag);
thr->nResFrames = frag;
thr->resFrames[frag--] = data;
while (frag >= 0)
{
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP);
frag = *(((u8*)data.mv_data)+sizeof(u64)*2);
// DBG("SUCCESS? %d frag=%d, size=%ld",pgno,frag,data.mv_size);
thr->resFrames[frag--] = data;
}
*piRead = 1;
break;
}
}
else
{
DBG("Find page no last dup");
*piRead = 0;
}
}
else if (rc == MDB_NOTFOUND)
{
DBG("Frame not found!");
*piRead = 0;
}
else
{
DBG("ERROR findframe: %d",rc);
*piRead = 0;
}
return SQLITE_OK;
}
/* Write a frame or frames to the log. */
int sqlite3WalFrames(Wal *pWal, int szPage, PgHdr *pList, Pgno nTruncate, int isCommit, int sync_flags)
{
PgHdr *p;
MDB_val key, data;
int rc;
mdbinf* mdb;
MDB_txn* txn;
#if ATOMIC
db_thread *thr = g_tsd_thread;
db_connection* pCon = g_tsd_conn;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
db_connection* pCon = enif_tsd_get(g_tsd_conn);
#endif
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
txn = mdb->txn;
if (!mdb)
return SQLITE_ERROR;
key.mv_size = sizeof(u64);
key.mv_data = (void*)&pWal->index;
// Term/evnum must always be increasing
if ((pWal->inProgressTerm > 0 && pWal->inProgressTerm < pWal->lastCompleteTerm) ||
(pWal->inProgressEvnum > 0 && pWal->inProgressEvnum < pWal->lastCompleteEvnum))
return SQLITE_ERROR;
track_time(2,thr);
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Fragment,CompressedPage/binary>>}
for(p=pList; p; p=p->pDirty)
{
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
u8 pagesBuf[PAGE_BUFF_SIZE];
int full_size = 0;
int page_size = LZ4_compress_default((char*)p->pData,(char*)pagesBuf+sizeof(u64)*2+1,szPage,sizeof(pagesBuf));
char fragment_index = 0;
int skipped = 0;
track_time(3,thr);
DBG("Insert frame, actor=%lld, pgno=%u, "
"term=%lld, evnum=%lld, commit=%d, truncate=%d, compressedsize=%d",
pWal->index,p->pgno,pWal->inProgressTerm,pWal->inProgressEvnum,
isCommit,nTruncate,page_size);
if (pCon->doReplicate)
{
u8 hdr[sizeof(u64)*2+sizeof(u32)*2];
put8byte(hdr, pWal->inProgressTerm);
put8byte(hdr+sizeof(u64), pWal->inProgressEvnum);
put4byte(hdr+sizeof(u64)*2, p->pgno);
if (p->pDirty)
put4byte(hdr+sizeof(u64)*2+sizeof(u32), 0);
else
put4byte(hdr+sizeof(u64)*2+sizeof(u32), nTruncate);
#ifndef _TESTAPP_
wal_page_hook(thr,pagesBuf+sizeof(u64)*2+1, page_size, hdr, sizeof(hdr));
#endif
}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
// Check if there are pages with the same or higher evnum/evterm. If there are, delete them.
// This can happen if sqlite flushed some page to disk before commiting, because there were
// so many pages that they could not be held in memory. Or it could happen if pages need to be
// overwritten because there was a write that did not pass raft consensus.
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY);
if (rc == MDB_SUCCESS)
{
size_t ndupl;
mdb_cursor_count(mdb->cursorPages,&ndupl);
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP);
if (rc == MDB_SUCCESS)
{
MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL};
u64 evnum, evterm;
u8 frag = *((u8*)data.mv_data+sizeof(u64)*2);
memcpy(&evterm, data.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64));
while ((evterm > pWal->inProgressTerm || evnum >= pWal->inProgressEvnum))
//(pWal->inProgressTerm + pWal->inProgressEvnum) > 0)
{
DBG("Deleting pages higher or equal to current. "
"Evterm=%llu, evnum=%llu, curterm=%llu, curevn=%llu, dupl=%ld",
evterm,evnum,pWal->inProgressTerm,pWal->inProgressEvnum,ndupl);
if (pgDelKey.mv_data != NULL)
{
if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS)
{
DBG("Unable to cleanup page from pagedb %d",rc);
break;
}
pgDelKey.mv_data = NULL;
}
mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT);
// if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS)
// {
// DBG("Cant delete!");
// break;
// }
if (frag == 0)
pWal->allPages--;
ndupl--;
if (!ndupl)
break;
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP);
if (rc != MDB_SUCCESS)
break;
memcpy(&evterm, data.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64));
frag = *((u8*)data.mv_data+sizeof(u64)*2);
}
if (pgDelKey.mv_data != NULL)
{
if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS)
{
DBG("Unable to cleanup page from pagedb %d",rc);
break;
}
pgDelKey.mv_data = NULL;
}
}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
}
track_time(4,thr);
memcpy(pagesBuf, &pWal->inProgressTerm, sizeof(u64));
memcpy(pagesBuf + sizeof(u64), &pWal->inProgressEvnum, sizeof(u64));
full_size = page_size + sizeof(u64)*2 + 1;
if (full_size < thr->maxvalsize)
fragment_index = 0;
else
{
full_size = page_size;
skipped = thr->maxvalsize - sizeof(u64)*2 - 1;
full_size -= skipped;
while(full_size > 0)
{
full_size -= (thr->maxvalsize - sizeof(u64)*2 - 1);
fragment_index++;
}
full_size = page_size + sizeof(u64)*2 +1;
}
pagesBuf[sizeof(u64)*2] = fragment_index;
data.mv_size = fragment_index == 0 ? full_size : thr->maxvalsize;
data.mv_data = pagesBuf;
// fragment_index == 0 ? MDB_APPENDDUP : 0
if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS)
{
// printf("Cursor put failed to pages %d",rc);
DBG("ERROR: cursor put failed=%d, datasize=%d",rc,full_size);
return SQLITE_ERROR;
}
fragment_index--;
skipped = data.mv_size;
while (fragment_index >= 0)
{
DBG("Insert fragment %d",(int)fragment_index);
if (fragment_index == 0)
data.mv_size = full_size - skipped + sizeof(u64)*2 + 1;
else
data.mv_size = thr->maxvalsize;
data.mv_data = pagesBuf + skipped - (sizeof(u64)*2+1);
memcpy(pagesBuf + skipped - (sizeof(u64)*2+1), &pWal->inProgressTerm, sizeof(u64));
memcpy(pagesBuf + skipped - (sizeof(u64)+1), &pWal->inProgressEvnum, sizeof(u64));
pagesBuf[skipped-1] = fragment_index;
if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS)
{
DBG("ERROR: cursor secondary put failed: err=%d, datasize=%d, skipped=%d, frag=%d",
rc,full_size, skipped, (int)fragment_index);
return SQLITE_ERROR;
}
skipped += data.mv_size - sizeof(u64)*2 - 1;
fragment_index--;
}
thr->pagesChanged++;
}
// printf("");
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
if (pWal->inProgressTerm > 0)
{
for(p=pList; p; p=p->pDirty)
{
u8 logKeyBuf[sizeof(u64)*3];
DBG("Inserting to log");
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum, sizeof(u64));
key.mv_size = sizeof(logKeyBuf);
key.mv_data = logKeyBuf;
data.mv_size = sizeof(u32);
data.mv_data = &p->pgno;
if (mdb_cursor_put(mdb->cursorLog,&key,&data,0) != MDB_SUCCESS)
{
// printf("Cursor put failed to log");
DBG("ERROR: cursor put to log failed: %d",rc);
return SQLITE_ERROR;
}
pWal->allPages++;
}
}
else
{
DBG("Skipping log");
for(p=pList; p; p=p->pDirty)
pWal->allPages++;
}
/** - Info DB: {<<ActorIndex:64>>, <<V,FirstCompleteTerm:64,FirstCompleteEvnum:64,
LastCompleteTerm:64,LastCompleteEvnum:64,
InprogressTerm:64,InProgressEvnum:64>>} */
{
if (isCommit)
{
DBG("Commit actor=%llu fct=%llu, fcev=%llu, lct=%llu, lcev=%llu, int=%llu, inev=%llu",
pWal->index,
pWal->firstCompleteTerm, pWal->firstCompleteEvnum, pWal->lastCompleteTerm,
pWal->lastCompleteEvnum, pWal->inProgressTerm,pWal->inProgressEvnum);
#ifndef _TESTAPP_
enif_mutex_lock(pWal->mtx);
#endif
pWal->lastCompleteTerm = pWal->inProgressTerm > 0 ?
pWal->inProgressTerm : pWal->lastCompleteTerm;
pWal->lastCompleteEvnum = pWal->inProgressEvnum > 0 ?
pWal->inProgressEvnum : pWal->lastCompleteEvnum;
if (pWal->firstCompleteTerm == 0)
{
pWal->firstCompleteTerm = pWal->inProgressTerm;
pWal->firstCompleteEvnum = pWal->inProgressEvnum;
}
pWal->inProgressTerm = pWal->inProgressEvnum = 0;
pWal->mxPage = pWal->mxPage > nTruncate ? pWal->mxPage : nTruncate;
// pWal->changed = 0;
thr->forceCommit = 1;
pCon->dirty = 0;
#ifndef _TESTAPP_
enif_mutex_unlock(pWal->mtx);
#endif
DBG("cur mxpage=%u",pWal->mxPage);
}
else
{
// pWal->changed = 1;
pCon->dirty = 1;
}
thr->pagesChanged++;
rc = storeinfo(pWal,0,0,NULL);
if (rc != SQLITE_OK)
return rc;
track_time(5,thr);
}
return SQLITE_OK;
}
int cursor_count(cursor_t cursor, size_t *count){
return cursor->prev ? mdb_cursor_count(cursor->cursor, count) : MDB_BAD_TXN;
}
int
mdb_idl_insert_keys(
BackendDB *be,
MDB_cursor *cursor,
struct berval *keys,
ID id )
{
struct mdb_info *mdb = be->be_private;
MDB_val key, data;
ID lo, hi, *i;
char *err;
int rc = 0, k;
unsigned int flag = MDB_NODUPDATA;
#ifndef MISALIGNED_OK
int kbuf[2];
#endif
{
char buf[16];
Debug( LDAP_DEBUG_ARGS,
"mdb_idl_insert_keys: %lx %s\n",
(long) id, mdb_show_key( buf, keys->bv_val, keys->bv_len ), 0 );
}
assert( id != NOID );
#ifndef MISALIGNED_OK
if (keys[0].bv_len & ALIGNER)
kbuf[1] = 0;
#endif
for ( k=0; keys[k].bv_val; k++ ) {
/* Fetch the first data item for this key, to see if it
* exists and if it's a range.
*/
#ifndef MISALIGNED_OK
if (keys[k].bv_len & ALIGNER) {
key.mv_size = sizeof(kbuf);
key.mv_data = kbuf;
memcpy(key.mv_data, keys[k].bv_val, keys[k].bv_len);
} else
#endif
{
key.mv_size = keys[k].bv_len;
key.mv_data = keys[k].bv_val;
}
rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
err = "c_get";
if ( rc == 0 ) {
i = data.mv_data;
memcpy(&lo, data.mv_data, sizeof(ID));
if ( lo != 0 ) {
/* not a range, count the number of items */
size_t count;
rc = mdb_cursor_count( cursor, &count );
if ( rc != 0 ) {
err = "c_count";
goto fail;
}
if ( count >= MDB_IDL_DB_MAX ) {
/* No room, convert to a range */
lo = *i;
rc = mdb_cursor_get( cursor, &key, &data, MDB_LAST_DUP );
if ( rc != 0 && rc != MDB_NOTFOUND ) {
err = "c_get last_dup";
goto fail;
}
i = data.mv_data;
hi = *i;
/* Update hi/lo if needed */
if ( id < lo ) {
lo = id;
} else if ( id > hi ) {
hi = id;
}
/* delete the old key */
rc = mdb_cursor_del( cursor, MDB_NODUPDATA );
if ( rc != 0 ) {
err = "c_del dups";
goto fail;
}
/* Store the range */
data.mv_size = sizeof(ID);
data.mv_data = &id;
id = 0;
rc = mdb_cursor_put( cursor, &key, &data, 0 );
if ( rc != 0 ) {
err = "c_put range";
goto fail;
}
id = lo;
rc = mdb_cursor_put( cursor, &key, &data, 0 );
if ( rc != 0 ) {
err = "c_put lo";
goto fail;
}
id = hi;
rc = mdb_cursor_put( cursor, &key, &data, 0 );
if ( rc != 0 ) {
err = "c_put hi";
goto fail;
}
} else {
/* There's room, just store it */
if (id == mdb->mi_nextid)
flag |= MDB_APPENDDUP;
goto put1;
}
} else {
/* It's a range, see if we need to rewrite
* the boundaries
*/
lo = i[1];
hi = i[2];
if ( id < lo || id > hi ) {
/* position on lo */
rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP );
if ( rc != 0 ) {
err = "c_get lo";
goto fail;
}
if ( id > hi ) {
/* position on hi */
rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP );
if ( rc != 0 ) {
err = "c_get hi";
goto fail;
}
}
data.mv_size = sizeof(ID);
data.mv_data = &id;
/* Replace the current lo/hi */
rc = mdb_cursor_put( cursor, &key, &data, MDB_CURRENT );
if ( rc != 0 ) {
err = "c_put lo/hi";
goto fail;
}
}
}
} else if ( rc == MDB_NOTFOUND ) {
flag &= ~MDB_APPENDDUP;
put1: data.mv_data = &id;
data.mv_size = sizeof(ID);
rc = mdb_cursor_put( cursor, &key, &data, flag );
/* Don't worry if it's already there */
if ( rc == MDB_KEYEXIST )
rc = 0;
if ( rc ) {
err = "c_put id";
goto fail;
}
} else {
/* initial c_get failed, nothing was done */
fail:
Debug( LDAP_DEBUG_ANY, "=> mdb_idl_insert_keys: "
"%s failed: %s (%d)\n", err, mdb_strerror(rc), rc );
break;
}
}
return rc;
}
int mdb_tool_idl_add(
MDB_cursor *mc,
struct berval *keys,
ID id )
{
MDB_dbi dbi;
mdb_tool_idl_cache *ic, itmp;
mdb_tool_idl_cache_entry *ice;
int i, rc, lcount;
AttrInfo *ai = (AttrInfo *)mc;
mc = ai->ai_cursor;
dbi = ai->ai_dbi;
for (i=0; keys[i].bv_val; i++) {
itmp.kstr = keys[i];
ic = tavl_find( (Avlnode *)ai->ai_root, &itmp, mdb_tool_idl_cmp );
/* No entry yet, create one */
if ( !ic ) {
MDB_val key, data;
ID nid;
int rc;
if ( ai->ai_clist ) {
ic = ai->ai_clist;
ai->ai_clist = ic->head;
} else {
ic = ch_malloc( sizeof( mdb_tool_idl_cache ) + itmp.kstr.bv_len + 4 );
}
ic->kstr.bv_len = itmp.kstr.bv_len;
ic->kstr.bv_val = (char *)(ic+1);
memcpy( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len );
ic->head = ic->tail = NULL;
ic->last = 0;
ic->count = 0;
ic->offset = 0;
ic->flags = 0;
tavl_insert( (Avlnode **)&ai->ai_root, ic, mdb_tool_idl_cmp,
avl_dup_error );
/* load existing key count here */
key.mv_size = keys[i].bv_len;
key.mv_data = keys[i].bv_val;
rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
if ( rc == 0 ) {
ic->flags |= WAS_FOUND;
nid = *(ID *)data.mv_data;
if ( nid == 0 ) {
ic->count = MDB_IDL_DB_SIZE+1;
ic->flags |= WAS_RANGE;
} else {
size_t count;
mdb_cursor_count( mc, &count );
ic->count = count;
ic->first = nid;
ic->offset = count & (IDBLOCK-1);
}
}
}
/* are we a range already? */
if ( ic->count > MDB_IDL_DB_SIZE ) {
ic->last = id;
continue;
/* Are we at the limit, and converting to a range? */
} else if ( ic->count == MDB_IDL_DB_SIZE ) {
if ( ic->head ) {
ic->tail->next = ai->ai_flist;
ai->ai_flist = ic->head;
}
ic->head = ic->tail = NULL;
ic->last = id;
ic->count++;
continue;
}
/* No free block, create that too */
lcount = ic->count & (IDBLOCK-1);
if ( !ic->tail || lcount == 0) {
if ( ai->ai_flist ) {
ice = ai->ai_flist;
ai->ai_flist = ice->next;
} else {
ice = ch_malloc( sizeof( mdb_tool_idl_cache_entry ));
}
ice->next = NULL;
if ( !ic->head ) {
ic->head = ice;
} else {
ic->tail->next = ice;
}
ic->tail = ice;
if ( lcount )
ice->ids[lcount-1] = 0;
if ( !ic->count )
ic->first = id;
}
ice = ic->tail;
if (!lcount || ice->ids[lcount-1] != id)
ice->ids[lcount] = id;
ic->count++;
}
return 0;
}
static int
mdb_dn2id_upgrade( BackendDB *be ) {
struct mdb_info *mi = (struct mdb_info *) be->be_private;
MDB_txn *mt;
MDB_cursor *mc = NULL;
MDB_val key, data;
char *ptr;
int rc, writes=0, depth=0;
int enable_meter = 0;
ID id = 0, *num, count = 0;
rec *stack;
lutil_meter_t meter;
if (!(mi->mi_flags & MDB_NEED_UPGRADE)) {
Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n",
be->be_suffix[0].bv_val, 0, 0 );
return 0;
}
{
MDB_stat st;
mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st);
if (!st.ms_entries) {
/* Empty DB, nothing to upgrade? */
return 0;
}
if (isatty(2))
enable_meter = !lutil_meter_open(&meter,
&lutil_meter_text_display,
&lutil_meter_linear_estimator,
st.ms_entries);
}
num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec)));
stack = (rec *)(num + STACKSIZ);
rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
key.mv_size = sizeof(ID);
/* post-order depth-first update */
for(;;) {
size_t dkids;
unsigned char *ptr;
/* visit */
key.mv_data = &id;
stack[depth].id = id;
rc = mdb_cursor_get(mc, &key, &data, MDB_SET);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
num[depth] = 1;
rc = mdb_cursor_count(mc, &dkids);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
if (dkids > 1) {
rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
down:
ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID);
memcpy(&id, ptr, sizeof(ID));
depth++;
memcpy(stack[depth].rdn, data.mv_data, data.mv_size);
stack[depth].len = data.mv_size;
continue;
}
/* pop: write updated count, advance to next node */
pop:
/* update superior counts */
if (depth)
num[depth-1] += num[depth];
key.mv_data = &id;
id = stack[depth-1].id;
data.mv_data = stack[depth].rdn;
data.mv_size = stack[depth].len;
rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
data.mv_data = stack[depth].rdn;
ptr = (unsigned char *)data.mv_data + data.mv_size;
memcpy(ptr, &num[depth], sizeof(ID));
data.mv_size += sizeof(ID);
rc = mdb_cursor_del(mc, 0);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_cursor_put(mc, &key, &data, 0);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
count++;
#if 1
if (enable_meter)
lutil_meter_update(&meter, count, 0);
#else
{
int len;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2);
}
#endif
writes++;
if (writes == 1000) {
mdb_cursor_close(mc);
rc = mdb_txn_commit(mt);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
writes = 0;
}
depth--;
rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
if (rc == 0)
goto down;
rc = 0;
if (depth)
goto pop;
else
break;
}
leave:
mdb_cursor_close(mc);
if (mt) {
int r2;
r2 = mdb_txn_commit(mt);
if (r2) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n",
mdb_strerror(r2), r2, 0 );
if (!rc)
rc = r2;
}
}
ch_free(num);
if (enable_meter) {
lutil_meter_update(&meter, count, 1);
lutil_meter_close(&meter);
}
return rc;
}