int lmdbgo_mdb_cursor_putmulti(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, size_t vstride, unsigned int flags) {
MDB_val key, val[2];
LMDBGO_SET_VAL(&key, kn, kdata);
LMDBGO_SET_VAL(&(val[0]), vstride, vdata);
LMDBGO_SET_VAL(&(val[1]), vn, 0);
return mdb_cursor_put(cur, &key, &val[0], flags);
}
bool DBPrivWriteCursorEntry(DBCursorPriv *cursor, const void *value, int value_size)
{
MDB_val data;
int rc;
cursor->pending_delete = false;
data.mv_data = (void *)value;
data.mv_size = value_size;
if (cursor->curkv)
{
MDB_val curkey;
curkey.mv_data = cursor->curkv;
curkey.mv_size = sizeof(cursor->curkv);
if ((rc = mdb_cursor_put(cursor->mc, &curkey, &data, MDB_CURRENT)) != MDB_SUCCESS)
{
Log(LOG_LEVEL_ERR, "Could not write cursor entry: %s", mdb_strerror(rc));
}
}
else
{
Log(LOG_LEVEL_ERR, "Could not write cursor entry: cannot find current key");
rc = MDB_INVALID;
}
return rc == MDB_SUCCESS;
}
CAMLprim value caml_mdb_cursor_put(value cursor,value key,value data,value flags){
CAMLparam4(cursor,flags,key,data);
MDB_val key_,data_;
key_.mv_data=String_val(key);
key_.mv_size=caml_string_length(key);
data_.mv_data=String_val(data);
data_.mv_size=caml_string_length(data);
if(mdb_cursor_put((MDB_cursor*)cursor, &key_,&data_,Int_val(flags))){
caml_failwith("error in mdb_cursor_put");
}
CAMLreturn0;
}
auto except(std::exception_ptr ex)
{
ICY_EXCEPT(ex ? S_OK : E_INVALIDARG);
auto record = debug_record{ ex };
auto txn = database_txn{ m_db, database_flag::write };
{
const auto cursor = database_cursor{ txn, m_log };
auto now = clock::now();
auto key = MDB_val{ sizeof(now), &now };
auto val = MDB_val{ sizeof(record), &record };
ICY_DATABASE_EXCEPT(mdb_cursor_put(cursor, &key, &val, MDB_NOOVERWRITE));
}
txn.commit();
}
int setIntIndex(MDB_cursor *indexCursor, long long elementId, int propertyKeyId, int propertyValue) {
int rc;
MDB_val key, data;
IntIndexKeyStruct *intIndexKeyStruct = malloc(sizeof(IntIndexKeyStruct));
intIndexKeyStruct->propertyKeyId = (int) propertyKeyId;
intIndexKeyStruct->elementId = (long long) elementId;
intIndexKeyStruct->value = propertyValue;
key.mv_size = sizeof(IntIndexKeyStruct);
key.mv_data = intIndexKeyStruct;
data.mv_size = sizeof(long long);
data.mv_data = &elementId;
rc = mdb_cursor_put(indexCursor, &key, &data, 0);
free(intIndexKeyStruct);
return rc;
}
int setEdgePropertyInt(MDB_cursor *cursor, jlong edgeId, jint propertyKeyId, jint *propertyValue) {
if (propertyValue != NULL) {
MDB_val key, data;
EdgeDbId id;
id.edgeId = edgeId;
id.coreOrPropertyEnum = EPROPERTY_KEY;
id.propertykeyId = propertyKeyId;
key.mv_size = sizeof(EdgeDbId);
key.mv_data = &id;
data.mv_size = sizeof(jint);
data.mv_data = propertyValue;
return mdb_cursor_put(cursor, &key, &data, 0);
} else {
return GLMDB_WRITE_NULL;
}
}
auto insert_server(const GUID& id, const crypto_key& password)
{
auto txn = database_txn{ m_db, database_flag::write };
{
const auto cur = database_cursor{ txn, m_servers.dbi };
auto key = id;
auto val = server{ password };
auto db_val = MDB_val{ sizeof(val), &val };
auto db_key = MDB_val{ sizeof(key), &key };
const auto error = mdb_cursor_put(cur, &db_key, &db_val, MDB_NOOVERWRITE);
if (error == MDB_KEYEXIST)
return false;
ICY_DATABASE_EXCEPT(error);
}
txn.commit();
return true;
}
int setVertexPropertyInt(MDB_cursor *cursor, jlong vertexId, jint propertyKeyId, jint *propertyValue) {
if (propertyValue != NULL) {
MDB_val key, data;
VertexDbId id;
id.vertexId = vertexId;
id.coreOrPropertyEnum = VPROPERTY_KEY;
id.propertykeyId = propertyKeyId;
id.labelId = -1;
id.edgeId = -1LL;
key.mv_size = sizeof(VertexDbId);
key.mv_data = &id;
data.mv_size = sizeof(jint);
data.mv_data = propertyValue;
return mdb_cursor_put(cursor, &key, &data, 0);
} else {
return GLMDB_WRITE_NULL;
}
}
auto insert_client(const string_view login, const crypto_key& password)
{
if (login.bytes().size() > sizeof(client::login))
return false;
auto txn = database_txn{ m_db, database_flag::write };
{
const auto cur = database_cursor{ txn, m_clients.dbi };
auto key = crypto_login{ login };
auto val = client{ password, login };
auto db_val = MDB_val{ sizeof(val), &val };
auto db_key = MDB_val{ sizeof(key), &key };
const auto error = mdb_cursor_put(cur, &db_key, &db_val, MDB_NOOVERWRITE);
if (error == MDB_KEYEXIST)
return false;
ICY_DATABASE_EXCEPT(error);
}
txn.commit();
return true;
}
int cache_insert(MDB_txn *txn, MDB_dbi dbi, const knot_dname_t *name, struct entry *entry)
{
MDB_cursor *cursor = cursor_acquire(txn, dbi);
if (cursor == NULL) {
return KNOT_ERROR;
}
MDB_val key = pack_key(name);
MDB_val data = { 0, malloc(ENTRY_MAXLEN) };
int ret = pack_entry(&data, entry);
if (ret != KNOT_EOK) {
free(data.mv_data);
return ret;
}
ret = mdb_cursor_put(cursor, &key, &data, 0);
free(data.mv_data);
cursor_release(cursor);
return ret;
}
/*
* Class: jmdb_DatabaseWrapper
* Method: cursorPut
* Signature: (J[BII[BIII)V
*/
JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_cursorPut(JNIEnv *vm,
jclass clazz, jlong cursorL, jbyteArray keyA, jint kofs, jint klen,
jbyteArray valueA, jint vofs, jint vlen, jint flags) {
MDB_cursor *cursorC = (MDB_cursor*) cursorL;
enum {
NONE, OOM, MDB
} result = NONE;
jint ret;
MDB_val key, value;
jbyte *keyC = (*vm)->GetPrimitiveArrayCritical(vm, keyA, NULL);
jbyte *valueC = (*vm)->GetPrimitiveArrayCritical(vm, valueA, NULL);
if (keyC == NULL || valueC == NULL) {
result = OOM;
ret = -1;
} else {
key.mv_size = klen;
key.mv_data = keyC + kofs;
value.mv_size = vlen;
value.mv_data = valueC + vofs;
ret = mdb_cursor_put(cursorC, &key, &value, (unsigned int) flags);
if (ret) {
result = MDB;
}
}
(*vm)->ReleasePrimitiveArrayCritical(vm, valueA, valueC, 0);
(*vm)->ReleasePrimitiveArrayCritical(vm, keyA, keyC, JNI_ABORT);
switch (result) {
case NONE:
return;
case OOM:
throwNew(vm, "java/lang/OutOfMemoryError", "");
return;
case MDB:
throwDatabaseException(vm, ret);
return;
}
}
PageDBError
page_db_new(PageDB **db, const char *path) {
PageDB *p = *db = malloc(sizeof(*p));
if (p == 0)
return page_db_error_memory;
p->error = error_new();
if (p->error == 0) {
free(p);
return page_db_error_memory;
}
p->persist = PAGE_DB_DEFAULT_PERSIST;
p->domain_temp = 0;
// create directory if not present yet
const char *error = make_dir(path);
if ((p->path = strdup(path)) == 0) {
error = "could not duplicate path string";
}
if (error != 0) {
page_db_set_error(p, page_db_error_invalid_path, __func__);
page_db_add_error(p, error);
return p->error->code;
}
if (txn_manager_new(&p->txn_manager, 0) != 0) {
page_db_set_error(p, page_db_error_internal, __func__);
page_db_add_error(p, p->txn_manager?
p->txn_manager->error->message:
"NULL");
return p->error->code;
}
// initialize LMDB on the directory
MDB_txn *txn;
MDB_dbi dbi;
int mdb_rc = 0;
if ((mdb_rc = mdb_env_create(&p->txn_manager->env) != 0))
error = "creating environment";
else if ((mdb_rc = mdb_env_set_mapsize(
p->txn_manager->env, PAGE_DB_DEFAULT_SIZE)) != 0)
error = "setting map size";
else if ((mdb_rc = mdb_env_set_maxdbs(p->txn_manager->env, 5)) != 0)
error = "setting number of databases";
else if ((mdb_rc = mdb_env_open(
p->txn_manager->env,
path,
MDB_NOTLS | MDB_NOSYNC, 0664) != 0))
error = "opening environment";
else if ((mdb_rc = txn_manager_begin(p->txn_manager, 0, &txn)) != 0)
error = "starting transaction";
// create all database
else if ((mdb_rc = mdb_dbi_open(txn,
"hash2info",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating hash2info database";
else if ((mdb_rc = mdb_dbi_open(txn,
"hash2idx",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating hash2idx database";
else if ((mdb_rc = mdb_dbi_open(txn,
"links",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating links database";
else if ((mdb_rc = mdb_dbi_open(txn, "info", MDB_CREATE, &dbi)) != 0)
error = "creating info database";
else {
// initialize n_pages inside info database
size_t n_pages = 0;
MDB_val key = {
.mv_size = sizeof(info_n_pages),
.mv_data = info_n_pages
};
MDB_val val = {
.mv_size = sizeof(size_t),
.mv_data = &n_pages
};
switch (mdb_rc = mdb_put(txn, dbi, &key, &val, MDB_NOOVERWRITE)) {
case MDB_KEYEXIST:
case 0:
if (txn_manager_commit(p->txn_manager, txn) != 0)
error = p->txn_manager->error->message;
break; // we good
default:
error = "could not initialize info.n_pages";
}
}
if (error != 0) {
page_db_set_error(p, page_db_error_internal, __func__);
page_db_add_error(p, error);
page_db_add_error(p, mdb_strerror(mdb_rc));
mdb_env_close(p->txn_manager->env);
}
return p->error->code;
}
/** Store a new or updated @ref PageInfo for a crawled page.
*
* @param cur An open cursor to the hash2info database
* @param key The key (hash) to the page
* @param page
* @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter
* will be set with the error (otherwise is set to zero).
* @return 0 if success, -1 if failure.
*/
static int
page_db_add_crawled_page_info(MDB_cursor *cur,
MDB_val *key,
const CrawledPage *page,
PageInfo **page_info,
int *mdb_error) {
MDB_val val;
*page_info = 0;
int mdb_rc = mdb_cursor_get(cur, key, &val, MDB_SET);
int put_flags = 0;
switch (mdb_rc) {
case 0:
if (!(*page_info = page_info_load(&val)))
goto on_error;
if ((page_info_update(*page_info, page) != 0))
goto on_error;
put_flags = MDB_CURRENT;
break;
case MDB_NOTFOUND:
if (!(*page_info = page_info_new_crawled(page)))
goto on_error;
break;
default:
goto on_error;
}
if ((page_info_dump(*page_info, &val) != 0))
goto on_error;
if ((mdb_rc = mdb_cursor_put(cur, key, &val, put_flags)) != 0)
goto on_error;
free(val.mv_data);
val.mv_data = 0;
*mdb_error = 0;
return 0;
on_error:
if (val.mv_data)
free(val.mv_data);
*mdb_error = mdb_rc;
page_info_delete(*page_info);
return -1;
}
/** Store a new or updated @ref PageInfo for an uncrawled link.
*
* @param cur An open cursor to the hash2info database
* @param key The key (hash) to the page
* @param url
* @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter
* will be set with the error (otherwise is set to zero).
* @return 0 if success, -1 if failure.
*/
static int
page_db_add_link_page_info(MDB_cursor *cur,
MDB_val *key,
uint64_t linked_from,
uint64_t depth,
const LinkInfo *link,
PageInfo **page_info,
int *mdb_error) {
MDB_val val;
int mdb_rc = 0;
PageInfo *pi = *page_info =
page_info_new_link(link->url, linked_from, depth, link->score);
if (!pi)
goto on_error;
if ((page_info_dump(pi, &val) != 0))
goto on_error;
if ((mdb_rc = mdb_cursor_put(cur, key, &val, MDB_NOOVERWRITE)) != 0)
goto on_error;
free(val.mv_data);
val.mv_data = 0;
*mdb_error = 0;
return 0;
on_error:
if (val.mv_data)
free(val.mv_data);
*mdb_error = mdb_rc;
page_info_delete(pi);
return -1;
}
/* How new PageInfo are created:
page_db_add ---------------> page_db_add_crawled_page_info
| |
| |
v v
page_db_add_link_page_info page_info_new_crawled
| |
| |
| |
+--------> page_info_new_link <---+
*/
PageDBError
page_db_add(PageDB *db, const CrawledPage *page, PageInfoList **page_info_list) {
// check if page should be expanded
if (page_db_expand(db) != 0)
return db->error->code;
MDB_txn *txn;
MDB_cursor *cur_hash2info;
MDB_cursor *cur_hash2idx;
MDB_cursor *cur_links;
MDB_cursor *cur_info;
MDB_val key;
MDB_val val;
int mdb_rc;
char *error = 0;
uint64_t *diff_id = 0;
uint64_t *same_id = 0;
// start a new write transaction
txn = 0;
if ((txn_manager_begin(db->txn_manager, 0, &txn)) != 0)
error = db->txn_manager->error->message;
else if ((mdb_rc = page_db_open_hash2info(txn, &cur_hash2info)) != 0)
error = "opening hash2info cursor";
else if ((mdb_rc = page_db_open_hash2idx(txn, &cur_hash2idx)) != 0)
error = "opening hash2idx cursor";
else if ((mdb_rc = page_db_open_links(txn, &cur_links)) != 0)
error = "opening links cursor";
else if ((mdb_rc = page_db_open_info(txn, &cur_info)) != 0)
error = "opening info cursor";
if (error != 0)
goto on_error;
// get n_pages
key.mv_size = sizeof(info_n_pages);
key.mv_data = info_n_pages;
if ((mdb_rc = mdb_cursor_get(cur_info, &key, &val, MDB_SET)) != 0) {
error = "retrieving info.n_pages";
goto on_error;
}
size_t n_pages = *(size_t*)val.mv_data;
uint64_t cp_hash = page_db_hash(page->url);
key.mv_size = sizeof(uint64_t);
key.mv_data = &cp_hash;
if (db->domain_temp) {
domain_temp_update(db->domain_temp, (float)page->time);
domain_temp_heat(db->domain_temp, page_db_hash_get_domain(cp_hash));
}
PageInfo *pi;
if (page_db_add_crawled_page_info(cur_hash2info, &key, page, &pi, &mdb_rc) != 0) {
error = "adding/updating page info";
goto on_error;
}
uint64_t link_depth = pi->depth + 1;
if (page_info_list) {
*page_info_list = page_info_list_new(pi, cp_hash);
if (!*page_info_list) {
error = "allocating new PageInfo list";
goto on_error;
}
} else {
page_info_delete(pi);
pi = 0;
}
size_t n_links = crawled_page_n_links(page);
// store here links inside the same domain as the crawled page
same_id = malloc((n_links + 1)*sizeof(*same_id));
// store here links outside the domain of the crawled page
diff_id = malloc((n_links + 1)*sizeof(*diff_id));
// next link id is going to be written here
uint64_t *id = diff_id;
// number of id's in same_id and diff_id. The first element of diff_id
// array is reserved for the id of the crawled page, so we start at 1.
// The first element of same_id will be a copy of the last element of
// diff_id, so we start at 1 too.
uint64_t same_i = 1;
uint64_t diff_i = 1;
if (!same_id || !diff_id) {
error = "could not malloc";
goto on_error;
}
// hash of the current URL
uint64_t hash = cp_hash;
for (size_t i=0; i <= n_links; ++i) {
const LinkInfo *link = i > 0? crawled_page_get_link(page, i - 1): 0;
if (link) {
hash = page_db_hash(link->url);
key.mv_size = sizeof(uint64_t);
key.mv_data = &hash;
id = same_domain(page->url, link->url)?
same_id + same_i++:
diff_id + diff_i++;
}
val.mv_size = sizeof(uint64_t);
val.mv_data = &n_pages;
switch (mdb_rc = mdb_cursor_put(cur_hash2idx, &key, &val, MDB_NOOVERWRITE)) {
case MDB_KEYEXIST: // not really an error
*id = *(uint64_t*)val.mv_data;
break;
case 0:
*id = n_pages++;
if (link) {
if (page_db_add_link_page_info(
cur_hash2info,
&key,
cp_hash,
link_depth,
link,
&pi,
&mdb_rc) != 0) {
error = "adding/updating link info";
goto on_error;
}
if (page_info_list) {
if (!(*page_info_list = page_info_list_cons(*page_info_list, pi, hash))) {
error = "adding new PageInfo to list";
goto on_error;
}
}
else {
page_info_delete(pi);
}
}
break;
default:
goto on_error;
}
}
// store n_pages
key.mv_size = sizeof(info_n_pages);
key.mv_data = info_n_pages;
val.mv_size = sizeof(size_t);
val.mv_data = &n_pages;
if ((mdb_rc = mdb_cursor_put(cur_info, &key, &val, 0)) != 0) {
error = "storing n_pages";
goto on_error;
}
// store links and commit transaction
// The format for the links is the following:
//
// KEY = ID of crawled page
// VAL = Number of links to different domain,
// diff link id 1, diff link id 2, ...
// same link id 1, same link id 2, ...
key.mv_size = sizeof(uint64_t);
key.mv_data = diff_id; // remember that diff_id[0] is the id of the
// crawled page
// the links are stored as deltas starting from the 'from' page, encoded
// using varint.
uint8_t *buf = val.mv_data = malloc(MAX_VARINT_SIZE*(n_links + 1));
if (!buf) {
error = "allocating memory to store links";
goto on_error;
}
// write number of diff links (substract 1 to take into account this page id)
buf = varint_encode_uint64(diff_i - 1, buf);
// write diff links
for (size_t i=1; i<diff_i; ++i)
buf = varint_encode_int64((int64_t)diff_id[i] - (int64_t)diff_id[i-1], buf);
same_id[0] = diff_id[diff_i-1];
for (size_t i=1; i<same_i; ++i)
buf = varint_encode_int64((int64_t)same_id[i] - (int64_t)same_id[i-1], buf);
val.mv_size = (char*)buf - (char*)val.mv_data;
if ((mdb_rc = mdb_cursor_put(cur_links, &key, &val, 0)) != 0) {
error = "storing links";
goto on_error;
}
free(val.mv_data);
free(same_id);
free(diff_id);
same_id = diff_id = 0;
if (txn_manager_commit(db->txn_manager, txn) != 0) {
error = db->txn_manager->error->message;
goto on_error;
}
return db->error->code;
on_error:
if (same_id)
free(same_id);
if (diff_id)
free(diff_id);
if (txn)
txn_manager_abort(db->txn_manager, txn);
page_db_set_error(db, page_db_error_internal, __func__);
page_db_add_error(db, error);
if (mdb_rc != 0)
page_db_add_error(db, mdb_strerror(mdb_rc));
return db->error->code;
}
static int mdb_tool_next_id(
Operation *op,
MDB_txn *tid,
Entry *e,
struct berval *text,
int hole )
{
struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
struct berval dn = e->e_name;
struct berval ndn = e->e_nname;
struct berval pdn, npdn, nmatched;
ID id, pid = 0;
int rc;
if (ndn.bv_len == 0) {
e->e_id = 0;
return 0;
}
rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, &nmatched );
if ( rc == MDB_NOTFOUND ) {
if ( !be_issuffix( op->o_bd, &ndn ) ) {
ID eid = e->e_id;
dnParent( &ndn, &npdn );
if ( nmatched.bv_len != npdn.bv_len ) {
dnParent( &dn, &pdn );
e->e_name = pdn;
e->e_nname = npdn;
rc = mdb_tool_next_id( op, tid, e, text, 1 );
e->e_name = dn;
e->e_nname = ndn;
if ( rc ) {
return rc;
}
/* If parent didn't exist, it was created just now
* and its ID is now in e->e_id. Make sure the current
* entry gets added under the new parent ID.
*/
if ( eid != e->e_id ) {
pid = e->e_id;
}
} else {
pid = id;
}
}
rc = mdb_next_id( op->o_bd, idcursor, &e->e_id );
if ( rc ) {
snprintf( text->bv_val, text->bv_len,
"next_id failed: %s (%d)",
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
"=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
return rc;
}
rc = mdb_dn2id_add( op, mcp, mcd, pid, e );
if ( rc ) {
snprintf( text->bv_val, text->bv_len,
"dn2id_add failed: %s (%d)",
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
"=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
} else if ( hole ) {
MDB_val key, data;
if ( nholes == nhmax - 1 ) {
if ( holes == hbuf ) {
holes = ch_malloc( nhmax * sizeof(dn_id) * 2 );
AC_MEMCPY( holes, hbuf, sizeof(hbuf) );
} else {
holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 );
}
nhmax *= 2;
}
ber_dupbv( &holes[nholes].dn, &ndn );
holes[nholes++].id = e->e_id;
key.mv_size = sizeof(ID);
key.mv_data = &e->e_id;
data.mv_size = 0;
data.mv_data = NULL;
rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE );
if ( rc == MDB_KEYEXIST )
rc = 0;
if ( rc ) {
snprintf( text->bv_val, text->bv_len,
"dummy id2entry add failed: %s (%d)",
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
"=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 );
}
}
} else if ( !hole ) {
unsigned i, j;
e->e_id = id;
for ( i=0; i<nholes; i++) {
if ( holes[i].id == e->e_id ) {
free(holes[i].dn.bv_val);
for (j=i;j<nholes;j++) holes[j] = holes[j+1];
holes[j].id = 0;
nholes--;
break;
} else if ( holes[i].id > e->e_id ) {
break;
}
}
}
return rc;
}
static int
mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic )
{
mdb_tool_idl_cache_entry *ice;
MDB_val key, data[2];
int i, rc;
ID id, nid;
/* Freshly allocated, ignore it */
if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
return 0;
}
key.mv_data = ic->kstr.bv_val;
key.mv_size = ic->kstr.bv_len;
if ( ic->count > MDB_IDL_DB_SIZE ) {
while ( ic->flags & WAS_FOUND ) {
rc = mdb_cursor_get( mc, &key, data, MDB_SET );
if ( rc ) {
/* FIXME: find out why this happens */
ic->flags = 0;
break;
}
if ( ic->flags & WAS_RANGE ) {
/* Skip lo */
rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
/* Get hi */
rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP );
/* Store range hi */
data[0].mv_data = &ic->last;
rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT );
} else {
/* Delete old data, replace with range */
ic->first = *(ID *)data[0].mv_data;
mdb_cursor_del( mc, MDB_NODUPDATA );
}
break;
}
if ( !(ic->flags & WAS_RANGE)) {
/* range, didn't exist before */
nid = 0;
data[0].mv_size = sizeof(ID);
data[0].mv_data = &nid;
rc = mdb_cursor_put( mc, &key, data, 0 );
if ( rc == 0 ) {
data[0].mv_data = &ic->first;
rc = mdb_cursor_put( mc, &key, data, 0 );
if ( rc == 0 ) {
data[0].mv_data = &ic->last;
rc = mdb_cursor_put( mc, &key, data, 0 );
}
}
if ( rc ) {
rc = -1;
}
}
} else {
/* Normal write */
int n;
data[0].mv_size = sizeof(ID);
rc = 0;
i = ic->offset;
for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) {
int end;
if ( ice->next ) {
end = IDBLOCK;
} else {
end = ic->count & (IDBLOCK-1);
if ( !end )
end = IDBLOCK;
}
data[1].mv_size = end - i;
data[0].mv_data = &ice->ids[i];
i = 0;
rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE );
if ( rc ) {
if ( rc == MDB_KEYEXIST ) {
rc = 0;
continue;
}
rc = -1;
break;
}
}
if ( ic->head ) {
ic->tail->next = ai->ai_flist;
ai->ai_flist = ic->head;
}
}
ic->head = ai->ai_clist;
ai->ai_clist = ic;
return rc;
}
/* We add two elements to the DN2ID database - a data item under the parent's
* entryID containing the child's RDN and entryID, and an item under the
* child's entryID containing the parent's entryID.
*/
int
mdb_dn2id_add(
Operation *op,
MDB_cursor *mcp,
MDB_cursor *mcd,
ID pid,
ID nsubs,
int upsub,
Entry *e )
{
struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
MDB_val key, data;
ID nid;
int rc, rlen, nrlen;
diskNode *d;
char *ptr;
Debug( LDAP_DEBUG_TRACE, "=> mdb_dn2id_add 0x%lx: \"%s\"\n",
e->e_id, e->e_ndn ? e->e_ndn : "" );
nrlen = dn_rdnlen( op->o_bd, &e->e_nname );
if (nrlen) {
rlen = dn_rdnlen( op->o_bd, &e->e_name );
} else {
nrlen = e->e_nname.bv_len;
rlen = e->e_name.bv_len;
}
d = op->o_tmpalloc(sizeof(diskNode) + rlen + nrlen + sizeof(ID), op->o_tmpmemctx);
d->nrdnlen[1] = nrlen & 0xff;
d->nrdnlen[0] = (nrlen >> 8) | 0x80;
ptr = lutil_strncopy( d->nrdn, e->e_nname.bv_val, nrlen );
*ptr++ = '\0';
ptr = lutil_strncopy( ptr, e->e_name.bv_val, rlen );
*ptr++ = '\0';
memcpy( ptr, &e->e_id, sizeof( ID ));
ptr += sizeof( ID );
memcpy( ptr, &nsubs, sizeof( ID ));
key.mv_size = sizeof(ID);
key.mv_data = &nid;
nid = pid;
/* Need to make dummy root node once. Subsequent attempts
* will fail harmlessly.
*/
if ( pid == 0 ) {
diskNode dummy = {{0, 0}, "", "", ""};
data.mv_data = &dummy;
data.mv_size = sizeof(diskNode);
mdb_cursor_put( mcp, &key, &data, MDB_NODUPDATA );
}
data.mv_data = d;
data.mv_size = sizeof(diskNode) + rlen + nrlen + sizeof( ID );
/* Add our child node under parent's key */
rc = mdb_cursor_put( mcp, &key, &data, MDB_NODUPDATA );
/* Add our own node */
if (rc == 0) {
int flag = MDB_NODUPDATA;
nid = e->e_id;
/* drop subtree count */
data.mv_size -= sizeof( ID );
ptr -= sizeof( ID );
memcpy( ptr, &pid, sizeof( ID ));
d->nrdnlen[0] ^= 0x80;
if ((slapMode & SLAP_TOOL_MODE) || (e->e_id == mdb_read_nextid(mdb)))
flag |= MDB_APPEND;
rc = mdb_cursor_put( mcd, &key, &data, flag );
}
op->o_tmpfree( d, op->o_tmpmemctx );
/* Add our subtree count to all superiors */
if ( rc == 0 && upsub && pid ) {
ID subs;
nid = pid;
do {
/* Get parent's RDN */
rc = mdb_cursor_get( mcp, &key, &data, MDB_SET );
if ( !rc ) {
char *p2;
ptr = (char *)data.mv_data + data.mv_size - sizeof( ID );
memcpy( &nid, ptr, sizeof( ID ));
/* Get parent's node under grandparent */
d = data.mv_data;
rlen = ( d->nrdnlen[0] << 8 ) | d->nrdnlen[1];
p2 = op->o_tmpalloc( rlen + 2, op->o_tmpmemctx );
memcpy( p2, data.mv_data, rlen+2 );
*p2 ^= 0x80;
data.mv_data = p2;
rc = mdb_cursor_get( mcp, &key, &data, MDB_GET_BOTH );
op->o_tmpfree( p2, op->o_tmpmemctx );
if ( !rc ) {
/* Get parent's subtree count */
ptr = (char *)data.mv_data + data.mv_size - sizeof( ID );
memcpy( &subs, ptr, sizeof( ID ));
subs += nsubs;
p2 = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx );
memcpy( p2, data.mv_data, data.mv_size - sizeof( ID ));
memcpy( p2+data.mv_size - sizeof( ID ), &subs, sizeof( ID ));
data.mv_data = p2;
rc = mdb_cursor_put( mcp, &key, &data, MDB_CURRENT );
op->o_tmpfree( p2, op->o_tmpmemctx );
}
}
if ( rc )
break;
} while ( nid );
}
Debug( LDAP_DEBUG_TRACE, "<= mdb_dn2id_add 0x%lx: %d\n", e->e_id, rc );
return rc;
}
int
mdb_idl_insert_keys(
BackendDB *be,
MDB_cursor *cursor,
struct berval *keys,
ID id )
{
struct mdb_info *mdb = be->be_private;
MDB_val key, data;
ID lo, hi, *i;
char *err;
int rc = 0, k;
unsigned int flag = MDB_NODUPDATA;
#ifndef MISALIGNED_OK
int kbuf[2];
#endif
{
char buf[16];
Debug( LDAP_DEBUG_ARGS,
"mdb_idl_insert_keys: %lx %s\n",
(long) id, mdb_show_key( buf, keys->bv_val, keys->bv_len ), 0 );
}
assert( id != NOID );
#ifndef MISALIGNED_OK
if (keys[0].bv_len & ALIGNER)
kbuf[1] = 0;
#endif
for ( k=0; keys[k].bv_val; k++ ) {
/* Fetch the first data item for this key, to see if it
* exists and if it's a range.
*/
#ifndef MISALIGNED_OK
if (keys[k].bv_len & ALIGNER) {
key.mv_size = sizeof(kbuf);
key.mv_data = kbuf;
memcpy(key.mv_data, keys[k].bv_val, keys[k].bv_len);
} else
#endif
{
key.mv_size = keys[k].bv_len;
key.mv_data = keys[k].bv_val;
}
rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
err = "c_get";
if ( rc == 0 ) {
i = data.mv_data;
memcpy(&lo, data.mv_data, sizeof(ID));
if ( lo != 0 ) {
/* not a range, count the number of items */
size_t count;
rc = mdb_cursor_count( cursor, &count );
if ( rc != 0 ) {
err = "c_count";
goto fail;
}
if ( count >= MDB_IDL_DB_MAX ) {
/* No room, convert to a range */
lo = *i;
rc = mdb_cursor_get( cursor, &key, &data, MDB_LAST_DUP );
if ( rc != 0 && rc != MDB_NOTFOUND ) {
err = "c_get last_dup";
goto fail;
}
i = data.mv_data;
hi = *i;
/* Update hi/lo if needed */
if ( id < lo ) {
lo = id;
} else if ( id > hi ) {
hi = id;
}
/* delete the old key */
rc = mdb_cursor_del( cursor, MDB_NODUPDATA );
if ( rc != 0 ) {
err = "c_del dups";
goto fail;
}
/* Store the range */
data.mv_size = sizeof(ID);
data.mv_data = &id;
id = 0;
rc = mdb_cursor_put( cursor, &key, &data, 0 );
if ( rc != 0 ) {
err = "c_put range";
goto fail;
}
id = lo;
rc = mdb_cursor_put( cursor, &key, &data, 0 );
if ( rc != 0 ) {
err = "c_put lo";
goto fail;
}
id = hi;
rc = mdb_cursor_put( cursor, &key, &data, 0 );
if ( rc != 0 ) {
err = "c_put hi";
goto fail;
}
} else {
/* There's room, just store it */
if (id == mdb->mi_nextid)
flag |= MDB_APPENDDUP;
goto put1;
}
} else {
/* It's a range, see if we need to rewrite
* the boundaries
*/
lo = i[1];
hi = i[2];
if ( id < lo || id > hi ) {
/* position on lo */
rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP );
if ( rc != 0 ) {
err = "c_get lo";
goto fail;
}
if ( id > hi ) {
/* position on hi */
rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP );
if ( rc != 0 ) {
err = "c_get hi";
goto fail;
}
}
data.mv_size = sizeof(ID);
data.mv_data = &id;
/* Replace the current lo/hi */
rc = mdb_cursor_put( cursor, &key, &data, MDB_CURRENT );
if ( rc != 0 ) {
err = "c_put lo/hi";
goto fail;
}
}
}
} else if ( rc == MDB_NOTFOUND ) {
flag &= ~MDB_APPENDDUP;
put1: data.mv_data = &id;
data.mv_size = sizeof(ID);
rc = mdb_cursor_put( cursor, &key, &data, flag );
/* Don't worry if it's already there */
if ( rc == MDB_KEYEXIST )
rc = 0;
if ( rc ) {
err = "c_put id";
goto fail;
}
} else {
/* initial c_get failed, nothing was done */
fail:
Debug( LDAP_DEBUG_ANY, "=> mdb_idl_insert_keys: "
"%s failed: %s (%d)\n", err, mdb_strerror(rc), rc );
break;
}
}
return rc;
}
/* Write a frame or frames to the log. */
int sqlite3WalFrames(Wal *pWal, int szPage, PgHdr *pList, Pgno nTruncate, int isCommit, int sync_flags)
{
PgHdr *p;
MDB_val key, data;
int rc;
mdbinf* mdb;
MDB_txn* txn;
#if ATOMIC
db_thread *thr = g_tsd_thread;
db_connection* pCon = g_tsd_conn;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
db_connection* pCon = enif_tsd_get(g_tsd_conn);
#endif
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
txn = mdb->txn;
if (!mdb)
return SQLITE_ERROR;
key.mv_size = sizeof(u64);
key.mv_data = (void*)&pWal->index;
// Term/evnum must always be increasing
if ((pWal->inProgressTerm > 0 && pWal->inProgressTerm < pWal->lastCompleteTerm) ||
(pWal->inProgressEvnum > 0 && pWal->inProgressEvnum < pWal->lastCompleteEvnum))
return SQLITE_ERROR;
track_time(2,thr);
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Fragment,CompressedPage/binary>>}
for(p=pList; p; p=p->pDirty)
{
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
u8 pagesBuf[PAGE_BUFF_SIZE];
int full_size = 0;
int page_size = LZ4_compress_default((char*)p->pData,(char*)pagesBuf+sizeof(u64)*2+1,szPage,sizeof(pagesBuf));
char fragment_index = 0;
int skipped = 0;
track_time(3,thr);
DBG("Insert frame, actor=%lld, pgno=%u, "
"term=%lld, evnum=%lld, commit=%d, truncate=%d, compressedsize=%d",
pWal->index,p->pgno,pWal->inProgressTerm,pWal->inProgressEvnum,
isCommit,nTruncate,page_size);
if (pCon->doReplicate)
{
u8 hdr[sizeof(u64)*2+sizeof(u32)*2];
put8byte(hdr, pWal->inProgressTerm);
put8byte(hdr+sizeof(u64), pWal->inProgressEvnum);
put4byte(hdr+sizeof(u64)*2, p->pgno);
if (p->pDirty)
put4byte(hdr+sizeof(u64)*2+sizeof(u32), 0);
else
put4byte(hdr+sizeof(u64)*2+sizeof(u32), nTruncate);
#ifndef _TESTAPP_
wal_page_hook(thr,pagesBuf+sizeof(u64)*2+1, page_size, hdr, sizeof(hdr));
#endif
}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
// Check if there are pages with the same or higher evnum/evterm. If there are, delete them.
// This can happen if sqlite flushed some page to disk before commiting, because there were
// so many pages that they could not be held in memory. Or it could happen if pages need to be
// overwritten because there was a write that did not pass raft consensus.
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY);
if (rc == MDB_SUCCESS)
{
size_t ndupl;
mdb_cursor_count(mdb->cursorPages,&ndupl);
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP);
if (rc == MDB_SUCCESS)
{
MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL};
u64 evnum, evterm;
u8 frag = *((u8*)data.mv_data+sizeof(u64)*2);
memcpy(&evterm, data.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64));
while ((evterm > pWal->inProgressTerm || evnum >= pWal->inProgressEvnum))
//(pWal->inProgressTerm + pWal->inProgressEvnum) > 0)
{
DBG("Deleting pages higher or equal to current. "
"Evterm=%llu, evnum=%llu, curterm=%llu, curevn=%llu, dupl=%ld",
evterm,evnum,pWal->inProgressTerm,pWal->inProgressEvnum,ndupl);
if (pgDelKey.mv_data != NULL)
{
if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS)
{
DBG("Unable to cleanup page from pagedb %d",rc);
break;
}
pgDelKey.mv_data = NULL;
}
mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT);
// if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS)
// {
// DBG("Cant delete!");
// break;
// }
if (frag == 0)
pWal->allPages--;
ndupl--;
if (!ndupl)
break;
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP);
if (rc != MDB_SUCCESS)
break;
memcpy(&evterm, data.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64));
frag = *((u8*)data.mv_data+sizeof(u64)*2);
}
if (pgDelKey.mv_data != NULL)
{
if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS)
{
DBG("Unable to cleanup page from pagedb %d",rc);
break;
}
pgDelKey.mv_data = NULL;
}
}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
}
track_time(4,thr);
memcpy(pagesBuf, &pWal->inProgressTerm, sizeof(u64));
memcpy(pagesBuf + sizeof(u64), &pWal->inProgressEvnum, sizeof(u64));
full_size = page_size + sizeof(u64)*2 + 1;
if (full_size < thr->maxvalsize)
fragment_index = 0;
else
{
full_size = page_size;
skipped = thr->maxvalsize - sizeof(u64)*2 - 1;
full_size -= skipped;
while(full_size > 0)
{
full_size -= (thr->maxvalsize - sizeof(u64)*2 - 1);
fragment_index++;
}
full_size = page_size + sizeof(u64)*2 +1;
}
pagesBuf[sizeof(u64)*2] = fragment_index;
data.mv_size = fragment_index == 0 ? full_size : thr->maxvalsize;
data.mv_data = pagesBuf;
// fragment_index == 0 ? MDB_APPENDDUP : 0
if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS)
{
// printf("Cursor put failed to pages %d",rc);
DBG("ERROR: cursor put failed=%d, datasize=%d",rc,full_size);
return SQLITE_ERROR;
}
fragment_index--;
skipped = data.mv_size;
while (fragment_index >= 0)
{
DBG("Insert fragment %d",(int)fragment_index);
if (fragment_index == 0)
data.mv_size = full_size - skipped + sizeof(u64)*2 + 1;
else
data.mv_size = thr->maxvalsize;
data.mv_data = pagesBuf + skipped - (sizeof(u64)*2+1);
memcpy(pagesBuf + skipped - (sizeof(u64)*2+1), &pWal->inProgressTerm, sizeof(u64));
memcpy(pagesBuf + skipped - (sizeof(u64)+1), &pWal->inProgressEvnum, sizeof(u64));
pagesBuf[skipped-1] = fragment_index;
if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS)
{
DBG("ERROR: cursor secondary put failed: err=%d, datasize=%d, skipped=%d, frag=%d",
rc,full_size, skipped, (int)fragment_index);
return SQLITE_ERROR;
}
skipped += data.mv_size - sizeof(u64)*2 - 1;
fragment_index--;
}
thr->pagesChanged++;
}
// printf("");
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
if (pWal->inProgressTerm > 0)
{
for(p=pList; p; p=p->pDirty)
{
u8 logKeyBuf[sizeof(u64)*3];
DBG("Inserting to log");
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum, sizeof(u64));
key.mv_size = sizeof(logKeyBuf);
key.mv_data = logKeyBuf;
data.mv_size = sizeof(u32);
data.mv_data = &p->pgno;
if (mdb_cursor_put(mdb->cursorLog,&key,&data,0) != MDB_SUCCESS)
{
// printf("Cursor put failed to log");
DBG("ERROR: cursor put to log failed: %d",rc);
return SQLITE_ERROR;
}
pWal->allPages++;
}
}
else
{
DBG("Skipping log");
for(p=pList; p; p=p->pDirty)
pWal->allPages++;
}
/** - Info DB: {<<ActorIndex:64>>, <<V,FirstCompleteTerm:64,FirstCompleteEvnum:64,
LastCompleteTerm:64,LastCompleteEvnum:64,
InprogressTerm:64,InProgressEvnum:64>>} */
{
if (isCommit)
{
DBG("Commit actor=%llu fct=%llu, fcev=%llu, lct=%llu, lcev=%llu, int=%llu, inev=%llu",
pWal->index,
pWal->firstCompleteTerm, pWal->firstCompleteEvnum, pWal->lastCompleteTerm,
pWal->lastCompleteEvnum, pWal->inProgressTerm,pWal->inProgressEvnum);
#ifndef _TESTAPP_
enif_mutex_lock(pWal->mtx);
#endif
pWal->lastCompleteTerm = pWal->inProgressTerm > 0 ?
pWal->inProgressTerm : pWal->lastCompleteTerm;
pWal->lastCompleteEvnum = pWal->inProgressEvnum > 0 ?
pWal->inProgressEvnum : pWal->lastCompleteEvnum;
if (pWal->firstCompleteTerm == 0)
{
pWal->firstCompleteTerm = pWal->inProgressTerm;
pWal->firstCompleteEvnum = pWal->inProgressEvnum;
}
pWal->inProgressTerm = pWal->inProgressEvnum = 0;
pWal->mxPage = pWal->mxPage > nTruncate ? pWal->mxPage : nTruncate;
// pWal->changed = 0;
thr->forceCommit = 1;
pCon->dirty = 0;
#ifndef _TESTAPP_
enif_mutex_unlock(pWal->mtx);
#endif
DBG("cur mxpage=%u",pWal->mxPage);
}
else
{
// pWal->changed = 1;
pCon->dirty = 1;
}
thr->pagesChanged++;
rc = storeinfo(pWal,0,0,NULL);
if (rc != SQLITE_OK)
return rc;
track_time(5,thr);
}
return SQLITE_OK;
}
int cursor_put(cursor_t cursor, MDB_val *key, MDB_val *data, unsigned int flags){
int ret = cursor->prev ? mdb_cursor_put(cursor->cursor, key, data, flags) : MDB_BAD_TXN;
if(MDB_SUCCESS == ret)
cursor->txn->updated = 1;
return ret;
}
int
mdb_idl_delete_keys(
BackendDB *be,
MDB_cursor *cursor,
struct berval *keys,
ID id )
{
int rc = 0, k;
MDB_val key, data;
ID lo, hi, tmp, *i;
char *err;
#ifndef MISALIGNED_OK
int kbuf[2];
#endif
{
char buf[16];
Debug( LDAP_DEBUG_ARGS,
"mdb_idl_delete_keys: %lx %s\n",
(long) id, mdb_show_key( buf, keys->bv_val, keys->bv_len ), 0 );
}
assert( id != NOID );
#ifndef MISALIGNED_OK
if (keys[0].bv_len & ALIGNER)
kbuf[1] = 0;
#endif
for ( k=0; keys[k].bv_val; k++) {
/* Fetch the first data item for this key, to see if it
* exists and if it's a range.
*/
#ifndef MISALIGNED_OK
if (keys[k].bv_len & ALIGNER) {
key.mv_size = sizeof(kbuf);
key.mv_data = kbuf;
memcpy(key.mv_data, keys[k].bv_val, keys[k].bv_len);
} else
#endif
{
key.mv_size = keys[k].bv_len;
key.mv_data = keys[k].bv_val;
}
rc = mdb_cursor_get( cursor, &key, &data, MDB_SET );
err = "c_get";
if ( rc == 0 ) {
memcpy( &tmp, data.mv_data, sizeof(ID) );
i = data.mv_data;
if ( tmp != 0 ) {
/* Not a range, just delete it */
data.mv_data = &id;
rc = mdb_cursor_get( cursor, &key, &data, MDB_GET_BOTH );
if ( rc != 0 ) {
err = "c_get id";
goto fail;
}
rc = mdb_cursor_del( cursor, 0 );
if ( rc != 0 ) {
err = "c_del id";
goto fail;
}
} else {
/* It's a range, see if we need to rewrite
* the boundaries
*/
lo = i[1];
hi = i[2];
if ( id == lo || id == hi ) {
ID lo2 = lo, hi2 = hi;
if ( id == lo ) {
lo2++;
} else if ( id == hi ) {
hi2--;
}
if ( lo2 >= hi2 ) {
/* The range has collapsed... */
rc = mdb_cursor_del( cursor, MDB_NODUPDATA );
if ( rc != 0 ) {
err = "c_del dup";
goto fail;
}
} else {
/* position on lo */
rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP );
if ( id == lo )
data.mv_data = &lo2;
else {
/* position on hi */
rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP );
data.mv_data = &hi2;
}
/* Replace the current lo/hi */
data.mv_size = sizeof(ID);
rc = mdb_cursor_put( cursor, &key, &data, MDB_CURRENT );
if ( rc != 0 ) {
err = "c_put lo/hi";
goto fail;
}
}
}
}
} else {
/* initial c_get failed, nothing was done */
fail:
if ( rc == MDB_NOTFOUND )
rc = 0;
if ( rc ) {
Debug( LDAP_DEBUG_ANY, "=> mdb_idl_delete_key: "
"%s failed: %s (%d)\n", err, mdb_strerror(rc), rc );
break;
}
}
}
return rc;
}
static int storeinfo(Wal *pWal, u64 currentTerm, u8 votedForSize, u8 *votedFor)
{
MDB_val key = {0,NULL}, data = {0,NULL};
int rc;
#if ATOMIC
db_thread *thr = g_tsd_thread;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
#endif
mdbinf* mdb;
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
if (!mdb)
return SQLITE_ERROR;
key.mv_size = sizeof(u64);
key.mv_data = &pWal->index;
if (votedFor == NULL)
{
rc = mdb_cursor_get(mdb->cursorInfo,&key,&data,MDB_SET_KEY);
if (rc == MDB_SUCCESS && data.mv_size >= (1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1))
{
memcpy(¤tTerm, (u8*)data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2, sizeof(u64));
votedForSize = (u8)((u8*)data.mv_data)[1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)];
//votedFor = data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1;
votedFor = alloca(votedForSize);
memcpy(votedFor,(u8*)data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1, votedForSize);
// DBG("Voted for %.*s",(int)votedForSize,(char*)votedFor));
}
}
key.mv_size = sizeof(u64);
key.mv_data = &pWal->index;
data.mv_data = NULL;
data.mv_size = 1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1+votedForSize;
rc = mdb_cursor_put(mdb->cursorInfo,&key,&data,MDB_RESERVE);
if (rc == MDB_SUCCESS)
{
u8 *infoBuf = data.mv_data;
infoBuf[0] = 1;
memcpy(infoBuf+1, &pWal->firstCompleteTerm,sizeof(u64));
memcpy(infoBuf+1+sizeof(u64), &pWal->firstCompleteEvnum,sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*2, &pWal->lastCompleteTerm, sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*3, &pWal->lastCompleteEvnum,sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*4, &pWal->inProgressTerm, sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*5, &pWal->inProgressEvnum, sizeof(u64));
memcpy(infoBuf+1+sizeof(u64)*6, &pWal->mxPage, sizeof(u32));
memcpy(infoBuf+1+sizeof(u64)*6+sizeof(u32), &pWal->allPages,sizeof(u32));
memcpy(infoBuf+1+sizeof(u64)*6+sizeof(u32)*2, ¤tTerm, sizeof(u64));
infoBuf[1+sizeof(u64)*7+sizeof(u32)*2] = votedForSize;
memcpy(infoBuf+2+sizeof(u64)*7+sizeof(u32)*2, votedFor, votedForSize);
thr->forceCommit = 1;
return SQLITE_OK;
}
return SQLITE_ERROR;
}
static int
mdb_dn2id_upgrade( BackendDB *be ) {
struct mdb_info *mi = (struct mdb_info *) be->be_private;
MDB_txn *mt;
MDB_cursor *mc = NULL;
MDB_val key, data;
char *ptr;
int rc, writes=0, depth=0;
int enable_meter = 0;
ID id = 0, *num, count = 0;
rec *stack;
lutil_meter_t meter;
if (!(mi->mi_flags & MDB_NEED_UPGRADE)) {
Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n",
be->be_suffix[0].bv_val, 0, 0 );
return 0;
}
{
MDB_stat st;
mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st);
if (!st.ms_entries) {
/* Empty DB, nothing to upgrade? */
return 0;
}
if (isatty(2))
enable_meter = !lutil_meter_open(&meter,
&lutil_meter_text_display,
&lutil_meter_linear_estimator,
st.ms_entries);
}
num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec)));
stack = (rec *)(num + STACKSIZ);
rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
key.mv_size = sizeof(ID);
/* post-order depth-first update */
for(;;) {
size_t dkids;
unsigned char *ptr;
/* visit */
key.mv_data = &id;
stack[depth].id = id;
rc = mdb_cursor_get(mc, &key, &data, MDB_SET);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
num[depth] = 1;
rc = mdb_cursor_count(mc, &dkids);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
if (dkids > 1) {
rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
down:
ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID);
memcpy(&id, ptr, sizeof(ID));
depth++;
memcpy(stack[depth].rdn, data.mv_data, data.mv_size);
stack[depth].len = data.mv_size;
continue;
}
/* pop: write updated count, advance to next node */
pop:
/* update superior counts */
if (depth)
num[depth-1] += num[depth];
key.mv_data = &id;
id = stack[depth-1].id;
data.mv_data = stack[depth].rdn;
data.mv_size = stack[depth].len;
rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
data.mv_data = stack[depth].rdn;
ptr = (unsigned char *)data.mv_data + data.mv_size;
memcpy(ptr, &num[depth], sizeof(ID));
data.mv_size += sizeof(ID);
rc = mdb_cursor_del(mc, 0);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_cursor_put(mc, &key, &data, 0);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
count++;
#if 1
if (enable_meter)
lutil_meter_update(&meter, count, 0);
#else
{
int len;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2);
}
#endif
writes++;
if (writes == 1000) {
mdb_cursor_close(mc);
rc = mdb_txn_commit(mt);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH);
if (rc) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n",
mdb_strerror(rc), rc, 0 );
goto leave;
}
writes = 0;
}
depth--;
rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP);
if (rc == 0)
goto down;
rc = 0;
if (depth)
goto pop;
else
break;
}
leave:
mdb_cursor_close(mc);
if (mt) {
int r2;
r2 = mdb_txn_commit(mt);
if (r2) {
Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n",
mdb_strerror(r2), r2, 0 );
if (!rc)
rc = r2;
}
}
ch_free(num);
if (enable_meter) {
lutil_meter_update(&meter, count, 1);
lutil_meter_close(&meter);
}
return rc;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data, sdata;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
long kval;
char *sval;
srand(time(NULL));
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
sdata.mv_size = mst.ms_psize / 4 - 30;
sdata.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
do {
printf("key: %p %s, data: %p %.*s\n",
key.mv_data, mdb_dkey(&key, dkbuf),
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
} while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0);
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
#if 0
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
#endif
mdb_env_close(env);
return 0;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32];
char kval[sizeof(int)];
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
if (!(i & 0x0f))
sprintf(kval, "%03x", values[i]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
mdb_cursor_close(cursor);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_cursor *mc;
MDB_dbi dbi;
char *envname;
int envflags = 0, putflags = 0;
int dohdr = 0;
prog = argv[0];
if (argc < 2) {
usage();
}
/* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 's':
subname = strdup(optarg);
break;
case 'N':
putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
dbuf.mv_size = 4096;
dbuf.mv_data = malloc(dbuf.mv_size);
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
mdb_env_set_maxdbs(env, 2);
if (info.me_maxreaders)
mdb_env_set_maxreaders(env, info.me_maxreaders);
if (info.me_mapsize)
mdb_env_set_mapsize(env, info.me_mapsize);
if (info.me_mapaddr)
envflags |= MDB_FIXEDMAP;
rc = mdb_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
kbuf.mv_data = malloc(kbuf.mv_size);
while(!Eof) {
MDB_val key, data;
int batch = 0;
flags = 0;
if (!dohdr) {
dohdr = 1;
} else if (!(mode & NOHDR))
readhdr();
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while(1) {
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
goto txn_abort;
}
rc = mdb_cursor_put(mc, &key, &data, putflags);
if (rc == MDB_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdb_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
mdb_dbi_close(env, dbi);
}
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
/* mc must have been set by mdb_dn2id */
int
mdb_dn2id_delete(
Operation *op,
MDB_cursor *mc,
ID id,
ID nsubs )
{
ID nid;
char *ptr;
int rc;
Debug( LDAP_DEBUG_TRACE, "=> mdb_dn2id_delete 0x%lx\n", id );
/* Delete our ID from the parent's list */
rc = mdb_cursor_del( mc, 0 );
/* Delete our ID from the tree. With sorted duplicates, this
* will leave any child nodes still hanging around. This is OK
* for modrdn, which will add our info back in later.
*/
if ( rc == 0 ) {
MDB_val key, data;
if ( nsubs ) {
mdb_cursor_get( mc, &key, NULL, MDB_GET_CURRENT );
memcpy( &nid, key.mv_data, sizeof( ID ));
}
key.mv_size = sizeof(ID);
key.mv_data = &id;
rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
if ( rc == 0 )
rc = mdb_cursor_del( mc, 0 );
}
/* Delete our subtree count from all superiors */
if ( rc == 0 && nsubs && nid ) {
MDB_val key, data;
ID subs;
key.mv_data = &nid;
key.mv_size = sizeof( ID );
do {
rc = mdb_cursor_get( mc, &key, &data, MDB_SET );
if ( !rc ) {
char *p2;
diskNode *d;
int rlen;
ptr = (char *)data.mv_data + data.mv_size - sizeof( ID );
memcpy( &nid, ptr, sizeof( ID ));
/* Get parent's node under grandparent */
d = data.mv_data;
rlen = ( d->nrdnlen[0] << 8 ) | d->nrdnlen[1];
p2 = op->o_tmpalloc( rlen + 2, op->o_tmpmemctx );
memcpy( p2, data.mv_data, rlen+2 );
*p2 ^= 0x80;
data.mv_data = p2;
rc = mdb_cursor_get( mc, &key, &data, MDB_GET_BOTH );
op->o_tmpfree( p2, op->o_tmpmemctx );
if ( !rc ) {
/* Get parent's subtree count */
ptr = (char *)data.mv_data + data.mv_size - sizeof( ID );
memcpy( &subs, ptr, sizeof( ID ));
subs -= nsubs;
p2 = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx );
memcpy( p2, data.mv_data, data.mv_size - sizeof( ID ));
memcpy( p2+data.mv_size - sizeof( ID ), &subs, sizeof( ID ));
data.mv_data = p2;
rc = mdb_cursor_put( mc, &key, &data, MDB_CURRENT );
op->o_tmpfree( p2, op->o_tmpmemctx );
}
}
if ( rc )
break;
} while ( nid );
}
Debug( LDAP_DEBUG_TRACE, "<= mdb_dn2id_delete 0x%lx: %d\n", id, rc );
return rc;
}
int lmdbgo_mdb_cursor_put2(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, unsigned int flags) {
MDB_val key, val;
LMDBGO_SET_VAL(&key, kn, kdata);
LMDBGO_SET_VAL(&val, vn, vdata);
return mdb_cursor_put(cur, &key, &val, flags);
}
FreqSchedulerError
freq_scheduler_cursor_write(FreqScheduler *sch,
MDB_cursor *cursor,
uint64_t hash,
float freq) {
if (freq <= 0)
return 0;
ScheduleKey sk = {
.score = 0,
.hash = hash
};
MDB_val key = {
.mv_size = sizeof(sk),
.mv_data = &sk,
};
MDB_val val = {
.mv_size = sizeof(float),
.mv_data = &freq,
};
int mdb_rc;
if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) {
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, "adding page to schedule");
freq_scheduler_add_error(sch, mdb_strerror(mdb_rc));
}
return sch->error->code;
}
FreqSchedulerError
freq_scheduler_load_simple(FreqScheduler *sch,
float freq_default,
float freq_scale) {
char *error1 = 0;
char *error2 = 0;
MDB_cursor *cursor = 0;
HashInfoStream *st;
if (hashinfo_stream_new(&st, sch->page_db) != 0) {
error1 = "creating stream";
error2 = st? sch->page_db->error->message: "NULL";
goto on_error;
}
if (freq_scheduler_cursor_open(sch, &cursor) != 0)
goto on_error;
StreamState ss;
uint64_t hash;
PageInfo *pi;
while ((ss = hashinfo_stream_next(st, &hash, &pi)) == stream_state_next) {
if ((pi->n_crawls > 0) &&
((sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls)) &&
!page_info_is_seed(pi)){
float freq = freq_default;
if (freq_scale > 0) {
float rate = page_info_rate(pi);
if (rate > 0) {
freq = freq_scale * rate;
}
}
if (freq_scheduler_cursor_write(sch, cursor, hash, freq) != 0)
goto on_error;
}
page_info_delete(pi);
}
if (ss != stream_state_end) {
error1 = "incorrect stream state";
error2 = 0;
hashinfo_stream_delete(st);
goto on_error;
}
hashinfo_stream_delete(st);
if (freq_scheduler_cursor_commit(sch, cursor) != 0)
goto on_error;
return sch->error->code;
on_error:
freq_scheduler_cursor_abort(sch, cursor);
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, error1);
freq_scheduler_add_error(sch, error2);
return sch->error->code;
}
FreqSchedulerError
freq_scheduler_load_mmap(FreqScheduler *sch, MMapArray *freqs) {
char *error1 = 0;
char *error2 = 0;
MDB_cursor *cursor = 0;
if (txn_manager_expand(
sch->txn_manager,
2*freqs->n_elements*freqs->element_size) != 0) {
error1 = "resizing database";
error2 = sch->txn_manager->error->message;
goto on_error;
}
if (freq_scheduler_cursor_open(sch, &cursor) != 0)
goto on_error;
for (size_t i=0; i<freqs->n_elements; ++i) {
PageFreq *f = mmap_array_idx(freqs, i);
ScheduleKey sk = {
.score = 1.0/f->freq,
.hash = f->hash
};
MDB_val key = {
.mv_size = sizeof(sk),
.mv_data = &sk,
};
MDB_val val = {
.mv_size = sizeof(float),
.mv_data = &f->freq,
};
int mdb_rc;
if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) {
error1 = "adding page to schedule";
error2 = mdb_strerror(mdb_rc);
goto on_error;
}
}
if (freq_scheduler_cursor_commit(sch, cursor) != 0)
goto on_error;
return sch->error->code;
on_error:
freq_scheduler_cursor_abort(sch, cursor);
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, error1);
freq_scheduler_add_error(sch, error2);
return sch->error->code;
}
FreqSchedulerError
freq_scheduler_request(FreqScheduler *sch,
size_t max_requests,
PageRequest **request) {
char *error1 = 0;
char *error2 = 0;
MDB_cursor *cursor = 0;
if (freq_scheduler_cursor_open(sch, &cursor) != 0)
goto on_error;
PageRequest *req = *request = page_request_new(max_requests);
if (!req) {
error1 = "allocating memory";
goto on_error;
}
int interrupt_requests = 0;
while ((req->n_urls < max_requests) && !interrupt_requests) {
MDB_val key;
MDB_val val;
ScheduleKey sk;
float freq;
int mdb_rc;
int crawl = 0;
switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, MDB_FIRST)) {
case 0:
// copy data before deleting cursor
sk = *(ScheduleKey*)key.mv_data;
freq = *(float*)val.mv_data;
PageInfo *pi = 0;
if (page_db_get_info(sch->page_db, sk.hash, &pi) != 0) {
error1 = "retrieving PageInfo from PageDB";
error2 = sch->page_db->error->message;
goto on_error;
}
if (pi) {
if (sch->margin >= 0) {
double elapsed = difftime(time(0), 0) - pi->last_crawl;
if (elapsed < 1.0/(freq*(1.0 + sch->margin)))
interrupt_requests = 1;
}
crawl = (sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls);
}
if (!interrupt_requests) {
if ((mdb_rc = mdb_cursor_del(cursor, 0)) != 0) {
error1 = "deleting head of schedule";
error2 = mdb_strerror(mdb_rc);
goto on_error;
}
if (crawl) {
if (page_request_add_url(req, pi->url) != 0) {
error1 = "adding url to request";
goto on_error;
}
sk.score += 1.0/freq;
val.mv_data = &freq;
key.mv_data = &sk;
if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) {
error1 = "moving element inside schedule";
error2 = mdb_strerror(mdb_rc);
goto on_error;
}
}
}
page_info_delete(pi);
break;
case MDB_NOTFOUND: // no more pages left
interrupt_requests = 1;
break;
default:
error1 = "getting head of schedule";
error2 = mdb_strerror(mdb_rc);
goto on_error;
}
}
if (freq_scheduler_cursor_commit(sch, cursor) != 0)
goto on_error;
return sch->error->code;
on_error:
freq_scheduler_cursor_abort(sch, cursor);
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, error1);
freq_scheduler_add_error(sch, error2);
return sch->error->code;
}
FreqSchedulerError
freq_scheduler_add(FreqScheduler *sch, const CrawledPage *page) {
if (page_db_add(sch->page_db, page, 0) != 0) {
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, "adding crawled page");
freq_scheduler_add_error(sch, sch->page_db->error->message);
}
return sch->error->code;
}
void
freq_scheduler_delete(FreqScheduler *sch) {
mdb_env_close(sch->txn_manager->env);
(void)txn_manager_delete(sch->txn_manager);
if (!sch->persist) {
char *data = build_path(sch->path, "data.mdb");
char *lock = build_path(sch->path, "lock.mdb");
remove(data);
remove(lock);
free(data);
free(lock);
remove(sch->path);
}
free(sch->path);
error_delete(sch->error);
free(sch);
}
FreqSchedulerError
freq_scheduler_dump(FreqScheduler *sch, FILE *output) {
MDB_cursor *cursor;
if (freq_scheduler_cursor_open(sch, &cursor) != 0)
return sch->error->code;
int end = 0;
MDB_cursor_op cursor_op = MDB_FIRST;
do {
int mdb_rc;
MDB_val key;
MDB_val val;
ScheduleKey *key_data;
float *val_data;
switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, cursor_op)) {
case 0:
key_data = (ScheduleKey*)key.mv_data;
val_data = (float*)val.mv_data;
fprintf(output, "%.2e %016"PRIx64" %.2e\n",
key_data->score, key_data->hash, *val_data);
break;
case MDB_NOTFOUND:
end = 1;
break;
default:
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, "iterating over database");
freq_scheduler_add_error(sch, mdb_strerror(mdb_rc));
end = 1;
break;
}
cursor_op = MDB_NEXT;
} while (!end);
freq_scheduler_cursor_abort(sch, cursor);
return sch->error->code;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi db;
rc = mdb_env_create(&env);
if (rc != 0) {
return rc;
}
rc = mdb_env_set_mapsize(env, 107374182400); //100G
if (rc != 0) {
return rc;
}
rc = mdb_env_set_maxdbs(env, 27);
if (rc != 0) {
return rc;
}
rc = mdb_env_open(env, "/home/pieter/Downloads/thundergraph/src/main/native/testdb", MDB_NOSYNC, 0664);
if (rc != 0) {
return rc;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_open(txn, "testdb", MDB_CREATE | MDB_INTEGERKEY, &db);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_set_compare(txn, db, compareTestDbId);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_txn_commit(txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
int i= 0;
for (i = 0; i < 1000; i++) {
MDB_cursor *cursor;
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
//Create a core vertex
MDB_val key, data;
TestDbId testDbId;
testDbId.testId = i;
testDbId.coreOrPropertyEnum = 0;
testDbId.labelId = -1;
testDbId.propertykeyId = -1;
testDbId.edgeId = -1LL;
key.mv_size = sizeof(TestDbId);
key.mv_data = &testDbId;
data.mv_size = 50;
char *value = malloc(5);
char v[] = "12345";
memcpy(value, v, 5);
data.mv_data = &v;
mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE);
free(value);
//Create a vertex property
MDB_val propertyKey, propertyData;
TestDbId propertyTestDbId;
propertyTestDbId.testId = i;
propertyTestDbId.coreOrPropertyEnum = 1;
propertyTestDbId.labelId = -1;
propertyTestDbId.propertykeyId = -1;
propertyTestDbId.edgeId = -1LL;
propertyKey.mv_size = sizeof(TestDbId);
propertyKey.mv_data = &propertyTestDbId;
data.mv_size = 50;
char *propertyValue = malloc(5);
char propertyV[] = "12345";
memcpy(propertyValue, propertyV, 5);
data.mv_data = &propertyV;
mdb_cursor_put(cursor, &propertyKey, &propertyData, MDB_NOOVERWRITE);
free(propertyValue);
//Create a vertex in edge
MDB_val inEdgeKey, inEdgeData;
TestDbId inEdgeTestDbId;
inEdgeTestDbId.testId = i;
inEdgeTestDbId.coreOrPropertyEnum = 2;
inEdgeTestDbId.labelId = -1;
inEdgeTestDbId.propertykeyId = -1;
inEdgeTestDbId.edgeId = -1LL;
inEdgeKey.mv_size = sizeof(TestDbId);
inEdgeKey.mv_data = &inEdgeTestDbId;
data.mv_size = 50;
char *inEdgeValue = malloc(5);
char inEdgeV[] = "12345";
memcpy(inEdgeValue, inEdgeV, 5);
data.mv_data = &inEdgeV;
mdb_cursor_put(cursor, &inEdgeKey, &inEdgeData, MDB_NOOVERWRITE);
mdb_cursor_close(cursor);
free(inEdgeValue);
}
MDB_cursor *cursor;
mdb_cursor_close(cursor);
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
MDB_val key, data;
for (i = 0; i < 1000; i++) {
TestDbId testDbId;
testDbId.testId = i;
testDbId.coreOrPropertyEnum = 0;
testDbId.labelId = -1;
testDbId.propertykeyId = -1;
testDbId.edgeId = -1LL;
key.mv_size = sizeof(TestDbId);
key.mv_data = &testDbId;
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
printf("mdb_cursor_get index = %i result = %i\n", i, rc);
MDB_val keyDelete, dataDelete;
TestDbId deleteTestDbId;
deleteTestDbId.testId = i;
deleteTestDbId.coreOrPropertyEnum = 0;
deleteTestDbId.labelId = -1;
deleteTestDbId.propertykeyId = -1;
deleteTestDbId.edgeId = -1LL;
keyDelete.mv_size = sizeof(TestDbId);
keyDelete.mv_data = &deleteTestDbId;
rc = mdb_del(txn, db, &keyDelete, &dataDelete);
printf("mdb_del = %i\n", rc);
MDB_val propertyKeyDelete, propertyDataDelete;
TestDbId propertyDeleteTestDbId;
propertyDeleteTestDbId.testId = i;
propertyDeleteTestDbId.coreOrPropertyEnum = 1;
propertyDeleteTestDbId.labelId = -1;
propertyDeleteTestDbId.propertykeyId = -1;
propertyDeleteTestDbId.edgeId = -1LL;
propertyKeyDelete.mv_size = sizeof(TestDbId);
propertyKeyDelete.mv_data = &propertyDeleteTestDbId;
rc = mdb_del(txn, db, &propertyKeyDelete, &propertyDataDelete);
printf("mdb_del = %i\n", rc);
MDB_val inEdgeKeyDelete, inEdgeDataDelete;
TestDbId inEdgeDeleteTestDbId;
inEdgeDeleteTestDbId.testId = i;
inEdgeDeleteTestDbId.coreOrPropertyEnum = 2;
inEdgeDeleteTestDbId.labelId = -1;
inEdgeDeleteTestDbId.propertykeyId = -1;
inEdgeDeleteTestDbId.edgeId = -1LL;
inEdgeKeyDelete.mv_size = sizeof(TestDbId);
inEdgeKeyDelete.mv_data = &inEdgeDeleteTestDbId;
rc = mdb_del(txn, db, &inEdgeKeyDelete, &inEdgeDataDelete);
printf("mdb_del = %i\n", rc);
}
mdb_cursor_close(cursor);
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
TestDbId testDbId = *((TestDbId *) (key.mv_data));
printf("keyId = %llu\n", testDbId.testId);
}
mdb_cursor_close(cursor);
fail:
mdb_close(env, db);
mdb_env_close(env);
printf("closing graph!\n");
return 0;
}
int lmdbgo_mdb_cursor_put1(MDB_cursor *cur, char *kdata, size_t kn, MDB_val *val, unsigned int flags) {
MDB_val key;
LMDBGO_SET_VAL(&key, kn, kdata);
return mdb_cursor_put(cur, &key, val, flags);
}
