static void db_connect() {
MDB_dbi dbi_session;
MDB_dbi dbi_session_id;
MDB_dbi dbi_event;
MDB_dbi dbi_ip;
LMDB_CHECK(mdb_env_create(&env));
LMDB_CHECK(mdb_env_set_mapsize(env, 300000L * 4096L));
LMDB_CHECK(mdb_env_set_maxdbs(env, 30));
#if defined(MDBX_LIFORECLAIM)
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664));
#else
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664));
#endif
MDB_txn *txn;
// transaction init
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
// transaction commit
LMDB_CHECK(mdb_txn_commit(txn));
printf("Connection open\n");
}
int db_schema_open(MDB_txn *const txn, DB_schema *const schema) {
mdb_dbi_open(txn, "schema", MDB_CREATE | MDB_DUPSORT, &schema->schema);
mdb_dbi_open(txn, "stringByID", MDB_CREATE, &schema->stringByID);
mdb_dbi_open(txn, "stringIDByValue", MDB_CREATE, &schema->stringIDByValue);
mdb_dbi_open(txn, "stringIDByHash", MDB_CREATE, &schema->stringIDByHash);
return 0;
}
FreqSchedulerError
freq_scheduler_cursor_open(FreqScheduler *sch, MDB_cursor **cursor) {
char *error1 = 0;
char *error2 = 0;
MDB_txn *txn = 0;
if (txn_manager_begin(sch->txn_manager, 0, &txn) != 0) {
error1 = "starting transaction";
error2 = sch->txn_manager->error->message;
goto on_error;
}
MDB_dbi dbi;
int mdb_rc =
mdb_dbi_open(txn, "schedule", MDB_CREATE, &dbi) ||
mdb_set_compare(txn, dbi, schedule_entry_mdb_cmp_asc) ||
mdb_cursor_open(txn, dbi, cursor);
if (mdb_rc != 0) {
*cursor = 0;
error1 = "opening cursor";
error2 = mdb_strerror(mdb_rc);
}
return sch->error->code;
on_error:
if (txn != 0)
txn_manager_abort(sch->txn_manager, txn);
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, error1);
freq_scheduler_add_error(sch, error2);
return sch->error->code;
}
bool Context::BeginTxn(){
Error(0);
if (envHandle==nullptr){
return false;
}
else if (txnHandle!=nullptr){
return false;
}
else{
int txnret = mdb_txn_begin(envHandle,nullptr,0,&txnHandle);
if (txnret != 0) {
Error(txnret);
txnHandle=nullptr;
return false;
}
else{
int dbiret = mdb_dbi_open(txnHandle,"Entity",MDB_CREATE,&dbiHandle);
if (dbiret != 0){
Error(dbiret);
mdb_txn_abort(txnHandle);
txnHandle=nullptr;
return false;
}
else return true;
}
}
}
counter_t *counter_init(const char *path, uint64_t readers) {
counter_t *lc = NULL;
if((lc = calloc(1, sizeof(counter_t))) == NULL) {
perror("calloc");
return NULL;
}
// Setup and open the lmdb enviornment
MDB_CHECK(mdb_env_create(&lc->env), MDB_SUCCESS, NULL);
MDB_CHECK(mdb_env_set_maxreaders(lc->env, readers), MDB_SUCCESS, NULL);
MDB_CHECK(mdb_env_set_mapsize(lc->env, MDB_MAPSIZE), MDB_SUCCESS, NULL);
MDB_CHECK(mdb_env_open(lc->env, path, MDB_WRITEMAP | MDB_MAPASYNC | MDB_NOSUBDIR, 0664), MDB_SUCCESS, NULL);
MDB_txn *txn = NULL;
MDB_CHECK(mdb_txn_begin(lc->env, NULL, 0, &txn), MDB_SUCCESS, NULL);
if((lc->dbi = calloc(1, sizeof(MDB_dbi))) == NULL) {
perror("calloc");
return NULL;
}
MDB_CHECK(mdb_dbi_open(txn, NULL, 0, lc->dbi), MDB_SUCCESS, NULL);
mdb_txn_commit(txn);
return lc;
}
int main(void)
{
check(mdb_env_create(&env));
check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL));
check(mdb_env_set_maxreaders(env, 126));
check(mdb_env_set_maxdbs(env, 1));
if(! access(DB_PATH, X_OK)) {
system("rm -rf " DB_PATH);
}
check(mkdir(DB_PATH, 0777));
check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644));
new_txn();
check(mdb_dbi_open(txn, NULL, 0, &dbi));
put("a");
put("b");
put("baa");
put("d");
new_txn();
check(mdb_cursor_open(txn, dbi, &c1));
check(mdb_cursor_get(c1, &keyv, &valv, MDB_LAST));
check(mdb_cursor_del(c1, 0));
check(mdb_cursor_del(c1, 0));
new_txn();
}
int mail_cache_db_get_size(struct mail_cache_db * cache_db,
const void * key, size_t key_len, size_t * pvalue_len)
{
int r;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDB_val mdb_key;
MDB_val mdb_val;
env = cache_db->internal_database;
mdb_key.mv_size = key_len;
mdb_key.mv_data = (void *) key;
r = mdb_txn_begin(env, NULL, 0, &txn);
if (r != 0)
return -1;
r = mdb_dbi_open(txn, NULL, 0, &dbi);
if (r != 0)
goto error;
r = mdb_get(txn, dbi, &mdb_key, &mdb_val);
if (r != 0)
goto error;
* pvalue_len = mdb_val.mv_size;
mdb_txn_commit(txn);
return 0;
error:
mdb_txn_abort(txn);
return -1;
}
LMDBCursor* LMDB::NewCursor() {
MDB_txn* mdb_txn;
MDB_cursor* mdb_cursor;
MDB_CHECK(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn));
MDB_CHECK(mdb_dbi_open(mdb_txn, NULL, 0, &mdb_dbi_));
MDB_CHECK(mdb_cursor_open(mdb_txn, mdb_dbi_, &mdb_cursor));
return new LMDBCursor(mdb_txn, mdb_cursor);
}
static int begin_txn(struct dbengine *db, struct txn **tidptr, int readonly)
{
struct txn *tid = xzmalloc(sizeof(struct txn));
int mflags, mr, r;
struct MDB_txn *parent = NULL;
assert(db && tidptr);
PDEBUG("cyrusdb_lmdb(%s)[begin_txn] readonly=%d tidptr=%p *tidptr=%p",
db->fname, readonly, tidptr, tidptr ? *tidptr : NULL);
/* Read-only transactions may only be master transactions and do
* not allow nested transactions. */
readonly = !db->tid ? readonly : 0;
/*
* Hacky workaround, similar to skiplist
*
* If no transaction was passed, but we're in a transaction,
* then create a nested exception within the current main
* transaction.
*
* Note that transactions are always either the main transaction,
* or a direct descendant of it. There are no deeper transaction
* levels supported (although LMDB supports them).
*/
if (db->tid) {
parent = db->tid->mtxn;
}
/* Begin a new LMDB transaction */
mr = mdb_txn_begin(db->env, parent, readonly ? MDB_RDONLY : 0, &tid->mtxn);
if (mr) goto fail;
/* Open the database */
mflags = db->flags & CYRUSDB_CREATE ? MDB_CREATE : 0;
mr = mdb_dbi_open(tid->mtxn, NULL /*name*/, mflags, &tid->dbi);
if (mr) goto fail;
if (db->flags & CYRUSDB_MBOXSORT) {
/* Set mboxsort order */
mr = mdb_set_compare(tid->mtxn, tid->dbi, mboxcmp);
if (mr) goto fail;
}
if (!db->tid) {
/* Set the master transaction */
db->tid = tid;
}
*tidptr = tid;
return CYRUSDB_OK;
fail:
r = my_mdberror(mr);
if (tid->mtxn) abort_txn(db, tid);
syslog(LOG_ERR, "cryusdb_lmdb(%s): %s", db->fname, mdb_strerror(mr));
return r;
}
static PageDBError
page_db_link_stream_open_cursor(PageDBLinkStream *es) {
MDB_txn *txn = 0;
MDB_dbi dbi_links;
int mdb_rc = 0;
char *error = 0;
// start a new read transaction
if (txn_manager_begin(es->db->txn_manager, MDB_RDONLY, &txn) != 0) {
error = es->db->txn_manager->error->message;
goto mdb_error;
}
// open cursor to links database
switch (mdb_rc = mdb_dbi_open(
txn,
"links",
MDB_INTEGERKEY,
&dbi_links)) {
case 0:
if ((mdb_rc = mdb_cursor_open(
txn,
dbi_links,
&es->cur)) != 0) {
error = "opening links cursor";
goto mdb_error;
}
es->state = stream_state_init;
break;
case MDB_NOTFOUND:
txn_manager_abort(es->db->txn_manager, txn);
es->cur = 0;
es->state = stream_state_end;
break;
default:
error = "opening links database";
goto mdb_error;
}
return es->db->error->code;
mdb_error:
es->state = stream_state_error;
if (txn)
txn_manager_abort(es->db->txn_manager, txn);
es->cur = 0;
page_db_set_error(es->db, page_db_error_internal, __func__);
page_db_add_error(es->db, error);
if (mdb_rc != 0)
page_db_add_error(es->db, mdb_strerror(mdb_rc));
return es->db->error->code;
}
int mail_cache_db_get_keys(struct mail_cache_db * cache_db,
chash * keys)
{
int r;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDB_cursor *cursor;
MDB_val mdb_key;
MDB_val mdb_val;
env = cache_db->internal_database;
r = mdb_txn_begin(env, NULL, 0, &txn);
if (r != 0)
return -1;
r = mdb_dbi_open(txn, NULL, 0, &dbi);
if (r != 0)
goto error;
r = mdb_cursor_open(txn, dbi, &cursor);
if (r != 0)
goto error;
r = mdb_cursor_get(cursor, &mdb_key, &mdb_val, MDB_FIRST);
if (r != 0)
goto cursor_error;
while (r == 0) {
chashdatum hash_key;
chashdatum hash_data;
hash_key.data = mdb_key.mv_data;
hash_key.len = (unsigned int) mdb_key.mv_size;
hash_data.data = NULL;
hash_data.len = 0;
r = chash_set(keys, &hash_key, &hash_data, NULL);
if (r != 0)
goto cursor_error;
r = mdb_cursor_get(cursor, &mdb_key, &mdb_val, MDB_NEXT);
}
mdb_txn_commit(txn);
return 0;
cursor_error:
mdb_cursor_close(cursor);
error:
mdb_txn_abort(txn);
return -1;
}
/*
* Class: jmdb_DatabaseWrapper
* Method: dbiOpen
* Signature: (JLjava/lang/String;I)I
*/
JNIEXPORT jint JNICALL Java_jmdb_DatabaseWrapper_dbiOpen(JNIEnv *vm,
jclass clazz, jlong txnL, jstring name, jint flags) {
MDB_txn *txnC = (MDB_txn*) txnL;
const char *nameC = (*vm)->GetStringUTFChars(vm, name, NULL);
MDB_dbi dbi;
int code = mdb_dbi_open(txnC, nameC, flags, &dbi);
(*vm)->ReleaseStringUTFChars(vm, name, nameC);
if (code) {
throwDatabaseException(vm, code);
return -1;
}
return dbi;
}
CAMLprim value caml_mdb_dbi_open(value txn,value name,value flags){
CAMLparam3(txn,name,flags);
MDB_dbi dbi;
char*str=NULL;
if(caml_string_length(name))
str=String_val(name);
if(mdb_dbi_open((MDB_txn*)txn,str,Int_val(flags),&dbi)){
caml_failwith("error in mdb_dbi_open");
}
CAMLreturn(Val_int(dbi));
}
int db_dbi_open(MDB_txn *const txn, DB_schema *const schema, unsigned int opts, DB_column const *const cols, count_t const ncols, strarg_t const name, MDB_dbi *const dbi) {
int rc;
uint64_t const dbname_id = db_string_id(txn, schema, name);
if(!dbname_id) return -1;
DB_VAL(dbinfo_val, 2);
db_bind(dbinfo_val, 0);
db_bind(dbinfo_val, dbname_id);
DB_VAL(info_val, 1);
db_bind(info_val, 0xff & opts);
mdb_put(txn, schema->schema, dbinfo_val, info_val, MDB_NOOVERWRITE);
// TODO: Check opts
MDB_cursor *cur = NULL;
mdb_cursor_open(txn, schema->schema, &cur);
DB_VAL(dbcols_val, 2);
db_bind(dbcols_val, 1);
db_bind(dbcols_val, dbname_id);
MDB_val col_val;
mdb_cursor_get(cur, &dbcols_val, &col_val, MDB_GET);
for(; MDB_SUCCESS == rc; rc = mdb_cursor_get(cur, &dbcols_val, &col_val, MDB_NEXT_DUP)) {
uint64_t const col = db_column(col_val, 0);
uint64_t const type = db_column(col_val, 1);
strarg_t const colname = db_column_text(txn, schema, col_val, 2);
if(col >= ncols) break; // Extra columns are not an error.
if(type != cols[i].type || 0 != strcmp(colname, cols[i].name)) {
mdb_cursor_close(cur); cur = NULL;
return -1;
}
}
mdb_cursor_close(cur); cur = NULL;
for(index_t i = 0; i < ncols; ++i) {
uint64_t const colname_id = db_string_id(txn, schema, cols[i].name);
if(!colname_id) return -1;
DB_VAL(col_val, 3);
db_bind(col_val, i);
db_bind(col_val, cols[i].type);
db_bind(col_val, colname_id);
rc = mdb_put(txn, schema->schema, dbcols_val, col_val, MDB_NODUPDATA);
if(MDB_SUCCESS != rc && MDB_KEYEXIST != rc) return -1;
}
mdb_dbi_open(txn, name, MDB_CREATE | opts, dbi);
return 0;
}
// Remove a key from database.
void database_del(MDB_env *db, MDB_val *key) {
MDB_dbi dbi;
MDB_txn *txn;
// transaction init
mdb_txn_begin(db, NULL, 0, &txn);
// open database in read-write mode
mdb_dbi_open(txn, NULL, 0, &dbi);
// delete key
mdb_del(txn, dbi, key, NULL);
// close database
mdb_dbi_close(db, dbi);
// transaction commit
mdb_txn_commit(txn);
}
static int
page_db_open_cursor(MDB_txn *txn,
const char *db_name,
int flags,
MDB_cursor **cursor,
MDB_cmp_func *func) {
MDB_dbi dbi;
int mdb_rc =
mdb_dbi_open(txn, db_name, flags, &dbi) ||
(func && mdb_set_compare(txn, dbi, func)) ||
mdb_cursor_open(txn, dbi, cursor);
if (mdb_rc != 0)
*cursor = 0;
return mdb_rc;
}
// Add or update a key in database.
void database_put(MDB_env *db, MDB_val *key, MDB_val *value) {
MDB_dbi dbi;
MDB_txn *txn;
// transaction init
mdb_txn_begin(db, NULL, 0, &txn);
// open database in read-write mode
mdb_dbi_open(txn, NULL, 0, &dbi);
// put data
mdb_put(txn, dbi, key, value, 0);
// close database
mdb_dbi_close(db, dbi);
// transaction commit
mdb_txn_commit(txn);
}
// Return a value from database.
void database_get(MDB_env *db, MDB_val *key, MDB_val *value) {
MDB_dbi dbi;
MDB_txn *txn;
// transaction init
mdb_txn_begin(db, NULL, MDB_RDONLY, &txn);
// open database in read-write mode
mdb_dbi_open(txn, NULL, 0, &dbi);
// get data
if (mdb_get(txn, dbi, key, value))
bzero(value, sizeof(*value));
// end of transaction
mdb_txn_abort(txn);
// close database
mdb_dbi_close(db, dbi);
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_cursor *cursor;
char sval[MAX_DATA_ALLOCATE_SIZE], kval[MAX_KEY_ALLOCATE_SIZE];
/* Note: Most error checking omitted for simplicity */
rc = mdb_env_create(&env);
mdb_env_set_mapsize(env, MAX_DB_SIZE);
mdb_env_set_maxdbs(env, (MDB_dbi)10);
rc = mdb_env_open(env, "./demoDB", 0, 0664);
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_dbi_open(txn, "what", 0, &dbi);
// rc = mdb_put(txn, dbi, &key, &data, 0);
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc));
goto leave;
}
fprintf(stderr, "print out old data:\n");
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
rc = mdb_cursor_open(txn, dbi, &cursor);
key.mv_size = sizeof(kval);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
memset(kval, argv[1][0], sizeof(kval));
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET);
fprintf(stderr, "key: %s, data: %s\n",(char *) key.mv_data,(char *) data.mv_data);
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
leave:
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}
/*
bool Context::Write(size_t ksz, void* key,
size_t dsz, void* data){
Error(0);
if (envHandle==nullptr){
return false;
}
else{
MDB_txn *txn;
int txnret = mdb_txn_begin(envHandle,nullptr,0,&txn);
if (txnret != 0) {
Error(txnret);
return false;
}
else{
MDB_dbi dbi;
int dbiret = mdb_dbi_open(txn,"Entity",MDB_CREATE,&dbi);
if (dbiret != 0){
Error(dbiret);
mdb_txn_abort(txn);
return false;
}
else{
MDB_val kkey { ksz, key };
MDB_val ddata { dsz, data};
putReturn = mdb_put(txn,dbi,&kkey,&ddata,0);
if (putReturn !=0){
Error(putReturn);
mdb_txn_abort(txn);
return false;
}
else{
int commit = mdb_txn_commit(txn);
if (commit!=0){
Error(commit);
return false;
}
else return true;
}
}
}
}
}
*/
bool Context::GetCopy(size_t ksz, void* key,
void** data){
Error(0);
if (envHandle==nullptr){
return false;
}
else{
MDB_txn *txn;
int txnret = mdb_txn_begin(envHandle,nullptr,0,&txn);
if (txnret != 0){
Error(txnret);
return false;
}
else{
MDB_dbi dbi;
int dbiret = mdb_dbi_open(txn,"Entity",0,&dbi);
if (dbiret !=0){
Error(dbiret);
mdb_txn_abort(txn);
return false;
}
else{
MDB_val kkey { ksz, key };
MDB_val ddata;
int get = mdb_get(txn,dbi,&kkey,&ddata);
if (get!=0){
Error(get);
mdb_txn_abort(txn);
return false;
}
else {
*data = new char[ddata.mv_size];
memcpy(*data, (void*)ddata.mv_data, (ddata.mv_size)*sizeof(char));
int commit = mdb_txn_commit(txn);
if (commit!=0){
Error(commit);
delete (char*)(*data);
return false;
}
else return true;
}
}
}
}
}
uint64_t counter_get(counter_t *lc, const char *key) {
MDB_dbi dbi;
MDB_val mkey, data;
MDB_txn *txn;
mdb_txn_begin(lc->env, NULL, 0, &txn);
mdb_dbi_open(txn, NULL, 0, &dbi);
mkey.mv_size = strlen(key) * sizeof(char);
mkey.mv_data = (void *)key;
// First we get our data from the db
uint64_t stored_counter = 0;
if(mdb_get(txn, dbi, &mkey, &data) == MDB_SUCCESS) {
stored_counter = *(uint64_t *)data.mv_data;
}
mdb_dbi_close(lc->env, dbi);
return stored_counter;
}
int _db_open(DB *db, DB_TXN *txnid, const char *file, const char *database, DBTYPE type, uint32_t flags, int mode)
{
printf("%s\n", __FUNCTION__);
if (db == NULL) return EINVAL;
mdb_env_open(db->_internal->env, "./", /*MDB_FIXEDMAP*/0, mode);
if (txnid == NULL) {
if (mdb_txn_begin(db->_internal->env, NULL, 0, &db->_internal->txn) != 0) return EINVAL;
} else {
if (mdb_txn_begin(db->_internal->env, txnid->_internal->txn, 0, &db->_internal->txn) != 0) return EINVAL;
}
uint32_t _flags = 0;
if (flags & DB_CREATE) _flags |= MDB_CREATE;
if (mdb_dbi_open(db->_internal->txn, database, _flags, &db->_internal->dbi) !=0 ) {
mdb_txn_abort(db->_internal->txn);
return EINVAL;
}
mdb_txn_commit(db->_internal->txn);
return 0;
}
int LMDBStore::set(hcat_keypair* pair, void* transaction_context)
{
int rc;
lmdb_transaction_context* context = (lmdb_transaction_context*)transaction_context;
MDB_dbi db_instance = dbi;
if (pair->keyspace.length() > 0)
{
db_instance = keyspaces->operator[](pair->keyspace.data());
if (db_instance == NULL)
{
rc = mdb_dbi_open(context->transaction, pair->keyspace.data(), MDB_CREATE, &db_instance);
if (rc == MDB_SUCCESS)
{
keyspaces->operator[](pair->keyspace.data()) = db_instance;
}
else
{
// TODO: Return an error.
}
}
}
MDB_val mdb_key;
MDB_val mdb_value;
mdb_key.mv_size = pair->key.length() + 1;
mdb_key.mv_data = (void*)pair->key.data();
mdb_value.mv_size = pair->value_length + 1;
mdb_value.mv_data = pair->value;
rc = mdb_put(context->transaction, db_instance, &mdb_key, &mdb_value, 0);
return (rc == 0 ? HCAT_SUCCESS : HCAT_FAIL);
}
LMDBTransaction* LMDB::NewTransaction() {
MDB_txn* mdb_txn;
MDB_CHECK(mdb_txn_begin(mdb_env_, NULL, 0, &mdb_txn));
MDB_CHECK(mdb_dbi_open(mdb_txn, NULL, 0, &mdb_dbi_));
return new LMDBTransaction(&mdb_dbi_, mdb_txn);
}
bool B_ACCURATE_LMDB::init(JCR *jcr, uint32_t nbfile)
{
int result;
MDB_env *env;
size_t mapsize = 10485760;
if (!m_db_env) {
result = mdb_env_create(&env);
if (result) {
Jmsg1(jcr, M_FATAL, 0, _("Unable to create MDB environment: %s\n"), mdb_strerror(result));
return false;
}
if ((nbfile * AVG_NR_BYTES_PER_ENTRY) > mapsize) {
size_t pagesize;
#ifdef HAVE_GETPAGESIZE
pagesize = getpagesize();
#else
pagesize = B_PAGE_SIZE;
#endif
mapsize = (((nbfile * AVG_NR_BYTES_PER_ENTRY) / pagesize) + 1) * pagesize;
}
result = mdb_env_set_mapsize(env, mapsize);
if (result) {
Jmsg1(jcr, M_FATAL, 0, _("Unable to set MDB mapsize: %s\n"), mdb_strerror(result));
goto bail_out;
}
/*
* Explicitly set the number of readers to 1.
*/
result = mdb_env_set_maxreaders(env, 1);
if (result) {
Jmsg1(jcr, M_FATAL, 0, _("Unable to set MDB maxreaders: %s\n"), mdb_strerror(result));
goto bail_out;
}
Mmsg(m_lmdb_name, "%s/.accurate_lmdb.%d", me->working_directory, jcr->JobId);
result = mdb_env_open(env, m_lmdb_name, MDB_NOSUBDIR | MDB_NOLOCK | MDB_NOSYNC, 0600);
if (result) {
Jmsg2(jcr, M_FATAL, 0, _("Unable create LDMD database %s: %s\n"), m_lmdb_name, mdb_strerror(result));
goto bail_out;
}
result = mdb_txn_begin(env, NULL, 0, &m_db_rw_txn);
if (result) {
Jmsg1(jcr, M_FATAL, 0, _("Unable to start a write transaction: %s\n"), mdb_strerror(result));
goto bail_out;
}
result = mdb_dbi_open(m_db_rw_txn, NULL, MDB_CREATE, &m_db_dbi);
if (result) {
Jmsg1(jcr, M_FATAL, 0, _("Unable to open LMDB internal database: %s\n"), mdb_strerror(result));
mdb_txn_abort(m_db_rw_txn);
m_db_rw_txn = NULL;
goto bail_out;
}
m_db_env = env;
}
if (!m_pay_load) {
m_pay_load = get_pool_memory(PM_MESSAGE);
}
if (!m_lmdb_name) {
m_pay_load = get_pool_memory(PM_FNAME);
}
if (!m_seen_bitmap) {
m_seen_bitmap = (char *)malloc(nbytes_for_bits(nbfile));
clear_all_bits(nbfile, m_seen_bitmap);
}
return true;
bail_out:
if (env) {
mdb_env_close(env);
}
return false;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data, sdata;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
long kval;
char *sval;
srand(time(NULL));
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
sdata.mv_size = mst.ms_psize / 4 - 30;
sdata.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
do {
printf("key: %p %s, data: %p %.*s\n",
key.mv_data, mdb_dkey(&key, dkbuf),
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
} while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0);
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
#if 0
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
#endif
mdb_env_close(env);
return 0;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[8];
char kval[sizeof(int)];
memset(sval, 0, sizeof(sval));
count = 510;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = i*5;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
strcpy(kval, "001");
for (i=0;i<count;i++) {
sprintf(sval, "%07x", values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
/* there should be one full page of dups now.
*/
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
/* test all 3 branches of split code:
* 1: new key in lower half
* 2: new key at split point
* 3: new key in upper half
*/
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%07x", values[3]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[255]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[500]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
E(mdb_txn_commit(txn));
/* Try MDB_NEXT_MULTIPLE */
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%3)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%07x", values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n" );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
mdb_env_set_userctx(mdb->mi_dbenv, mdb);
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef MDBX_LIFORECLAIM
#if MDBX_MODE_ENABLED
rc = mdbx_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul);
#else
rc = mdb_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul);
#endif /* MDBX_MODE_ENABLED */
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_sync_threshold failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#if MDBX_MODE_ENABLED
mdbx_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler );
#else
mdb_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler );
#endif /* MDBX_MODE_ENABLED */
if ( (slapMode & SLAP_SERVER_MODE) && SLAP_MULTIMASTER(be) &&
((MDBX_OOM_YIELD & mdb->mi_oom_flags) == 0 || mdb->mi_renew_lag == 0)) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"for properly operation in multi-master mode"
" 'oom-handler yield' and 'dreamcatcher' are recommended",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg );
}
#endif /* MDBX_LIFORECLAIM */
dbhome = mdb->mi_dbenv_home;
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home);
flags = mdb->mi_dbenv_flags;
#ifdef MDBX_PAGEPERTURB
if (reopenldap_mode_check())
flags |= MDBX_PAGEPERTURB;
#endif
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC|MDB_WRITEMAP;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_dbi_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_dbi_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID ) {
MDB_cursor *mc;
MDB_val key, data;
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
/* check for old dn2id format */
rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
/* first record is always ID 0 */
rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
if ( rc == 0 ) {
rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
if ( rc == 0 ) {
int len;
unsigned char *ptr;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
snprintf( cr->msg, sizeof(cr->msg),
"database \"%s\": DN index needs upgrade, "
"run \"slapindex entryDN\".",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg );
if ( !(slapMode & SLAP_TOOL_READMAIN ))
rc = LDAP_OTHER;
mdb->mi_flags |= MDB_NEED_UPGRADE;
}
}
}
mdb_cursor_close( mc );
if ( rc == LDAP_OTHER )
goto fail;
}
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
/* slapcat doesn't need indexes. avoid a failure if
* a configured index wasn't created yet.
*/
if ( !(slapMode & SLAP_TOOL_READONLY) ) {
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database %s: "
"txn_commit failed: %s (%d)\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}
PageDBError
page_db_new(PageDB **db, const char *path) {
PageDB *p = *db = malloc(sizeof(*p));
if (p == 0)
return page_db_error_memory;
p->error = error_new();
if (p->error == 0) {
free(p);
return page_db_error_memory;
}
p->persist = PAGE_DB_DEFAULT_PERSIST;
p->domain_temp = 0;
// create directory if not present yet
const char *error = make_dir(path);
if ((p->path = strdup(path)) == 0) {
error = "could not duplicate path string";
}
if (error != 0) {
page_db_set_error(p, page_db_error_invalid_path, __func__);
page_db_add_error(p, error);
return p->error->code;
}
if (txn_manager_new(&p->txn_manager, 0) != 0) {
page_db_set_error(p, page_db_error_internal, __func__);
page_db_add_error(p, p->txn_manager?
p->txn_manager->error->message:
"NULL");
return p->error->code;
}
// initialize LMDB on the directory
MDB_txn *txn;
MDB_dbi dbi;
int mdb_rc = 0;
if ((mdb_rc = mdb_env_create(&p->txn_manager->env) != 0))
error = "creating environment";
else if ((mdb_rc = mdb_env_set_mapsize(
p->txn_manager->env, PAGE_DB_DEFAULT_SIZE)) != 0)
error = "setting map size";
else if ((mdb_rc = mdb_env_set_maxdbs(p->txn_manager->env, 5)) != 0)
error = "setting number of databases";
else if ((mdb_rc = mdb_env_open(
p->txn_manager->env,
path,
MDB_NOTLS | MDB_NOSYNC, 0664) != 0))
error = "opening environment";
else if ((mdb_rc = txn_manager_begin(p->txn_manager, 0, &txn)) != 0)
error = "starting transaction";
// create all database
else if ((mdb_rc = mdb_dbi_open(txn,
"hash2info",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating hash2info database";
else if ((mdb_rc = mdb_dbi_open(txn,
"hash2idx",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating hash2idx database";
else if ((mdb_rc = mdb_dbi_open(txn,
"links",
MDB_CREATE | MDB_INTEGERKEY,
&dbi)) != 0)
error = "creating links database";
else if ((mdb_rc = mdb_dbi_open(txn, "info", MDB_CREATE, &dbi)) != 0)
error = "creating info database";
else {
// initialize n_pages inside info database
size_t n_pages = 0;
MDB_val key = {
.mv_size = sizeof(info_n_pages),
.mv_data = info_n_pages
};
MDB_val val = {
.mv_size = sizeof(size_t),
.mv_data = &n_pages
};
switch (mdb_rc = mdb_put(txn, dbi, &key, &val, MDB_NOOVERWRITE)) {
case MDB_KEYEXIST:
case 0:
if (txn_manager_commit(p->txn_manager, txn) != 0)
error = p->txn_manager->error->message;
break; // we good
default:
error = "could not initialize info.n_pages";
}
}
if (error != 0) {
page_db_set_error(p, page_db_error_internal, __func__);
page_db_add_error(p, error);
page_db_add_error(p, mdb_strerror(mdb_rc));
mdb_env_close(p->txn_manager->env);
}
return p->error->code;
}
/** Store a new or updated @ref PageInfo for a crawled page.
*
* @param cur An open cursor to the hash2info database
* @param key The key (hash) to the page
* @param page
* @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter
* will be set with the error (otherwise is set to zero).
* @return 0 if success, -1 if failure.
*/
static int
page_db_add_crawled_page_info(MDB_cursor *cur,
MDB_val *key,
const CrawledPage *page,
PageInfo **page_info,
int *mdb_error) {
MDB_val val;
*page_info = 0;
int mdb_rc = mdb_cursor_get(cur, key, &val, MDB_SET);
int put_flags = 0;
switch (mdb_rc) {
case 0:
if (!(*page_info = page_info_load(&val)))
goto on_error;
if ((page_info_update(*page_info, page) != 0))
goto on_error;
put_flags = MDB_CURRENT;
break;
case MDB_NOTFOUND:
if (!(*page_info = page_info_new_crawled(page)))
goto on_error;
break;
default:
goto on_error;
}
if ((page_info_dump(*page_info, &val) != 0))
goto on_error;
if ((mdb_rc = mdb_cursor_put(cur, key, &val, put_flags)) != 0)
goto on_error;
free(val.mv_data);
val.mv_data = 0;
*mdb_error = 0;
return 0;
on_error:
if (val.mv_data)
free(val.mv_data);
*mdb_error = mdb_rc;
page_info_delete(*page_info);
return -1;
}
/** Store a new or updated @ref PageInfo for an uncrawled link.
*
* @param cur An open cursor to the hash2info database
* @param key The key (hash) to the page
* @param url
* @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter
* will be set with the error (otherwise is set to zero).
* @return 0 if success, -1 if failure.
*/
static int
page_db_add_link_page_info(MDB_cursor *cur,
MDB_val *key,
uint64_t linked_from,
uint64_t depth,
const LinkInfo *link,
PageInfo **page_info,
int *mdb_error) {
MDB_val val;
int mdb_rc = 0;
PageInfo *pi = *page_info =
page_info_new_link(link->url, linked_from, depth, link->score);
if (!pi)
goto on_error;
if ((page_info_dump(pi, &val) != 0))
goto on_error;
if ((mdb_rc = mdb_cursor_put(cur, key, &val, MDB_NOOVERWRITE)) != 0)
goto on_error;
free(val.mv_data);
val.mv_data = 0;
*mdb_error = 0;
return 0;
on_error:
if (val.mv_data)
free(val.mv_data);
*mdb_error = mdb_rc;
page_info_delete(pi);
return -1;
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n",
1, 0, 0 );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val, 0, 0 );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef HAVE_EBCDIC
strcpy( path, mdb->mi_dbenv_home );
__atoe( path );
dbhome = path;
#else
dbhome = mdb->mi_dbenv_home;
#endif
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0);
flags = mdb->mi_dbenv_flags;
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC|MDB_WRITEMAP;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_dbi_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_dbi_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID ) {
MDB_cursor *mc;
MDB_val key, data;
ID id;
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
/* check for old dn2id format */
rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
/* first record is always ID 0 */
rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
if ( rc == 0 ) {
rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
if ( rc == 0 ) {
int len;
unsigned char *ptr;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
snprintf( cr->msg, sizeof(cr->msg),
"database \"%s\": DN index needs upgrade, "
"run \"slapindex entryDN\".",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
if ( !(slapMode & SLAP_TOOL_READMAIN ))
rc = LDAP_OTHER;
mdb->mi_flags |= MDB_NEED_UPGRADE;
}
}
}
mdb_cursor_close( mc );
if ( rc == LDAP_OTHER )
goto fail;
}
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
/* slapcat doesn't need indexes. avoid a failure if
* a configured index wasn't created yet.
*/
if ( !(slapMode & SLAP_TOOL_READONLY) ) {
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database %s: "
"txn_commit failed: %s (%d)\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}