FreqSchedulerError
freq_scheduler_cursor_open(FreqScheduler *sch, MDB_cursor **cursor) {
char *error1 = 0;
char *error2 = 0;
MDB_txn *txn = 0;
if (txn_manager_begin(sch->txn_manager, 0, &txn) != 0) {
error1 = "starting transaction";
error2 = sch->txn_manager->error->message;
goto on_error;
}
MDB_dbi dbi;
int mdb_rc =
mdb_dbi_open(txn, "schedule", MDB_CREATE, &dbi) ||
mdb_set_compare(txn, dbi, schedule_entry_mdb_cmp_asc) ||
mdb_cursor_open(txn, dbi, cursor);
if (mdb_rc != 0) {
*cursor = 0;
error1 = "opening cursor";
error2 = mdb_strerror(mdb_rc);
}
return sch->error->code;
on_error:
if (txn != 0)
txn_manager_abort(sch->txn_manager, txn);
freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__);
freq_scheduler_add_error(sch, error1);
freq_scheduler_add_error(sch, error2);
return sch->error->code;
}
static int begin_txn(struct dbengine *db, struct txn **tidptr, int readonly)
{
struct txn *tid = xzmalloc(sizeof(struct txn));
int mflags, mr, r;
struct MDB_txn *parent = NULL;
assert(db && tidptr);
PDEBUG("cyrusdb_lmdb(%s)[begin_txn] readonly=%d tidptr=%p *tidptr=%p",
db->fname, readonly, tidptr, tidptr ? *tidptr : NULL);
/* Read-only transactions may only be master transactions and do
* not allow nested transactions. */
readonly = !db->tid ? readonly : 0;
/*
* Hacky workaround, similar to skiplist
*
* If no transaction was passed, but we're in a transaction,
* then create a nested exception within the current main
* transaction.
*
* Note that transactions are always either the main transaction,
* or a direct descendant of it. There are no deeper transaction
* levels supported (although LMDB supports them).
*/
if (db->tid) {
parent = db->tid->mtxn;
}
/* Begin a new LMDB transaction */
mr = mdb_txn_begin(db->env, parent, readonly ? MDB_RDONLY : 0, &tid->mtxn);
if (mr) goto fail;
/* Open the database */
mflags = db->flags & CYRUSDB_CREATE ? MDB_CREATE : 0;
mr = mdb_dbi_open(tid->mtxn, NULL /*name*/, mflags, &tid->dbi);
if (mr) goto fail;
if (db->flags & CYRUSDB_MBOXSORT) {
/* Set mboxsort order */
mr = mdb_set_compare(tid->mtxn, tid->dbi, mboxcmp);
if (mr) goto fail;
}
if (!db->tid) {
/* Set the master transaction */
db->tid = tid;
}
*tidptr = tid;
return CYRUSDB_OK;
fail:
r = my_mdberror(mr);
if (tid->mtxn) abort_txn(db, tid);
syslog(LOG_ERR, "cryusdb_lmdb(%s): %s", db->fname, mdb_strerror(mr));
return r;
}
static int
page_db_open_cursor(MDB_txn *txn,
const char *db_name,
int flags,
MDB_cursor **cursor,
MDB_cmp_func *func) {
MDB_dbi dbi;
int mdb_rc =
mdb_dbi_open(txn, db_name, flags, &dbi) ||
(func && mdb_set_compare(txn, dbi, func)) ||
mdb_cursor_open(txn, dbi, cursor);
if (mdb_rc != 0)
*cursor = 0;
return mdb_rc;
}
int LMDBEngine::Init(const LMDBConfig& cfg, MDB_env *env,
const std::string& name)
{
m_env = env;
MDB_txn *txn;
int rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, NULL, MDB_CREATE, &m_dbi);
if (rc != 0)
{
ERROR_LOG(
"Failed to open mdb:%s for reason:%s\n", name.c_str(), mdb_strerror(rc));
return -1;
}
mdb_set_compare(txn, m_dbi, LMDBCompareFunc);
mdb_txn_commit(txn);
return 0;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi db;
rc = mdb_env_create(&env);
if (rc != 0) {
return rc;
}
rc = mdb_env_set_mapsize(env, 107374182400); //100G
if (rc != 0) {
return rc;
}
rc = mdb_env_set_maxdbs(env, 27);
if (rc != 0) {
return rc;
}
rc = mdb_env_open(env, "/home/pieter/Downloads/thundergraph/src/main/native/testdb", MDB_NOSYNC, 0664);
if (rc != 0) {
return rc;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_open(txn, "testdb", MDB_CREATE | MDB_INTEGERKEY, &db);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_set_compare(txn, db, compareTestDbId);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_txn_commit(txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
int i= 0;
for (i = 0; i < 1000; i++) {
MDB_cursor *cursor;
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
//Create a core vertex
MDB_val key, data;
TestDbId testDbId;
testDbId.testId = i;
testDbId.coreOrPropertyEnum = 0;
testDbId.labelId = -1;
testDbId.propertykeyId = -1;
testDbId.edgeId = -1LL;
key.mv_size = sizeof(TestDbId);
key.mv_data = &testDbId;
data.mv_size = 50;
char *value = malloc(5);
char v[] = "12345";
memcpy(value, v, 5);
data.mv_data = &v;
mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE);
free(value);
//Create a vertex property
MDB_val propertyKey, propertyData;
TestDbId propertyTestDbId;
propertyTestDbId.testId = i;
propertyTestDbId.coreOrPropertyEnum = 1;
propertyTestDbId.labelId = -1;
propertyTestDbId.propertykeyId = -1;
propertyTestDbId.edgeId = -1LL;
propertyKey.mv_size = sizeof(TestDbId);
propertyKey.mv_data = &propertyTestDbId;
data.mv_size = 50;
char *propertyValue = malloc(5);
char propertyV[] = "12345";
memcpy(propertyValue, propertyV, 5);
data.mv_data = &propertyV;
mdb_cursor_put(cursor, &propertyKey, &propertyData, MDB_NOOVERWRITE);
free(propertyValue);
//Create a vertex in edge
MDB_val inEdgeKey, inEdgeData;
TestDbId inEdgeTestDbId;
inEdgeTestDbId.testId = i;
inEdgeTestDbId.coreOrPropertyEnum = 2;
inEdgeTestDbId.labelId = -1;
inEdgeTestDbId.propertykeyId = -1;
inEdgeTestDbId.edgeId = -1LL;
inEdgeKey.mv_size = sizeof(TestDbId);
inEdgeKey.mv_data = &inEdgeTestDbId;
data.mv_size = 50;
char *inEdgeValue = malloc(5);
char inEdgeV[] = "12345";
memcpy(inEdgeValue, inEdgeV, 5);
data.mv_data = &inEdgeV;
mdb_cursor_put(cursor, &inEdgeKey, &inEdgeData, MDB_NOOVERWRITE);
mdb_cursor_close(cursor);
free(inEdgeValue);
}
MDB_cursor *cursor;
mdb_cursor_close(cursor);
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
MDB_val key, data;
for (i = 0; i < 1000; i++) {
TestDbId testDbId;
testDbId.testId = i;
testDbId.coreOrPropertyEnum = 0;
testDbId.labelId = -1;
testDbId.propertykeyId = -1;
testDbId.edgeId = -1LL;
key.mv_size = sizeof(TestDbId);
key.mv_data = &testDbId;
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
printf("mdb_cursor_get index = %i result = %i\n", i, rc);
MDB_val keyDelete, dataDelete;
TestDbId deleteTestDbId;
deleteTestDbId.testId = i;
deleteTestDbId.coreOrPropertyEnum = 0;
deleteTestDbId.labelId = -1;
deleteTestDbId.propertykeyId = -1;
deleteTestDbId.edgeId = -1LL;
keyDelete.mv_size = sizeof(TestDbId);
keyDelete.mv_data = &deleteTestDbId;
rc = mdb_del(txn, db, &keyDelete, &dataDelete);
printf("mdb_del = %i\n", rc);
MDB_val propertyKeyDelete, propertyDataDelete;
TestDbId propertyDeleteTestDbId;
propertyDeleteTestDbId.testId = i;
propertyDeleteTestDbId.coreOrPropertyEnum = 1;
propertyDeleteTestDbId.labelId = -1;
propertyDeleteTestDbId.propertykeyId = -1;
propertyDeleteTestDbId.edgeId = -1LL;
propertyKeyDelete.mv_size = sizeof(TestDbId);
propertyKeyDelete.mv_data = &propertyDeleteTestDbId;
rc = mdb_del(txn, db, &propertyKeyDelete, &propertyDataDelete);
printf("mdb_del = %i\n", rc);
MDB_val inEdgeKeyDelete, inEdgeDataDelete;
TestDbId inEdgeDeleteTestDbId;
inEdgeDeleteTestDbId.testId = i;
inEdgeDeleteTestDbId.coreOrPropertyEnum = 2;
inEdgeDeleteTestDbId.labelId = -1;
inEdgeDeleteTestDbId.propertykeyId = -1;
inEdgeDeleteTestDbId.edgeId = -1LL;
inEdgeKeyDelete.mv_size = sizeof(TestDbId);
inEdgeKeyDelete.mv_data = &inEdgeDeleteTestDbId;
rc = mdb_del(txn, db, &inEdgeKeyDelete, &inEdgeDataDelete);
printf("mdb_del = %i\n", rc);
}
mdb_cursor_close(cursor);
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
TestDbId testDbId = *((TestDbId *) (key.mv_data));
printf("keyId = %llu\n", testDbId.testId);
}
mdb_cursor_close(cursor);
fail:
mdb_close(env, db);
mdb_env_close(env);
printf("closing graph!\n");
return 0;
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n" );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
mdb_env_set_userctx(mdb->mi_dbenv, mdb);
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef MDBX_LIFORECLAIM
#if MDBX_MODE_ENABLED
rc = mdbx_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul);
#else
rc = mdb_env_set_syncbytes( mdb->mi_dbenv, mdb->mi_txn_cp_kbyte * 1024ul);
#endif /* MDBX_MODE_ENABLED */
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_sync_threshold failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#if MDBX_MODE_ENABLED
mdbx_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler );
#else
mdb_env_set_oomfunc( mdb->mi_dbenv, mdb_oom_handler );
#endif /* MDBX_MODE_ENABLED */
if ( (slapMode & SLAP_SERVER_MODE) && SLAP_MULTIMASTER(be) &&
((MDBX_OOM_YIELD & mdb->mi_oom_flags) == 0 || mdb->mi_renew_lag == 0)) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"for properly operation in multi-master mode"
" 'oom-handler yield' and 'dreamcatcher' are recommended",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY, LDAP_XSTRING(mdb_db_open) ": %s\n", cr->msg );
}
#endif /* MDBX_LIFORECLAIM */
dbhome = mdb->mi_dbenv_home;
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home);
flags = mdb->mi_dbenv_flags;
#ifdef MDBX_PAGEPERTURB
if (reopenldap_mode_check())
flags |= MDBX_PAGEPERTURB;
#endif
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC|MDB_WRITEMAP;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_dbi_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_dbi_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID ) {
MDB_cursor *mc;
MDB_val key, data;
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
/* check for old dn2id format */
rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
/* first record is always ID 0 */
rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
if ( rc == 0 ) {
rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
if ( rc == 0 ) {
int len;
unsigned char *ptr;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
snprintf( cr->msg, sizeof(cr->msg),
"database \"%s\": DN index needs upgrade, "
"run \"slapindex entryDN\".",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg );
if ( !(slapMode & SLAP_TOOL_READMAIN ))
rc = LDAP_OTHER;
mdb->mi_flags |= MDB_NEED_UPGRADE;
}
}
}
mdb_cursor_close( mc );
if ( rc == LDAP_OTHER )
goto fail;
}
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
/* slapcat doesn't need indexes. avoid a failure if
* a configured index wasn't created yet.
*/
if ( !(slapMode & SLAP_TOOL_READONLY) ) {
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database %s: "
"txn_commit failed: %s (%d)\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n",
1, 0, 0 );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val, 0, 0 );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef HAVE_EBCDIC
strcpy( path, mdb->mi_dbenv_home );
__atoe( path );
dbhome = path;
#else
dbhome = mdb->mi_dbenv_home;
#endif
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0);
flags = mdb->mi_dbenv_flags;
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC|MDB_WRITEMAP;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, flags & MDB_RDONLY, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened: %s (%d). "
"Restore from backup!\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_dbi_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_dbi_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID ) {
MDB_cursor *mc;
MDB_val key, data;
ID id;
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
/* check for old dn2id format */
rc = mdb_cursor_open( txn, mdb->mi_dbis[i], &mc );
/* first record is always ID 0 */
rc = mdb_cursor_get( mc, &key, &data, MDB_FIRST );
if ( rc == 0 ) {
rc = mdb_cursor_get( mc, &key, &data, MDB_NEXT );
if ( rc == 0 ) {
int len;
unsigned char *ptr;
ptr = data.mv_data;
len = (ptr[0] & 0x7f) << 8 | ptr[1];
if (data.mv_size < 2*len + 4 + 2*sizeof(ID)) {
snprintf( cr->msg, sizeof(cr->msg),
"database \"%s\": DN index needs upgrade, "
"run \"slapindex entryDN\".",
be->be_suffix[0].bv_val );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
if ( !(slapMode & SLAP_TOOL_READMAIN ))
rc = LDAP_OTHER;
mdb->mi_flags |= MDB_NEED_UPGRADE;
}
}
}
mdb_cursor_close( mc );
if ( rc == LDAP_OTHER )
goto fail;
}
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
/* slapcat doesn't need indexes. avoid a failure if
* a configured index wasn't created yet.
*/
if ( !(slapMode & SLAP_TOOL_READONLY) ) {
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database %s: "
"txn_commit failed: %s (%d)\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}
DWORD
MDBOpenDB(
PVDIR_MDB_DB pDB,
PVDIR_DB pmdbDBi,
const char * dbName,
const char * fileName,
PFN_BT_KEY_CMP btKeyCmpFcn,
unsigned int extraFlags)
{
DWORD dwError = 0;
MDB_txn* pTxn = NULL;
VDIR_DB mdbDBi = 0;
VmDirLog( LDAP_DEBUG_TRACE, "MdbOpenDB: Begin, DN name = %s", fileName );
assert(pDB && pmdbDBi);
extraFlags |= MDB_CREATE;
dwError = mdb_txn_begin( pDB->mdbEnv, NULL, BE_DB_FLAGS_ZERO, &pTxn );
BAIL_ON_VMDIR_ERROR(dwError);
dwError = mdb_open( pTxn, dbName, extraFlags, &mdbDBi);
BAIL_ON_VMDIR_ERROR(dwError);
if (NULL != btKeyCmpFcn)
{
// set customize "key" compare function.
dwError = mdb_set_compare( pTxn, mdbDBi, btKeyCmpFcn);
BAIL_ON_VMDIR_ERROR(dwError);
// if db is opened with MDB_DUPSORT flag, you can set customize "data" compare function.
// we use default lexical comparison.
//dwError = mdb_set_dupsort( pTxn, mdbDBi, btKeyCmpFcn);
//BAIL_ON_VMDIR_ERROR(dwError);
}
dwError = mdb_txn_commit(pTxn);
// regardless of commit result, pTxn should not be accessed anymore
// see mdb-back/init.c mdb_db_open example.
// this is consistent with BDB DB_TXN->commit() man page.
pTxn = NULL;
BAIL_ON_VMDIR_ERROR(dwError);
*pmdbDBi = mdbDBi;
cleanup:
VmDirLog( LDAP_DEBUG_TRACE, "MdbOpenDB: End" );
return dwError;
error:
if (pTxn)
{
mdb_txn_abort(pTxn);
pTxn = NULL;
}
VmDirLog( LDAP_DEBUG_ANY, "MdbOpenDB failed with error code: %d, error string: %s", dwError, mdb_strerror(dwError) );
goto cleanup;
}
int
main(int argc, char *argv[])
{
int n, c, rc, ec = EXIT_FAILURE, fd_in = -1, fd_out = -1, verbosity = 0;
uint32_t ts_start = 0, ts_end = 0, count = 0, nmsg_cnt = 0;
MDB_env *env = NULL;
MDB_txn *txn = NULL;
MDB_dbi dbi;
MDB_val key, data;
MDB_cursor *cursor = NULL;
struct timespec ts, msg_ts;
off_t *offset;
nmsg_input_t nmsg_in = NULL;
nmsg_output_t nmsg_out = NULL;
nmsg_res res;
nmsg_message_t msg;
char *json;
bool input_json = false, input_nmsg = false, is_counting = false, need_exact = false;
const char *lmdb_filename = NULL, *nmsg_filename_in = NULL;
char nmsg_filename_out[BUFSIZ] = {0};
while ((c = getopt(argc, argv, "c:e:f:j:r:s:hvx")) != EOF) {
switch (c) {
case 'c':
count = atoi(optarg);
is_counting = true;
break;
case 'e':
ts_end = atoi(optarg);
break;
case 'f':
lmdb_filename = optarg;
break;
case 'j':
nmsg_filename_in = optarg;
input_json = true;
break;
case 'r':
nmsg_filename_in = optarg;
input_nmsg = true;
break;
case 's':
ts_start = atoi(optarg);
break;
case 'v':
verbosity++;
break;
case 'x':
need_exact = true;
break;
case 'h':
default:
usage(argv[0], NULL);
goto done;
}
}
if (ts_start == 0) {
usage(argv[0], "Need a starting timestamp (-s).");
goto done;
}
if (lmdb_filename == NULL) {
usage(argv[0], "Need a tsindex file (-f).");
goto done;
return (EXIT_FAILURE);
}
if ((input_json == false && input_nmsg == false) ||
(input_json && input_nmsg)) {
usage(argv[0], "Need either an nmsg json file (-j) or binary nmsg file (-r).");
goto done;
return (EXIT_FAILURE);
}
if ((ts_end == 0 && count == 0) ||
(ts_end != 0 && count != 0)) {
usage(argv[0], "Need either an ending timestamp (-e) or a count (-c).");
goto done;
return (EXIT_FAILURE);
}
res = nmsg_init();
if (res != nmsg_res_success) {
fprintf(stderr, "Error initializing NMSG library: %s\n",
nmsg_res_lookup(res));
goto done;
return (EXIT_FAILURE);
}
fd_in = open(nmsg_filename_in, O_RDONLY);
if (fd_in < 0) {
fprintf(stderr, "Can't open nmsg input file \"%s\": %s\n",
nmsg_filename_in, strerror(errno));
goto done;
return (EXIT_FAILURE);
}
n = strlcpy(nmsg_filename_out, nmsg_filename_in,
sizeof (nmsg_filename_out));
snprintf(nmsg_filename_out + n, sizeof (nmsg_filename_out) - n,
"-tsindex.%u.%s", getpid(),
input_json ? "json" : "nmsg");
fd_out = open(nmsg_filename_out, O_CREAT | O_WRONLY, 0644);
if (fd_out < 0) {
fprintf(stderr, "Can't open nmsg output file \"%s\": %s\n",
nmsg_filename_out, strerror(errno));
goto done;
return (EXIT_FAILURE);
}
if (input_json) {
nmsg_in = nmsg_input_open_json(fd_in);
if (nmsg_in == NULL) {
fprintf(stderr, "nmsg_input_open_json() failed\n");
goto done;
}
nmsg_out = nmsg_output_open_json(fd_out);
if (nmsg_out == NULL) {
fprintf(stderr, "nmsg_ouput_open_json() failed\n");
goto done;
}
}
else if (input_nmsg) {
nmsg_in = nmsg_input_open_file(fd_in);
if (nmsg_in == NULL) {
fprintf(stderr, "nmsg_input_open_file() failed\n");
goto done;
}
nmsg_out = nmsg_output_open_file(fd_out, NMSG_WBUFSZ_MAX);
if (nmsg_out == NULL) {
fprintf(stderr, "nmsg_ouput_open_file() failed\n");
goto done;
}
}
rc = mdb_env_create(&env);
if (rc != 0) {
fprintf(stderr, "mdb_create() failed: %s\n", mdb_strerror(rc));
goto done;
}
rc = mdb_env_open(env, lmdb_filename, MDB_NOSUBDIR | MDB_RDONLY, 0664);
if (rc != 0) {
fprintf(stderr, "mdb_env_open failed(): %s\n",
mdb_strerror(rc));
goto done;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc != 0) {
fprintf(stderr, "mdb_txn_begin failed(): %s\n",
mdb_strerror(rc));
goto done;
}
rc = mdb_open(txn, NULL, MDB_INTEGERKEY, &dbi);
if (rc) {
fprintf(stderr, "mdb_open(): %s\n", mdb_strerror(rc));
goto done;
}
rc = mdb_set_compare(txn, dbi, axa_tsi_mdb_cmp);
if (rc) {
fprintf(stderr, "mdb_set_compare(): %s\n", mdb_strerror(rc));
goto done;
}
ts.tv_sec = ts_start;
ts.tv_nsec = 0;
key.mv_size = sizeof (ts);
key.mv_data = &ts;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open(): %s\n", mdb_strerror(rc));
goto done;
}
rc = mdb_cursor_get(cursor, &key, &data,
need_exact ? MDB_SET : MDB_SET_RANGE);
if (rc == MDB_NOTFOUND) {
printf("Did not find starting timestamp %u in %s.\n",
ts_start, lmdb_filename);
goto done;
}
if (rc) {
fprintf(stderr, "mdb_cursor_get(): %s\n", mdb_strerror(rc));
goto done;
}
(void) mdb_cursor_close(cursor);
offset = (off_t *)data.mv_data;
if (verbosity > 0)
printf("Found %u at offset 0x%"PRIu64".\n", ts_start, *offset);
if (lseek(fd_in, *offset, SEEK_SET) == sizeof (off_t) - 1) {
fprintf(stderr, "lseek(): %s\n", strerror(errno));
goto done;
}
while (1) {
if (is_counting) {
if (count-- <= 0)
break;
}
res = nmsg_input_read(nmsg_in, &msg);
if (res == nmsg_res_eof) {
if (verbosity > 0)
printf("End of file reached.\n");
break;
}
if (res != nmsg_res_success) {
fprintf(stderr, "nmsg_input_read(): %s\n", nmsg_res_lookup(res));
goto done;
}
if (is_counting == false) {
nmsg_message_get_time(msg, &msg_ts);
if (msg_ts.tv_sec >= ts_end) {
nmsg_message_destroy(&msg);
break;
}
}
res = nmsg_output_write(nmsg_out, msg);
if (res != nmsg_res_success) {
fprintf(stderr, "nmsg_output_write(): %s\n", nmsg_res_lookup(res));
nmsg_message_destroy(&msg);
goto done;
}
if (verbosity > 1) {
res = nmsg_message_to_json(msg, &json);
if (res != nmsg_res_success) {
fprintf(stderr, "nmsg_message_to_pres(): %s\n", nmsg_res_lookup(res));
nmsg_message_destroy(&msg);
goto done;
}
printf("%s\n", json);
free(json);
}
nmsg_cnt++;
nmsg_message_destroy(&msg);
}
ec = EXIT_SUCCESS;
printf("Wrote %u nmsgs to %s.\n", nmsg_cnt, nmsg_filename_out);
done:
if (fd_in != -1)
close(fd_in);
if (fd_out != -1)
close(fd_out);
if (nmsg_in != NULL)
nmsg_input_close(&nmsg_in);
if (nmsg_out != NULL)
nmsg_output_close(&nmsg_out);
if (txn != NULL)
mdb_txn_abort(txn);
if (env != NULL)
mdb_env_close(env);
return (ec);
}
static int
mdb_db_open( BackendDB *be, ConfigReply *cr )
{
int rc, i;
struct mdb_info *mdb = (struct mdb_info *) be->be_private;
struct stat stat1;
uint32_t flags;
char *dbhome;
MDB_txn *txn;
if ( be->be_suffix == NULL ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": need suffix.\n",
1, 0, 0 );
return -1;
}
Debug( LDAP_DEBUG_ARGS,
LDAP_XSTRING(mdb_db_open) ": \"%s\"\n",
be->be_suffix[0].bv_val, 0, 0 );
/* Check existence of dbenv_home. Any error means trouble */
rc = stat( mdb->mi_dbenv_home, &stat1 );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"cannot access database directory \"%s\" (%d).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, errno );
return -1;
}
/* mdb is always clean */
be->be_flags |= SLAP_DBFLAG_CLEAN;
rc = mdb_env_create( &mdb->mi_dbenv );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_create failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
if ( mdb->mi_readers ) {
rc = mdb_env_set_maxreaders( mdb->mi_dbenv, mdb->mi_readers );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxreaders failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
}
rc = mdb_env_set_mapsize( mdb->mi_dbenv, mdb->mi_mapsize );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_mapsize failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
rc = mdb_env_set_maxdbs( mdb->mi_dbenv, MDB_INDICES );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"mdb_env_set_maxdbs failed: %s (%d).\n",
be->be_suffix[0].bv_val, mdb_strerror(rc), rc );
goto fail;
}
#ifdef HAVE_EBCDIC
strcpy( path, mdb->mi_dbenv_home );
__atoe( path );
dbhome = path;
#else
dbhome = mdb->mi_dbenv_home;
#endif
Debug( LDAP_DEBUG_TRACE,
LDAP_XSTRING(mdb_db_open) ": database \"%s\": "
"dbenv_open(%s).\n",
be->be_suffix[0].bv_val, mdb->mi_dbenv_home, 0);
flags = mdb->mi_dbenv_flags;
if ( slapMode & SLAP_TOOL_QUICK )
flags |= MDB_NOSYNC;
if ( slapMode & SLAP_TOOL_READONLY)
flags |= MDB_RDONLY;
rc = mdb_env_open( mdb->mi_dbenv, dbhome,
flags, mdb->mi_dbenv_mode );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. "
"Restore from backup!\n",
be->be_suffix[0].bv_val, rc, 0 );
goto fail;
}
rc = mdb_txn_begin( mdb->mi_dbenv, NULL, 0, &txn );
if ( rc ) {
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": database \"%s\" cannot be opened, err %d. "
"Restore from backup!\n",
be->be_suffix[0].bv_val, rc, 0 );
goto fail;
}
/* open (and create) main databases */
for( i = 0; mdmi_databases[i].bv_val; i++ ) {
flags = MDB_INTEGERKEY;
if( i == MDB_ID2ENTRY ) {
if ( !(slapMode & (SLAP_TOOL_READMAIN|SLAP_TOOL_READONLY) ))
flags |= MDB_CREATE;
} else {
if ( i == MDB_DN2ID )
flags |= MDB_DUPSORT;
if ( !(slapMode & SLAP_TOOL_READONLY) )
flags |= MDB_CREATE;
}
rc = mdb_open( txn,
mdmi_databases[i].bv_val,
flags,
&mdb->mi_dbis[i] );
if ( rc != 0 ) {
snprintf( cr->msg, sizeof(cr->msg), "database \"%s\": "
"mdb_open(%s/%s) failed: %s (%d).",
be->be_suffix[0].bv_val,
mdb->mi_dbenv_home, mdmi_databases[i].bv_val,
mdb_strerror(rc), rc );
Debug( LDAP_DEBUG_ANY,
LDAP_XSTRING(mdb_db_open) ": %s\n",
cr->msg, 0, 0 );
goto fail;
}
if ( i == MDB_ID2ENTRY )
mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare );
else if ( i == MDB_DN2ID )
mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare );
}
rc = mdb_ad_read( mdb, txn );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
rc = mdb_attr_dbs_open( be, txn, cr );
if ( rc ) {
mdb_txn_abort( txn );
goto fail;
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
goto fail;
}
/* monitor setup */
rc = mdb_monitor_db_open( be );
if ( rc != 0 ) {
goto fail;
}
mdb->mi_flags |= MDB_IS_OPEN;
return 0;
fail:
mdb_db_close( be, NULL );
return rc;
}
static int
lmdb_storage_init(struct lmdb_storage* s, char* db_env_path)
{
int result;
MDB_env* env = NULL;
MDB_txn* txn = NULL;
MDB_dbi dbi = 0;
if ((result = mdb_env_create(&env)) != 0) {
paxos_log_error("Could not create lmdb environment. %s",
mdb_strerror(result));
goto error;
}
if ((result = mdb_env_set_mapsize(env, paxos_config.lmdb_mapsize)) != 0) {
paxos_log_error("Could not set lmdb map size. %s", mdb_strerror(result));
goto error;
}
if ((result = mdb_env_open(env, db_env_path,
!paxos_config.lmdb_sync ? MDB_NOSYNC : 0 | MDB_INTEGERKEY,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH)) != 0) {
paxos_log_error("Could not open lmdb environment at %s. %s",
db_env_path, mdb_strerror(result));
goto error;
}
if ((result = mdb_txn_begin(env, NULL, 0, &txn)) != 0) {
paxos_log_error("Could not start txn on lmdb environment at %s. %s",
db_env_path, mdb_strerror(result));
goto error;
}
if ((result = mdb_open(txn, NULL, 0, &dbi)) != 0) {
paxos_log_error("Could not open db on lmdb environment at %s. %s",
db_env_path, mdb_strerror(result));
goto error;
}
if ((result = mdb_set_compare(txn, dbi, lmdb_compare_iid)) != 0) {
paxos_log_error("Could setup compare function on lmdb "
"environment at %s. %s", db_env_path, mdb_strerror(result));
goto error;
}
if ((result = mdb_txn_commit(txn)) != 0) {
paxos_log_error("Could commit txn on lmdb environment at %s. %s",
db_env_path, mdb_strerror(result));
goto error;
}
s->env = env;
s->dbi = dbi;
return 0;
error:
if (txn) {
mdb_txn_abort(txn);
}
if (dbi) {
mdb_close(env, dbi);
}
if (env) {
mdb_env_close(env);
}
return -1;
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_cursor *mc;
MDB_dbi dbi;
char *envname;
int envflags = MDB_NOSYNC, putflags = 0;
int dohdr = 0, append = 0;
MDB_val prevk;
prog = argv[0];
if (argc < 2) {
usage();
}
/* -a: append records in input order
* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'a':
append = 1;
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 's':
subname = strdup(optarg);
break;
case 'N':
putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
dbuf.mv_size = 4096;
dbuf.mv_data = malloc(dbuf.mv_size);
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
mdb_env_set_maxdbs(env, 2);
if (info.me_maxreaders)
mdb_env_set_maxreaders(env, info.me_maxreaders);
if (info.me_mapsize)
mdb_env_set_mapsize(env, info.me_mapsize);
if (info.me_mapaddr)
envflags |= MDB_FIXEDMAP;
rc = mdb_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
kbuf.mv_data = malloc(kbuf.mv_size * 2);
k0buf.mv_size = kbuf.mv_size;
k0buf.mv_data = (char *)kbuf.mv_data + kbuf.mv_size;
prevk.mv_data = k0buf.mv_data;
while(!Eof) {
MDB_val key, data;
int batch = 0;
int appflag;
if (!dohdr) {
dohdr = 1;
} else if (!(mode & NOHDR))
readhdr();
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prevk.mv_size = 0;
if (append) {
mdb_set_compare(txn, dbi, greater);
if (flags & MDB_DUPSORT)
mdb_set_dupsort(txn, dbi, greater);
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while(1) {
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %"Yu": failed to read key value\n", prog, lineno);
goto txn_abort;
}
if (append) {
appflag = MDB_APPEND;
if (flags & MDB_DUPSORT) {
if (prevk.mv_size == key.mv_size && !memcmp(prevk.mv_data, key.mv_data, key.mv_size))
appflag = MDB_CURRENT|MDB_APPENDDUP;
else {
memcpy(prevk.mv_data, key.mv_data, key.mv_size);
prevk.mv_size = key.mv_size;
}
}
} else {
appflag = 0;
}
rc = mdb_cursor_put(mc, &key, &data, putflags|appflag);
if (rc == MDB_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %"Yu": txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
if (appflag & MDB_APPENDDUP) {
MDB_val k, d;
mdb_cursor_get(mc, &k, &d, MDB_LAST);
}
batch = 0;
}
}
rc = mdb_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %"Yu": txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
mdb_dbi_close(env, dbi);
}
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
