/*
* Get or create an existing transaction from tidptr, and store it in _tidptr.
*/
static int getorset_txn(struct dbengine *db,
struct txn **tidptr,
struct txn **_tidptr,
int readonly)
{
struct txn *tid;
int r;
assert(_tidptr);
tid = *_tidptr;
if (!tidptr || !*tidptr) {
/* Create a new transaction */
if ((r = begin_txn(db, &tid, readonly))) {
*_tidptr = NULL;
return r;
}
} else {
/* Reuse the existing transaction */
tid = *tidptr;
}
if (tidptr) {
/* Refresh tidptr */
*tidptr = tid;
}
/* Return the transaction */
*_tidptr = tid;
PDEBUG("cyrusdb_lmdb(%s)[getorset_txn] tidptr=%p *tidptr=%p _tidptr=%p *_tidptr=%p ",
db->fname, tidptr, tidptr ? *tidptr : NULL, _tidptr, _tidptr ? *_tidptr : NULL);
return CYRUSDB_OK;
}
int64 LMDBFileIndex::get(const LMDBFileIndex::SIndexKey& key)
{
begin_txn(MDB_RDONLY);
MDB_val mdb_tkey;
mdb_tkey.mv_data=const_cast<void*>(static_cast<const void*>(&key));
mdb_tkey.mv_size=sizeof(SIndexKey);
MDB_val mdb_tvalue;
int rc=mdb_get(txn, dbi, &mdb_tkey, &mdb_tvalue);
int64 ret = 0;
if(rc==MDB_NOTFOUND)
{
}
else if(rc)
{
Server->Log("LMDB: Failed to read ("+(std::string)mdb_strerror(rc)+")", LL_ERROR);
_has_error=true;
}
else
{
CRData data((const char*)mdb_tvalue.mv_data, mdb_tvalue.mv_size);
data.getVarInt(&ret);
}
abort_transaction();
return ret;
}
int64 LMDBFileIndex::get_prefer_client( const SIndexKey& key )
{
begin_txn(MDB_RDONLY);
MDB_cursor* cursor;
mdb_cursor_open(txn, dbi, &cursor);
SIndexKey orig_key = key;
MDB_val mdb_tkey;
mdb_tkey.mv_data=const_cast<void*>(static_cast<const void*>(&key));
mdb_tkey.mv_size=sizeof(SIndexKey);
MDB_val mdb_tvalue;
int rc=mdb_cursor_get(cursor,&mdb_tkey, &mdb_tvalue, MDB_SET_RANGE);
int64 ret = 0;
int retry_prev=2;
while(rc==0 && retry_prev>0 && !_has_error)
{
SIndexKey* curr_key = reinterpret_cast<SIndexKey*>(mdb_tkey.mv_data);
if(rc==MDB_NOTFOUND)
{
retry_prev=0;
}
else if(rc)
{
Server->Log("LMDB: Failed to read ("+(std::string)mdb_strerror(rc)+")", LL_ERROR);
_has_error=true;
}
else if( curr_key->isEqualWithoutClientid(orig_key))
{
CRData data((const char*)mdb_tvalue.mv_data, mdb_tvalue.mv_size);
data.getVarInt(&ret);
retry_prev=0;
}
else
{
rc=mdb_cursor_get(cursor, &mdb_tkey, &mdb_tvalue, MDB_PREV);
--retry_prev;
}
}
mdb_cursor_close(cursor);
abort_transaction();
return ret;
}
std::map<int, int64> LMDBFileIndex::get_all_clients( const SIndexKey& key )
{
begin_txn(MDB_RDONLY);
MDB_cursor* cursor;
mdb_cursor_open(txn, dbi, &cursor);
SIndexKey orig_key = key;
MDB_val mdb_tkey;
mdb_tkey.mv_data=const_cast<void*>(static_cast<const void*>(&key));
mdb_tkey.mv_size=sizeof(SIndexKey);
MDB_val mdb_tvalue;
int rc=mdb_cursor_get(cursor,&mdb_tkey, &mdb_tvalue, MDB_SET_RANGE);
std::map<int, int64> ret;
SIndexKey* curr_key = reinterpret_cast<SIndexKey*>(mdb_tkey.mv_data);
while(rc==0 &&
orig_key.isEqualWithoutClientid(*curr_key))
{
CRData data((const char*)mdb_tvalue.mv_data, mdb_tvalue.mv_size);
int64 entryid;
data.getVarInt(&entryid);
ret[curr_key->getClientid()] = entryid;
rc=mdb_cursor_get(cursor, &mdb_tkey, &mdb_tvalue, MDB_NEXT);
curr_key = reinterpret_cast<SIndexKey*>(mdb_tkey.mv_data);
}
if(rc && rc!=MDB_NOTFOUND)
{
Server->Log("LMDB: Failed to read ("+(std::string)mdb_strerror(rc)+")", LL_ERROR);
_has_error=true;
}
mdb_cursor_close(cursor);
abort_transaction();
return ret;
}
// std::string persistent test.
void StlAdvancedFeaturesExample::storing_std_strings()
{
string kstring = "hello world", *sstring = new string("hi there");
if (explicit_txn)
begin_txn(0, penv);
db_map<string, string> pmap(dmstringdb, NULL);
pmap[kstring] = *sstring + "!";
*sstring = pmap[kstring];
map<string, string> spmap;
spmap.insert(make_pair(kstring, *sstring));
cout<<"sstring append ! is : "<<pmap[kstring]
<<" ; *sstring is : "<<*sstring;
delete sstring;
for (db_map<string, string>::iterator ii = pmap.begin();
ii != pmap.end();
++ii) {
cout << (*ii).first << ": " << (*ii).second << endl;
}
close_db(dmstringdb);
dmstringdb = dbstl::open_db(penv, "db_map_stringdb.db",
dbtype, DB_CREATE | dboflags, 0);
db_map<string, string> pmap2(dmstringdb, NULL);
for (db_map<string, string>::iterator ii = pmap2.begin();
ii != pmap2.end(); ++ii) {
cout << (*ii).first << ": " << (*ii).second << endl;
// assert key/data pair set equal
check_expr((spmap.count(ii->first) == 1) &&
(spmap[ii->first] == ii->second));
}
if (explicit_txn)
commit_txn(penv);
db_vector<string> strvctor(10);
vector<string> sstrvctor(10);
for (int i = 0; i < 10; i++) {
strvctor[i] = "abc";
sstrvctor[i] = strvctor[i];
}
check_expr(is_equal(strvctor, sstrvctor));
}
/* Initiates the non-blocking capture of a consistent snapshot of the database,
* using the exported snapshot context->repl.snapshot_name. */
int snapshot_start(client_context_t context) {
if (!context->repl.snapshot_name || context->repl.snapshot_name[0] == '\0') {
client_error(context, "snapshot_name must be set in client context");
return EINVAL;
}
int err = 0;
check(err, exec_sql(context, "BEGIN"));
check(err, exec_sql(context, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"));
PQExpBuffer query = createPQExpBuffer();
appendPQExpBuffer(query, "SET TRANSACTION SNAPSHOT '%s'", context->repl.snapshot_name);
check(err, exec_sql(context, query->data));
destroyPQExpBuffer(query);
Oid argtypes[] = { 25, 16 }; // 25 == TEXTOID, 16 == BOOLOID
const char *args[] = { "%", context->allow_unkeyed ? "t" : "f" };
if (!PQsendQueryParams(context->sql_conn,
"SELECT bottledwater_export(table_pattern := $1, allow_unkeyed := $2)",
2, argtypes, args, NULL, NULL, 1)) { // The final 1 requests results in binary format
client_error(context, "Could not dispatch snapshot fetch: %s",
PQerrorMessage(context->sql_conn));
return EIO;
}
if (!PQsetSingleRowMode(context->sql_conn)) {
client_error(context, "Could not activate single-row mode");
return EIO;
}
// Invoke the begin-transaction callback with xid==0 to indicate start of snapshot
begin_txn_cb begin_txn = context->repl.frame_reader->on_begin_txn;
void *cb_context = context->repl.frame_reader->cb_context;
if (begin_txn) {
check(err, begin_txn(cb_context, context->repl.start_lsn, 0));
}
return 0;
}
static int myopen(const char *fname, int flags, struct dbengine **ret,
struct txn **tidptr)
{
struct dbengine *db;
int r, mr = 0;
PDEBUG("cyrusdb_lmdb(%s)[open] flags=0x%x tidptr=%p *tidptr=%p",
fname, flags, tidptr, tidptr ? *tidptr : NULL);
assert(fname && ret);
/* Create a new database engine */
db = (struct dbengine *) xzmalloc(sizeof(struct dbengine));
db->fname = xstrdup(fname);
db->flags = flags;
/* Assert that either the parent directory or the database exists */
r = flags & CYRUSDB_CREATE ? my_mkparentdir(fname) : my_stat(fname);
if (r) {
/* Both my_stat and my_mkparentdir preserve errno */
r = errno == ENOENT ? CYRUSDB_NOTFOUND : CYRUSDB_IOERROR;
goto fail;
}
/* Create the environment for this database */
mr = mdb_env_create(&db->env);
if (mr) {
r = CYRUSDB_INTERNAL;
goto fail;
}
/* Size (or resize) the environment */
mr = mdb_env_set_mapsize(db->env, maxdbsize);
if (mr) {
r = CYRUSDB_INTERNAL;
goto fail;
}
/* Open the environment */
mr = mdb_env_open(db->env, fname, MDB_NOSUBDIR, 0600);
if (mr) {
r = CYRUSDB_IOERROR;
goto fail;
}
/* Lock the database, if requested */
if (tidptr) {
r = begin_txn(db, tidptr, 0);
if (r) goto fail;
}
/* Keep database for internal bookkeeping */
register_db(db);
/* Return the database handle */
*ret = db;
return CYRUSDB_OK;
fail:
if (mr)
syslog(LOG_ERR, "cryusdb_lmdb(%s): %s", db->fname, mdb_strerror(mr));
if (db->env)
mdb_env_close(db->env);
free(db->fname);
free(db);
return r;
}
void StlAdvancedFeaturesExample::secondary_containers()
{
int i;
if (explicit_txn)
begin_txn(0, penv);
// test secondary db
cout<<"\ndb container backed by secondary database.";
dbp3->associate(dbstl::current_txn(penv), dbp3sec,
get_dest_secdb_callback, DB_CREATE);
typedef db_multimap<int, BaseMsg> sec_mmap_t;
sec_mmap_t secmmap(dbp3sec, penv);// index "to" field
db_map<int, BaseMsg> basemsgs(dbp3, penv);
basemsgs.clear();
BaseMsg tmpmsg;
multiset<BaseMsg> bsmsgs, bsmsgs2;
multiset<BaseMsg>::iterator bsitr1, bsitr2;
// populate primary and sec db
for (i = 0; i < 10; i++) {
tmpmsg.when = time(NULL);
tmpmsg.to = 100 - i % 3;// sec index multiple
tmpmsg.from = i + 20;
bsmsgs.insert( tmpmsg);
basemsgs.insert(make_pair(i, tmpmsg));
}
check_expr(basemsgs.size() == 10);
// check retrieved data is identical to those fed in
sec_mmap_t::iterator itrsec;
for (itrsec = secmmap.begin(
ReadModifyWriteOption::no_read_modify_write(), true);
itrsec != secmmap.end(); itrsec++) {
bsmsgs2.insert(itrsec->second);
}
for (bsitr1 = bsmsgs.begin(), bsitr2 = bsmsgs2.begin();
bsitr1 != bsmsgs.end() && bsitr2 != bsmsgs2.end();
bsitr1++, bsitr2++) {
check_expr(*bsitr1 == *bsitr2);
}
check_expr(bsitr1 == bsmsgs.end() && bsitr2 == bsmsgs2.end());
// search using sec index, check the retrieved data is expected
// and exists in bsmsgs
check_expr(secmmap.size() == 10);
pair<sec_mmap_t::iterator, sec_mmap_t::iterator> secrg =
secmmap.equal_range(98);
for (itrsec = secrg.first; itrsec != secrg.second; itrsec++) {
check_expr(itrsec->second.to == 98 &&
bsmsgs.count(itrsec->second) > 0);
}
// delete via sec db
size_t nersd = secmmap.erase(98);
check_expr(10 - nersd == basemsgs.size());
secrg = secmmap.equal_range(98);
check_expr(secrg.first == secrg.second);
if (explicit_txn)
dbstl::commit_txn(penv);
} // secondary_containers
void StlAdvancedFeaturesExample::arbitrary_object_storage()
{
int i;
if (explicit_txn)
begin_txn(0, penv);
// varying length objects test
cout<<"\nArbitary object storage using Dbt..\n";
rand_str_dbt smsdbt;
DbstlDbt dbt, dbtmsg;
string msgstr;
SMSMsg *smsmsgs[10];
dbtmsg.set_flags(DB_DBT_USERMEM);
dbt.set_data(DbstlMalloc(256));
dbt.set_flags(DB_DBT_USERMEM);
dbt.set_ulen(256);
db_map<int, DbstlDbt> msgmap(dbp3, penv);
for (i = 0; i < 10; i++) {
smsdbt(dbt, msgstr, 10, 200);
SMSMsg *pmsg = SMSMsg::make_sms_msg(time(NULL),
(char *)dbt.get_data(), i);
smsmsgs[i] = SMSMsg::make_sms_msg(time(NULL),
(char *)dbt.get_data(), i);
dbtmsg.set_data(pmsg);
dbtmsg.set_ulen((u_int32_t)(pmsg->mysize));
dbtmsg.set_size((u_int32_t)(pmsg->mysize));
dbtmsg.set_flags(DB_DBT_USERMEM);
msgmap.insert(make_pair(i, dbtmsg));
free(pmsg);
memset(&dbtmsg, 0, sizeof(dbtmsg));
}
dbtmsg.set_data(NULL);
SMSMsg *psmsmsg;
for (i = 0; i < 10; i++) {
db_map<int, DbstlDbt>::data_type_wrap msgref = msgmap[i];
psmsmsg = (SMSMsg *)msgref.get_data();
check_expr(memcmp(smsmsgs[i], psmsmsg,
smsmsgs[i]->mysize) == 0);
}
i = 0;
for (db_map<int, DbstlDbt>::iterator msgitr =
msgmap.begin(ReadModifyWriteOption::
read_modify_write()); msgitr != msgmap.end(); ++msgitr, i++) {
db_map<int, DbstlDbt>::reference smsmsg = *msgitr;
(((SMSMsg*)(smsmsg.second.get_data())))->when = time(NULL);
smsmsg.second._DB_STL_StoreElement();
}
for (i = 0; i < 10; i++)
free(smsmsgs[i]);
msgmap.clear();
cout<<"\nArbitary object(sparse, varying length) storage support using registered callbacks.\n";
db_map<int, SMSMsg2> msgmap2(dbp3, penv);
SMSMsg2 smsmsgs2[10];
DbstlElemTraits<SMSMsg2>::instance()->set_copy_function(SMSMsgCopy);
DbstlElemTraits<SMSMsg2>::instance()->set_size_function(SMSMsgSize);
DbstlElemTraits<SMSMsg2>::instance()->set_restore_function(SMSMsgRestore);
// use new technique to store varying length and inconsecutive objs
for (i = 0; i < 10; i++) {
smsdbt(dbt, msgstr, 10, 200);
SMSMsg2 msg2(time(NULL), msgstr.c_str(), i);
smsmsgs2[i] = msg2;
msgmap2.insert(make_pair(i, msg2));
}
// check that retrieved data is identical to stored data
SMSMsg2 tmpmsg2;
for (i = 0; i < 10; i++) {
tmpmsg2 = msgmap2[i];
check_expr(smsmsgs2[i] == tmpmsg2);
}
for (db_map<int, SMSMsg2>::iterator msgitr =
msgmap2.begin(ReadModifyWriteOption::
read_modify_write()); msgitr != msgmap2.end(); msgitr++) {
db_map<int, SMSMsg2>::reference smsmsg = *msgitr;
smsmsg.second.when = time(NULL);
smsmsg.second._DB_STL_StoreElement();
}
msgmap2.clear();
if (explicit_txn)
commit_txn(penv);
} // arbitrary_object_storage
void LMDBFileIndex::start_transaction(void)
{
begin_txn(0);
}
void LMDBFileIndex::create(get_data_callback_t get_data_callback, void *userdata)
{
begin_txn(0);
IDatabase *db=Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_FILES_NEW);
ServerFilesDao filesdao(db);
size_t n_done=0;
size_t n_rows=0;
SIndexKey last;
int64 last_prev_entry;
int64 last_id;
db_results res;
do
{
res=get_data_callback(n_done, n_rows, userdata);
++n_rows;
for(size_t i=0;i<res.size();++i)
{
const std::string& shahash=res[i]["shahash"];
int64 id = watoi64(res[i]["id"]);
SIndexKey key(reinterpret_cast<const char*>(shahash.c_str()), watoi64(res[i]["filesize"]), watoi(res[i]["clientid"]));
int64 next_entry = watoi64(res[i]["next_entry"]);
int64 prev_entry = watoi64(res[i]["prev_entry"]);
int pointed_to = watoi(res[i]["pointed_to"]);
assert(memcmp(&last, &key, sizeof(SIndexKey))!=1);
if(key==last)
{
if(last_prev_entry==0)
{
filesdao.setPrevEntry(id, last_id);
}
if(next_entry==0
&& (last_prev_entry==0 || last_prev_entry==id) )
{
filesdao.setNextEntry(last_id, id);
}
if(pointed_to)
{
filesdao.setPointedTo(0, id);
}
last=key;
last_id=id;
last_prev_entry=prev_entry;
continue;
}
else
{
if(!pointed_to)
{
filesdao.setPointedTo(1, id);
}
}
put(key, id, MDB_APPEND);
if(_has_error)
{
Server->Log("LMDB error after putting element. Error state interrupting..", LL_ERROR);
return;
}
if(n_done % 1000 == 0 && n_done>0)
{
if ((Server->getFailBits() & IServer::FAIL_DATABASE_CORRUPTED) ||
(Server->getFailBits() & IServer::FAIL_DATABASE_IOERR) ||
(Server->getFailBits() & IServer::FAIL_DATABASE_FULL))
{
Server->Log("Database error. Stopping.", LL_ERROR);
return;
}
Server->Log("File entry index contains "+convert(n_done)+" entries now.", LL_INFO);
}
if(n_done % c_create_commit_n == 0 && n_done>0)
{
commit_transaction();
begin_txn(0);
}
++n_done;
last=key;
last_id=id;
last_prev_entry=prev_entry;
}
}
while(!res.empty());
commit_transaction();
}