TEST_F(ReqSuite, deleteUpdateRollback)
{
buildSample();
mark1();
{
MojDbReq req;
// start transaction
req.begin(&db, false);
checkMarkWithUpdate(50ul, -1, req);
checkMarkWithUpdate(0ul, -3, req);
deleteMark(50ul, -1, req);
checkMarkWithUpdate(0ul, -1, req);
checkMarkWithUpdate(0ul, -3, req);
mark3(req);
checkMarkWithUpdate(0ul, -1, req);
checkMarkWithUpdate(33ul, -3, req);
}
checkMarkWithUpdate(50ul, -1);
checkMarkWithUpdate(0ul, -3);
}
MojErr MojDb::updateLocale(const MojChar* locale, MojDbReqRef req)
{
MojAssert(locale);
MojLogTrace(s_log);
MojErr err = beginReq(req, true);
MojErrCheck(err);
MojString oldLocale;
err = getLocale(oldLocale, req);
MojErrCheck(err);
MojString newLocale;
err = newLocale.assign(locale);
MojErrCheck(err);
MojErr updateErr = err = updateLocaleImpl(oldLocale, newLocale, req);
MojErrCatchAll(err) {
err = req->abort();
MojErrCheck(err);
err = m_kindEngine.close();
MojErrCheck(err);
MojDbReq openReq;
err = beginReq(openReq, true);
MojErrCheck(err);
err = m_kindEngine.open(this, openReq);
MojErrCheck(err);
err = openReq.end();
MojErrCheck(err);
MojErrThrow(updateErr);
}
return MojErrNone;
}
MojErr MojDb::delObj(const MojObject& id, const MojObject& obj, MojDbStorageItem* item, MojObject& foundObjOut, MojDbReq& req, MojUInt32 flags)
{
MojAssert(item);
MojLogTrace(s_log);
if (MojFlagGet(flags, FlagPurge)) {
// update indexes
MojTokenSet tokenSet;
// we want purge to force delete
req.fixmode(true);
MojErr err = m_kindEngine.update(NULL, &obj, req, OpDelete, tokenSet);
MojErrCheck(err);
// gross layering violation
err = req.txn()->offsetQuota(-(MojInt64) item->size());
MojErrCheck(err);
// permanently delete
bool found = false;
err = m_objDb->del(id, req.txn(), found);
MojErrCheck(err);
if (!found)
MojErrThrow(MojErrDbCorruptDatabase);
err = foundObjOut.put(IdKey, id);
MojErrCheck(err);
} else {
// set deleted flag and put if we are not purging
MojObject newObj = obj;
MojErr err = newObj.putBool(DelKey, true);
MojErrCheck(err);
err = putObj(id, newObj, &obj, item, req, OpDelete);
MojErrCheck(err);
foundObjOut = newObj;
}
return MojErrNone;
}
MojErr MojDbIndex::find(MojDbCursor& cursor, MojDbWatcher* watcher, MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(isOpen());
MojAutoPtr<MojDbQueryPlan> plan(new MojDbQueryPlan(*m_kindEngine));
MojAllocCheck(plan.get());
MojErr err = plan->init(cursor.query(), *this);
MojErrCheck(err);
if (watcher) {
// we have to add the watch before beginning the txn or we may miss events
MojAssert(cursor.txn() == NULL);
err = addWatch(*plan, cursor, watcher, req);
MojErrCheck(err);
}
if (!cursor.txn()) {
MojDbStorageTxn* txn = req.txn();
bool cursorOwnsTxn = !(req.batch() || txn);
if (txn) {
cursor.txn(txn, cursorOwnsTxn);
} else {
MojRefCountedPtr<MojDbStorageTxn> localTxn;
err = m_collection->beginTxn(localTxn);
MojErrCheck(err);
cursor.txn(localTxn.get(), cursorOwnsTxn);
req.txn(localTxn.get());
}
}
cursor.m_dbIndex = this; // for debugging
err = m_collection->find(plan, cursor.txn(), cursor.m_storageQuery);
MojErrCheck(err);
cursor.m_watcher = watcher;
return MojErrNone;
}
MojErr MojDbIndex::addWatch(const MojDbQueryPlan& plan, MojDbCursor& cursor, MojDbWatcher* watcher, MojDbReq& req)
{
MojAssert(watcher);
MojLogTrace(s_log);
// TODO: use interval tree instead of vector for watches
MojThreadWriteGuard guard(m_lock);
MojErr err = m_watcherVec.push(watcher);
MojErrCheck(err);
// update count map
watcher->domain(req.domain());
WatcherMap::Iterator iter;
err = m_watcherMap.find(req.domain(), iter);
MojErrCheck(err);
if (iter == m_watcherMap.end()) {
err = m_watcherMap.put(req.domain(), 1);
MojErrCheck(err);
} else {
iter.value() += 1;
if (iter.value() > WatchWarningThreshold) {
MojLogWarning(s_log, _T("db:'%s' has %zd watches open on index '%s - %s'"),
req.domain().data(), iter.value(), m_kind->id().data(), m_name.data());
}
}
MojLogInfo(s_log, _T("DbIndex_addWatch - '%s' on index '%s - %s'"),
req.domain().data(), m_kind->id().data(), m_name.data());
// drop lock before acquiring watcher mutex in init
guard.unlock();
watcher->init(this, plan.ranges(), plan.desc(), false);
return MojErrNone;
}
TEST_F(ReqSuite, original)
{
buildSample();
mark1();
// test visibility with update
{
MojDbReq req;
// start transaction
req.begin(&db, false);
mark2(req);
// visible within transaction
checkMarkWithUpdate(50ul, -2, req);
}
// invisible after aborted transaction
checkMarkWithUpdate(0ul, -2);
// test visibility with delete
mark1();
{
MojDbReq req;
// start transaction
req.begin(&db, false);
deleteMark(50ul, -1, req);
// visible within transaction
{
MojDbQuery query;
MojAssertNoErr( query.from(_T("Test:1")) );
MojAssertNoErr( query.where(_T("bar"), MojDbQuery::OpLessThan, 2) );
MojObject update;
MojUInt32 count = 0xbaddcafe;
MojAssertNoErr( db.merge(query, update, count, MojDb::FlagNone, req) );
EXPECT_EQ( 33ul, count);
}
}
// invisible after aborted transaction
{
MojDbQuery query;
MojAssertNoErr( query.from(_T("Test:1")) );
MojAssertNoErr( query.where(_T("bar"), MojDbQuery::OpLessThan, 2) );
MojObject update;
// Note: should not set value to something that will introduce double-update
MojUInt32 count = 0xbaddcafe;
MojAssertNoErr( db.merge(query, update, count) );
EXPECT_EQ( 83ul, count);
}
}
MojErr MojDbIndex::updateLocale(const MojChar* locale, MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(isOpen());
MojAssert(locale);
bool haveCollate = false;
for (PropVec::ConstIterator i = m_props.begin(); i != m_props.end(); ++i) {
if ((*i)->collation() != MojDbCollationInvalid) {
haveCollate = true;
MojErr err = (*i)->updateLocale(locale);
MojErrCheck(err);
}
}
if (haveCollate) {
// drop and reindex
MojErr err = drop(req);
MojErrCheck(err);
err = build(req.txn());
MojErrCheck(err);
}
m_locale.assign(locale);
return MojErrNone;
}
MojErr MojDbKindState::writeIds(const MojChar* key, const MojObject& obj, MojDbReq& req, MojRefCountedPtr<MojDbStorageItem>& oldItem)
{
MojErr err = writeObj(key, obj, m_kindEngine->indexIdDb(), req.txn(), oldItem);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbIndex::stats(MojObject& objOut, MojSize& usageOut, MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(isOpen());
MojSize count = 0;
MojSize size = 0;
MojErr err = m_index->stats(req.txn(), count, size);
LOG_DEBUG("[db_mojodb] IndexStats: Kind: %s;Index: %s; Id: %zX; count= %zu; size= %zu; delMisses = %d, err= %d \n",
m_kind->name().data(), m_name.data(), idIndex(), count, size, m_delMisses, err);
MojErrCheck(err);
usageOut += size;
err = objOut.put(SizeKey, (MojInt64) size);
MojErrCheck(err);
err = objOut.put(CountKey, (MojInt64) count);
MojErrCheck(err);
err = objOut.put(DelMissesKey, (MojInt64) m_delMisses); // cumulative since start
MojErrCheck(err);
MojThreadReadGuard guard(m_lock);
if (!m_watcherMap.empty()) {
MojObject watcherInfo;
for (WatcherMap::ConstIterator i = m_watcherMap.begin(); i != m_watcherMap.end(); ++i) {
err = watcherInfo.put(i.key(), (MojInt64) i.value());
MojErrCheck(err);
}
err = objOut.put(WatchesKey, watcherInfo);
MojErrCheck(err);
}
return MojErrNone;
}
MojErr MojDbIndex::open(MojDbStorageIndex* index, const MojObject& id, MojDbReq& req, bool created)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(!isOpen() && !m_props.empty());
MojAssert(index);
// we don't want to take built-in props into account when sorting indexes,
m_sortKey = m_propNames;
m_id = id;
MojDbKey idKey;
MojErr err = idKey.assign(id);
MojErrCheck(err);
err = m_idSet.put(idKey);
MojErrCheck(err);
err = addBuiltinProps();
MojErrCheck(err);
if (created && !isIdIndex()) {
// if this index was just created, we need to re-index before committing the transaction
MojDbStorageTxn* txn = req.txn();
txn->notifyPreCommit(m_preCommitSlot);
txn->notifyPostCommit(m_postCommitSlot);
} else {
// otherwise it's ready
m_ready = true;
}
// and we're open
m_index.reset(index);
m_collection = m_index.get();
return MojErrNone;
}
MojErr MojDbKindState::readIds(const MojChar* key, MojDbReq& req, MojObject& objOut, MojRefCountedPtr<MojDbStorageItem>& itemOut)
{
MojErr err = readObj(key, objOut, m_kindEngine->indexIdDb(), req.txn(), itemOut);
MojErrCheck(err);
return MojErrNone;
}
TEST_F(ReqSuite, visibility)
{
buildSample();
mark1();
MojDbReq req;
// start transaction
req.begin(&db, false);
checkMarkWithUpdate(50ul, -1, req);
checkMarkWithUpdate(0ul, -2, req);
mark2(req);
checkMarkWithUpdate(50ul, -2, req);
}
MojErr MojDbKind::updateOwnIndexes(const MojObject* newObj, const MojObject* oldObj, const MojDbReq& req, MojInt32& idxcount)
{
MojInt32 count = 0;
for (IndexVec::ConstIterator i = m_indexes.begin();
i != m_indexes.end(); ++i) {
count++;
MojErr err = (*i)->update(newObj, oldObj, req.txn(), req.fixmode());
MojErrCheck(err);
}
MojLogInfo(s_log, _T("Kind_UpdateOwnIndexes: %s; count: %d \n"), this->id().data(), count);
idxcount += count;
return MojErrNone;
}
MojErr MojDbKind::update(MojObject* newObj, const MojObject* oldObj, MojDbOp op, MojDbReq& req, bool checkSchema)
{
MojLogTrace(s_log);
MojErr err = checkPermission(op, req);
MojErrCheck(err);
err = req.curKind(this);
MojErrCheck(err);
#if defined(TESTDBKIND)
MojString s;
MojErr e2;
if (oldObj) {
e2 = oldObj->toJson(s);
MojLogInfo(s_log, _T("Kind_Update_OldObj: %s ;\n"), s.data());
}
if (newObj) {
e2 = newObj->toJson(s);
MojLogInfo(s_log, _T("Kind_Update_NewObj: %s ;\n"), s.data());
}
#endif
if (newObj) {
// add the _backup property if not set
if (m_backup && !newObj->contains(MojDb::SyncKey)) {
err = newObj->putBool(MojDb::SyncKey, true);
MojErrCheck(err);
}
// TEMPORARY!!! This should be done in pre-update to also check parent kinds
// warning message comes from preUpdate
if(checkSchema)
{
MojSchema::Result res;
err = m_schema.validate(*newObj, res);
MojErrCheck(err);
if (!res.valid())
{
MojErrThrowMsg(MojErrSchemaValidation, _T("schema validation failed for kind '%s': %s"),
m_id.data(), res.msg().data());
}
}
}
// update revSets and validate schema
err = preUpdate(newObj, oldObj, req);
MojErrCheck(err);
// update indexes
MojVector<MojDbKind*> kindVec;
MojInt32 idxcount = 0;
err = updateIndexes(newObj, oldObj, req, op, kindVec, idxcount);
MojLogInfo(s_log, _T("Kind_UpdateIndexes_End: %s; supers = %zu; indexcount = %zu; updated = %d \n"), this->id().data(),
m_supers.size(), m_indexes.size(), idxcount);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbQuotaEngine::initUsage(MojDbKind* kind, MojDbReq& req)
{
MojAssert(kind);
MojRefCountedPtr<MojDbStorageItem> item;
MojErr err = m_usageDb->get(kind->id(), req.txn(), false, item);
MojErrCheck(err);
if (!item.get()) {
MojObject stats;
MojSize usage = 0;
err = kind->stats(stats, usage, req, false);
MojErrCheck(err);
err = insertUsage(kind->id(), (MojInt64) usage, req.txn());
MojErrCheck(err);
}
return MojErrNone;
}
MojErr MojDbKind::deny(MojDbReq& req)
{
MojLogWarning(s_log, _T("db: permission denied for caller '%s' on kind '%s'"), req.domain().data(), m_id.data());
if (m_kindEngine->permissionEngine()->enabled()) {
// don't leak any information in an error message
MojErrThrow(MojErrDbPermissionDenied);
}
return MojErrNone;
}
MojDbPermissionEngine::Value MojDbKind::objectPermission(const MojChar* op, MojDbReq& req)
{
MojDbPermissionEngine::Value val = m_kindEngine->permissionEngine()->
check(PermissionType, m_id, req.domain(), op);
if (val == MojDbPermissionEngine::ValueUndefined && !m_supers.empty()) {
val = m_supers[0]->objectPermission(op, req);
}
return val;
}
MojErr MojDbIndex::drop(MojDbReq& req)
{
MojAssert(isOpen());
MojLogTrace(s_log);
MojErr err = m_index->drop(req.txn());
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbIndex::drop(MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(isOpen());
MojErr err = m_index->drop(req.txn());
MojErrCheck(err);
return MojErrNone;
}
TEST_F(ReqSuite, updateRollback)
{
buildSample();
mark1();
{
MojDbReq req;
// start transaction
req.begin(&db, false);
checkMarkWithUpdate(50ul, -1, req);
checkMarkWithUpdate(0ul, -2, req);
mark2(req);
checkMarkWithUpdate(50ul, -2, req);
}
checkMarkWithUpdate(50ul, -1);
checkMarkWithUpdate(0ul, -2);
}
MojErr MojDb::drop(const MojChar* path)
{
MojAssert(path);
MojLogTrace(s_log);
MojErr err = requireOpen();
MojErrCheck(err);
MojDbReq req;
err = req.begin(this, true);
MojErrCheck(err);
err = m_storageEngine->drop(path, req.txn());
MojErrCheck(err);
err = req.end();
MojErrCheck(err);
err = close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbKind::updateLocale(const MojChar* locale, MojDbReq& req)
{
MojLogTrace(s_log);
MojAssert(locale);
MojErr err = req.curKind(this);
MojErrCheck(err);
for (IndexVec::ConstIterator i = m_indexes.begin(); i != m_indexes.end(); ++i) {
err = (*i)->updateLocale(locale, req);
MojErrCheck(err);
}
return MojErrNone;
}
MojErr MojDb::updateLocaleImpl(const MojString& oldLocale, const MojString& newLocale, MojDbReq& req)
{
if (oldLocale != newLocale) {
MojErr err = m_kindEngine.updateLocale(newLocale, req);
MojErrCheck(err);
err = updateState(LocaleKey, newLocale, req);
MojErrCheck(err);
}
MojErr err = req.end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbQuotaEngine::open(const MojObject& conf, MojDb* db, MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(db);
MojErr err = db->storageEngine()->openDatabase(_T("UsageDbName"), req.txn(), m_usageDb);
MojErrCheck(err);
err = MojDbPutHandler::open(conf, db, req);
MojErrCheck(err);
MojDbKindEngine::KindMap& kinds = db->kindEngine()->kindMap();
for (MojDbKindEngine::KindMap::ConstIterator i = kinds.begin();
i != kinds.end(); ++i) {
err = initUsage(i.value().get(), req);
MojErrCheck(err);
}
err = refreshImpl(req.txn());
MojErrCheck(err);
m_isOpen = true;
return MojErrNone;
}
MojErr MojDbKindState::initTokens(MojDbReq& req, const StringSet& strings)
{
// TODO: bug inside this function. (latest strace step)
MojAssertMutexLocked(m_lock);
// TODO: filing load tokens. Go inside readObj
// load tokens
MojErr err = readObj(TokensKey, m_tokensObj, m_kindEngine->kindDb(), req.txn(), m_oldTokensItem);
MojErrCheck(err);
// populate token vec
MojUInt8 maxToken = 0;
err = m_tokenVec.resize(m_tokensObj.size());
MojErrCheck(err);
for (MojObject::ConstIterator i = m_tokensObj.begin(); i != m_tokensObj.end(); ++i) {
MojString key = i.key();
MojInt64 value = i.value().intValue();
MojSize idx = (MojSize) (value - MojObjectWriter::TokenStartMarker);
if (value < MojObjectWriter::TokenStartMarker || value >= MojUInt8Max || idx >= m_tokenVec.size()) {
MojErrThrow(MojErrDbInvalidToken);
}
if (value > maxToken) {
maxToken = (MojUInt8) value;
}
err = m_tokenVec.setAt(idx, key);
MojErrCheck(err);
}
if (maxToken > 0) {
m_nextToken = (MojUInt8) (maxToken + 1);
}
// add strings
bool updated = false;
for (StringSet::ConstIterator i = strings.begin(); i != strings.end(); ++i) {
if (!m_tokensObj.contains(*i)) {
updated = true;
MojUInt8 token = 0;
TokenVec tokenVec;
MojObject tokenObj;
err = addPropImpl(*i, false, token, tokenVec, tokenObj);
MojErrCheck(err);
}
}
if (updated) {
err = writeTokens(m_tokensObj);
MojErrCheck(err);
}
return MojErrNone;
}
TEST_F(ReqSuite, originalEq)
{
buildSample();
mark1();
// test visibility with update
{
MojDbReq req;
// start transaction
req.begin(&db, false);
mark2(req);
// visible within transaction
checkMarkWithUpdate(50ul, -2, req);
}
// invisible after aborted transaction
checkMarkWithUpdate(0ul, -2);
// test visibility with delete
mark1();
{
MojDbReq req;
// start transaction
req.begin(&db, false);
deleteMark(50ul, -1, req);
// visible within transaction
checkMarkWithUpdate(0ul, -1, req);
}
// invisible after aborted transaction
checkMarkWithUpdate(50ul, -1);
}
MojErr MojDbIndex::addWatch(const MojDbQueryPlan& plan, MojDbCursor& cursor, MojDbWatcher* watcher, MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(watcher);
// TODO: use interval tree instead of vector for watches
MojThreadWriteGuard guard(m_lock);
MojErr err = m_watcherVec.push(watcher);
MojErrCheck(err);
// update count map
watcher->domain(req.domain());
WatcherMap::Iterator iter;
err = m_watcherMap.find(req.domain(), iter);
MojErrCheck(err);
if (iter == m_watcherMap.end()) {
err = m_watcherMap.put(req.domain(), 1);
MojErrCheck(err);
} else {
iter.value() += 1;
if (iter.value() > WatchWarningThreshold) {
LOG_WARNING(MSGID_MOJ_DB_INDEX_WARNING, 4,
PMLOGKS("domain", req.domain().data()),
PMLOGKFV("iter", "%zd", iter.value()),
PMLOGKS("kindId", m_kind->id().data()),
PMLOGKS("name", m_name.data()),
"db:'domain' has 'iter' watches open on index 'kindId - name'");
}
}
LOG_DEBUG("[db_mojodb] DbIndex_addWatch - '%s' on index '%s - %s'",
req.domain().data(), m_kind->id().data(), m_name.data());
// drop lock before acquiring watcher mutex in init
guard.unlock();
watcher->init(this, plan.ranges(), plan.desc(), false);
return MojErrNone;
}
MojErr MojDbQuotaEngine::stats(MojObject& objOut, MojDbReq& req)
{
// check for admin permission
if (!req.admin()) {
MojErrThrow(MojErrDbPermissionDenied);
}
for (QuotaMap::ConstIterator i = m_quotas.begin(); i != m_quotas.end(); ++i) {
MojObject quota;
MojErr err = quota.put(MojDbServiceDefs::SizeKey, i.value()->size());
MojErrCheck(err);
err = quota.put(MojDbServiceDefs::UsedKey, i.value()->usage());
MojErrCheck(err);
err = objOut.put(i.key(), quota);
MojErrCheck(err);
}
return MojErrNone;
}
MojErr MojDbQuotaEngine::put(MojObject& obj, MojDbReq& req, bool putObj)
{
MojLogTrace(s_log);
MojAssertWriteLocked(m_db->schemaLock());
// check for admin permission
if (!req.admin()) {
MojErrThrow(MojErrDbPermissionDenied);
}
// pull params out of object
MojString owner;
MojErr err = obj.getRequired(MojDbServiceDefs::OwnerKey, owner);
MojErrCheck(err);
MojInt64 size = 0;
err = obj.getRequired(MojDbServiceDefs::SizeKey, size);
MojErrCheck(err);
// validate owner
err = validateWildcard(owner, MojErrDbInvalidOwner);
MojErrCheck(err);
// put object
if (putObj) {
MojString id;
err = id.format(_T("_quotas/%s"), owner.data());
MojErrCheck(err);
err = obj.put(MojDb::IdKey, id);
MojErrCheck(err);
err = obj.putString(MojDb::KindKey, MojDbKindEngine::QuotaId);
MojErrCheck(err);
MojDbAdminGuard adminGuard(req);
err = m_db->put(obj, MojDb::FlagForce, req);
MojErrCheck(err);
// defer commit of quota until txn commit
MojRefCountedPtr<MojDbQuotaCommitHandler> handler(new MojDbQuotaCommitHandler(this, owner, size, req.txn()));
MojAllocCheck(handler.get());
} else {
err = commitQuota(owner, size);
MojErrCheck(err);
}
return MojErrNone;
}
MojErr MojDb::getImpl(const MojObject& id, MojObjectVisitor& visitor, MojDbOp op, MojDbReq& req)
{
MojRefCountedPtr<MojDbStorageItem> item;
MojErr err = m_objDb->get(id, req.txn(), false, item);
MojErrCheck(err);
if (item.get()) {
MojString kindId;
err = item->kindId(kindId, m_kindEngine);
MojErrCheck(err);
err = m_kindEngine.checkPermission(kindId, op, req);
MojErrCheck(err);
err = item->visit(visitor, m_kindEngine);
MojErrCheck(err);
err = item->close();
MojErrCheck(err);
}
return MojErrNone;
}