MojErr MojDbServiceHandler::handleWatch(MojServiceMessage* msg, MojObject& payload, MojDbReq& req)
{
MojAssert(msg);
MojLogTrace(s_log);
MojObject queryObj;
MojErr err = payload.getRequired(MojDbServiceDefs::QueryKey, queryObj);
MojErrCheck(err);
MojRefCountedPtr<Watcher> watcher(new Watcher(msg));
MojAllocCheck(watcher.get());
MojDbQuery query;
err = query.fromObject(queryObj);
MojErrCheck(err);
bool fired = false;
MojDbCursor cursor;
err = m_db.watch(query, cursor, watcher->m_watchSlot, fired, req);
MojErrCheck(err);
MojLogInfo(s_log, _T("handleWatch: %s, err: (%d); sender= %s;\n fired=%d; \n"),
msg->method(), (int)err, msg->senderName(), (int)fired);
if (!fired) {
err = msg->replySuccess();
MojErrCheck(err);
}
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::watch(const MojDbQuery& query, MojDbCursor& cursor, WatchSignal::SlotRef watchHandler, bool& firedOut, MojDbReqRef req)
{
MojLogTrace(s_log);
firedOut = false;
MojErr err = beginReq(req);
MojErrCheck(err);
MojRefCountedPtr<MojDbWatcher> watcher(new MojDbWatcher(watchHandler));
MojAllocCheck(watcher.get());
MojDbQuery limitedQuery = query;
limitedQuery.limit(1);
err = findImpl(limitedQuery, cursor, watcher.get(), req, OpRead);
MojErrCheck(err);
MojDbStorageItem* item = NULL;
bool found = false;
cursor.verifymode(false);
err = cursor.get(item, found);
MojErrCheck(err);
if (found) {
const MojDbKey& key = cursor.storageQuery()->endKey();
err = watcher->fire(key);
MojErrCheck(err);
firedOut = true;
}
err = req->end(false);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbPutHandler::open(const MojObject& conf, MojDb* db, MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssertWriteLocked(db->schemaLock());
m_db = db;
// configure
MojErr err = configure(conf, req);
MojErrCheck(err);
// load permissions from db
MojDbQuery query;
err = query.from(m_kindId);
MojErrCheck(err);
MojDbCursor cursor;
err = db->find(query, cursor, req);
MojErrCheck(err);
for (;;) {
bool found = false;
MojObject perm;
err = cursor.get(perm, found);
MojErrCheck(err);
if (!found)
break;
err = put(perm, req, false);
MojErrCheck(err);
}
return MojErrNone;
}
/**
* verifyRecords
*/
MojErr MojDbShardManagerTest::verifyRecords (const MojChar* strKind, MojDb& db, const MojDbShardInfo&, MojUInt32& count)
{
MojDbQuery query;
MojDbCursor cursor;
count = 0;
MojErr err = query.from(strKind);
MojErrCheck(err);
err = db.find(query, cursor);
MojErrCheck(err);
while (true)
{
bool found;
MojObject dbObj;
err = cursor.get(dbObj, found);
MojErrCheck(err);
if (!found)
break;
++count;
}
return MojErrNone;
}
void checkCount()
{
MojDbQuery query;
MojAssertNoErr( query.from(_T("LoadTest:1")) );
MojDbCursor cursor;
MojAssertNoErr( db.find(query, cursor) );
MojUInt32 count = 0;
MojAssertNoErr( cursor.count(count) );
EXPECT_EQ( 10LL, count );
}
MojErr MojDb::merge(const MojDbQuery& query, const MojObject& props, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
MojErr err = beginReq(req);
MojErrCheck(err);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpUpdate);
MojErrCheck(err);
MojAssert(cursor.txn());
MojUInt32 count = 0;
MojUInt32 warns = 0;
bool found = false;
MojObject prev;
for (;;) {
// get prev rev from cursor
MojDbStorageItem* prevItem = NULL;
err = cursor.get(prevItem, found);
if (err == MojErrInternalIndexOnFind) {
warns++;
continue;
}
MojErrCheck(err);
if (!found)
break;
err = prevItem->toObject(prev, m_kindEngine);
MojErrCheck(err);
// merge obj into prev
MojObject merged;
err = mergeInto(merged, props, prev);
MojErrCheck(err);
// and update the db
const MojObject& id = prevItem->id();
err = putObj(id, merged, &prev, prevItem, req, OpUpdate);
MojErrCheck(err);
++count;
}
if (warns > 0)
MojLogWarning(s_log, _T("Merge index_warnings: %s; count: %d\n"), query.from().data(), warns);
err = cursor.close();
MojErrCheck(err);
err = req->end();
MojErrCheck(err);
countOut = count;
return MojErrNone;
}
MojErr MojDbWatchTest::cancelTest(MojDb& db)
{
// cancel find
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpLessThanEq, 45);
MojTestErrCheck(err);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
watcher->m_slot.cancel();
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
watcher->m_slot.cancel();
MojTestAssert(watcher->m_count == 0);
MojObject id;
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
// cancel watch
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
MojDbQuery queryWithRev;
err = queryWithRev.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = queryWithRev.where(_T("foo"), MojDbQuery::OpEq, 45);
MojTestErrCheck(err);
err = queryWithRev.where(_T("_rev"), MojDbQuery::OpGreaterThan, m_rev);
MojTestErrCheck(err);
bool fired = false;
err = db.watch(queryWithRev, cursor, watcher->m_slot, fired);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(!fired);
MojTestAssert(watcher->m_count == 0);
watcher->m_slot.cancel();
MojTestAssert(watcher->m_count == 0);
err = put(db, 45, 45, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
return MojErrNone;
}
MojErr MojDbDistinctTest::simpleTest(MojDb& db)
{
MojErr err;
MojDbQuery query;
const MojChar* queryString;
const MojChar* expectedIdsJson;
MojString str;
MojDbSearchCursor searchCursor(str);
MojDbCursor cursor;
//1st test
queryString = _T("bar");
expectedIdsJson = _T("[\"a\",\"b\",\"c\",\"d\"]");
err = initQuery(query, queryString);
MojTestErrCheck(err);
err = check(db, query, searchCursor, queryString, expectedIdsJson);
MojTestErrCheck(err);
searchCursor.close();
//test for find
err = check(db, query, cursor, queryString, expectedIdsJson);
MojTestErrCheck(err);
cursor.close();
//2nd test
queryString = _T("foo");
expectedIdsJson = _T("[\"e\",\"f\",\"g\"]");
err = initQuery(query, queryString);
MojTestErrCheck(err);
err = check(db, query, searchCursor, queryString, expectedIdsJson);
MojTestErrCheck(err);
searchCursor.close();
//test for find
err = check(db, query, cursor, queryString, expectedIdsJson);
MojTestErrCheck(err);
cursor.close();
return MojErrNone;
}
MojErr MojDbWatchTest::rangeTest(MojDb& db)
{
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpGreaterThan, 5);
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpLessThan, 100);
MojTestErrCheck(err);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojObject id;
MojInt64 rev;
err = put(db, 5, 5, id, rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = put(db, 100, 100, id, rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = put(db, 6, 6, id, rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = put(db, 99, 99, id, rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
return MojErrNone;
}
MojErr MojDb::findImpl(const MojDbQuery& query, MojDbCursor& cursor, MojDbWatcher* watcher, MojDbReq& req, MojDbOp op)
{
MojLogTrace(s_log);
if (cursor.isOpen())
MojErrThrow(MojErrDbCursorAlreadyOpen);
MojErr err = m_kindEngine.find(query, cursor, watcher, req, op);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbIndex::find(MojDbCursor& cursor, MojDbWatcher* watcher, MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(isOpen());
MojAutoPtr<MojDbQueryPlan> plan(new MojDbQueryPlan(*m_kindEngine));
MojAllocCheck(plan.get());
MojErr err = plan->init(cursor.query(), *this);
MojErrCheck(err);
if (watcher) {
// we have to add the watch before beginning the txn or we may miss events
MojAssert(cursor.txn() == NULL);
err = addWatch(*plan, cursor, watcher, req);
MojErrCheck(err);
}
if (!cursor.txn()) {
MojDbStorageTxn* txn = req.txn();
bool cursorOwnsTxn = !(req.batch() || txn);
if (txn) {
cursor.txn(txn, cursorOwnsTxn);
} else {
MojRefCountedPtr<MojDbStorageTxn> localTxn;
err = m_collection->beginTxn(localTxn);
MojErrCheck(err);
cursor.txn(localTxn.get(), cursorOwnsTxn);
req.txn(localTxn.get());
}
}
cursor.m_dbIndex = this; // for debugging
err = m_collection->find(plan, cursor.txn(), cursor.m_storageQuery);
MojErrCheck(err);
cursor.m_watcher = watcher;
return MojErrNone;
}
MojErr MojDbKindEngine::find(const MojDbQuery& query, MojDbCursor& cursor, MojDbWatcher* watcher, MojDbReq& req, MojDbOp op)
{
MojAssert(isOpen());
MojLogTrace(s_log);
MojErr err = query.validate();
MojErrCheck(err);
// In order to find collate index, cursor needs kindEngine
cursor.kindEngine(this);
err = cursor.init(query);
MojErrCheck(err);
MojDbKind* kind = NULL;
err = getKind(query.from().data(), kind);
MojErrCheck(err);
MojAssert(kind);
err = kind->find(cursor, watcher, req, op);
MojErrCheck(err);
cursor.kindEngine(this);
return MojErrNone;
}
MojErr MojDbIndex::build(MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(isOpen());
MojAssert(m_kind && m_kindEngine);
MojAssert(m_props.size() > 1);
// query for all existing objects of this type and add them to the index.
MojDbQuery query;
MojErr err = query.from(m_kind->id());
MojErrCheck(err);
if (m_includeDeleted) {
err = query.includeDeleted();
MojErrCheck(err);
}
MojDbCursor cursor;
MojDbReq adminRequest(true);
adminRequest.txn(txn);
err = m_kindEngine->find(query, cursor, NULL, adminRequest, OpRead);
MojErrCheck(err);
for (;;) {
MojObject obj;
bool found = false;
err = cursor.get(obj, found);
MojErrCheck(err);
if (!found)
break;
// add this object to the index
err = update(&obj, NULL, txn, false);
MojErrCheck(err);
}
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbDistinctTest::check(MojDb& db, const MojDbQuery& query, MojDbCursor& cursor, const MojChar* queryString, const MojChar* expectedIdsJson)
{
MojErr err = db.find(query, cursor);
MojTestErrCheck(err);
MojObjectBuilder builder;
err = builder.beginArray();
MojTestErrCheck(err);
err = cursor.visit(builder);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = builder.endArray();
MojTestErrCheck(err);
MojObject results = builder.object();
MojString json;
err = results.toJson(json);
MojTestErrCheck(err);
MojObject expected;
err = expected.fromJson(expectedIdsJson);
MojTestErrCheck(err);
// size check
MojTestAssert(expected.size() == results.size());
// value check
MojObject::ConstArrayIterator j = results.arrayBegin();
for (MojObject::ConstArrayIterator i = expected.arrayBegin();
i != expected.arrayEnd(); ++i, ++j) {
MojObject value;
err = j->getRequired(queryString, value);
MojTestErrCheck(err);
MojTestAssert(*i == value);
}
return MojErrNone;
}
MojErr MojDbKindEngine::loadKinds(MojDbReq& req)
{
MojAssert(isOpen());
MojAssertWriteLocked(m_db->m_schemaLock);
MojLogTrace(s_log);
MojDbQuery query;
MojErr err = query.from(KindKindId);
MojErrCheck(err);
MojDbCursor cursor;
err = m_db->find(query, cursor, req);
MojErrCheck(err);
for (;;) {
MojObject obj;
bool found = false;
err = cursor.get(obj, found);
MojErrCheck(err);
if (!found)
break;
// load kind
MojErr loadErr = err = putKind(obj, req);
MojErrCatchAll(err) {
MojString id;
bool found = false;
MojErr err = obj.get(MojDbServiceDefs::IdKey, id, found);
MojErrCheck(err);
MojString errStr;
MojErrToString(loadErr, errStr);
MojLogError(s_log, _T("error loading kind '%s' - %s"), id.data(), errStr.data());
}
}
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbKind::find(MojDbCursor& cursor, MojDbWatcher* watcher, MojDbReq& req, MojDbOp op)
{
MojLogTrace(s_log);
MojErr err = checkPermission(op, req);
MojErrCheck(err);
const MojDbQuery& query = cursor.query();
MojDbIndex* index = indexForQuery(query);
if (index == NULL)
MojErrThrow(MojErrDbNoIndexForQuery);
cursor.m_dbIndex = index;
MojLogInfo(s_log, _T("Dbkind_find: Kind: %s, UsingIndex: %s, order: %s, limit: %d \n"), m_id.data(), index->name().data(),
query.order().data(), (int)query.limit());
err = index->find(cursor, watcher, req);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbPurgeTest::checkObjectsPurged(MojDb& db, const MojUInt32& count, const MojSize& expectedCount,
const MojSize& expectedNumObjects, const MojSize& expectedNumRevTimestampObjects, const MojObject& expectedLastPurgeRevNum)
{
//check number of objects purged
MojTestAssert(count == expectedCount);
//there should still be expectedNumObjects test objects
MojDbQuery query;
MojErr err = query.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor);
MojTestErrCheck(err);
MojUInt32 objCount;
err = cursor.count(objCount);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(objCount == expectedNumObjects);
//there should be expectedNumRevTimestampObjects RevTimestamp objects
MojDbQuery revQuery;
err = revQuery.from(MojDbKindEngine::RevTimestampId);
MojTestErrCheck(err);
MojDbCursor revCursor;
err = db.find(revQuery, revCursor);
MojTestErrCheck(err);
MojUInt32 revTimestampObjCount;
err = revCursor.count(revTimestampObjCount);
MojTestErrCheck(err);
err = revCursor.close();
MojTestErrCheck(err);
MojTestAssert(revTimestampObjCount == expectedNumRevTimestampObjects);
//lastPurgedRevNum should be equal to the expectedLastPurgeRevNum
MojObject revNum;
err = db.purgeStatus(revNum);
MojTestErrCheck(err);
MojTestAssert(revNum == expectedLastPurgeRevNum);
return MojErrNone;
}
MojErr MojDbWatchTest::eqTest(MojDb& db)
{
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpEq, 1);
MojTestErrCheck(err);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojRefCountedPtr<TestWatcher> watcher2(new TestWatcher);
MojTestAssert(watcher2.get());
bool fired = false;
err = db.watch(query, cursor, watcher2->m_slot, fired);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(!fired);
// puts
MojObject id;
err = put(db, 0, 0, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
err = put(db, 2, 2, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
// put, changing property not in index
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = put(db, 1, 2, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
// dels
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
watcher2.reset(new TestWatcher);
MojTestAssert(watcher2.get());
MojDbQuery queryWithRev;
err = queryWithRev.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = queryWithRev.where(_T("foo"), MojDbQuery::OpEq, 1);
MojTestErrCheck(err);
err = queryWithRev.where(_T("_rev"), MojDbQuery::OpGreaterThan, m_rev);
MojTestErrCheck(err);
err = queryWithRev.includeDeleted();
MojTestErrCheck(err);
fired = false;
err = db.watch(queryWithRev, cursor, watcher2->m_slot, fired);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(!fired);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
bool found;
err = db.del(id, found);
MojTestErrCheck(err);
MojTestAssert(found);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
// ordering
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
return MojErrNone;
}
MojErr MojDbWatchTest::pageTest(MojDb& db)
{
MojObject id;
MojObject idFirst;
MojObject idFourth;
MojObject idLast;
MojInt64 rev;
for (int i = 100; i < 150; ++i) {
MojErr err = put(db, 100, i, id, rev);
MojTestErrCheck(err);
if (i == 100) {
idFirst = id;
} else if (i == 103) {
idFourth = id;
} else if (i == 149) {
idLast = id;
}
}
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpGreaterThanEq, 100);
MojTestErrCheck(err);
query.limit(3);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
bool found = false;
MojUInt32 count = 0;
do {
MojObject obj;
err = cursor.get(obj, found);
MojTestErrCheck(err);
if (found)
++count;
} while (found);
MojTestAssert(count == 3);
MojDbQuery::Page page;
err = cursor.nextPage(page);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = merge(db, idFourth, 53);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
query.page(page);
MojRefCountedPtr<TestWatcher> watcher2(new TestWatcher);
MojTestAssert(watcher2.get());
err = db.find(query, cursor, watcher2->m_slot);
MojTestErrCheck(err);
found = false;
count = 0;
do {
MojObject obj;
err = cursor.get(obj, found);
MojTestErrCheck(err);
if (found)
++count;
} while (found);
MojTestAssert(count == 3);
err = cursor.close();
MojTestErrCheck(err);
err = db.del(idFirst, found);
MojTestErrCheck(err);
MojTestAssert(found);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 0);
err = db.del(idFourth, found);
MojTestErrCheck(err);
MojTestAssert(found);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
// desc order
query.page(MojDbQuery::Page());
query.desc(true);
MojRefCountedPtr<TestWatcher> watcher3(new TestWatcher);
MojTestAssert(watcher3.get());
err = db.find(query, cursor, watcher3->m_slot);
MojTestErrCheck(err);
found = false;
count = 0;
do {
MojObject obj;
err = cursor.get(obj, found);
MojTestErrCheck(err);
if (found)
++count;
} while (found);
MojTestAssert(count == 3);
err = cursor.close();
MojTestErrCheck(err);
err = merge(db, idLast, 53);
MojTestErrCheck(err);
MojTestAssert(watcher3->m_count == 1);
MojRefCountedPtr<TestWatcher> watcher4(new TestWatcher);
MojTestAssert(watcher4.get());
err = db.find(query, cursor, watcher4->m_slot);
MojTestErrCheck(err);
found = false;
count = 0;
do {
MojObject obj;
err = cursor.get(obj, found);
MojTestErrCheck(err);
if (found)
++count;
} while (found);
MojTestAssert(count == 3);
err = cursor.close();
MojTestErrCheck(err);
err = merge(db, idLast, 54);
MojTestErrCheck(err);
MojTestAssert(watcher4->m_count == 1);
return MojErrNone;
}
MojErr MojDb::dumpImpl(MojFile& file, bool backup, bool incDel, const MojObject& revParam, const MojObject& delRevParam, bool skipKinds, MojUInt32& countOut, MojDbReq& req,
MojObject* response, const MojChar* keyName, MojSize& bytesWritten, MojSize& warns, MojUInt32 maxBytes)
{
// query for objects, adding the backup key and rev key if necessary
MojDbQuery query;
MojErr err = query.from(MojDbKindEngine::RootKindId);
MojErrCheck(err);
err = query.where(MojDb::DelKey, MojDbQuery::OpEq, incDel);
MojErrCheck(err);
if (backup) {
err = query.where(MojDb::SyncKey, MojDbQuery::OpEq, true);
MojErrCheck(err);
if (incDel) {
err = query.where(MojDb::RevKey, MojDbQuery::OpGreaterThan, delRevParam);
MojErrCheck(err);
} else {
err = query.where(MojDb::RevKey, MojDbQuery::OpGreaterThan, revParam);
MojErrCheck(err);
}
}
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpRead);
MojErrCheck(err);
warns = 0;
MojObject curRev;
for(;;) {
bool found = false;
MojObject obj;
err = cursor.get(obj, found);
// So that we can get as much data as possible from a corrupt database
// We simply skip ghost keys and continue
if (err == MojErrInternalIndexOnFind) {
warns++;
continue;
}
MojErrCheck(err);
if (!found)
break;
if (skipKinds) {
MojString kind;
err = obj.getRequired(KindKey, kind);
MojErrCheck(err);
if (kind == MojDbKindEngine::KindKindId) {
continue;
}
}
// write out each object, if the backup is full, insert the appropriate incremental key
err = dumpObj(file, obj, bytesWritten, maxBytes);
MojErrCatch(err, MojErrDbBackupFull) {
if (response) {
MojErr errBackup = MojErrNone;
if (!curRev.undefined()) {
errBackup = insertIncrementalKey(*response, keyName, curRev);
MojErrCheck(errBackup);
} else {
errBackup = insertIncrementalKey(*response, keyName, incDel ? delRevParam: revParam);
MojErrCheck(errBackup);
}
errBackup = handleBackupFull(revParam, delRevParam, *response, keyName);
MojErrCheck(errBackup);
}
return MojErrNone;
}
MojErrCheck(err);
err = obj.getRequired(MojDb::RevKey, curRev);
MojErrCheck(err);
countOut++;
}
err = cursor.close();
MojErrCheck(err);
if (warns > 0) {
MojLogWarning(s_log, _T("Finished Backup with %d warnings \n"), (int)warns);
} else {
MojLogDebug(s_log, _T("Finished Backup with no warnings \n"));
}
// construct the next incremental key
if (response && !curRev.undefined()) {
err = insertIncrementalKey(*response, keyName, curRev);
MojErrCheck(err);
}
return MojErrNone;
}
MojErr MojDbKind::verifyIndex(MojDbIndex *pIndex, MojObject &iinfo, MojDbReq& req)
{
// Goes throudh each index entry and verifies that it points to a valid object
// For debugging purposes as stats for indexes does not access the target objects
// Index->stats function does not have enough context to find the object
// db/stats usage '{"verify":true,"kind":"xyz"}' - each optional
MojDbQuery query;
MojErr err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
query.m_forceIndex = pIndex; // Important: Otherwise, it will pick the default index
cursor.verifymode(true); // to get the errors
err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
MojLogInfo(s_log, _T("Kind_verifyIndex: Kind: %s; Index: %s; idIndex: %zX; size: %zu; CursorIndex: %s \n"), m_name.data(),
pIndex->name().data(), pIndex->idIndex(), pIndex->size(), cursor.m_dbIndex->name().data());
MojErrCheck(err);
MojSize count = 0;
MojSize delCount = 0;
MojSize warnCount = 0;
char s[1024];
for (;;) {
MojDbStorageItem* item = NULL;
bool found = false;
err = cursor.get(item, found);
if (err == MojErrInternalIndexOnFind) {
warnCount++;
MojDbIsamQuery *iquery = (MojDbIsamQuery *)cursor.m_storageQuery.get();
MojErr err2 = MojByteArrayToHex(iquery->m_keyData, iquery->m_keySize, s);
MojErrCheck(err2);
MojChar *ids = (iquery->m_keySize > 18) ? (MojChar *)(iquery->m_keyData + iquery->m_keySize - 17) : NULL;
MojLogInfo(s_log, _T("VerifyIndex Warning: %s; KeySize: %zu; %s ;id: %s \n"),
cursor.m_dbIndex->name().data(), iquery->m_keySize, s, ids);
continue;
}
MojErrCheck(err);
if (!found)
break;
MojObject obj;
err = item->toObject(obj, *m_kindEngine, true);
MojErrCheck(err);
bool deleted = false;
if (obj.get(MojDb::DelKey, deleted) && deleted) {
delCount++;
} else {
count++;
}
}
MojLogInfo(s_log, _T("Kind_verifyIndex Counts: Kind: %s; Index: %s; count: %zu; delcount: %zu; warnings: %zu \n"), m_name.data(),
pIndex->name().data(), count, delCount, warnCount);
err = iinfo.put(VerifyCountKey, (MojInt64)count);
MojErrCheck(err);
err = iinfo.put(VerifyWarnCountKey, (MojInt64) warnCount);
MojErrCheck(err);
err = iinfo.put(VerifyDelCountKey, (MojInt64) delCount);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbKind::updateSupers(const KindMap& map, const StringVec& superIds, bool updating, MojDbReq& req)
{
MojLogTrace(s_log);
MojInt32 indexes = 0;
if (updating) {
KindVec addedSupers;
MojErr err = diffSupers(map, superIds, m_superIds, addedSupers);
MojErrCheck(err);
KindVec removedSupers;
err = diffSupers(map, m_superIds, superIds, removedSupers);
MojErrCheck(err);
// remove/add our objects from new/removed supers
if (!addedSupers.empty() || !removedSupers.empty()) {
MojDbQuery query;
err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
cursor.kindEngine(m_kindEngine);
err = find(cursor, NULL, req, OpKindUpdate);
MojErrCheck(err);
for (;;) {
MojObject obj;
bool found = false;
err = cursor.get(obj, found);
MojErrCheck(err);
if (!found)
break;
for (KindVec::ConstIterator i = addedSupers.begin(); i != addedSupers.end(); ++i) {
err = (*i)->updateOwnIndexes(&obj, NULL, req, indexes);
MojErrCheck(err);
}
for (KindVec::ConstIterator i = removedSupers.begin(); i != removedSupers.end(); ++i) {
err = (*i)->updateOwnIndexes(NULL, &obj, req, indexes);
MojErrCheck(err);
}
}
}
}
// remove old supers
m_superIds = superIds;
MojErr err = clearSupers();
MojErrCheck(err);
// look for new supers
for (StringVec::ConstIterator i = m_superIds.begin(); i != m_superIds.end(); ++i) {
KindMap::ConstIterator mapIter = map.find(*i);
if (mapIter != map.end()) {
err = addSuper(mapIter->get());
MojErrCheck(err);
}
}
// look for kinds that extend us
for (KindMap::ConstIterator i = map.begin(); i != map.end(); ++i) {
if ((*i)->m_superIds.find(m_id) != MojInvalidIndex) {
err = (*i)->addSuper(this);
MojErrCheck(err);
}
}
// add root kind if we have no supers
if (m_supers.empty()) {
KindMap::ConstIterator mapIter = map.find(MojDbKindEngine::RootKindId);
if (mapIter != map.end()) {
err = addSuper(mapIter->get());
MojErrCheck(err);
}
}
return MojErrNone;
}
MojErr MojDbPermissionTest::checkPermissions(MojDb& db)
{
MojDbReq reqOwner(false);
MojErr err = reqOwner.domain(_T("com.foo"));
MojTestErrCheck(err);
MojDbReq reqFooBar(false);
err = reqFooBar.domain(_T("com.foo.bar"));
MojTestErrCheck(err);
MojDbReq reqFooBarBad(false);
err = reqFooBarBad.domain(_T("com.foobar"));
MojTestErrCheck(err);
MojDbReq reqGranted(false);
err = reqGranted.domain("com.granted");
MojTestErrCheck(err);
MojDbReq reqEmpty(false);
MojDbReq reqBad(false);
err = reqBad.domain("com.bad");
MojTestErrCheck(err);
MojObject objNoId;
err = objNoId.putString(MojDb::KindKey, _T("PermissionTest:1"));
MojTestErrCheck(err);
MojObject obj2NoId;
err = obj2NoId.putString(MojDb::KindKey, _T("PermissionTest2:1"));
MojTestErrCheck(err);
// put new obj
MojObject obj1 = objNoId;
err = db.put(obj1, MojDb::FlagNone, reqOwner);
MojTestErrCheck(err);
MojObject obj2 = objNoId;
err = db.put(obj2, MojDb::FlagNone, reqGranted);
MojTestErrCheck(err);
MojObject obj3 = objNoId;
err = db.put(obj3, MojDb::FlagNone, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
obj3 = objNoId;
err = db.put(obj3, MojDb::FlagNone, reqFooBarBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
obj3 = objNoId;
err = db.put(obj3, MojDb::FlagNone, reqFooBar);
MojTestErrCheck(err);
err = db.put(obj2NoId, MojDb::FlagNone, reqEmpty);
MojTestErrCheck(err);
// put existing obj
err = obj1.put(_T("foo"), 1);
MojTestErrCheck(err);
err = db.put(obj1, MojDb::FlagNone, reqOwner);
MojTestErrCheck(err);
err = obj2.put(_T("foo"), 2);
MojTestErrCheck(err);
err = db.put(obj2, MojDb::FlagNone, reqGranted);
MojTestErrCheck(err);
err = obj2.put(_T("foo"), 3);
MojTestErrCheck(err);
err = db.put(obj2, MojDb::FlagNone, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
// find
MojDbQuery query;
err = query.from(_T("PermissionTest:1"));
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor, reqOwner);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = db.find(query, cursor, reqGranted);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = db.find(query, cursor, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
err = cursor.close();
MojTestErrCheck(err);
// get
MojObject id1;
MojTestAssert(obj1.get(MojDb::IdKey, id1));
bool found = false;
MojObject gotten;
err = db.get(id1, gotten, found, reqOwner);
MojTestErrCheck(err);
err = db.get(id1, gotten, found, reqGranted);
MojTestErrCheck(err);
err = db.get(id1, gotten, found, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
// del by id
err = db.del(id1, found, MojDb::FlagNone, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
MojTestAssert(!found);
err = db.del(id1, found, MojDb::FlagNone, reqOwner);
MojTestErrCheck(err);
MojTestAssert(found);
MojObject id2;
MojTestAssert(obj2.get(MojDb::IdKey, id2));
err = db.del(id2, found, MojDb::FlagNone, reqGranted);
MojTestErrCheck(err);
MojTestAssert(found);
// del query
MojUInt32 count = 0;
err = db.del(query, count, MojDb::FlagNone, reqOwner);
MojTestErrCheck(err);
err = db.del(query, count, MojDb::FlagNone, reqGranted);
MojTestErrCheck(err);
err = db.del(query, count, MojDb::FlagNone, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
return MojErrNone;
}
MojErr MojDbKind::stats(MojObject& objOut, MojSize& usageOut, MojDbReq& req, bool verify)
{
MojLogTrace(s_log);
#if defined(TESTDBKIND)
MojLogInfo(s_log, _T("Subkinds for - %s ; count = %d\n"), m_id.data(), m_subs.size());
int n = 0;
for (KindVec::ConstIterator i = m_subs.begin(); i != m_subs.end(); ++i) {
MojLogInfo(s_log, _T("SubKind %d: %s"), n++, (*i)->id().data());
}
MojLogInfo(s_log, _T("Supers for - %s ; count = %d\n"), m_id.data(), m_supers.size());
n = 0;
for (KindVec::ConstIterator i = m_supers.begin(); i != m_supers.end(); ++i) {
MojLogInfo(s_log, _T("Super %d: %s"), n++, (*i)->id().data());
}
#endif
// analyze objects
MojDbQuery query;
MojErr err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
MojLogInfo(s_log, _T("KindStats start: %s ; Indexes = %zu; Using Index: %s; \n"),
m_id.data(), m_indexes.size(), cursor.m_dbIndex->name().data());
MojErrCheck(err);
MojSize count = 0;
MojSize size = 0;
MojSize delCount = 0;
MojSize delSize = 0;
MojSize warnings = 0;
for (;;) {
MojDbStorageItem* item = NULL;
bool found = false;
cursor.verifymode(true);
err = cursor.get(item, found);
if (err == MojErrInternalIndexOnFind) {
warnings++;
continue;
}
if (err != MojErrNone) // for all other errors break and dump current stats
break;
if (!found)
break;
MojObject obj;
err = item->toObject(obj, *m_kindEngine, true);
if (err != MojErrNone)
break;
bool deleted = false;
if (obj.get(MojDb::DelKey, deleted) && deleted) {
delSize += item->size();
delCount++;
} else {
size += item->size();
count++;
}
}
MojLogInfo(s_log, _T("KindStats Summary: %s : Count: %zu; delCount: %zu; warnings: %zu \n"), m_id.data(), count, delCount, warnings);
usageOut += size + delSize;
MojObject info;
err = info.put(SizeKey, (MojInt64) size);
MojErrCheck(err);
err = info.put(CountKey, (MojInt64) count);
MojErrCheck(err);
if (delCount > 0) {
err = info.put(DelSizeKey, (MojInt64) delSize);
MojErrCheck(err);
err = info.put(DelCountKey, (MojInt64) delCount);
MojErrCheck(err);
}
if (warnings > 0) {
err = info.put(WarnKey, (MojInt64) warnings);
MojErrCheck(err);
}
err = objOut.put(ObjectsKey, info);
MojErrCheck(err);
// and indexes
MojObject indexes;
for (IndexVec::ConstIterator i = m_indexes.begin(); i != m_indexes.end(); ++i) {
MojObject indexInfo;
err = (*i)->stats(indexInfo, usageOut, req);
MojErrCheck(err);
if (verify) {
MojDbIndex *pi = i->get();
MojErr err2 = verifyIndex(pi, indexInfo, req);
MojErrCheck(err2);
}
err = indexes.put((*i)->name(), indexInfo);
MojErrCheck(err);
}
err = objOut.put(IndexesKey, indexes);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbPurgeTest::run()
{
MojDb db;
MojErr err = db.open(MojDbTestDir);
MojTestErrCheck(err);
// put type
MojObject obj;
err = obj.fromJson(MojKindStr);
MojTestErrCheck(err);
err = db.putKind(obj);
MojTestErrCheck(err);
MojObject revNums[10];
MojObject ids[10];
//put 10 objects in the db
for(int i = 0; i < 10; i++) {
err = obj.fromJson(MojTestObjStr1);
MojTestErrCheck(err);
err = db.put(obj);
MojTestErrCheck(err);
// get _rev and id
MojObject rev;
err = obj.getRequired(MojDb::RevKey, rev);
MojTestErrCheck(err);
revNums[i] = rev;
MojObject id;
err = obj.getRequired(MojDb::IdKey, id);
MojTestErrCheck(err);
ids[i] = id;
}
//purge now, there are no RevTimestamp entries
MojUInt32 count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 1, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry - that's not more than PurgeNumDays days ago
MojTime time;
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[0], time.microsecs());
MojTestErrCheck(err);
//purge now, there are no RevTimestamp entries that are more than
//PurgeNumDays ago, so nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 3, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry for more than PurgeNumDays days ago
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[9], time.microsecs() - (((MojInt64)40) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//purge now, since nothing has been deleted, nothing should be purged,
//but the RevTimestamp object should be deleted
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 4, -1);
MojTestErrCheck(err);
//delete something - this will set its revision number higher
bool found;
err = db.del(ids[0], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//purge now, since nothing has been deleted prior to the revision
//number, nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 9, 5, -1);
MojTestErrCheck(err);
//delete another object
err = db.del(ids[1], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//create a RevTimestamp entry for more than PurgeNumDays days ago,
//with the rev number of the 1st obj we deleted
MojDbQuery query;
err = query.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
err = query.where(MojDb::IdKey, MojDbQuery::OpEq, ids[0]);
MojTestErrCheck(err);
err = query.includeDeleted();
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor);
MojTestErrCheck(err);
MojObject objFromDb;
err = cursor.get(objFromDb, found);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(found == true);
MojObject revFromDb;
err = objFromDb.getRequired(MojDb::RevKey, revFromDb);
MojTestErrCheck(err);
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revFromDb, time.microsecs() - (((MojInt64)35) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//now purge, only id[0] should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 1, 8, 6, revFromDb);
MojTestErrCheck(err);
//TODO 2.12.10 - this test does not pass yet, we need to fix calling delKind after a purge
//err = delKindTest(db);
//MojTestErrCheck(err);
err = db.close();
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDb::dump(const MojChar* path, MojUInt32& countOut, bool incDel, MojDbReqRef req, bool backup,
MojUInt32 maxBytes, const MojObject* incrementalKey, MojObject* backupResponse)
{
MojAssert(path);
MojLogTrace(s_log);
MojErr err = beginReq(req);
MojErrCheck(err);
if (!req->admin()) {
MojLogError(s_log, _T("access denied: '%s' cannot dump db to path: '%s'"), req->domain().data(), path);
MojErrThrow(MojErrDbAccessDenied);
}
MojFile file;
err = file.open(path, MOJ_O_WRONLY | MOJ_O_CREAT | MOJ_O_TRUNC, MOJ_S_IRUSR | MOJ_S_IWUSR);
MojErrCheck(err);
// write out kinds first, then existing objects, then deleted objects
MojSize bytesWritten = 0;
MojSize totalwarns = 0;
MojSize newwarns = 0;
MojDbQuery objQuery;
MojVector<MojObject> kindVec;
MojObject revParam = -1;
MojObject delRevParam = -1;
// if we were given an incremental key, pull out the revs now
if (incrementalKey) {
incrementalKey->get(MojDbServiceDefs::RevKey, revParam);
incrementalKey->get(MojDbServiceDefs::DeletedRevKey, delRevParam);
}
err = m_kindEngine.getKinds(kindVec);
MojErrCheck(err);
// write kinds - if incremental, only write the kinds that have changed since the respective revs
MojString countStr;
for (MojVector<MojObject>::ConstIterator i = kindVec.begin(); i != kindVec.end(); ++i) {
if (backup) {
bool backupKind = false;
i->get(MojDbKind::SyncKey, backupKind);
if (!backupKind)
continue;
MojString id;
err = i->getRequired(MojDbServiceDefs::IdKey, id);
MojErrCheck(err);
MojDbQuery countQuery;
err = countQuery.from(id);
MojErrCheck(err);
MojDbCursor cursor;
err = find(countQuery, cursor, req);
MojErrCheck(err);
MojUInt32 count = 0;
err = cursor.count(count);
MojErrCheck(err);
if (count > 0) {
if (i != kindVec.begin()) {
err = countStr.appendFormat(_T(", "));
MojErrCheck(err);
}
err = countStr.appendFormat("%s=%u", id.data(), count);
MojErrCheck(err);
}
}
bool deleted = false;
i->get(DelKey, deleted);
MojObject kindRev;
err = i->getRequired(RevKey, kindRev);
MojErrCheck(err);
if ((deleted && kindRev > delRevParam) || (!deleted && kindRev > revParam)) {
err = dumpObj(file, (*i), bytesWritten, maxBytes);
MojErrCheck(err);
countOut++;
}
}
// dump all the non-deleted objects
err = dumpImpl(file, backup, false, revParam, delRevParam, true, countOut, req, backupResponse, MojDbServiceDefs::RevKey, bytesWritten, newwarns, maxBytes);
MojErrCheck(err);
totalwarns += newwarns;
// If we're supposed to include deleted objects, dump the deleted objects now.
// There's a chance that we may have run out of space in our backup. If that's the case,
// we don't want to try to dump deleted objects - we can detect this by looking for the HasMoreKey
if (incDel && backupResponse && !backupResponse->contains(MojDbServiceDefs::HasMoreKey)) {
err = dumpImpl(file, backup, true, revParam, delRevParam, false, countOut, req, backupResponse, MojDbServiceDefs::DeletedRevKey, bytesWritten, newwarns, maxBytes);
MojErrCheck(err);
}
totalwarns += newwarns;
// Add the Full and Version keys
if (backup && backupResponse) {
bool incremental = (incrementalKey != NULL);
err = backupResponse->putBool(MojDbServiceDefs::FullKey, !incremental);
MojErrCheck(err);
err = backupResponse->put(MojDbServiceDefs::VersionKey, DatabaseVersion);
MojErrCheck(err);
err = backupResponse->put(MojDbServiceDefs::WarningsKey, (MojInt32)totalwarns);
MojErrCheck(err);
MojString description;
err = description.format(_T("incremental=%u"), countOut);
MojErrCheck(err);
if (!countStr.empty()) {
err = description.appendFormat(_T(", %s"), countStr.data());
MojErrCheck(err);
}
err = backupResponse->put(_T("description"), description);
MojErrCheck(err);
}
err = req->end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbServiceHandler::findImpl(MojServiceMessage* msg, MojObject& payload, MojDbReq& req, MojDbCursor& cursor, bool doCount)
{
MojAssert(msg);
MojLogTrace(s_log);
MojObject queryObj;
MojErr err = payload.getRequired(MojDbServiceDefs::QueryKey, queryObj);
MojErrCheck(err);
bool doWatch = false;
payload.get(MojDbServiceDefs::WatchKey, doWatch);
MojDbQuery query;
err = query.fromObject(queryObj);
MojErrCheck(err);
MojUInt32 limit = query.limit();
if (limit == MojDbQuery::LimitDefault){
query.limit(MaxQueryLimit);
} else if (limit > MaxQueryLimit) {
MojErrThrowMsg(MojErrDbInvalidQuery, _T("db: limit greater than %d not allowed"), MaxQueryLimit);
}
if (doWatch) {
MojRefCountedPtr<Watcher> watcher(new Watcher(msg));
MojAllocCheck(watcher.get());
err = m_db.find(query, cursor, watcher->m_watchSlot, req);
MojErrCheck(err);
} else {
err = m_db.find(query, cursor, req);
MojErrCheck(err);
}
// append results
MojObjectVisitor& writer = msg->writer();
err = writer.beginObject();
MojErrCheck(err);
err = writer.boolProp(MojServiceMessage::ReturnValueKey, true);
MojErrCheck(err);
err = writer.propName(MojDbServiceDefs::ResultsKey);
MojErrCheck(err);
err = writer.beginArray();
MojErrCheck(err);
err = cursor.visit(writer);
MojErrCheck(err);
err = writer.endArray();
MojErrCheck(err);
// append next page
MojDbQuery::Page page;
err = cursor.nextPage(page);
MojErrCheck(err);
if (!page.empty()) {
MojObject pageObj;
err = page.toObject(pageObj);
MojErrCheck(err);
err = writer.objectProp(MojDbServiceDefs::NextKey, pageObj);
MojErrCheck(err);
}
// append count
if (doCount) {
MojUInt32 count = 0;
err = cursor.count(count);
MojErrCheck(err);
err = writer.intProp(MojDbServiceDefs::CountKey, (MojInt64) count);
MojErrCheck(err);
}
err = writer.endObject();
MojErrCheck(err);
if (doWatch) {
// if this is a watched query, it cannot be part of a batch so it's safe to reply here
err = msg->reply();
MojErrCheck(err);
}
// notifications can fire any time after the cursor is closed,
// so don't close it until after sending the reply.
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::delImpl(const MojDbQuery& quer, MojUInt32& countOut, MojDbReq& req, MojUInt32 flags)
{
MojLogTrace(s_log);
countOut = 0;
MojInt32 warns = 0;
MojDbQuery newQuery = quer;
MojUInt32 queryLimit = newQuery.limit();
if(newQuery.limit() == MojDbQuery::LimitDefault)
newQuery.limit(AutoBatchSize);
while(queryLimit > 0)
{
MojDbCursor cursor;
MojErr err = findImpl(newQuery, cursor, NULL, req, OpDelete);
MojErrCheck(err);
MojAssert(cursor.txn());
MojUInt32 count = 0;
MojUInt32 numberInBatch = 0;
bool found = false;
MojObject obj;
for (;;) {
MojDbStorageItem* item = NULL;
err = cursor.get(item, found);
// We simply skip ghost keys and continue; A warning is already written to the system log earlier
if (err == MojErrInternalIndexOnFind) {
warns++;
numberInBatch++;
continue;
}
MojErrCheck(err);
if (!found)
break;
err = item->toObject(obj, m_kindEngine);
MojErrCheck(err);
const MojObject& id = item->id();
MojObject deleted;
err = delObj(id, obj, item, deleted, req, flags);
MojErrCheck(err);
++count;
numberInBatch++;
}
if (warns > 0)
MojLogInfo(s_log, _T("delquery index_warnings: %s, count: %d\n"), newQuery.from().data(), warns);
countOut += count;
err = cursor.close();
MojErrCheck(err);
if(numberInBatch >= AutoBatchSize) // sing > - just in case something messed up
{
err = commitBatch(req);
MojErrCheck(err);
}
if(count == 0)
break;
queryLimit -= newQuery.limit();
if(queryLimit > AutoBatchSize)
newQuery.limit(AutoBatchSize);
else
newQuery.limit(queryLimit);
}
return MojErrNone;
}
MojErr MojDbPurgeTest::delKindTest(MojDb& db)
{
// start from scratch - purge everything
MojUInt32 count = 0;
MojErr err = db.purge(count, 0);
MojTestErrCheck(err);
MojTestAssert(count > 0);
// purge again, make sure nothing is left
err = db.purge(count, 0);
MojTestErrCheck(err);
MojTestAssert(count == 0);
// make sure at least 2 objects exist
MojDbQuery q;
err = q.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(q, cursor);
MojTestErrCheck(err);
MojUInt32 objCount;
err = cursor.count(objCount);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
if (objCount <= 1) {
for (int i = objCount; i < 2; i++) {
MojObject obj;
err = obj.fromJson(MojTestObjStr1);
MojTestErrCheck(err);
err = db.put(obj);
MojTestErrCheck(err);
objCount++;
}
}
// delete half the objects
q.limit(objCount / 2);
MojUInt32 delCount;
err = db.del(q, delCount);
MojTestErrCheck(err);
// delete the kind
MojString idStr;
err = idStr.assign(_T("PurgeTest:1"));
MojTestErrCheck(err);
MojObject id(idStr);
bool found;
err = db.delKind(id, found);
MojTestErrCheck(err);
MojTestAssert(found);
/*MojUInt32 deletedObjCount;
err = q.where(_T("_del"), MojDbQuery::OpEq, true);
MojTestErrCheck(err);
q.includeDeleted(true);
err = db.find(q, cursor);
MojTestErrCheck(err);
err = cursor.count(deletedObjCount);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
// now all the objects should be deleted
MojTestAssert(deletedObjCount == objCount);*/
// purge now
err = db.purge(count, 0);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDb::purge(MojUInt32& countOut, MojInt64 numDays, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
if (numDays <= -1) {
numDays = m_purgeWindow;
}
MojErr err = beginReq(req);
MojErrCheck(err);
MojLogDebug(s_log, _T("purging objects deleted more than %lld days ago..."), numDays);
MojTime time;
err = MojGetCurrentTime(time);
MojErrCheck(err);
// store the revision number to current timestamp mapping
MojObject revTimeMapping;
MojInt64 rev;
err = nextId(rev);
MojErrCheck(err);
err = revTimeMapping.put(RevNumKey, rev);
MojErrCheck(err);
err = revTimeMapping.put(TimestampKey, time.microsecs());
MojErrCheck(err);
err = revTimeMapping.putString(KindKey, MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
err = putImpl(revTimeMapping, MojDb::FlagNone, req);
MojErrCheck(err);
// find the revision number for numDays prior to now
MojInt64 purgeTime = time.microsecs() - (MojTime::UnitsPerDay * numDays);
MojDbQuery query;
err = query.from(MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
query.limit(1);
err = query.where(TimestampKey, MojDbQuery::OpLessThanEq, purgeTime);
MojErrCheck(err);
err = query.order(TimestampKey);
MojErrCheck(err);
query.desc(true);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpDelete);
MojErrCheck(err);
bool found = false;
MojObject obj;
err = cursor.get(obj, found);
MojErrCheck(err);
err = cursor.close();
MojErrCheck(err);
MojUInt32 batchCount = 0;
MojUInt32 totalCount = 0;
while ((found))
{
// Do it in AutoBatchSize batches
batchCount = 0;
req->fixmode(true); // purge even if index mis-matches
err = purgeImpl(obj, batchCount, req);
MojLogDebug(s_log, _T("purge batch processed: batch: %d; total: %d; err = %d\n"),
batchCount, (totalCount + batchCount), err);
MojErrCheck(err);
totalCount += batchCount;
countOut = totalCount;
if (batchCount < AutoBatchSize) // last batch
break;
err = commitBatch(req);
MojErrCheck(err);
continue;
}
// end request
err = req->end();
MojErrCheck(err);
MojLogDebug(s_log, _T("purged %d objects"), countOut);
return MojErrNone;
}