MojErr MojDbPurgeTest::checkObjectsPurged(MojDb& db, const MojUInt32& count, const MojSize& expectedCount,
const MojSize& expectedNumObjects, const MojSize& expectedNumRevTimestampObjects, const MojObject& expectedLastPurgeRevNum)
{
//check number of objects purged
MojTestAssert(count == expectedCount);
//there should still be expectedNumObjects test objects
MojDbQuery query;
MojErr err = query.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor);
MojTestErrCheck(err);
MojUInt32 objCount;
err = cursor.count(objCount);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(objCount == expectedNumObjects);
//there should be expectedNumRevTimestampObjects RevTimestamp objects
MojDbQuery revQuery;
err = revQuery.from(MojDbKindEngine::RevTimestampId);
MojTestErrCheck(err);
MojDbCursor revCursor;
err = db.find(revQuery, revCursor);
MojTestErrCheck(err);
MojUInt32 revTimestampObjCount;
err = revCursor.count(revTimestampObjCount);
MojTestErrCheck(err);
err = revCursor.close();
MojTestErrCheck(err);
MojTestAssert(revTimestampObjCount == expectedNumRevTimestampObjects);
//lastPurgedRevNum should be equal to the expectedLastPurgeRevNum
MojObject revNum;
err = db.purgeStatus(revNum);
MojTestErrCheck(err);
MojTestAssert(revNum == expectedLastPurgeRevNum);
return MojErrNone;
}
MojErr MojDbWatchTest::cancelTest(MojDb& db)
{
// cancel find
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpLessThanEq, 45);
MojTestErrCheck(err);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
watcher->m_slot.cancel();
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
watcher->m_slot.cancel();
MojTestAssert(watcher->m_count == 0);
MojObject id;
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
// cancel watch
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
MojDbQuery queryWithRev;
err = queryWithRev.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = queryWithRev.where(_T("foo"), MojDbQuery::OpEq, 45);
MojTestErrCheck(err);
err = queryWithRev.where(_T("_rev"), MojDbQuery::OpGreaterThan, m_rev);
MojTestErrCheck(err);
bool fired = false;
err = db.watch(queryWithRev, cursor, watcher->m_slot, fired);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(!fired);
MojTestAssert(watcher->m_count == 0);
watcher->m_slot.cancel();
MojTestAssert(watcher->m_count == 0);
err = put(db, 45, 45, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
return MojErrNone;
}
MojErr MojDbServiceHandler::handleWatch(MojServiceMessage* msg, MojObject& payload, MojDbReq& req)
{
MojAssert(msg);
MojLogTrace(s_log);
MojObject queryObj;
MojErr err = payload.getRequired(MojDbServiceDefs::QueryKey, queryObj);
MojErrCheck(err);
MojRefCountedPtr<Watcher> watcher(new Watcher(msg));
MojAllocCheck(watcher.get());
MojDbQuery query;
err = query.fromObject(queryObj);
MojErrCheck(err);
bool fired = false;
MojDbCursor cursor;
err = m_db.watch(query, cursor, watcher->m_watchSlot, fired, req);
MojErrCheck(err);
MojLogInfo(s_log, _T("handleWatch: %s, err: (%d); sender= %s;\n fired=%d; \n"),
msg->method(), (int)err, msg->senderName(), (int)fired);
if (!fired) {
err = msg->replySuccess();
MojErrCheck(err);
}
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbWatchTest::rangeTest(MojDb& db)
{
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpGreaterThan, 5);
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpLessThan, 100);
MojTestErrCheck(err);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojObject id;
MojInt64 rev;
err = put(db, 5, 5, id, rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = put(db, 100, 100, id, rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = put(db, 6, 6, id, rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = put(db, 99, 99, id, rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
return MojErrNone;
}
MojErr MojDbDistinctTest::simpleTest(MojDb& db)
{
MojErr err;
MojDbQuery query;
const MojChar* queryString;
const MojChar* expectedIdsJson;
MojString str;
MojDbSearchCursor searchCursor(str);
MojDbCursor cursor;
//1st test
queryString = _T("bar");
expectedIdsJson = _T("[\"a\",\"b\",\"c\",\"d\"]");
err = initQuery(query, queryString);
MojTestErrCheck(err);
err = check(db, query, searchCursor, queryString, expectedIdsJson);
MojTestErrCheck(err);
searchCursor.close();
//test for find
err = check(db, query, cursor, queryString, expectedIdsJson);
MojTestErrCheck(err);
cursor.close();
//2nd test
queryString = _T("foo");
expectedIdsJson = _T("[\"e\",\"f\",\"g\"]");
err = initQuery(query, queryString);
MojTestErrCheck(err);
err = check(db, query, searchCursor, queryString, expectedIdsJson);
MojTestErrCheck(err);
searchCursor.close();
//test for find
err = check(db, query, cursor, queryString, expectedIdsJson);
MojTestErrCheck(err);
cursor.close();
return MojErrNone;
}
MojErr MojDb::merge(const MojDbQuery& query, const MojObject& props, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
MojErr err = beginReq(req);
MojErrCheck(err);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpUpdate);
MojErrCheck(err);
MojAssert(cursor.txn());
MojUInt32 count = 0;
MojUInt32 warns = 0;
bool found = false;
MojObject prev;
for (;;) {
// get prev rev from cursor
MojDbStorageItem* prevItem = NULL;
err = cursor.get(prevItem, found);
if (err == MojErrInternalIndexOnFind) {
warns++;
continue;
}
MojErrCheck(err);
if (!found)
break;
err = prevItem->toObject(prev, m_kindEngine);
MojErrCheck(err);
// merge obj into prev
MojObject merged;
err = mergeInto(merged, props, prev);
MojErrCheck(err);
// and update the db
const MojObject& id = prevItem->id();
err = putObj(id, merged, &prev, prevItem, req, OpUpdate);
MojErrCheck(err);
++count;
}
if (warns > 0)
MojLogWarning(s_log, _T("Merge index_warnings: %s; count: %d\n"), query.from().data(), warns);
err = cursor.close();
MojErrCheck(err);
err = req->end();
MojErrCheck(err);
countOut = count;
return MojErrNone;
}
MojErr MojDbDistinctTest::check(MojDb& db, const MojDbQuery& query, MojDbCursor& cursor, const MojChar* queryString, const MojChar* expectedIdsJson)
{
MojErr err = db.find(query, cursor);
MojTestErrCheck(err);
MojObjectBuilder builder;
err = builder.beginArray();
MojTestErrCheck(err);
err = cursor.visit(builder);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = builder.endArray();
MojTestErrCheck(err);
MojObject results = builder.object();
MojString json;
err = results.toJson(json);
MojTestErrCheck(err);
MojObject expected;
err = expected.fromJson(expectedIdsJson);
MojTestErrCheck(err);
// size check
MojTestAssert(expected.size() == results.size());
// value check
MojObject::ConstArrayIterator j = results.arrayBegin();
for (MojObject::ConstArrayIterator i = expected.arrayBegin();
i != expected.arrayEnd(); ++i, ++j) {
MojObject value;
err = j->getRequired(queryString, value);
MojTestErrCheck(err);
MojTestAssert(*i == value);
}
return MojErrNone;
}
MojErr MojDbIndex::build(MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(isOpen());
MojAssert(m_kind && m_kindEngine);
MojAssert(m_props.size() > 1);
// query for all existing objects of this type and add them to the index.
MojDbQuery query;
MojErr err = query.from(m_kind->id());
MojErrCheck(err);
if (m_includeDeleted) {
err = query.includeDeleted();
MojErrCheck(err);
}
MojDbCursor cursor;
MojDbReq adminRequest(true);
adminRequest.txn(txn);
err = m_kindEngine->find(query, cursor, NULL, adminRequest, OpRead);
MojErrCheck(err);
for (;;) {
MojObject obj;
bool found = false;
err = cursor.get(obj, found);
MojErrCheck(err);
if (!found)
break;
// add this object to the index
err = update(&obj, NULL, txn, false);
MojErrCheck(err);
}
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbKindEngine::loadKinds(MojDbReq& req)
{
MojAssert(isOpen());
MojAssertWriteLocked(m_db->m_schemaLock);
MojLogTrace(s_log);
MojDbQuery query;
MojErr err = query.from(KindKindId);
MojErrCheck(err);
MojDbCursor cursor;
err = m_db->find(query, cursor, req);
MojErrCheck(err);
for (;;) {
MojObject obj;
bool found = false;
err = cursor.get(obj, found);
MojErrCheck(err);
if (!found)
break;
// load kind
MojErr loadErr = err = putKind(obj, req);
MojErrCatchAll(err) {
MojString id;
bool found = false;
MojErr err = obj.get(MojDbServiceDefs::IdKey, id, found);
MojErrCheck(err);
MojString errStr;
MojErrToString(loadErr, errStr);
MojLogError(s_log, _T("error loading kind '%s' - %s"), id.data(), errStr.data());
}
}
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbWatchTest::pageTest(MojDb& db)
{
MojObject id;
MojObject idFirst;
MojObject idFourth;
MojObject idLast;
MojInt64 rev;
for (int i = 100; i < 150; ++i) {
MojErr err = put(db, 100, i, id, rev);
MojTestErrCheck(err);
if (i == 100) {
idFirst = id;
} else if (i == 103) {
idFourth = id;
} else if (i == 149) {
idLast = id;
}
}
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpGreaterThanEq, 100);
MojTestErrCheck(err);
query.limit(3);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
bool found = false;
MojUInt32 count = 0;
do {
MojObject obj;
err = cursor.get(obj, found);
MojTestErrCheck(err);
if (found)
++count;
} while (found);
MojTestAssert(count == 3);
MojDbQuery::Page page;
err = cursor.nextPage(page);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = merge(db, idFourth, 53);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
query.page(page);
MojRefCountedPtr<TestWatcher> watcher2(new TestWatcher);
MojTestAssert(watcher2.get());
err = db.find(query, cursor, watcher2->m_slot);
MojTestErrCheck(err);
found = false;
count = 0;
do {
MojObject obj;
err = cursor.get(obj, found);
MojTestErrCheck(err);
if (found)
++count;
} while (found);
MojTestAssert(count == 3);
err = cursor.close();
MojTestErrCheck(err);
err = db.del(idFirst, found);
MojTestErrCheck(err);
MojTestAssert(found);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 0);
err = db.del(idFourth, found);
MojTestErrCheck(err);
MojTestAssert(found);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
// desc order
query.page(MojDbQuery::Page());
query.desc(true);
MojRefCountedPtr<TestWatcher> watcher3(new TestWatcher);
MojTestAssert(watcher3.get());
err = db.find(query, cursor, watcher3->m_slot);
MojTestErrCheck(err);
found = false;
count = 0;
do {
MojObject obj;
err = cursor.get(obj, found);
MojTestErrCheck(err);
if (found)
++count;
} while (found);
MojTestAssert(count == 3);
err = cursor.close();
MojTestErrCheck(err);
err = merge(db, idLast, 53);
MojTestErrCheck(err);
MojTestAssert(watcher3->m_count == 1);
MojRefCountedPtr<TestWatcher> watcher4(new TestWatcher);
MojTestAssert(watcher4.get());
err = db.find(query, cursor, watcher4->m_slot);
MojTestErrCheck(err);
found = false;
count = 0;
do {
MojObject obj;
err = cursor.get(obj, found);
MojTestErrCheck(err);
if (found)
++count;
} while (found);
MojTestAssert(count == 3);
err = cursor.close();
MojTestErrCheck(err);
err = merge(db, idLast, 54);
MojTestErrCheck(err);
MojTestAssert(watcher4->m_count == 1);
return MojErrNone;
}
MojErr MojDbPermissionTest::checkPermissions(MojDb& db)
{
MojDbReq reqOwner(false);
MojErr err = reqOwner.domain(_T("com.foo"));
MojTestErrCheck(err);
MojDbReq reqFooBar(false);
err = reqFooBar.domain(_T("com.foo.bar"));
MojTestErrCheck(err);
MojDbReq reqFooBarBad(false);
err = reqFooBarBad.domain(_T("com.foobar"));
MojTestErrCheck(err);
MojDbReq reqGranted(false);
err = reqGranted.domain("com.granted");
MojTestErrCheck(err);
MojDbReq reqEmpty(false);
MojDbReq reqBad(false);
err = reqBad.domain("com.bad");
MojTestErrCheck(err);
MojObject objNoId;
err = objNoId.putString(MojDb::KindKey, _T("PermissionTest:1"));
MojTestErrCheck(err);
MojObject obj2NoId;
err = obj2NoId.putString(MojDb::KindKey, _T("PermissionTest2:1"));
MojTestErrCheck(err);
// put new obj
MojObject obj1 = objNoId;
err = db.put(obj1, MojDb::FlagNone, reqOwner);
MojTestErrCheck(err);
MojObject obj2 = objNoId;
err = db.put(obj2, MojDb::FlagNone, reqGranted);
MojTestErrCheck(err);
MojObject obj3 = objNoId;
err = db.put(obj3, MojDb::FlagNone, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
obj3 = objNoId;
err = db.put(obj3, MojDb::FlagNone, reqFooBarBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
obj3 = objNoId;
err = db.put(obj3, MojDb::FlagNone, reqFooBar);
MojTestErrCheck(err);
err = db.put(obj2NoId, MojDb::FlagNone, reqEmpty);
MojTestErrCheck(err);
// put existing obj
err = obj1.put(_T("foo"), 1);
MojTestErrCheck(err);
err = db.put(obj1, MojDb::FlagNone, reqOwner);
MojTestErrCheck(err);
err = obj2.put(_T("foo"), 2);
MojTestErrCheck(err);
err = db.put(obj2, MojDb::FlagNone, reqGranted);
MojTestErrCheck(err);
err = obj2.put(_T("foo"), 3);
MojTestErrCheck(err);
err = db.put(obj2, MojDb::FlagNone, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
// find
MojDbQuery query;
err = query.from(_T("PermissionTest:1"));
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor, reqOwner);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = db.find(query, cursor, reqGranted);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = db.find(query, cursor, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
err = cursor.close();
MojTestErrCheck(err);
// get
MojObject id1;
MojTestAssert(obj1.get(MojDb::IdKey, id1));
bool found = false;
MojObject gotten;
err = db.get(id1, gotten, found, reqOwner);
MojTestErrCheck(err);
err = db.get(id1, gotten, found, reqGranted);
MojTestErrCheck(err);
err = db.get(id1, gotten, found, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
// del by id
err = db.del(id1, found, MojDb::FlagNone, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
MojTestAssert(!found);
err = db.del(id1, found, MojDb::FlagNone, reqOwner);
MojTestErrCheck(err);
MojTestAssert(found);
MojObject id2;
MojTestAssert(obj2.get(MojDb::IdKey, id2));
err = db.del(id2, found, MojDb::FlagNone, reqGranted);
MojTestErrCheck(err);
MojTestAssert(found);
// del query
MojUInt32 count = 0;
err = db.del(query, count, MojDb::FlagNone, reqOwner);
MojTestErrCheck(err);
err = db.del(query, count, MojDb::FlagNone, reqGranted);
MojTestErrCheck(err);
err = db.del(query, count, MojDb::FlagNone, reqBad);
MojTestErrExpected(err, MojErrDbPermissionDenied);
return MojErrNone;
}
MojErr MojDb::dumpImpl(MojFile& file, bool backup, bool incDel, const MojObject& revParam, const MojObject& delRevParam, bool skipKinds, MojUInt32& countOut, MojDbReq& req,
MojObject* response, const MojChar* keyName, MojSize& bytesWritten, MojSize& warns, MojUInt32 maxBytes)
{
// query for objects, adding the backup key and rev key if necessary
MojDbQuery query;
MojErr err = query.from(MojDbKindEngine::RootKindId);
MojErrCheck(err);
err = query.where(MojDb::DelKey, MojDbQuery::OpEq, incDel);
MojErrCheck(err);
if (backup) {
err = query.where(MojDb::SyncKey, MojDbQuery::OpEq, true);
MojErrCheck(err);
if (incDel) {
err = query.where(MojDb::RevKey, MojDbQuery::OpGreaterThan, delRevParam);
MojErrCheck(err);
} else {
err = query.where(MojDb::RevKey, MojDbQuery::OpGreaterThan, revParam);
MojErrCheck(err);
}
}
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpRead);
MojErrCheck(err);
warns = 0;
MojObject curRev;
for(;;) {
bool found = false;
MojObject obj;
err = cursor.get(obj, found);
// So that we can get as much data as possible from a corrupt database
// We simply skip ghost keys and continue
if (err == MojErrInternalIndexOnFind) {
warns++;
continue;
}
MojErrCheck(err);
if (!found)
break;
if (skipKinds) {
MojString kind;
err = obj.getRequired(KindKey, kind);
MojErrCheck(err);
if (kind == MojDbKindEngine::KindKindId) {
continue;
}
}
// write out each object, if the backup is full, insert the appropriate incremental key
err = dumpObj(file, obj, bytesWritten, maxBytes);
MojErrCatch(err, MojErrDbBackupFull) {
if (response) {
MojErr errBackup = MojErrNone;
if (!curRev.undefined()) {
errBackup = insertIncrementalKey(*response, keyName, curRev);
MojErrCheck(errBackup);
} else {
errBackup = insertIncrementalKey(*response, keyName, incDel ? delRevParam: revParam);
MojErrCheck(errBackup);
}
errBackup = handleBackupFull(revParam, delRevParam, *response, keyName);
MojErrCheck(errBackup);
}
return MojErrNone;
}
MojErrCheck(err);
err = obj.getRequired(MojDb::RevKey, curRev);
MojErrCheck(err);
countOut++;
}
err = cursor.close();
MojErrCheck(err);
if (warns > 0) {
MojLogWarning(s_log, _T("Finished Backup with %d warnings \n"), (int)warns);
} else {
MojLogDebug(s_log, _T("Finished Backup with no warnings \n"));
}
// construct the next incremental key
if (response && !curRev.undefined()) {
err = insertIncrementalKey(*response, keyName, curRev);
MojErrCheck(err);
}
return MojErrNone;
}
MojErr MojDbPurgeTest::delKindTest(MojDb& db)
{
// start from scratch - purge everything
MojUInt32 count = 0;
MojErr err = db.purge(count, 0);
MojTestErrCheck(err);
MojTestAssert(count > 0);
// purge again, make sure nothing is left
err = db.purge(count, 0);
MojTestErrCheck(err);
MojTestAssert(count == 0);
// make sure at least 2 objects exist
MojDbQuery q;
err = q.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(q, cursor);
MojTestErrCheck(err);
MojUInt32 objCount;
err = cursor.count(objCount);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
if (objCount <= 1) {
for (int i = objCount; i < 2; i++) {
MojObject obj;
err = obj.fromJson(MojTestObjStr1);
MojTestErrCheck(err);
err = db.put(obj);
MojTestErrCheck(err);
objCount++;
}
}
// delete half the objects
q.limit(objCount / 2);
MojUInt32 delCount;
err = db.del(q, delCount);
MojTestErrCheck(err);
// delete the kind
MojString idStr;
err = idStr.assign(_T("PurgeTest:1"));
MojTestErrCheck(err);
MojObject id(idStr);
bool found;
err = db.delKind(id, found);
MojTestErrCheck(err);
MojTestAssert(found);
/*MojUInt32 deletedObjCount;
err = q.where(_T("_del"), MojDbQuery::OpEq, true);
MojTestErrCheck(err);
q.includeDeleted(true);
err = db.find(q, cursor);
MojTestErrCheck(err);
err = cursor.count(deletedObjCount);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
// now all the objects should be deleted
MojTestAssert(deletedObjCount == objCount);*/
// purge now
err = db.purge(count, 0);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDb::delImpl(const MojDbQuery& quer, MojUInt32& countOut, MojDbReq& req, MojUInt32 flags)
{
MojLogTrace(s_log);
countOut = 0;
MojInt32 warns = 0;
MojDbQuery newQuery = quer;
MojUInt32 queryLimit = newQuery.limit();
if(newQuery.limit() == MojDbQuery::LimitDefault)
newQuery.limit(AutoBatchSize);
while(queryLimit > 0)
{
MojDbCursor cursor;
MojErr err = findImpl(newQuery, cursor, NULL, req, OpDelete);
MojErrCheck(err);
MojAssert(cursor.txn());
MojUInt32 count = 0;
MojUInt32 numberInBatch = 0;
bool found = false;
MojObject obj;
for (;;) {
MojDbStorageItem* item = NULL;
err = cursor.get(item, found);
// We simply skip ghost keys and continue; A warning is already written to the system log earlier
if (err == MojErrInternalIndexOnFind) {
warns++;
numberInBatch++;
continue;
}
MojErrCheck(err);
if (!found)
break;
err = item->toObject(obj, m_kindEngine);
MojErrCheck(err);
const MojObject& id = item->id();
MojObject deleted;
err = delObj(id, obj, item, deleted, req, flags);
MojErrCheck(err);
++count;
numberInBatch++;
}
if (warns > 0)
MojLogInfo(s_log, _T("delquery index_warnings: %s, count: %d\n"), newQuery.from().data(), warns);
countOut += count;
err = cursor.close();
MojErrCheck(err);
if(numberInBatch >= AutoBatchSize) // sing > - just in case something messed up
{
err = commitBatch(req);
MojErrCheck(err);
}
if(count == 0)
break;
queryLimit -= newQuery.limit();
if(queryLimit > AutoBatchSize)
newQuery.limit(AutoBatchSize);
else
newQuery.limit(queryLimit);
}
return MojErrNone;
}
MojErr MojDbServiceHandler::findImpl(MojServiceMessage* msg, MojObject& payload, MojDbReq& req, MojDbCursor& cursor, bool doCount)
{
MojAssert(msg);
MojLogTrace(s_log);
MojObject queryObj;
MojErr err = payload.getRequired(MojDbServiceDefs::QueryKey, queryObj);
MojErrCheck(err);
bool doWatch = false;
payload.get(MojDbServiceDefs::WatchKey, doWatch);
MojDbQuery query;
err = query.fromObject(queryObj);
MojErrCheck(err);
MojUInt32 limit = query.limit();
if (limit == MojDbQuery::LimitDefault){
query.limit(MaxQueryLimit);
} else if (limit > MaxQueryLimit) {
MojErrThrowMsg(MojErrDbInvalidQuery, _T("db: limit greater than %d not allowed"), MaxQueryLimit);
}
if (doWatch) {
MojRefCountedPtr<Watcher> watcher(new Watcher(msg));
MojAllocCheck(watcher.get());
err = m_db.find(query, cursor, watcher->m_watchSlot, req);
MojErrCheck(err);
} else {
err = m_db.find(query, cursor, req);
MojErrCheck(err);
}
// append results
MojObjectVisitor& writer = msg->writer();
err = writer.beginObject();
MojErrCheck(err);
err = writer.boolProp(MojServiceMessage::ReturnValueKey, true);
MojErrCheck(err);
err = writer.propName(MojDbServiceDefs::ResultsKey);
MojErrCheck(err);
err = writer.beginArray();
MojErrCheck(err);
err = cursor.visit(writer);
MojErrCheck(err);
err = writer.endArray();
MojErrCheck(err);
// append next page
MojDbQuery::Page page;
err = cursor.nextPage(page);
MojErrCheck(err);
if (!page.empty()) {
MojObject pageObj;
err = page.toObject(pageObj);
MojErrCheck(err);
err = writer.objectProp(MojDbServiceDefs::NextKey, pageObj);
MojErrCheck(err);
}
// append count
if (doCount) {
MojUInt32 count = 0;
err = cursor.count(count);
MojErrCheck(err);
err = writer.intProp(MojDbServiceDefs::CountKey, (MojInt64) count);
MojErrCheck(err);
}
err = writer.endObject();
MojErrCheck(err);
if (doWatch) {
// if this is a watched query, it cannot be part of a batch so it's safe to reply here
err = msg->reply();
MojErrCheck(err);
}
// notifications can fire any time after the cursor is closed,
// so don't close it until after sending the reply.
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbWatchTest::eqTest(MojDb& db)
{
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpEq, 1);
MojTestErrCheck(err);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojRefCountedPtr<TestWatcher> watcher2(new TestWatcher);
MojTestAssert(watcher2.get());
bool fired = false;
err = db.watch(query, cursor, watcher2->m_slot, fired);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(!fired);
// puts
MojObject id;
err = put(db, 0, 0, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
err = put(db, 2, 2, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
// put, changing property not in index
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = put(db, 1, 2, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
// dels
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
watcher2.reset(new TestWatcher);
MojTestAssert(watcher2.get());
MojDbQuery queryWithRev;
err = queryWithRev.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = queryWithRev.where(_T("foo"), MojDbQuery::OpEq, 1);
MojTestErrCheck(err);
err = queryWithRev.where(_T("_rev"), MojDbQuery::OpGreaterThan, m_rev);
MojTestErrCheck(err);
err = queryWithRev.includeDeleted();
MojTestErrCheck(err);
fired = false;
err = db.watch(queryWithRev, cursor, watcher2->m_slot, fired);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(!fired);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
bool found;
err = db.del(id, found);
MojTestErrCheck(err);
MojTestAssert(found);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
// ordering
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
return MojErrNone;
}
MojErr MojDbPurgeTest::run()
{
MojDb db;
MojErr err = db.open(MojDbTestDir);
MojTestErrCheck(err);
// put type
MojObject obj;
err = obj.fromJson(MojKindStr);
MojTestErrCheck(err);
err = db.putKind(obj);
MojTestErrCheck(err);
MojObject revNums[10];
MojObject ids[10];
//put 10 objects in the db
for(int i = 0; i < 10; i++) {
err = obj.fromJson(MojTestObjStr1);
MojTestErrCheck(err);
err = db.put(obj);
MojTestErrCheck(err);
// get _rev and id
MojObject rev;
err = obj.getRequired(MojDb::RevKey, rev);
MojTestErrCheck(err);
revNums[i] = rev;
MojObject id;
err = obj.getRequired(MojDb::IdKey, id);
MojTestErrCheck(err);
ids[i] = id;
}
//purge now, there are no RevTimestamp entries
MojUInt32 count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 1, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry - that's not more than PurgeNumDays days ago
MojTime time;
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[0], time.microsecs());
MojTestErrCheck(err);
//purge now, there are no RevTimestamp entries that are more than
//PurgeNumDays ago, so nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 3, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry for more than PurgeNumDays days ago
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[9], time.microsecs() - (((MojInt64)40) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//purge now, since nothing has been deleted, nothing should be purged,
//but the RevTimestamp object should be deleted
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 4, -1);
MojTestErrCheck(err);
//delete something - this will set its revision number higher
bool found;
err = db.del(ids[0], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//purge now, since nothing has been deleted prior to the revision
//number, nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 9, 5, -1);
MojTestErrCheck(err);
//delete another object
err = db.del(ids[1], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//create a RevTimestamp entry for more than PurgeNumDays days ago,
//with the rev number of the 1st obj we deleted
MojDbQuery query;
err = query.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
err = query.where(MojDb::IdKey, MojDbQuery::OpEq, ids[0]);
MojTestErrCheck(err);
err = query.includeDeleted();
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor);
MojTestErrCheck(err);
MojObject objFromDb;
err = cursor.get(objFromDb, found);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(found == true);
MojObject revFromDb;
err = objFromDb.getRequired(MojDb::RevKey, revFromDb);
MojTestErrCheck(err);
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revFromDb, time.microsecs() - (((MojInt64)35) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//now purge, only id[0] should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 1, 8, 6, revFromDb);
MojTestErrCheck(err);
//TODO 2.12.10 - this test does not pass yet, we need to fix calling delKind after a purge
//err = delKindTest(db);
//MojTestErrCheck(err);
err = db.close();
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDb::purge(MojUInt32& countOut, MojInt64 numDays, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
if (numDays <= -1) {
numDays = m_purgeWindow;
}
MojErr err = beginReq(req);
MojErrCheck(err);
MojLogDebug(s_log, _T("purging objects deleted more than %lld days ago..."), numDays);
MojTime time;
err = MojGetCurrentTime(time);
MojErrCheck(err);
// store the revision number to current timestamp mapping
MojObject revTimeMapping;
MojInt64 rev;
err = nextId(rev);
MojErrCheck(err);
err = revTimeMapping.put(RevNumKey, rev);
MojErrCheck(err);
err = revTimeMapping.put(TimestampKey, time.microsecs());
MojErrCheck(err);
err = revTimeMapping.putString(KindKey, MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
err = putImpl(revTimeMapping, MojDb::FlagNone, req);
MojErrCheck(err);
// find the revision number for numDays prior to now
MojInt64 purgeTime = time.microsecs() - (MojTime::UnitsPerDay * numDays);
MojDbQuery query;
err = query.from(MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
query.limit(1);
err = query.where(TimestampKey, MojDbQuery::OpLessThanEq, purgeTime);
MojErrCheck(err);
err = query.order(TimestampKey);
MojErrCheck(err);
query.desc(true);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpDelete);
MojErrCheck(err);
bool found = false;
MojObject obj;
err = cursor.get(obj, found);
MojErrCheck(err);
err = cursor.close();
MojErrCheck(err);
MojUInt32 batchCount = 0;
MojUInt32 totalCount = 0;
while ((found))
{
// Do it in AutoBatchSize batches
batchCount = 0;
req->fixmode(true); // purge even if index mis-matches
err = purgeImpl(obj, batchCount, req);
MojLogDebug(s_log, _T("purge batch processed: batch: %d; total: %d; err = %d\n"),
batchCount, (totalCount + batchCount), err);
MojErrCheck(err);
totalCount += batchCount;
countOut = totalCount;
if (batchCount < AutoBatchSize) // last batch
break;
err = commitBatch(req);
MojErrCheck(err);
continue;
}
// end request
err = req->end();
MojErrCheck(err);
MojLogDebug(s_log, _T("purged %d objects"), countOut);
return MojErrNone;
}