MojErr MojService::CategoryHandler::invoke(const MojChar* method, MojServiceMessage* msg, MojObject& payload)
{
MojAssert(method && msg);
MojLogTrace(s_log);
MojTime startTime;
MojErr err = MojGetCurrentTime(startTime);
MojErrCheck(err);
// lookup callback
CallbackInfo cb;
if (!m_callbackMap.get(method, cb)) {
MojErrThrow(MojErrMethodNotFound);
}
// validate schema
if (cb.m_schema.get()) {
MojSchema::Result res;
err = cb.m_schema->validate(payload, res);
MojErrCheck(err);
if (!res.valid()) {
MojErrThrowMsg(MojErrInvalidArg, _T("invalid parameters: caller='%s' error='%s'"), msg->senderName(), res.msg().data());
}
}
// invoke method
err = invoke(cb.m_callback, msg, payload);
MojErrCheck(err);
// log timing
MojTime endTime;
err = MojGetCurrentTime(endTime);
MojErrCheck(err);
MojLogInfo(s_log, _T("%s invoked: %.3fms"), method, (double) (endTime.microsecs() - startTime.microsecs()) / 1000);
return MojErrNone;
}
MojErr MojDbPerfUpdateTest::timeUpdateKind(MojDb& db, const MojChar* kindJson, MojObject& kindObj, MojTime& addIndexTime, MojTime& dropIndexTime)
{
MojTime startTime;
MojTime endTime;
MojObject origKindObj;
MojErr err = origKindObj.fromJson(kindJson);
MojTestErrCheck(err);
for (MojUInt64 i = 0; i < numUpdateKindIterations; i++) {
err = MojGetCurrentTime(startTime);
MojTestErrCheck(err);
bool found;
err = kindObj.del(MojDb::RevKey, found);
MojTestErrCheck(err);
err = db.putKind(kindObj);
MojTestErrCheck(err);
err = MojGetCurrentTime(endTime);
MojTestErrCheck(err);
addIndexTime += (endTime - startTime);
totalTestTime += (endTime - startTime);
err = MojGetCurrentTime(startTime);
MojTestErrCheck(err);
err = origKindObj.del(MojDb::RevKey, found);
MojTestErrCheck(err);
err = db.putKind(origKindObj);
MojTestErrCheck(err);
err = MojGetCurrentTime(endTime);
MojTestErrCheck(err);
dropIndexTime += (endTime - startTime);
totalTestTime += (endTime - startTime);
}
return MojErrNone;
}
MojErr MojDbPerfCreateTest::batchPutLargeArrayObj(MojDb& db, const MojChar* kindId, MojTime& lgArrayObjTime)
{
MojTime startTime;
MojTime endTime;
MojObject objArray;
for (MojUInt64 i = 0; i < numInsert; i++) {
MojObject obj;
MojErr err = obj.putString(MojDb::KindKey, kindId);
MojTestErrCheck(err);
err = createLargeArrayObj(obj, i);
MojTestErrCheck(err);
err = objArray.push(obj);
MojTestErrCheck(err);
}
MojObject::ArrayIterator begin;
MojErr err = objArray.arrayBegin(begin);
MojTestErrCheck(err);
MojObject::ConstArrayIterator end = objArray.arrayEnd();
err = MojGetCurrentTime(startTime);
MojTestErrCheck(err);
err = db.put(begin, end);
MojTestErrCheck(err);
err = MojGetCurrentTime(endTime);
MojTestErrCheck(err);
lgArrayObjTime += (endTime - startTime);
totalTestTime += (endTime - startTime);
return MojErrNone;
}
MojErr MojDbIdGenerator::id(MojObject& idOut)
{
// an id consists of a timestamp (in microsecs) concatenated with a 32-bit random number.
// The goal is to have a very low likelihood of id collision among multiple devices owned
// by the same user (so they can can share an id-space) while still keeping them sequential
// to minimize db fragmentation.
MojThreadGuard guard(m_mutex);
MojUInt32 randNum;
MojErr err = MojRandom(&m_randBuf, &randNum);
MojErrCheck(err);
guard.unlock();
MojTime time;
err = MojGetCurrentTime(time);
MojErrCheck(err);
MojBuffer buf;
MojDataWriter writer(buf);
err = writer.writeInt64(time.microsecs());
MojErrCheck(err);
err = writer.writeUInt32(randNum);
MojErrCheck(err);
MojVector<MojByte> byteVec;
err = buf.toByteVec(byteVec);
MojErrCheck(err);
MojString str;
err = str.base64Encode(byteVec, false);
MojErrCheck(err);
idOut = str;
return MojErrNone;
}
MojErr MojDbPerfUpdateTest::batchPutObj(MojDb& db, MojObject* begin, const MojObject* end, MojTime& objTime)
{
MojTime startTime;
MojTime endTime;
for (MojUInt64 i = 0; i < numBatchPutIterations; i++) {
MojErr err = MojGetCurrentTime(startTime);
MojTestErrCheck(err);
err = db.put(begin, end);
MojTestErrCheck(err);
err = MojGetCurrentTime(endTime);
MojTestErrCheck(err);
objTime += (endTime - startTime);
totalTestTime += (endTime - startTime);
}
return MojErrNone;
}
MojErr MojDbPerfUpdateTest::queryMergeObj(MojDb& db, MojDbQuery& query, MojObject& props, MojUInt32& count, MojTime& objTime)
{
MojTime startTime;
MojTime endTime;
for (MojUInt64 i = 0; i < numBatchMergeIterations; i++) {
MojErr err = MojGetCurrentTime(startTime);
MojTestErrCheck(err);
err = db.merge(query, props, count);
MojTestErrCheck(err);
err = MojGetCurrentTime(endTime);
MojTestErrCheck(err);
objTime += (endTime - startTime);
totalTestTime += (endTime - startTime);
}
return MojErrNone;
}
MojErr MojDbPerfUpdateTest::mergeObj(MojDb& db, MojObject& obj, MojTime& objTime)
{
MojTime startTime;
MojTime endTime;
for (MojUInt64 i = 0; i < numMergeIterations; i++) {
MojErr err = MojGetCurrentTime(startTime);
MojTestErrCheck(err);
err = obj.putInt(_T("newProp"), i);
MojTestErrCheck(err);
err = db.merge(obj);
MojTestErrCheck(err);
err = MojGetCurrentTime(endTime);
MojTestErrCheck(err);
objTime += (endTime - startTime);
totalTestTime += (endTime - startTime);
}
return MojErrNone;
}
MojErr MojDbIdGenerator::init()
{
MojThreadGuard guard(m_mutex);
MojTime time;
MojErr err = MojGetCurrentTime(time);
MojErrCheck(err);
MojZero(&m_randBuf, sizeof(m_randBuf));
err = MojInitRandom((MojUInt32) time.microsecs(), m_randStateBuf, sizeof(m_randStateBuf), &m_randBuf);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbPerfCreateTest::putLargeArrayObj(MojDb& db, const MojChar* kindId, MojTime& lgArrayObjTime)
{
MojTime startTime;
MojTime endTime;
for (MojUInt64 i = 0; i < numInsert; i++) {
MojObject obj;
MojErr err = obj.putString(MojDb::KindKey, kindId);
MojTestErrCheck(err);
err = createLargeArrayObj(obj, i);
MojTestErrCheck(err);
err = MojGetCurrentTime(startTime);
MojTestErrCheck(err);
err = db.put(obj);
MojTestErrCheck(err);
err = MojGetCurrentTime(endTime);
MojTestErrCheck(err);
lgArrayObjTime += (endTime - startTime);
totalTestTime += (endTime - startTime);
}
return MojErrNone;
}
MojErr MojFileAppender::append(MojLogger::Level level, MojLogger* logger, const MojChar* format, va_list args)
{
MojAssertNoLog(format);
MojAssertNoLog(m_file != MojInvalidFile);
// get current time
MojTime time = 0;
MojErr err = MojGetCurrentTime(time);
MojErrCheckNoLog(err);
MojTmT tm;
err = MojLocalTime(time, tm);
MojErrCheckNoLog(err);
// format message
err = m_buf.format(_T("[%04d-%02d-%02d %02d:%02d:%02d:%03d] [%p] [%s]"),
tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour,
tm.tm_min, tm.tm_sec, time.millisecsPart(),
(void*) (MojIntPtr) MojThreadCurrentId(),
MojLogger::stringFromLevel(level));
MojErrCheckNoLog(err);
if (logger) {
err = m_buf.appendFormat(_T(" [%s]: "), logger->name());
MojErrCheckNoLog(err);
} else {
err = m_buf.append(_T(": "));
MojErrCheckNoLog(err);
}
err = m_buf.appendVFormat(format, args);
MojErrCheckNoLog(err);
err = m_buf.append(_T('\n'));
MojErrCheckNoLog(err);
// append to file
const MojByte* begin = (const MojByte*) m_buf.begin();
const MojByte* end = (const MojByte*) m_buf.end();
while (begin < end) {
MojSize written = 0;
err = MojFileWrite(m_file, begin, end - begin, written);
MojErrCheckNoLog(err);
begin += written;
}
return MojErrNone;
}
MojErr MojDbPurgeTest::run()
{
MojDb db;
MojErr err = db.open(MojDbTestDir);
MojTestErrCheck(err);
// put type
MojObject obj;
err = obj.fromJson(MojKindStr);
MojTestErrCheck(err);
err = db.putKind(obj);
MojTestErrCheck(err);
MojObject revNums[10];
MojObject ids[10];
//put 10 objects in the db
for(int i = 0; i < 10; i++) {
err = obj.fromJson(MojTestObjStr1);
MojTestErrCheck(err);
err = db.put(obj);
MojTestErrCheck(err);
// get _rev and id
MojObject rev;
err = obj.getRequired(MojDb::RevKey, rev);
MojTestErrCheck(err);
revNums[i] = rev;
MojObject id;
err = obj.getRequired(MojDb::IdKey, id);
MojTestErrCheck(err);
ids[i] = id;
}
//purge now, there are no RevTimestamp entries
MojUInt32 count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 1, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry - that's not more than PurgeNumDays days ago
MojTime time;
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[0], time.microsecs());
MojTestErrCheck(err);
//purge now, there are no RevTimestamp entries that are more than
//PurgeNumDays ago, so nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 3, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry for more than PurgeNumDays days ago
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[9], time.microsecs() - (((MojInt64)40) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//purge now, since nothing has been deleted, nothing should be purged,
//but the RevTimestamp object should be deleted
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 4, -1);
MojTestErrCheck(err);
//delete something - this will set its revision number higher
bool found;
err = db.del(ids[0], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//purge now, since nothing has been deleted prior to the revision
//number, nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 9, 5, -1);
MojTestErrCheck(err);
//delete another object
err = db.del(ids[1], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//create a RevTimestamp entry for more than PurgeNumDays days ago,
//with the rev number of the 1st obj we deleted
MojDbQuery query;
err = query.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
err = query.where(MojDb::IdKey, MojDbQuery::OpEq, ids[0]);
MojTestErrCheck(err);
err = query.includeDeleted();
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor);
MojTestErrCheck(err);
MojObject objFromDb;
err = cursor.get(objFromDb, found);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(found == true);
MojObject revFromDb;
err = objFromDb.getRequired(MojDb::RevKey, revFromDb);
MojTestErrCheck(err);
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revFromDb, time.microsecs() - (((MojInt64)35) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//now purge, only id[0] should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 1, 8, 6, revFromDb);
MojTestErrCheck(err);
//TODO 2.12.10 - this test does not pass yet, we need to fix calling delKind after a purge
//err = delKindTest(db);
//MojTestErrCheck(err);
err = db.close();
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDb::purge(MojUInt32& countOut, MojInt64 numDays, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
if (numDays <= -1) {
numDays = m_purgeWindow;
}
MojErr err = beginReq(req);
MojErrCheck(err);
MojLogDebug(s_log, _T("purging objects deleted more than %lld days ago..."), numDays);
MojTime time;
err = MojGetCurrentTime(time);
MojErrCheck(err);
// store the revision number to current timestamp mapping
MojObject revTimeMapping;
MojInt64 rev;
err = nextId(rev);
MojErrCheck(err);
err = revTimeMapping.put(RevNumKey, rev);
MojErrCheck(err);
err = revTimeMapping.put(TimestampKey, time.microsecs());
MojErrCheck(err);
err = revTimeMapping.putString(KindKey, MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
err = putImpl(revTimeMapping, MojDb::FlagNone, req);
MojErrCheck(err);
// find the revision number for numDays prior to now
MojInt64 purgeTime = time.microsecs() - (MojTime::UnitsPerDay * numDays);
MojDbQuery query;
err = query.from(MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
query.limit(1);
err = query.where(TimestampKey, MojDbQuery::OpLessThanEq, purgeTime);
MojErrCheck(err);
err = query.order(TimestampKey);
MojErrCheck(err);
query.desc(true);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpDelete);
MojErrCheck(err);
bool found = false;
MojObject obj;
err = cursor.get(obj, found);
MojErrCheck(err);
err = cursor.close();
MojErrCheck(err);
MojUInt32 batchCount = 0;
MojUInt32 totalCount = 0;
while ((found))
{
// Do it in AutoBatchSize batches
batchCount = 0;
req->fixmode(true); // purge even if index mis-matches
err = purgeImpl(obj, batchCount, req);
MojLogDebug(s_log, _T("purge batch processed: batch: %d; total: %d; err = %d\n"),
batchCount, (totalCount + batchCount), err);
MojErrCheck(err);
totalCount += batchCount;
countOut = totalCount;
if (batchCount < AutoBatchSize) // last batch
break;
err = commitBatch(req);
MojErrCheck(err);
continue;
}
// end request
err = req->end();
MojErrCheck(err);
MojLogDebug(s_log, _T("purged %d objects"), countOut);
return MojErrNone;
}
