MojErr MojService::CategoryHandler::invoke(const MojChar* method, MojServiceMessage* msg, MojObject& payload)
{
MojAssert(method && msg);
MojLogTrace(s_log);
MojTime startTime;
MojErr err = MojGetCurrentTime(startTime);
MojErrCheck(err);
// lookup callback
CallbackInfo cb;
if (!m_callbackMap.get(method, cb)) {
MojErrThrow(MojErrMethodNotFound);
}
// validate schema
if (cb.m_schema.get()) {
MojSchema::Result res;
err = cb.m_schema->validate(payload, res);
MojErrCheck(err);
if (!res.valid()) {
MojErrThrowMsg(MojErrInvalidArg, _T("invalid parameters: caller='%s' error='%s'"), msg->senderName(), res.msg().data());
}
}
// invoke method
err = invoke(cb.m_callback, msg, payload);
MojErrCheck(err);
// log timing
MojTime endTime;
err = MojGetCurrentTime(endTime);
MojErrCheck(err);
MojLogInfo(s_log, _T("%s invoked: %.3fms"), method, (double) (endTime.microsecs() - startTime.microsecs()) / 1000);
return MojErrNone;
}
MojErr MojDbPerfCreateTest::run()
{
MojErr err = file.open(CreateTestFileName, MOJ_O_RDWR | MOJ_O_CREAT | MOJ_O_TRUNC, MOJ_S_IRUSR | MOJ_S_IWUSR);
MojString buf;
err = buf.format("MojoDb Create Performance Test,,,,,\n\nOperation,Kind,Total Time,Time Per Iteration,Time Per Object\n");
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
err = testCreate();
MojTestErrCheck(err);
err = MojPrintF("\n\n TOTAL TEST TIME: %llu microseconds\n\n", totalTestTime.microsecs());
MojTestErrCheck(err);
err = MojPrintF("\n-------\n");
MojTestErrCheck(err);
err = buf.format("\n\nTOTAL TEST TIME,,%llu,,,", totalTestTime.microsecs());
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
err = file.close();
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDbIdGenerator::id(MojObject& idOut)
{
// an id consists of a timestamp (in microsecs) concatenated with a 32-bit random number.
// The goal is to have a very low likelihood of id collision among multiple devices owned
// by the same user (so they can can share an id-space) while still keeping them sequential
// to minimize db fragmentation.
MojThreadGuard guard(m_mutex);
MojUInt32 randNum;
MojErr err = MojRandom(&m_randBuf, &randNum);
MojErrCheck(err);
guard.unlock();
MojTime time;
err = MojGetCurrentTime(time);
MojErrCheck(err);
MojBuffer buf;
MojDataWriter writer(buf);
err = writer.writeInt64(time.microsecs());
MojErrCheck(err);
err = writer.writeUInt32(randNum);
MojErrCheck(err);
MojVector<MojByte> byteVec;
err = buf.toByteVec(byteVec);
MojErrCheck(err);
MojString str;
err = str.base64Encode(byteVec, false);
MojErrCheck(err);
idOut = str;
return MojErrNone;
}
MojErr MojDbPerfUpdateTest::updateKind(MojDb& db, const MojChar* kindId, const MojChar* kindJson, const MojChar* extraIdxJson, MojErr (MojDbPerfTest::*createFn) (MojObject&, MojUInt64))
{
// register all the kinds
MojTime time;
MojErr err = putKinds(db, time);
MojTestErrCheck(err);
// put objects using createFn
MojObject objs;
err = putObjs(db, kindId, numObjectsBeforeUpdateKind, createFn, objs);
MojTestErrCheck(err);
// add an index
MojObject kindObj;
err = kindObj.fromJson(kindJson);
MojTestErrCheck(err);
MojObject indexes;
kindObj.get(_T("indexes"), indexes);
MojTestErrCheck(err);
MojObject extraIdx;
err = extraIdx.fromJson(extraIdxJson);
MojTestErrCheck(err);
indexes.push(extraIdx);
err = kindObj.put(_T("indexes"), indexes);
MojTestErrCheck(err);
MojTime addIndexTime;
MojTime dropIndexTime;
err = timeUpdateKind(db, kindJson, kindObj, addIndexTime, dropIndexTime);
MojTestErrCheck(err);
MojUInt64 addTime = addIndexTime.microsecs();
MojUInt64 dropTime = dropIndexTime.microsecs();
err = MojPrintF("\n -------------------- \n");
MojTestErrCheck(err);
err = MojPrintF(" updating kind %s - adding index %s %llu times took: %llu microsecs\n", kindId, extraIdxJson, numUpdateKindIterations, addTime);
MojTestErrCheck(err);
err = MojPrintF(" time per add/reindex: %llu microsecs\n", (addTime) / (numUpdateKindIterations));
MojTestErrCheck(err);
err = MojPrintF(" updating kind %s - dropping index %s %llu times took: %llu microsecs\n", kindId, extraIdxJson, numUpdateKindIterations, dropTime);
MojTestErrCheck(err);
err = MojPrintF(" time per drop: %llu microsecs", (dropTime) / (numUpdateKindIterations));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
MojString buf;
err = buf.format("Updating kind %s - adding index %s %llu times,%s,%llu,%llu,%llu,\nUpdating kind %s - dropping index %s %llu times,%s,%llu,%llu,%llu,\n",
kindId, extraIdxJson, numUpdateKindIterations, kindId, addTime, addTime/numUpdateKindIterations, addTime/(1*numUpdateKindIterations),
kindId, extraIdxJson, numUpdateKindIterations, kindId, dropTime, dropTime/numUpdateKindIterations, dropTime/(1*numUpdateKindIterations));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDbPerfUpdateTest::updateObjsViaPut(MojDb& db, const MojChar* kindId, MojErr (MojDbPerfTest::*createFn) (MojObject&, MojUInt64))
{
// register all the kinds
MojTime time;
MojErr err = putKinds(db, time);
MojTestErrCheck(err);
// put objects using createFn
MojObject objs;
err = putObjs(db, kindId, numInsertForPut, createFn, objs);
MojTestErrCheck(err);
MojObject midObj;
bool found = objs.at(numInsertForPut/2, midObj);
MojTestAssert(found);
MojTime objTime;
err = putObj(db, midObj, objTime);
MojTestErrCheck(err);
MojUInt64 putTime = objTime.microsecs();
err = MojPrintF("\n -------------------- \n");
MojTestErrCheck(err);
err = MojPrintF(" putting single object - index %llu - of kind %s %llu times took: %llu microsecs\n", numInsertForPut/2, kindId, numPutIterations, putTime);
MojTestErrCheck(err);
err = MojPrintF(" time per put: %llu microsecs", (putTime) / (numPutIterations));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
MojString buf;
err = buf.format("Put single object - index %llu - %llu times,%s,%llu,%llu,%llu,\n", numInsertForPut/2, numPutIterations, kindId, putTime, putTime/numPutIterations, putTime/(1*numPutIterations));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
MojTime batchTime;
MojObject::ArrayIterator beginArr;
err = objs.arrayBegin(beginArr);
MojErrCheck(err);
err = batchPutObj(db, beginArr, beginArr + (numInsertForPut / 10), batchTime);
putTime = batchTime.microsecs();
MojTestErrCheck(err);
err = MojPrintF(" putting batch - %llu objects - of kind %s %llu times took: %llu microsecs\n", numInsertForPut/10, kindId, numBatchPutIterations, putTime);
MojTestErrCheck(err);
err = MojPrintF(" time per batch put: %llu microsecs\n", (putTime) / (numBatchPutIterations));
MojTestErrCheck(err);
err = MojPrintF(" time per object: %llu microsecs", (putTime) / (numInsertForPut/10 * numBatchPutIterations));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
err = buf.format("Batch put %llu objects %llu times,%s,%llu,%llu,%llu,\n", numInsertForPut/10, numBatchPutIterations, kindId, putTime, putTime/numBatchPutIterations, putTime/(numInsertForPut/10*numBatchPutIterations));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojTimeTest::run()
{
MojTimevalT tv;
tv.tv_sec = 400;
tv.tv_usec = 54321;
MojTime time = -8;
time.fromTimeval(&tv);
MojTestAssert(time == 400054321);
MojTestAssert(time.secs() == 400);
MojTestAssert(time.millisecs() == 400054);
MojTestAssert(time.microsecs() == 400054321);
MojTestAssert(time.millisecsPart() == 54);
MojTestAssert(time.microsecsPart() == 54321);
MojTimevalT tv2;
time.toTimeval(&tv2);
MojTestAssert(tv.tv_sec == tv2.tv_sec && tv.tv_usec == tv2.tv_usec);
MojTimespecT ts;
ts.tv_sec = 400;
ts.tv_nsec = 54321;
time.fromTimespec(&ts);
MojTestAssert(time == 400000054);
MojTimespecT ts2;
time.toTimespec(&ts2);
MojTestAssert(ts2.tv_sec = 400 && ts2.tv_nsec == 54000);
time = MojSecs(3);
MojTestAssert(time == 3000000);
time = MojMillisecs(45);
MojTestAssert(time == 45000);
time = MojMicrosecs(5);
MojTestAssert(time == 5);
time = MojSecs(1) + MojMillisecs(2) + MojMicrosecs(3);
MojTestAssert(time == 1002003);
time = 1;
MojTestAssert(time++ == 1);
MojTestAssert(time == 2);
MojTestAssert(++time == 3);
MojTestAssert(--time == 2);
MojTestAssert(time-- == 2);
MojTestAssert(time == 1);
MojTestAssert((time += 2) == 3);
MojTestAssert((time -= 4) == -1);
MojTestAssert((time *= -10) == 10);
MojTestAssert((time /= 2) == 5);
time = MojTime(1) + MojTime(2);
MojTestAssert(time == 3);
time = MojTime(8) - MojTime(6);
MojTestAssert(time == 2);
time = MojTime(8) * MojTime(6);
MojTestAssert(time == 48);
time = MojTime(8) / MojTime(4);
MojTestAssert(time == 2);
return MojErrNone;
}
MojErr MojDbIdGenerator::init()
{
MojThreadGuard guard(m_mutex);
MojTime time;
MojErr err = MojGetCurrentTime(time);
MojErrCheck(err);
MojZero(&m_randBuf, sizeof(m_randBuf));
err = MojInitRandom((MojUInt32) time.microsecs(), m_randStateBuf, sizeof(m_randStateBuf), &m_randBuf);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojLocalTime(const MojTime& time, MojTmT& tmOut)
{
time_t timet = (time_t) time.secs();
if (localtime_r(&timet, &tmOut) == NULL)
MojErrThrowErrno(_T("localtime_r"));
return MojErrNone;
}
MojErr MojSleep(const MojTime& time)
{
MojTimespecT ts;
time.toTimespec(&ts);
if (nanosleep(&ts, NULL) < 0)
MojErrThrowErrno(_T("nanosleep"));
return MojErrNone;
}
MojErr MojFileAppender::append(MojLogger::Level level, MojLogger* logger, const MojChar* format, va_list args)
{
MojAssertNoLog(format);
MojAssertNoLog(m_file != MojInvalidFile);
// get current time
MojTime time = 0;
MojErr err = MojGetCurrentTime(time);
MojErrCheckNoLog(err);
MojTmT tm;
err = MojLocalTime(time, tm);
MojErrCheckNoLog(err);
// format message
err = m_buf.format(_T("[%04d-%02d-%02d %02d:%02d:%02d:%03d] [%p] [%s]"),
tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour,
tm.tm_min, tm.tm_sec, time.millisecsPart(),
(void*) (MojIntPtr) MojThreadCurrentId(),
MojLogger::stringFromLevel(level));
MojErrCheckNoLog(err);
if (logger) {
err = m_buf.appendFormat(_T(" [%s]: "), logger->name());
MojErrCheckNoLog(err);
} else {
err = m_buf.append(_T(": "));
MojErrCheckNoLog(err);
}
err = m_buf.appendVFormat(format, args);
MojErrCheckNoLog(err);
err = m_buf.append(_T('\n'));
MojErrCheckNoLog(err);
// append to file
const MojByte* begin = (const MojByte*) m_buf.begin();
const MojByte* end = (const MojByte*) m_buf.end();
while (begin < end) {
MojSize written = 0;
err = MojFileWrite(m_file, begin, end - begin, written);
MojErrCheckNoLog(err);
begin += written;
}
return MojErrNone;
}
MojErr MojGetCurrentTime(MojTime& timeOut)
{
MojTimevalT tv;
int ret = gettimeofday(&tv, NULL);
if(ret < 0)
MojErrThrowErrno(_T("gettimeofday"));
timeOut.fromTimeval(&tv);
return MojErrNone;
}
MojErr MojDbPerfCreateTest::testBatchInsertLgArrayObj(MojDb& db, const MojChar* kindId)
{
// register all the kinds again
MojTime time;
MojErr err = putKinds(db, time);
MojTestErrCheck(err);
// time put with large array objects (20 properties + 2 arrays of 5 elements each)
MojTime batchLgArrayObjTime = 0;
for (int i = 0; i < numRepetitions; i++) {
err = putLargeArrayObj(db, kindId, batchLgArrayObjTime);
MojTestErrCheck(err);
MojDbQuery q;
err = q.from(kindId);
MojTestErrCheck(err);
MojUInt32 count = 0;
err = db.del(q, count);
MojTestErrCheck(err);
}
MojUInt64 putTime = batchLgArrayObjTime.microsecs();
err = MojPrintF("\n -------------------- \n");
MojTestErrCheck(err);
err = MojPrintF(" time to batch put %llu %s objects %d times: %llu microsecs\n", numInsert, kindId, numRepetitions, putTime);
MojTestErrCheck(err);
err = MojPrintF(" time per batch: %llu microsecs", (putTime) / (numRepetitions));
MojTestErrCheck(err);
err = MojPrintF(" time per object: %llu microsecs", (putTime) / (numInsert * numRepetitions));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
MojString buf;
err = buf.format("batch put %llu objects %d times,%s,%llu,%llu,%llu,\n", numInsert, numRepetitions, kindId, putTime, putTime/numRepetitions, putTime / (numInsert * numRepetitions));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDbPerfCreateTest::testInsertMedArrayObj(MojDb& db, const MojChar* kindId)
{
// register all the kinds again
MojTime time;
MojErr err = putKinds(db, time);
MojTestErrCheck(err);
// time put with med array objects (10 properties + 1 array of 5 elements)
MojTime medArrayObjTime;
for (int i = 0; i < numRepetitions; i++) {
err = putMedArrayObj(db, kindId, medArrayObjTime);
MojTestErrCheck(err);
MojDbQuery q;
err = q.from(kindId);
MojTestErrCheck(err);
MojUInt32 count = 0;
err = db.del(q, count, MojDb::FlagPurge);
MojTestErrCheck(err);
}
MojUInt64 putTime = medArrayObjTime.microsecs();
err = MojPrintF("\n -------------------- \n");
MojTestErrCheck(err);
err = MojPrintF(" time to put %llu %s objects %d times: %llu microsecs\n", numInsert, kindId, numRepetitions, putTime);
MojTestErrCheck(err);
err = MojPrintF(" time per object: %llu microsecs", (putTime) / (numInsert * numRepetitions));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
MojString buf;
err = buf.format("put %llu objects %d times,%s,%llu,%llu,%llu,\n", numInsert, numRepetitions, kindId, putTime, putTime/numRepetitions, putTime / (numInsert * numRepetitions));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojSelect(int& nfdsOut, int nfds, MojFdSetT* readfds, MojFdSetT* writefds,
MojFdSetT* exceptfds, const MojTime& timeout)
{
MojTimevalT tv;
timeout.toTimeval(&tv);
int ret = select(nfds, readfds, writefds, exceptfds, &tv);
if (ret < 0) {
nfdsOut = 0;
MojErrThrowErrno(_T("select"));
}
nfdsOut = ret;
return MojErrNone;
}
MojErr MojDbPerfUpdateTest::updateObjsViaMerge(MojDb& db, const MojChar* kindId, MojErr (MojDbPerfTest::*createFn) (MojObject&, MojUInt64))
{
// register all the kinds
MojTime time;
MojErr err = putKinds(db, time);
MojTestErrCheck(err);
// put objects using createFn
MojObject objs;
err = putObjs(db, kindId, numInsertForPut, createFn, objs);
MojTestErrCheck(err);
MojObject midObj;
bool found = objs.at(numInsertForPut/2, midObj);
MojTestAssert(found);
MojTime objTime;
err = mergeObj(db, midObj, objTime);
MojTestErrCheck(err);
MojUInt64 mergeTime = objTime.microsecs();
err = MojPrintF("\n -------------------- \n");
MojTestErrCheck(err);
err = MojPrintF(" merging single object - index %llu - of kind %s %llu times took: %llu microsecs\n", numInsertForPut/2, kindId, numMergeIterations, mergeTime);
MojTestErrCheck(err);
err = MojPrintF(" time per merge: %llu microsecs", (mergeTime) / (numMergeIterations));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
MojString buf;
err = buf.format("Merge single object - index %llu - %llu times,%s,%llu,%llu,%llu,\n", numInsertForPut/2, numMergeIterations, kindId, mergeTime, mergeTime/numMergeIterations, mergeTime/(1*numMergeIterations));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
MojTime batchTime;
MojObject::ArrayIterator beginArr;
err = objs.arrayBegin(beginArr);
MojErrCheck(err);
err = batchMergeObj(db, beginArr, beginArr + (numInsertForPut / 10), batchTime);
MojTestErrCheck(err);
mergeTime = batchTime.microsecs();
err = MojPrintF(" merging batch - %llu objects - of kind %s %llu times took: %llu microsecs\n", numInsertForPut/10, kindId, numBatchMergeIterations, mergeTime);
MojTestErrCheck(err);
err = MojPrintF(" time per batch merge: %llu microsecs\n", (mergeTime) / (numBatchMergeIterations));
MojTestErrCheck(err);
err = MojPrintF(" time per object: %llu microsecs", (mergeTime) / (numInsertForPut/10 * numBatchMergeIterations));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
err = buf.format("Batch merge %llu objects %llu times,%s,%llu,%llu,%llu,\n", numInsertForPut/10, numBatchMergeIterations, kindId, mergeTime, mergeTime/numBatchMergeIterations, mergeTime/(numInsertForPut/10*numBatchMergeIterations));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
MojTime mergeQueryTime;
MojTestErrCheck(err);
MojDbQuery query;
err = query.from(kindId);
MojTestErrCheck(err);
query.limit(numInsertForPut/5);
query.desc(true);
MojObject queryObj;
err = query.toObject(queryObj);
MojTestErrCheck(err);
MojString queryStr;
err = queryObj.stringValue(queryStr);
MojTestErrCheck(err);
MojObject props;
err = props.putString(_T("newKey"), _T("here's a new value"));
MojTestErrCheck(err);
MojUInt32 count;
err = queryMergeObj(db, query, props, count, mergeQueryTime);
MojTestErrCheck(err);
mergeTime = mergeQueryTime.microsecs();
err = MojPrintF(" merging with query - %d objects - of kind %s %llu times took: %llu microsecs\n", count, kindId, numBatchMergeIterations, mergeTime);
MojTestErrCheck(err);
err = MojPrintF(" time per merge: %llu microsecs\n", (mergeTime) / (numBatchMergeIterations));
MojTestErrCheck(err);
err = MojPrintF(" time per object: %llu microsecs", (mergeTime) / (count * numBatchMergeIterations));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
err = buf.format("Merge with query %s,,,,,\nMerge with query - %d objects - %llu times,%s,%llu,%llu,%llu,\n", queryStr.data(), count, numBatchMergeIterations, kindId, mergeTime, mergeTime/numBatchMergeIterations, mergeTime/(count*numBatchMergeIterations));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDbPerfCreateTest::testCreate()
{
//setup the test storage engine
#ifdef MOJ_USE_BDB
MojRefCountedPtr<MojDbStorageEngine> engine(new MojDbBerkeleyEngine());
#elif MOJ_USE_LDB
MojRefCountedPtr<MojDbStorageEngine> engine(new MojDbLevelEngine());
#else
MojRefCountedPtr<MojDbStorageEngine> engine;
#endif
MojAllocCheck(engine.get());
MojDb db;
MojErr err = db.open(MojDbTestDir, engine.get());
MojTestErrCheck(err);
// time put kind
MojTime putKindTime;
for (int i = 0; i < numRepetitions; i++) {
err = putKinds(db, putKindTime);
MojTestErrCheck(err);
err = delKinds(db);
MojTestErrCheck(err);
}
MojUInt64 putKind = putKindTime.microsecs();
err = MojPrintF("\n -------------------- \n");
MojTestErrCheck(err);
err = MojPrintF(" putKind took: %llu microsecs", (putKind / (numKinds * numRepetitions)));
MojTestErrCheck(err);
err = MojPrintF("\n\n");
MojTestErrCheck(err);
MojString buf;
err = buf.format("put Kind,all %llu kinds,%llu,%llu,%llu,\n", numKinds, putKind, putKind/numRepetitions, putKind / (numKinds * numRepetitions));
MojTestErrCheck(err);
err = fileWrite(file, buf);
MojTestErrCheck(err);
// insert objects with one index
err = testInsertSmallObj(db, MojPerfSmKindId);
MojTestErrCheck(err);
err = testInsertMedObj(db, MojPerfMedKindId);
MojTestErrCheck(err);
err = testInsertLgObj(db, MojPerfLgKindId);
MojTestErrCheck(err);
err = testInsertMedNestedObj(db, MojPerfMedNestedKindId);
MojTestErrCheck(err);
err = testInsertLgNestedObj(db, MojPerfLgNestedKindId);
MojTestErrCheck(err);
err = testInsertMedArrayObj(db, MojPerfMedArrayKindId);
MojTestErrCheck(err);
err = testInsertLgArrayObj(db, MojPerfLgArrayKindId);
MojTestErrCheck(err);
// insert objects with two indices
err = testInsertSmallObj(db, MojPerfSmKind2Id);
MojTestErrCheck(err);
err = testInsertMedObj(db, MojPerfMedKind2Id);
MojTestErrCheck(err);
err = testInsertLgObj(db, MojPerfLgKind2Id);
MojTestErrCheck(err);
err = testInsertMedNestedObj(db, MojPerfMedNestedKind2Id);
MojTestErrCheck(err);
err = testInsertLgNestedObj(db, MojPerfLgNestedKind2Id);
MojTestErrCheck(err);
err = testInsertMedArrayObj(db, MojPerfMedArrayKind2Id);
MojTestErrCheck(err);
err = testInsertLgArrayObj(db, MojPerfLgArrayKind2Id);
MojTestErrCheck(err);
// batch insert with one index
err = testBatchInsertLgObj(db, MojPerfLgKindId);
MojTestErrCheck(err);
err = testBatchInsertLgNestedObj(db, MojPerfLgNestedKindId);
MojTestErrCheck(err);
err = testBatchInsertLgArrayObj(db, MojPerfLgArrayKindId);
MojTestErrCheck(err);
// batch insert with two indices
err = testBatchInsertLgObj(db, MojPerfLgKind2Id);
MojTestErrCheck(err);
err = testBatchInsertLgNestedObj(db, MojPerfLgNestedKind2Id);
MojTestErrCheck(err);
err = testBatchInsertLgArrayObj(db, MojPerfLgArrayKind2Id);
MojTestErrCheck(err);
err = db.close();
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDbPurgeTest::run()
{
MojDb db;
MojErr err = db.open(MojDbTestDir);
MojTestErrCheck(err);
// put type
MojObject obj;
err = obj.fromJson(MojKindStr);
MojTestErrCheck(err);
err = db.putKind(obj);
MojTestErrCheck(err);
MojObject revNums[10];
MojObject ids[10];
//put 10 objects in the db
for(int i = 0; i < 10; i++) {
err = obj.fromJson(MojTestObjStr1);
MojTestErrCheck(err);
err = db.put(obj);
MojTestErrCheck(err);
// get _rev and id
MojObject rev;
err = obj.getRequired(MojDb::RevKey, rev);
MojTestErrCheck(err);
revNums[i] = rev;
MojObject id;
err = obj.getRequired(MojDb::IdKey, id);
MojTestErrCheck(err);
ids[i] = id;
}
//purge now, there are no RevTimestamp entries
MojUInt32 count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 1, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry - that's not more than PurgeNumDays days ago
MojTime time;
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[0], time.microsecs());
MojTestErrCheck(err);
//purge now, there are no RevTimestamp entries that are more than
//PurgeNumDays ago, so nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 3, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry for more than PurgeNumDays days ago
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[9], time.microsecs() - (((MojInt64)40) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//purge now, since nothing has been deleted, nothing should be purged,
//but the RevTimestamp object should be deleted
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 4, -1);
MojTestErrCheck(err);
//delete something - this will set its revision number higher
bool found;
err = db.del(ids[0], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//purge now, since nothing has been deleted prior to the revision
//number, nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 9, 5, -1);
MojTestErrCheck(err);
//delete another object
err = db.del(ids[1], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//create a RevTimestamp entry for more than PurgeNumDays days ago,
//with the rev number of the 1st obj we deleted
MojDbQuery query;
err = query.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
err = query.where(MojDb::IdKey, MojDbQuery::OpEq, ids[0]);
MojTestErrCheck(err);
err = query.includeDeleted();
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor);
MojTestErrCheck(err);
MojObject objFromDb;
err = cursor.get(objFromDb, found);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(found == true);
MojObject revFromDb;
err = objFromDb.getRequired(MojDb::RevKey, revFromDb);
MojTestErrCheck(err);
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revFromDb, time.microsecs() - (((MojInt64)35) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//now purge, only id[0] should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 1, 8, 6, revFromDb);
MojTestErrCheck(err);
//TODO 2.12.10 - this test does not pass yet, we need to fix calling delKind after a purge
//err = delKindTest(db);
//MojTestErrCheck(err);
err = db.close();
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDb::purge(MojUInt32& countOut, MojInt64 numDays, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
if (numDays <= -1) {
numDays = m_purgeWindow;
}
MojErr err = beginReq(req);
MojErrCheck(err);
MojLogDebug(s_log, _T("purging objects deleted more than %lld days ago..."), numDays);
MojTime time;
err = MojGetCurrentTime(time);
MojErrCheck(err);
// store the revision number to current timestamp mapping
MojObject revTimeMapping;
MojInt64 rev;
err = nextId(rev);
MojErrCheck(err);
err = revTimeMapping.put(RevNumKey, rev);
MojErrCheck(err);
err = revTimeMapping.put(TimestampKey, time.microsecs());
MojErrCheck(err);
err = revTimeMapping.putString(KindKey, MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
err = putImpl(revTimeMapping, MojDb::FlagNone, req);
MojErrCheck(err);
// find the revision number for numDays prior to now
MojInt64 purgeTime = time.microsecs() - (MojTime::UnitsPerDay * numDays);
MojDbQuery query;
err = query.from(MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
query.limit(1);
err = query.where(TimestampKey, MojDbQuery::OpLessThanEq, purgeTime);
MojErrCheck(err);
err = query.order(TimestampKey);
MojErrCheck(err);
query.desc(true);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpDelete);
MojErrCheck(err);
bool found = false;
MojObject obj;
err = cursor.get(obj, found);
MojErrCheck(err);
err = cursor.close();
MojErrCheck(err);
MojUInt32 batchCount = 0;
MojUInt32 totalCount = 0;
while ((found))
{
// Do it in AutoBatchSize batches
batchCount = 0;
req->fixmode(true); // purge even if index mis-matches
err = purgeImpl(obj, batchCount, req);
MojLogDebug(s_log, _T("purge batch processed: batch: %d; total: %d; err = %d\n"),
batchCount, (totalCount + batchCount), err);
MojErrCheck(err);
totalCount += batchCount;
countOut = totalCount;
if (batchCount < AutoBatchSize) // last batch
break;
err = commitBatch(req);
MojErrCheck(err);
continue;
}
// end request
err = req->end();
MojErrCheck(err);
MojLogDebug(s_log, _T("purged %d objects"), countOut);
return MojErrNone;
}