void MojoDatabase::GetEmailsToDelete(Signal::SlotRef slot)
{
MojErr err;
MojDbQuery query;
err = query.from(PopEmailAdapter::POP_EMAIL_KIND);
ErrorToException(err);
err = query.select(PopEmailAdapter::ID);
ErrorToException(err);
err = query.select(PopEmailAdapter::SERVER_UID);
ErrorToException(err);
err = query.select(EmailSchema::FOLDER_ID);
ErrorToException(err);
err = query.where("_del", MojDbQuery::OpEq, true);
ErrorToException(err);
query.includeDeleted(true);
slot.cancel();
err = m_dbClient.find(slot, query);
ErrorToException(err);
}
void MojoDatabase::GetLocalEmailChanges(Signal::SlotRef slot, const MojObject& folderId, const MojInt64& rev, MojDbQuery::Page& page, MojInt32 limit)
{
MojDbQuery query;
MojErr err = query.from(PopEmailAdapter::POP_EMAIL_KIND);
ErrorToException(err);
err = query.where(EmailSchema::FOLDER_ID, MojDbQuery::OpEq, folderId);
ErrorToException(err);
if (rev > 0) {
err = query.where(PopFolderAdapter::LAST_SYNC_REV, MojDbQuery::OpGreaterThan, rev);
ErrorToException(err);
}
query.includeDeleted(true);
query.page(page);
slot.cancel();
err = m_dbClient.find(slot, query);
ErrorToException(err);
}
MojErr MojDbIndex::build(MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(isOpen());
MojAssert(m_kind && m_kindEngine);
MojAssert(m_props.size() > 1);
// query for all existing objects of this type and add them to the index.
MojDbQuery query;
MojErr err = query.from(m_kind->id());
MojErrCheck(err);
if (m_includeDeleted) {
err = query.includeDeleted();
MojErrCheck(err);
}
MojDbCursor cursor;
MojDbReq adminRequest(true);
adminRequest.txn(txn);
err = m_kindEngine->find(query, cursor, NULL, adminRequest, OpRead);
MojErrCheck(err);
for (;;) {
MojObject obj;
bool found = false;
err = cursor.get(obj, found);
MojErrCheck(err);
if (!found)
break;
// add this object to the index
err = update(&obj, NULL, txn, false);
MojErrCheck(err);
}
err = cursor.close();
MojErrCheck(err);
return MojErrNone;
}
void ImapActivityFactory::BuildFolderWatch(ActivityBuilder& ab, const MojObject& accountId, const MojObject& folderId, MojInt64 rev)
{
MojErr err;
MojDbQuery query;
ab.SetName( GetFolderWatchName(accountId, folderId) );
ab.SetDescription("Watches for updates to emails");
ab.SetExplicit(true);
ab.SetPersist(true);
SetNetworkRequirements(ab, true);
ab.SetImmediate(true, "low");
// Metadata
MojObject metadata;
SetMetadata(metadata, FOLDER_WATCH_NAME, accountId, folderId);
ab.SetMetadata(metadata);
// Query
err = query.from(ImapEmailAdapter::IMAP_EMAIL_KIND);
ErrorToException(err);
err = query.where(EmailSchema::FOLDER_ID, MojDbQuery::OpEq, folderId);
ErrorToException(err);
err = query.where(ImapEmailAdapter::UPSYNC_REV, MojDbQuery::OpGreaterThan, rev);
ErrorToException(err);
err = query.includeDeleted(true);
ErrorToException(err);
ab.SetDatabaseWatchTrigger(query);
// Callback
MojObject callbackParams;
err = callbackParams.put("accountId", accountId);
ErrorToException(err);
err = callbackParams.put("folderId", folderId);
ErrorToException(err);
ab.SetCallback(FOLDER_WATCH_CALLBACK, callbackParams);
}
void MojoDatabase::GetDeletedEmails(Signal::SlotRef slot, const MojObject& folderId, const MojInt64& rev, MojDbQuery::Page& page, MojInt32 limit)
{
MojErr err;
MojDbQuery query;
err = query.from(PopEmailAdapter::POP_EMAIL_KIND);
ErrorToException(err);
// Select relevant fields
err = query.select(PopEmailAdapter::ID);
ErrorToException(err);
err = query.select(PopEmailAdapter::SERVER_UID);
ErrorToException(err);
err = query.where(EmailSchema::FOLDER_ID, MojDbQuery::OpEq, folderId);
ErrorToException(err);
err = query.where("_del", MojDbQuery::OpEq, true);
ErrorToException(err);
// Set limit
if(limit > 0) {
query.limit(limit);
}
// Make sure deleted objects are returned
query.includeDeleted(true);
// Set page
query.page(page);
slot.cancel(); // cancel existing slot in case we're in a callback
err = m_dbClient.find(slot, query);
ErrorToException(err);
}
MojErr MojDbWatchTest::eqTest(MojDb& db)
{
MojDbQuery query;
MojErr err = query.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = query.where(_T("foo"), MojDbQuery::OpEq, 1);
MojTestErrCheck(err);
MojRefCountedPtr<TestWatcher> watcher(new TestWatcher);
MojTestAssert(watcher.get());
MojDbCursor cursor;
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojRefCountedPtr<TestWatcher> watcher2(new TestWatcher);
MojTestAssert(watcher2.get());
bool fired = false;
err = db.watch(query, cursor, watcher2->m_slot, fired);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(!fired);
// puts
MojObject id;
err = put(db, 0, 0, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
err = put(db, 2, 2, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
// put, changing property not in index
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
err = put(db, 1, 2, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
// dels
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
watcher2.reset(new TestWatcher);
MojTestAssert(watcher2.get());
MojDbQuery queryWithRev;
err = queryWithRev.from(_T("WatchTest:1"));
MojTestErrCheck(err);
err = queryWithRev.where(_T("foo"), MojDbQuery::OpEq, 1);
MojTestErrCheck(err);
err = queryWithRev.where(_T("_rev"), MojDbQuery::OpGreaterThan, m_rev);
MojTestErrCheck(err);
err = queryWithRev.includeDeleted();
MojTestErrCheck(err);
fired = false;
err = db.watch(queryWithRev, cursor, watcher2->m_slot, fired);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(!fired);
MojTestAssert(watcher->m_count == 0);
MojTestAssert(watcher2->m_count == 0);
bool found;
err = db.del(id, found);
MojTestErrCheck(err);
MojTestAssert(found);
MojTestAssert(watcher->m_count == 1);
MojTestAssert(watcher2->m_count == 1);
// ordering
watcher.reset(new TestWatcher);
MojTestAssert(watcher.get());
err = db.find(query, cursor, watcher->m_slot);
MojTestErrCheck(err);
err = put(db, 1, 1, id, m_rev);
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 0);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(watcher->m_count == 1);
return MojErrNone;
}
MojErr MojDbPurgeTest::run()
{
MojDb db;
MojErr err = db.open(MojDbTestDir);
MojTestErrCheck(err);
// put type
MojObject obj;
err = obj.fromJson(MojKindStr);
MojTestErrCheck(err);
err = db.putKind(obj);
MojTestErrCheck(err);
MojObject revNums[10];
MojObject ids[10];
//put 10 objects in the db
for(int i = 0; i < 10; i++) {
err = obj.fromJson(MojTestObjStr1);
MojTestErrCheck(err);
err = db.put(obj);
MojTestErrCheck(err);
// get _rev and id
MojObject rev;
err = obj.getRequired(MojDb::RevKey, rev);
MojTestErrCheck(err);
revNums[i] = rev;
MojObject id;
err = obj.getRequired(MojDb::IdKey, id);
MojTestErrCheck(err);
ids[i] = id;
}
//purge now, there are no RevTimestamp entries
MojUInt32 count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 1, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry - that's not more than PurgeNumDays days ago
MojTime time;
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[0], time.microsecs());
MojTestErrCheck(err);
//purge now, there are no RevTimestamp entries that are more than
//PurgeNumDays ago, so nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 3, -1);
MojTestErrCheck(err);
//create a RevTimestamp entry for more than PurgeNumDays days ago
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revNums[9], time.microsecs() - (((MojInt64)40) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//purge now, since nothing has been deleted, nothing should be purged,
//but the RevTimestamp object should be deleted
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 10, 4, -1);
MojTestErrCheck(err);
//delete something - this will set its revision number higher
bool found;
err = db.del(ids[0], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//purge now, since nothing has been deleted prior to the revision
//number, nothing should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 0, 9, 5, -1);
MojTestErrCheck(err);
//delete another object
err = db.del(ids[1], found);
MojTestErrCheck(err);
MojTestAssert(found == true);
//create a RevTimestamp entry for more than PurgeNumDays days ago,
//with the rev number of the 1st obj we deleted
MojDbQuery query;
err = query.from(_T("PurgeTest:1"));
MojTestErrCheck(err);
err = query.where(MojDb::IdKey, MojDbQuery::OpEq, ids[0]);
MojTestErrCheck(err);
err = query.includeDeleted();
MojTestErrCheck(err);
MojDbCursor cursor;
err = db.find(query, cursor);
MojTestErrCheck(err);
MojObject objFromDb;
err = cursor.get(objFromDb, found);
MojTestErrCheck(err);
err = cursor.close();
MojTestErrCheck(err);
MojTestAssert(found == true);
MojObject revFromDb;
err = objFromDb.getRequired(MojDb::RevKey, revFromDb);
MojTestErrCheck(err);
err = MojGetCurrentTime(time);
MojTestErrCheck(err);
err = createRevTimestamp(db, revFromDb, time.microsecs() - (((MojInt64)35) * MojTime::UnitsPerDay));
MojTestErrCheck(err);
//now purge, only id[0] should be purged
count = 0;
err = db.purge(count, 30);
MojTestErrCheck(err);
err = checkObjectsPurged(db, count, 1, 8, 6, revFromDb);
MojTestErrCheck(err);
//TODO 2.12.10 - this test does not pass yet, we need to fix calling delKind after a purge
//err = delKindTest(db);
//MojTestErrCheck(err);
err = db.close();
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDb::delKind(const MojObject& id, bool& foundOut, MojUInt32 flags, MojDbReqRef req)
{
MojLogTrace(s_log);
foundOut = false;
MojErr err = beginReq(req, true);
MojErrCheck(err);
// del kind obj
MojString idStr;
err = id.stringValue(idStr);
MojErrCheck(err);
MojDbKind *pk = NULL;
// If Kinds has sub-kinds, we give an error
err = m_kindEngine.getKind(idStr.data(), pk);
MojErrCheck(err);
if (pk->nsubkinds() > 0) {
MojLogWarning(s_log, _T("delKind_error: %s has %d subkinds \n"), idStr.data(), pk->nsubkinds());
MojErrThrow(MojErrDbKindHasSubKinds);
}
//MojLogInfo(s_log, _T("delKind: %s \n"), idStr.data());
MojLogWarning(s_log, _T("delKind: %s \n"), idStr.data());
err = m_kindEngine.checkOwnerPermission(idStr, req);
MojErrCheck(err);
MojString dbId;
err = formatKindId(idStr, dbId);
MojErrCheck(err);
MojObject deleted;
bool found = false;
MojDbAdminGuard adminGuard(req);
err = delImpl(dbId, found, deleted, req, flags);
MojErrCheck(err);
if (found) {
// del objects
MojDbQuery query;
err = query.from(idStr);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojUInt32 count;
req->fixmode(true);
err = delImpl(query, count, req, flags | FlagPurge);
MojErrCheck(err);
// del associated permissions
query.clear();
err = query.from(MojDbKindEngine::PermissionId);
MojErrCheck(err);
err = query.where(MojDbServiceDefs::ObjectKey, MojDbQuery::OpEq, idStr);
MojErrCheck(err);
req->fixmode(true);
err = delImpl(query, count, req, flags);
MojErrCheck(err);
// del kind
MojErr errAcc = m_kindEngine.delKind(idStr, req);
MojErrAccumulate(err, errAcc);
}
err = commitKind(idStr, req, err);
MojErrCheck(err);
foundOut = found;
return MojErrNone;
}
MojErr MojDbKind::stats(MojObject& objOut, MojSize& usageOut, MojDbReq& req, bool verify)
{
MojLogTrace(s_log);
#if defined(TESTDBKIND)
MojLogInfo(s_log, _T("Subkinds for - %s ; count = %d\n"), m_id.data(), m_subs.size());
int n = 0;
for (KindVec::ConstIterator i = m_subs.begin(); i != m_subs.end(); ++i) {
MojLogInfo(s_log, _T("SubKind %d: %s"), n++, (*i)->id().data());
}
MojLogInfo(s_log, _T("Supers for - %s ; count = %d\n"), m_id.data(), m_supers.size());
n = 0;
for (KindVec::ConstIterator i = m_supers.begin(); i != m_supers.end(); ++i) {
MojLogInfo(s_log, _T("Super %d: %s"), n++, (*i)->id().data());
}
#endif
// analyze objects
MojDbQuery query;
MojErr err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
MojLogInfo(s_log, _T("KindStats start: %s ; Indexes = %zu; Using Index: %s; \n"),
m_id.data(), m_indexes.size(), cursor.m_dbIndex->name().data());
MojErrCheck(err);
MojSize count = 0;
MojSize size = 0;
MojSize delCount = 0;
MojSize delSize = 0;
MojSize warnings = 0;
for (;;) {
MojDbStorageItem* item = NULL;
bool found = false;
cursor.verifymode(true);
err = cursor.get(item, found);
if (err == MojErrInternalIndexOnFind) {
warnings++;
continue;
}
if (err != MojErrNone) // for all other errors break and dump current stats
break;
if (!found)
break;
MojObject obj;
err = item->toObject(obj, *m_kindEngine, true);
if (err != MojErrNone)
break;
bool deleted = false;
if (obj.get(MojDb::DelKey, deleted) && deleted) {
delSize += item->size();
delCount++;
} else {
size += item->size();
count++;
}
}
MojLogInfo(s_log, _T("KindStats Summary: %s : Count: %zu; delCount: %zu; warnings: %zu \n"), m_id.data(), count, delCount, warnings);
usageOut += size + delSize;
MojObject info;
err = info.put(SizeKey, (MojInt64) size);
MojErrCheck(err);
err = info.put(CountKey, (MojInt64) count);
MojErrCheck(err);
if (delCount > 0) {
err = info.put(DelSizeKey, (MojInt64) delSize);
MojErrCheck(err);
err = info.put(DelCountKey, (MojInt64) delCount);
MojErrCheck(err);
}
if (warnings > 0) {
err = info.put(WarnKey, (MojInt64) warnings);
MojErrCheck(err);
}
err = objOut.put(ObjectsKey, info);
MojErrCheck(err);
// and indexes
MojObject indexes;
for (IndexVec::ConstIterator i = m_indexes.begin(); i != m_indexes.end(); ++i) {
MojObject indexInfo;
err = (*i)->stats(indexInfo, usageOut, req);
MojErrCheck(err);
if (verify) {
MojDbIndex *pi = i->get();
MojErr err2 = verifyIndex(pi, indexInfo, req);
MojErrCheck(err2);
}
err = indexes.put((*i)->name(), indexInfo);
MojErrCheck(err);
}
err = objOut.put(IndexesKey, indexes);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbKind::updateSupers(const KindMap& map, const StringVec& superIds, bool updating, MojDbReq& req)
{
MojLogTrace(s_log);
MojInt32 indexes = 0;
if (updating) {
KindVec addedSupers;
MojErr err = diffSupers(map, superIds, m_superIds, addedSupers);
MojErrCheck(err);
KindVec removedSupers;
err = diffSupers(map, m_superIds, superIds, removedSupers);
MojErrCheck(err);
// remove/add our objects from new/removed supers
if (!addedSupers.empty() || !removedSupers.empty()) {
MojDbQuery query;
err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
cursor.kindEngine(m_kindEngine);
err = find(cursor, NULL, req, OpKindUpdate);
MojErrCheck(err);
for (;;) {
MojObject obj;
bool found = false;
err = cursor.get(obj, found);
MojErrCheck(err);
if (!found)
break;
for (KindVec::ConstIterator i = addedSupers.begin(); i != addedSupers.end(); ++i) {
err = (*i)->updateOwnIndexes(&obj, NULL, req, indexes);
MojErrCheck(err);
}
for (KindVec::ConstIterator i = removedSupers.begin(); i != removedSupers.end(); ++i) {
err = (*i)->updateOwnIndexes(NULL, &obj, req, indexes);
MojErrCheck(err);
}
}
}
}
// remove old supers
m_superIds = superIds;
MojErr err = clearSupers();
MojErrCheck(err);
// look for new supers
for (StringVec::ConstIterator i = m_superIds.begin(); i != m_superIds.end(); ++i) {
KindMap::ConstIterator mapIter = map.find(*i);
if (mapIter != map.end()) {
err = addSuper(mapIter->get());
MojErrCheck(err);
}
}
// look for kinds that extend us
for (KindMap::ConstIterator i = map.begin(); i != map.end(); ++i) {
if ((*i)->m_superIds.find(m_id) != MojInvalidIndex) {
err = (*i)->addSuper(this);
MojErrCheck(err);
}
}
// add root kind if we have no supers
if (m_supers.empty()) {
KindMap::ConstIterator mapIter = map.find(MojDbKindEngine::RootKindId);
if (mapIter != map.end()) {
err = addSuper(mapIter->get());
MojErrCheck(err);
}
}
return MojErrNone;
}
MojErr MojDbKind::verifyIndex(MojDbIndex *pIndex, MojObject &iinfo, MojDbReq& req)
{
// Goes throudh each index entry and verifies that it points to a valid object
// For debugging purposes as stats for indexes does not access the target objects
// Index->stats function does not have enough context to find the object
// db/stats usage '{"verify":true,"kind":"xyz"}' - each optional
MojDbQuery query;
MojErr err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
query.m_forceIndex = pIndex; // Important: Otherwise, it will pick the default index
cursor.verifymode(true); // to get the errors
err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
MojLogInfo(s_log, _T("Kind_verifyIndex: Kind: %s; Index: %s; idIndex: %zX; size: %zu; CursorIndex: %s \n"), m_name.data(),
pIndex->name().data(), pIndex->idIndex(), pIndex->size(), cursor.m_dbIndex->name().data());
MojErrCheck(err);
MojSize count = 0;
MojSize delCount = 0;
MojSize warnCount = 0;
char s[1024];
for (;;) {
MojDbStorageItem* item = NULL;
bool found = false;
err = cursor.get(item, found);
if (err == MojErrInternalIndexOnFind) {
warnCount++;
MojDbIsamQuery *iquery = (MojDbIsamQuery *)cursor.m_storageQuery.get();
MojErr err2 = MojByteArrayToHex(iquery->m_keyData, iquery->m_keySize, s);
MojErrCheck(err2);
MojChar *ids = (iquery->m_keySize > 18) ? (MojChar *)(iquery->m_keyData + iquery->m_keySize - 17) : NULL;
MojLogInfo(s_log, _T("VerifyIndex Warning: %s; KeySize: %zu; %s ;id: %s \n"),
cursor.m_dbIndex->name().data(), iquery->m_keySize, s, ids);
continue;
}
MojErrCheck(err);
if (!found)
break;
MojObject obj;
err = item->toObject(obj, *m_kindEngine, true);
MojErrCheck(err);
bool deleted = false;
if (obj.get(MojDb::DelKey, deleted) && deleted) {
delCount++;
} else {
count++;
}
}
MojLogInfo(s_log, _T("Kind_verifyIndex Counts: Kind: %s; Index: %s; count: %zu; delcount: %zu; warnings: %zu \n"), m_name.data(),
pIndex->name().data(), count, delCount, warnCount);
err = iinfo.put(VerifyCountKey, (MojInt64)count);
MojErrCheck(err);
err = iinfo.put(VerifyWarnCountKey, (MojInt64) warnCount);
MojErrCheck(err);
err = iinfo.put(VerifyDelCountKey, (MojInt64) delCount);
MojErrCheck(err);
return MojErrNone;
}
