MojErr MojDbQueryPlan::init(const MojDbQuery& query, const MojDbIndex& index) { m_query = query; m_locale = index.locale(); m_idPropIndex = index.idIndex(); m_ranges.clear(); MojErr err = MojErrNone; if (index.includeDeleted() && !m_query.where().contains(MojDb::DelKey)) { err = m_query.where(MojDb::DelKey, MojDbQuery::OpEq, false); MojErrCheck(err); } // build ranges from where clauses err = buildRanges(index); MojErrCheck(err); if (query.desc()) { // reverse ranges if descending err = m_ranges.reverse(); MojErrCheck(err); } return MojErrNone; }
void MojoDatabase::GetEmails(Signal::SlotRef slot, const MojObject& folderId, MojInt32 limit) { MojDbQuery q; MojErr err = q.from(PopEmailAdapter::POP_EMAIL_KIND); ErrorToException(err); err = q.where(EmailSchema::FOLDER_ID, MojDbQuery::OpEq, folderId); ErrorToException(err); err = q.order(EmailSchema::TIMESTAMP); ErrorToException(err); q.limit(limit); q.desc(true); err = m_dbClient.find(slot, q); ErrorToException(err); }
MojErr MojDbSearchTest::initQuery(MojDbQuery& query, const MojChar* queryStr, const MojChar* orderBy, const MojObject& barVal, bool desc) { query.clear(); MojErr err = query.from(_T("SearchTest:1")); MojTestErrCheck(err); MojString val; err = val.assign(queryStr); MojTestErrCheck(err); err = query.where(_T("foo"), MojDbQuery::OpSearch, val, MojDbCollationPrimary); MojTestErrCheck(err); query.desc(desc); if (!barVal.undefined()) { err = query.where(_T("bar"), MojDbQuery::OpEq, barVal); MojTestErrCheck(err); } if (orderBy) { err = query.order(orderBy); MojTestErrCheck(err); } return MojErrNone; }
void MojoDatabase::GetLocalEmailChanges(Signal::SlotRef slot, const MojObject& folderId, const MojInt64& rev, MojDbQuery::Page& page, MojInt32 limit) { MojDbQuery query; MojErr err = query.from(PopEmailAdapter::POP_EMAIL_KIND); ErrorToException(err); err = query.where(EmailSchema::FOLDER_ID, MojDbQuery::OpEq, folderId); ErrorToException(err); if (rev > 0) { err = query.where(PopFolderAdapter::LAST_SYNC_REV, MojDbQuery::OpGreaterThan, rev); ErrorToException(err); } query.includeDeleted(true); query.page(page); slot.cancel(); err = m_dbClient.find(slot, query); ErrorToException(err); }
MojErr MojDbPurgeTest::run() { MojDb db; MojErr err = db.open(MojDbTestDir); MojTestErrCheck(err); // put type MojObject obj; err = obj.fromJson(MojKindStr); MojTestErrCheck(err); err = db.putKind(obj); MojTestErrCheck(err); MojObject revNums[10]; MojObject ids[10]; //put 10 objects in the db for(int i = 0; i < 10; i++) { err = obj.fromJson(MojTestObjStr1); MojTestErrCheck(err); err = db.put(obj); MojTestErrCheck(err); // get _rev and id MojObject rev; err = obj.getRequired(MojDb::RevKey, rev); MojTestErrCheck(err); revNums[i] = rev; MojObject id; err = obj.getRequired(MojDb::IdKey, id); MojTestErrCheck(err); ids[i] = id; } //purge now, there are no RevTimestamp entries MojUInt32 count = 0; err = db.purge(count, 30); MojTestErrCheck(err); err = checkObjectsPurged(db, count, 0, 10, 1, -1); MojTestErrCheck(err); //create a RevTimestamp entry - that's not more than PurgeNumDays days ago MojTime time; err = MojGetCurrentTime(time); MojTestErrCheck(err); err = createRevTimestamp(db, revNums[0], time.microsecs()); MojTestErrCheck(err); //purge now, there are no RevTimestamp entries that are more than //PurgeNumDays ago, so nothing should be purged count = 0; err = db.purge(count, 30); MojTestErrCheck(err); err = checkObjectsPurged(db, count, 0, 10, 3, -1); MojTestErrCheck(err); //create a RevTimestamp entry for more than PurgeNumDays days ago err = MojGetCurrentTime(time); MojTestErrCheck(err); err = createRevTimestamp(db, revNums[9], time.microsecs() - (((MojInt64)40) * MojTime::UnitsPerDay)); MojTestErrCheck(err); //purge now, since nothing has been deleted, nothing should be purged, //but the RevTimestamp object should be deleted count = 0; err = db.purge(count, 30); MojTestErrCheck(err); err = checkObjectsPurged(db, count, 0, 10, 4, -1); MojTestErrCheck(err); //delete something - this will set its revision number higher bool found; err = db.del(ids[0], found); MojTestErrCheck(err); MojTestAssert(found == true); //purge now, since nothing has been deleted prior to the revision //number, nothing should be purged count = 0; err = db.purge(count, 30); MojTestErrCheck(err); err = checkObjectsPurged(db, count, 0, 9, 5, -1); MojTestErrCheck(err); //delete another object err = db.del(ids[1], found); MojTestErrCheck(err); MojTestAssert(found == true); //create a RevTimestamp entry for more than PurgeNumDays days ago, //with the rev number of the 1st obj we deleted MojDbQuery query; err = query.from(_T("PurgeTest:1")); MojTestErrCheck(err); err = query.where(MojDb::IdKey, MojDbQuery::OpEq, ids[0]); MojTestErrCheck(err); err = query.includeDeleted(); MojTestErrCheck(err); MojDbCursor cursor; err = db.find(query, cursor); MojTestErrCheck(err); MojObject objFromDb; err = cursor.get(objFromDb, found); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); MojTestAssert(found == true); MojObject revFromDb; err = objFromDb.getRequired(MojDb::RevKey, revFromDb); MojTestErrCheck(err); err = MojGetCurrentTime(time); MojTestErrCheck(err); err = createRevTimestamp(db, revFromDb, time.microsecs() - (((MojInt64)35) * MojTime::UnitsPerDay)); MojTestErrCheck(err); //now purge, only id[0] should be purged count = 0; err = db.purge(count, 30); MojTestErrCheck(err); err = checkObjectsPurged(db, count, 1, 8, 6, revFromDb); MojTestErrCheck(err); //TODO 2.12.10 - this test does not pass yet, we need to fix calling delKind after a purge //err = delKindTest(db); //MojTestErrCheck(err); err = db.close(); MojTestErrCheck(err); return MojErrNone; }
MojErr MojDbTxnTest::run() { MojErr err; MojDb db; err = MojDbTestEnv::run(MojDbTestDir); MojTestErrCheck(err); // open err = db.open(MojDbTestDir, env()); MojTestErrCheck(err); // add type MojObject obj; err = obj.fromJson(MojKindStr); MojTestErrCheck(err); err = db.putKind(obj); MojTestErrCheck(err); for (int i = 0; i < 100; ++i) { MojObject obj; MojErr err = obj.putString(MojDb::KindKey, _T("Test:1")); MojTestErrCheck(err); err = obj.put(_T("foo"), (i + 25) % 100); MojTestErrCheck(err); err = obj.put(_T("bar"), i % 3); MojTestErrCheck(err); err = db.put(obj); MojTestErrCheck(err); } // db: x0 = (25, 0), (26, 1), (27, 2), (28, 0) .. x74 = (99,2), x75 = (0,0) .. x99 = (24,0) { MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("foo"), MojDbQuery::OpLessThan, 50); MojTestErrCheck(err); MojObject update; err = update.put(_T("bar"), -1); MojTestErrCheck(err); MojUInt32 count = 0; err = db.merge(query, update, count); MojTestErrCheck(err); MojTestAssert(count == 50); } // db: x0 = (25, -1) .. x24 = (49,-1), x25 = (50,1)i .. x74 = (99,2), x75 = (0,-1) .. x99 = (24, -1) // test visibility with update { MojDbReq req; // start transaction req.begin(&db, false); MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("bar"), MojDbQuery::OpEq, -1); MojTestErrCheck(err); MojObject update; err = update.put(_T("bar"), -2); MojTestErrCheck(err); MojUInt32 count = 0; err = db.merge(query, update, count, MojDb::FlagNone, req); MojTestErrCheck(err); MojTestAssert(count == 50); // txn: x0 = (25, -2) .. x24 = (49,-2), x25 = (50,1) .. x74 = (99,2), x75 = (0,-2) .. x99 = (24, -2) // visible within transaction { MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("bar"), MojDbQuery::OpEq, -2); MojTestErrCheck(err); MojObject update; err = update.put(_T("bar"), -2); MojTestErrCheck(err); MojUInt32 count = 0; err = db.merge(query, update, count, MojDb::FlagNone, req); MojTestErrCheck(err); MojTestAssert(count == 50); } // With BerkeleyDB parallel transaction is locked // invisible outside of transaction if (engineName().compare(_T("leveldb")) == 0) { MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("bar"), MojDbQuery::OpEq, -2); MojTestErrCheck(err); MojObject update; err = update.put(_T("bar"), -2); MojTestErrCheck(err); MojUInt32 count = 0; err = db.merge(query, update, count); MojTestErrCheck(err); MojTestAssert(count == 0); } } // invisible after aborted transaction { MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("bar"), MojDbQuery::OpEq, -2); MojTestErrCheck(err); MojObject update; err = update.put(_T("bar"), -2); MojTestErrCheck(err); MojUInt32 count = 0; err = db.merge(query, update, count); MojTestErrCheck(err); MojTestAssert(count == 0); } // test visibility with delete { MojDbReq req; // start transaction req.begin(&db, false); MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("bar"), MojDbQuery::OpEq, -1); MojTestErrCheck(err); MojUInt32 count = 0; err = db.del(query, count, MojDb::FlagNone, req); MojTestErrCheck(err); MojTestAssert(count == 50); // txn: x25 = (50,1) .. x74 = (99,2) // visible within transaction { MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("bar"), MojDbQuery::OpLessThan, 2); MojTestErrCheck(err); MojObject update; err = update.put(_T("bar"), -3); MojTestErrCheck(err); MojUInt32 count = 0; err = db.merge(query, update, count, MojDb::FlagNone, req); MojTestErrCheck(err); MojTestAssert(count == 33); } // With BerkeleyDB parallel transaction is locked // invisible outside of transaction if (engineName().compare(_T("leveldb")) == 0) { MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("bar"), MojDbQuery::OpLessThan, 2); MojTestErrCheck(err); MojObject update; err = update.put(_T("bar"), -3); MojTestErrCheck(err); MojUInt32 count = 0; err = db.merge(query, update, count); MojTestErrCheck(err); MojTestAssert(count == 83); } } // invisible after aborted transaction { MojDbQuery query; err = query.from(_T("Test:1")); MojTestErrCheck(err); err = query.where(_T("bar"), MojDbQuery::OpLessThan, 2); MojTestErrCheck(err); MojObject update; // Note that if we change bar=1 here we might get double-update when // record we just updated moved into records range ahead of our current // cursor position MojUInt32 count = 0; err = db.merge(query, update, count); MojTestErrCheck(err); MojTestAssert(count == 83); } err = db.close(); MojTestErrCheck(err); return MojErrNone; }
MojErr MojDbWatchTest::eqTest(MojDb& db) { MojDbQuery query; MojErr err = query.from(_T("WatchTest:1")); MojTestErrCheck(err); err = query.where(_T("foo"), MojDbQuery::OpEq, 1); MojTestErrCheck(err); MojRefCountedPtr<TestWatcher> watcher(new TestWatcher); MojTestAssert(watcher.get()); MojDbCursor cursor; err = db.find(query, cursor, watcher->m_slot); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 0); MojRefCountedPtr<TestWatcher> watcher2(new TestWatcher); MojTestAssert(watcher2.get()); bool fired = false; err = db.watch(query, cursor, watcher2->m_slot, fired); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); MojTestAssert(!fired); // puts MojObject id; err = put(db, 0, 0, id, m_rev); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 0); MojTestAssert(watcher2->m_count == 0); err = put(db, 2, 2, id, m_rev); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 0); MojTestAssert(watcher2->m_count == 0); err = put(db, 1, 1, id, m_rev); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 1); MojTestAssert(watcher2->m_count == 1); err = put(db, 1, 1, id, m_rev); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 1); MojTestAssert(watcher2->m_count == 1); // put, changing property not in index watcher.reset(new TestWatcher); MojTestAssert(watcher.get()); err = db.find(query, cursor, watcher->m_slot); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); err = put(db, 1, 2, id, m_rev); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 1); // dels watcher.reset(new TestWatcher); MojTestAssert(watcher.get()); err = db.find(query, cursor, watcher->m_slot); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); watcher2.reset(new TestWatcher); MojTestAssert(watcher2.get()); MojDbQuery queryWithRev; err = queryWithRev.from(_T("WatchTest:1")); MojTestErrCheck(err); err = queryWithRev.where(_T("foo"), MojDbQuery::OpEq, 1); MojTestErrCheck(err); err = queryWithRev.where(_T("_rev"), MojDbQuery::OpGreaterThan, m_rev); MojTestErrCheck(err); err = queryWithRev.includeDeleted(); MojTestErrCheck(err); fired = false; err = db.watch(queryWithRev, cursor, watcher2->m_slot, fired); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); MojTestAssert(!fired); MojTestAssert(watcher->m_count == 0); MojTestAssert(watcher2->m_count == 0); bool found; err = db.del(id, found); MojTestErrCheck(err); MojTestAssert(found); MojTestAssert(watcher->m_count == 1); MojTestAssert(watcher2->m_count == 1); // ordering watcher.reset(new TestWatcher); MojTestAssert(watcher.get()); err = db.find(query, cursor, watcher->m_slot); MojTestErrCheck(err); err = put(db, 1, 1, id, m_rev); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 0); err = cursor.close(); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 1); return MojErrNone; }
/* * Disabling an account means doing the following * delete com.palm.imloginstate.libpurple record * delete com.palm.immessage.libpurple records * delete com.palm.imcommand.libpurple records * delete com.palm.contact.libpurple records * delete com.palm.imbuddystatus.libpurple records * delete com.palm.imgroupchat.libpurple records -- groupchats not currently supported * Note: the ChatThreader service takes care of removing empty chats */ MojErr OnEnabledHandler::accountDisabled() { MojLogTrace(IMServiceApp::s_log); MojLogInfo(IMServiceApp::s_log, _T("accountDisabled id=%s, serviceName=%s"), m_accountId.data(), m_serviceName.data()); // delete com.palm.imloginstate.libpurple record MojDbQuery queryLoginState; queryLoginState.from(IM_LOGINSTATE_KIND); // we can get multiple calls to onEnabled(false) even after we have deleted the account records, so search by account Id so // we don't delete a new account with same username/servicename (see DFISH-16682) // queryLoginState.where(_T("serviceName"), MojDbQuery::OpEq, m_serviceName); // queryLoginState.where(_T("username"), MojDbQuery::OpEq, m_username); queryLoginState.where(_T("accountId"), MojDbQuery::OpEq, m_accountId); MojErr err = m_dbClient.del(m_deleteImLoginStateSlot, queryLoginState); if (err != MojErrNone) { MojString error; MojErrToString(err, error); MojLogError(IMServiceApp::s_log, _T("OnEnabledHandler db.del(imloginstate) failed: error %d - %s"), err, error.data()); } // delete com.palm.immessage.libpurple records //TODO: need to query both from & recipient addresses. Simplify this??? MojDbQuery queryMessage; queryMessage.from(IM_IMMESSAGE_KIND); queryMessage.where(_T("serviceName"), MojDbQuery::OpEq, m_serviceName); queryMessage.where(_T("username"), MojDbQuery::OpEq, m_username); err = m_dbClient.del(m_deleteImMessagesSlot, queryMessage); if (err != MojErrNone) { MojString error; MojErrToString(err, error); MojLogError(IMServiceApp::s_log, _T("OnEnabledHandler db.del(immessage) failed: error %d - %s"), err, error.data()); } // delete com.palm.imcommand.libpurple records MojDbQuery queryCommand; queryCommand.from(IM_IMCOMMAND_KIND); queryCommand.where(_T("serviceName"), MojDbQuery::OpEq, m_serviceName); queryCommand.where(_T("fromUsername"), MojDbQuery::OpEq, m_username); err = m_dbClient.del(m_deleteImCommandsSlot, queryCommand); if (err != MojErrNone) { MojString error; MojErrToString(err, error); MojLogError(IMServiceApp::s_log, _T("OnEnabledHandler db.del(imcommand) failed: error %d - %s"), err, error.data()); } // delete com.palm.contact.libpurple record MojDbQuery queryContact; queryContact.from(IM_CONTACT_KIND); queryContact.where(_T("accountId"), MojDbQuery::OpEq, m_accountId); err = m_dbClient.del(m_deleteContactsSlot, queryContact); if (err != MojErrNone) { MojString error; MojErrToString(err, error); MojLogError(IMServiceApp::s_log, _T("OnEnabledHandler db.del(contact) failed: error %d - %s"), err, error.data()); } // delete com.palm.imbuddystatus.libpurple record MojDbQuery queryBuddyStatus; queryBuddyStatus.from(IM_BUDDYSTATUS_KIND); queryBuddyStatus.where(_T("accountId"), MojDbQuery::OpEq, m_accountId); err = m_tempdbClient.del(m_deleteImBuddyStatusSlot, queryBuddyStatus); if (err != MojErrNone) { MojString error; MojErrToString(err, error); MojLogError(IMServiceApp::s_log, _T("OnEnabledHandler db.del(imbuddystatus) failed: error %d - %s"), err, error.data()); } // now we need to tell libpurple to disconnect the account so we don't get more messages for it // LoginCallbackInterface is null because we don't need to do any processing on the callback - all the DB kinds are already gone. LibpurpleAdapter::logout(m_serviceName, m_username, NULL); return MojErrNone; }
MojErr MojDbKind::updateSupers(const KindMap& map, const StringVec& superIds, bool updating, MojDbReq& req) { MojLogTrace(s_log); MojInt32 indexes = 0; if (updating) { KindVec addedSupers; MojErr err = diffSupers(map, superIds, m_superIds, addedSupers); MojErrCheck(err); KindVec removedSupers; err = diffSupers(map, m_superIds, superIds, removedSupers); MojErrCheck(err); // remove/add our objects from new/removed supers if (!addedSupers.empty() || !removedSupers.empty()) { MojDbQuery query; err = query.from(m_id); MojErrCheck(err); err = query.includeDeleted(true); MojErrCheck(err); MojDbCursor cursor; cursor.kindEngine(m_kindEngine); err = find(cursor, NULL, req, OpKindUpdate); MojErrCheck(err); for (;;) { MojObject obj; bool found = false; err = cursor.get(obj, found); MojErrCheck(err); if (!found) break; for (KindVec::ConstIterator i = addedSupers.begin(); i != addedSupers.end(); ++i) { err = (*i)->updateOwnIndexes(&obj, NULL, req, indexes); MojErrCheck(err); } for (KindVec::ConstIterator i = removedSupers.begin(); i != removedSupers.end(); ++i) { err = (*i)->updateOwnIndexes(NULL, &obj, req, indexes); MojErrCheck(err); } } } } // remove old supers m_superIds = superIds; MojErr err = clearSupers(); MojErrCheck(err); // look for new supers for (StringVec::ConstIterator i = m_superIds.begin(); i != m_superIds.end(); ++i) { KindMap::ConstIterator mapIter = map.find(*i); if (mapIter != map.end()) { err = addSuper(mapIter->get()); MojErrCheck(err); } } // look for kinds that extend us for (KindMap::ConstIterator i = map.begin(); i != map.end(); ++i) { if ((*i)->m_superIds.find(m_id) != MojInvalidIndex) { err = (*i)->addSuper(this); MojErrCheck(err); } } // add root kind if we have no supers if (m_supers.empty()) { KindMap::ConstIterator mapIter = map.find(MojDbKindEngine::RootKindId); if (mapIter != map.end()) { err = addSuper(mapIter->get()); MojErrCheck(err); } } return MojErrNone; }
MojErr MojDbKind::verifyIndex(MojDbIndex *pIndex, MojObject &iinfo, MojDbReq& req) { // Goes throudh each index entry and verifies that it points to a valid object // For debugging purposes as stats for indexes does not access the target objects // Index->stats function does not have enough context to find the object // db/stats usage '{"verify":true,"kind":"xyz"}' - each optional MojDbQuery query; MojErr err = query.from(m_id); MojErrCheck(err); err = query.includeDeleted(true); MojErrCheck(err); MojDbCursor cursor; query.m_forceIndex = pIndex; // Important: Otherwise, it will pick the default index cursor.verifymode(true); // to get the errors err = m_kindEngine->find(query, cursor, NULL, req, OpRead); MojLogInfo(s_log, _T("Kind_verifyIndex: Kind: %s; Index: %s; idIndex: %zX; size: %zu; CursorIndex: %s \n"), m_name.data(), pIndex->name().data(), pIndex->idIndex(), pIndex->size(), cursor.m_dbIndex->name().data()); MojErrCheck(err); MojSize count = 0; MojSize delCount = 0; MojSize warnCount = 0; char s[1024]; for (;;) { MojDbStorageItem* item = NULL; bool found = false; err = cursor.get(item, found); if (err == MojErrInternalIndexOnFind) { warnCount++; MojDbIsamQuery *iquery = (MojDbIsamQuery *)cursor.m_storageQuery.get(); MojErr err2 = MojByteArrayToHex(iquery->m_keyData, iquery->m_keySize, s); MojErrCheck(err2); MojChar *ids = (iquery->m_keySize > 18) ? (MojChar *)(iquery->m_keyData + iquery->m_keySize - 17) : NULL; MojLogInfo(s_log, _T("VerifyIndex Warning: %s; KeySize: %zu; %s ;id: %s \n"), cursor.m_dbIndex->name().data(), iquery->m_keySize, s, ids); continue; } MojErrCheck(err); if (!found) break; MojObject obj; err = item->toObject(obj, *m_kindEngine, true); MojErrCheck(err); bool deleted = false; if (obj.get(MojDb::DelKey, deleted) && deleted) { delCount++; } else { count++; } } MojLogInfo(s_log, _T("Kind_verifyIndex Counts: Kind: %s; Index: %s; count: %zu; delcount: %zu; warnings: %zu \n"), m_name.data(), pIndex->name().data(), count, delCount, warnCount); err = iinfo.put(VerifyCountKey, (MojInt64)count); MojErrCheck(err); err = iinfo.put(VerifyWarnCountKey, (MojInt64) warnCount); MojErrCheck(err); err = iinfo.put(VerifyDelCountKey, (MojInt64) delCount); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::purgeImpl(MojObject& obj, MojUInt32& countOut, MojDbReq& req) { MojLogTrace(s_log); MojObject val; MojErr err = obj.getRequired(RevNumKey, val); MojErrCheck(err); MojObject timestamp; err = obj.getRequired(TimestampKey, timestamp); MojErrCheck(err); // purge all objects that were deleted on or prior to this rev num // query for objects in two passes - once where backup is true and once where backup is false MojDbQuery objQuery; err = objQuery.from(MojDbKindEngine::RootKindId); MojErrCheck(err); err = objQuery.where(DelKey, MojDbQuery::OpEq, true); MojErrCheck(err); err = objQuery.where(SyncKey, MojDbQuery::OpEq, true); MojErrCheck(err); err = objQuery.where(RevKey, MojDbQuery::OpLessThanEq, val); MojErrCheck(err); MojUInt32 backupCount = 0; req.autobatch(true); req.fixmode(true); objQuery.limit(AutoBatchSize); err = delImpl(objQuery, backupCount, req, FlagPurge); MojErrCheck(err); MojDbQuery objQuery2; err = objQuery2.from(MojDbKindEngine::RootKindId); MojErrCheck(err); err = objQuery2.where(DelKey, MojDbQuery::OpEq, true); MojErrCheck(err); err = objQuery2.where(SyncKey, MojDbQuery::OpEq, false); MojErrCheck(err); err = objQuery2.where(RevKey, MojDbQuery::OpLessThanEq, val); MojErrCheck(err); MojUInt32 count = 0; MojUInt32 batchRemain = 0; if (backupCount <= AutoBatchSize) batchRemain = AutoBatchSize - backupCount; req.autobatch(true); // enable auto batch req.fixmode(true); // force deletion of bad entries if (batchRemain > 0) { objQuery2.limit(batchRemain); err = delImpl(objQuery2, count, req, FlagPurge); MojErrCheck(err); } countOut = count + backupCount; req.autobatch(false); // if we actually deleted objects, store this rev num as the last purge rev if (countOut > 0) { err = updateState(LastPurgedRevKey, val, req); MojErrCheck(err); } // delete all the RevTimestamp entries prior to this one MojDbQuery revTimestampQuery; err = revTimestampQuery.from(MojDbKindEngine::RevTimestampId); MojErrCheck(err); err = revTimestampQuery.where(TimestampKey, MojDbQuery::OpLessThanEq, timestamp); MojErrCheck(err); count = 0; err = delImpl(revTimestampQuery, count, req, FlagPurge); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::dumpImpl(MojFile& file, bool backup, bool incDel, const MojObject& revParam, const MojObject& delRevParam, bool skipKinds, MojUInt32& countOut, MojDbReq& req, MojObject* response, const MojChar* keyName, MojSize& bytesWritten, MojSize& warns, MojUInt32 maxBytes) { // query for objects, adding the backup key and rev key if necessary MojDbQuery query; MojErr err = query.from(MojDbKindEngine::RootKindId); MojErrCheck(err); err = query.where(MojDb::DelKey, MojDbQuery::OpEq, incDel); MojErrCheck(err); if (backup) { err = query.where(MojDb::SyncKey, MojDbQuery::OpEq, true); MojErrCheck(err); if (incDel) { err = query.where(MojDb::RevKey, MojDbQuery::OpGreaterThan, delRevParam); MojErrCheck(err); } else { err = query.where(MojDb::RevKey, MojDbQuery::OpGreaterThan, revParam); MojErrCheck(err); } } MojDbCursor cursor; err = findImpl(query, cursor, NULL, req, OpRead); MojErrCheck(err); warns = 0; MojObject curRev; for(;;) { bool found = false; MojObject obj; err = cursor.get(obj, found); // So that we can get as much data as possible from a corrupt database // We simply skip ghost keys and continue if (err == MojErrInternalIndexOnFind) { warns++; continue; } MojErrCheck(err); if (!found) break; if (skipKinds) { MojString kind; err = obj.getRequired(KindKey, kind); MojErrCheck(err); if (kind == MojDbKindEngine::KindKindId) { continue; } } // write out each object, if the backup is full, insert the appropriate incremental key err = dumpObj(file, obj, bytesWritten, maxBytes); MojErrCatch(err, MojErrDbBackupFull) { if (response) { MojErr errBackup = MojErrNone; if (!curRev.undefined()) { errBackup = insertIncrementalKey(*response, keyName, curRev); MojErrCheck(errBackup); } else { errBackup = insertIncrementalKey(*response, keyName, incDel ? delRevParam: revParam); MojErrCheck(errBackup); } errBackup = handleBackupFull(revParam, delRevParam, *response, keyName); MojErrCheck(errBackup); } return MojErrNone; } MojErrCheck(err); err = obj.getRequired(MojDb::RevKey, curRev); MojErrCheck(err); countOut++; } err = cursor.close(); MojErrCheck(err); if (warns > 0) { MojLogWarning(s_log, _T("Finished Backup with %d warnings \n"), (int)warns); } else { MojLogDebug(s_log, _T("Finished Backup with no warnings \n")); } // construct the next incremental key if (response && !curRev.undefined()) { err = insertIncrementalKey(*response, keyName, curRev); MojErrCheck(err); } return MojErrNone; }
MojErr MojDb::dump(const MojChar* path, MojUInt32& countOut, bool incDel, MojDbReqRef req, bool backup, MojUInt32 maxBytes, const MojObject* incrementalKey, MojObject* backupResponse) { MojAssert(path); MojLogTrace(s_log); MojErr err = beginReq(req); MojErrCheck(err); if (!req->admin()) { MojLogError(s_log, _T("access denied: '%s' cannot dump db to path: '%s'"), req->domain().data(), path); MojErrThrow(MojErrDbAccessDenied); } MojFile file; err = file.open(path, MOJ_O_WRONLY | MOJ_O_CREAT | MOJ_O_TRUNC, MOJ_S_IRUSR | MOJ_S_IWUSR); MojErrCheck(err); // write out kinds first, then existing objects, then deleted objects MojSize bytesWritten = 0; MojSize totalwarns = 0; MojSize newwarns = 0; MojDbQuery objQuery; MojVector<MojObject> kindVec; MojObject revParam = -1; MojObject delRevParam = -1; // if we were given an incremental key, pull out the revs now if (incrementalKey) { incrementalKey->get(MojDbServiceDefs::RevKey, revParam); incrementalKey->get(MojDbServiceDefs::DeletedRevKey, delRevParam); } err = m_kindEngine.getKinds(kindVec); MojErrCheck(err); // write kinds - if incremental, only write the kinds that have changed since the respective revs MojString countStr; for (MojVector<MojObject>::ConstIterator i = kindVec.begin(); i != kindVec.end(); ++i) { if (backup) { bool backupKind = false; i->get(MojDbKind::SyncKey, backupKind); if (!backupKind) continue; MojString id; err = i->getRequired(MojDbServiceDefs::IdKey, id); MojErrCheck(err); MojDbQuery countQuery; err = countQuery.from(id); MojErrCheck(err); MojDbCursor cursor; err = find(countQuery, cursor, req); MojErrCheck(err); MojUInt32 count = 0; err = cursor.count(count); MojErrCheck(err); if (count > 0) { if (i != kindVec.begin()) { err = countStr.appendFormat(_T(", ")); MojErrCheck(err); } err = countStr.appendFormat("%s=%u", id.data(), count); MojErrCheck(err); } } bool deleted = false; i->get(DelKey, deleted); MojObject kindRev; err = i->getRequired(RevKey, kindRev); MojErrCheck(err); if ((deleted && kindRev > delRevParam) || (!deleted && kindRev > revParam)) { err = dumpObj(file, (*i), bytesWritten, maxBytes); MojErrCheck(err); countOut++; } } // dump all the non-deleted objects err = dumpImpl(file, backup, false, revParam, delRevParam, true, countOut, req, backupResponse, MojDbServiceDefs::RevKey, bytesWritten, newwarns, maxBytes); MojErrCheck(err); totalwarns += newwarns; // If we're supposed to include deleted objects, dump the deleted objects now. // There's a chance that we may have run out of space in our backup. If that's the case, // we don't want to try to dump deleted objects - we can detect this by looking for the HasMoreKey if (incDel && backupResponse && !backupResponse->contains(MojDbServiceDefs::HasMoreKey)) { err = dumpImpl(file, backup, true, revParam, delRevParam, false, countOut, req, backupResponse, MojDbServiceDefs::DeletedRevKey, bytesWritten, newwarns, maxBytes); MojErrCheck(err); } totalwarns += newwarns; // Add the Full and Version keys if (backup && backupResponse) { bool incremental = (incrementalKey != NULL); err = backupResponse->putBool(MojDbServiceDefs::FullKey, !incremental); MojErrCheck(err); err = backupResponse->put(MojDbServiceDefs::VersionKey, DatabaseVersion); MojErrCheck(err); err = backupResponse->put(MojDbServiceDefs::WarningsKey, (MojInt32)totalwarns); MojErrCheck(err); MojString description; err = description.format(_T("incremental=%u"), countOut); MojErrCheck(err); if (!countStr.empty()) { err = description.appendFormat(_T(", %s"), countStr.data()); MojErrCheck(err); } err = backupResponse->put(_T("description"), description); MojErrCheck(err); } err = req->end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::purge(MojUInt32& countOut, MojInt64 numDays, MojDbReqRef req) { MojLogTrace(s_log); countOut = 0; if (numDays <= -1) { numDays = m_purgeWindow; } MojErr err = beginReq(req); MojErrCheck(err); MojLogDebug(s_log, _T("purging objects deleted more than %lld days ago..."), numDays); MojTime time; err = MojGetCurrentTime(time); MojErrCheck(err); // store the revision number to current timestamp mapping MojObject revTimeMapping; MojInt64 rev; err = nextId(rev); MojErrCheck(err); err = revTimeMapping.put(RevNumKey, rev); MojErrCheck(err); err = revTimeMapping.put(TimestampKey, time.microsecs()); MojErrCheck(err); err = revTimeMapping.putString(KindKey, MojDbKindEngine::RevTimestampId); MojErrCheck(err); err = putImpl(revTimeMapping, MojDb::FlagNone, req); MojErrCheck(err); // find the revision number for numDays prior to now MojInt64 purgeTime = time.microsecs() - (MojTime::UnitsPerDay * numDays); MojDbQuery query; err = query.from(MojDbKindEngine::RevTimestampId); MojErrCheck(err); query.limit(1); err = query.where(TimestampKey, MojDbQuery::OpLessThanEq, purgeTime); MojErrCheck(err); err = query.order(TimestampKey); MojErrCheck(err); query.desc(true); MojDbCursor cursor; err = findImpl(query, cursor, NULL, req, OpDelete); MojErrCheck(err); bool found = false; MojObject obj; err = cursor.get(obj, found); MojErrCheck(err); err = cursor.close(); MojErrCheck(err); MojUInt32 batchCount = 0; MojUInt32 totalCount = 0; while ((found)) { // Do it in AutoBatchSize batches batchCount = 0; req->fixmode(true); // purge even if index mis-matches err = purgeImpl(obj, batchCount, req); MojLogDebug(s_log, _T("purge batch processed: batch: %d; total: %d; err = %d\n"), batchCount, (totalCount + batchCount), err); MojErrCheck(err); totalCount += batchCount; countOut = totalCount; if (batchCount < AutoBatchSize) // last batch break; err = commitBatch(req); MojErrCheck(err); continue; } // end request err = req->end(); MojErrCheck(err); MojLogDebug(s_log, _T("purged %d objects"), countOut); return MojErrNone; }
void SyncStateUpdater::UpdateSyncState(MojSignal<>::SlotRef slot, const SyncState& syncState, bool clearSyncState) { m_doneSignal.connect(slot); MojErr err; MojObject batchPayload; MojObject batchOperations; // Delete old sync state MojObject delPayload; MojDbQuery query; err = query.from(SyncStateAdapter::SYNC_STATE_KIND); ErrorToException(err); MojString capabilityProvider; capabilityProvider.assign(m_capabilityProvider.c_str()); err = query.where(SyncStateAdapter::CAPABILITY_PROVIDER, MojDbQuery::OpEq, capabilityProvider); ErrorToException(err); err = query.where(SyncStateAdapter::ACCOUNT_ID, MojDbQuery::OpEq, syncState.GetAccountId()); ErrorToException(err); err = query.where(SyncStateAdapter::COLLECTION_ID, MojDbQuery::OpEq, syncState.GetCollectionId()); ErrorToException(err); MojObject queryObj; err = query.toObject(queryObj); ErrorToException(err); err = delPayload.put("query", queryObj); ErrorToException(err); MojObject delOperation; err = delOperation.putString("method", "del"); ErrorToException(err); err = delOperation.put("params", delPayload); ErrorToException(err); err = batchOperations.push(delOperation); ErrorToException(err); if(!clearSyncState) { // Store new sync state MojObject putPayload; MojObject syncStateObj; SyncStateAdapter::SerializeToDatabaseObject(syncState, syncStateObj, m_capabilityProvider.c_str(), m_busAddress.c_str()); MojObject objects; err = objects.push(syncStateObj); ErrorToException(err); err = putPayload.put("objects", objects); ErrorToException(err); MojObject putOperation; err = putOperation.putString("method", "put"); ErrorToException(err); err = putOperation.put("params", putPayload); ErrorToException(err); err = batchOperations.push(putOperation); ErrorToException(err); } err = batchPayload.put("operations", batchOperations); ErrorToException(err); // Note: batch operations are not atomic, this is just for convenience and performance m_busClient.SendRequest(m_updateSlot, "com.palm.tempdb", "batch", batchPayload); }
MojErr MojDbKind::stats(MojObject& objOut, MojSize& usageOut, MojDbReq& req, bool verify) { MojLogTrace(s_log); #if defined(TESTDBKIND) MojLogInfo(s_log, _T("Subkinds for - %s ; count = %d\n"), m_id.data(), m_subs.size()); int n = 0; for (KindVec::ConstIterator i = m_subs.begin(); i != m_subs.end(); ++i) { MojLogInfo(s_log, _T("SubKind %d: %s"), n++, (*i)->id().data()); } MojLogInfo(s_log, _T("Supers for - %s ; count = %d\n"), m_id.data(), m_supers.size()); n = 0; for (KindVec::ConstIterator i = m_supers.begin(); i != m_supers.end(); ++i) { MojLogInfo(s_log, _T("Super %d: %s"), n++, (*i)->id().data()); } #endif // analyze objects MojDbQuery query; MojErr err = query.from(m_id); MojErrCheck(err); err = query.includeDeleted(true); MojErrCheck(err); MojDbCursor cursor; err = m_kindEngine->find(query, cursor, NULL, req, OpRead); MojLogInfo(s_log, _T("KindStats start: %s ; Indexes = %zu; Using Index: %s; \n"), m_id.data(), m_indexes.size(), cursor.m_dbIndex->name().data()); MojErrCheck(err); MojSize count = 0; MojSize size = 0; MojSize delCount = 0; MojSize delSize = 0; MojSize warnings = 0; for (;;) { MojDbStorageItem* item = NULL; bool found = false; cursor.verifymode(true); err = cursor.get(item, found); if (err == MojErrInternalIndexOnFind) { warnings++; continue; } if (err != MojErrNone) // for all other errors break and dump current stats break; if (!found) break; MojObject obj; err = item->toObject(obj, *m_kindEngine, true); if (err != MojErrNone) break; bool deleted = false; if (obj.get(MojDb::DelKey, deleted) && deleted) { delSize += item->size(); delCount++; } else { size += item->size(); count++; } } MojLogInfo(s_log, _T("KindStats Summary: %s : Count: %zu; delCount: %zu; warnings: %zu \n"), m_id.data(), count, delCount, warnings); usageOut += size + delSize; MojObject info; err = info.put(SizeKey, (MojInt64) size); MojErrCheck(err); err = info.put(CountKey, (MojInt64) count); MojErrCheck(err); if (delCount > 0) { err = info.put(DelSizeKey, (MojInt64) delSize); MojErrCheck(err); err = info.put(DelCountKey, (MojInt64) delCount); MojErrCheck(err); } if (warnings > 0) { err = info.put(WarnKey, (MojInt64) warnings); MojErrCheck(err); } err = objOut.put(ObjectsKey, info); MojErrCheck(err); // and indexes MojObject indexes; for (IndexVec::ConstIterator i = m_indexes.begin(); i != m_indexes.end(); ++i) { MojObject indexInfo; err = (*i)->stats(indexInfo, usageOut, req); MojErrCheck(err); if (verify) { MojDbIndex *pi = i->get(); MojErr err2 = verifyIndex(pi, indexInfo, req); MojErrCheck(err2); } err = indexes.put((*i)->name(), indexInfo); MojErrCheck(err); } err = objOut.put(IndexesKey, indexes); MojErrCheck(err); return MojErrNone; }
MojErr MojDbSearchTest::simpleTest(MojDb& db) { // just foo MojErr err = check(db, _T("bogus"), _T("[]")); MojTestErrCheck(err); err = check(db, _T("f"), _T("[1,2,3,4]")); MojTestErrCheck(err); err = check(db, _T("F"), _T("[1,2,3,4]")); MojTestErrCheck(err); err = check(db, _T("fo"), _T("[1,2,3,4]")); MojTestErrCheck(err); err = check(db, _T("four"), _T("[1,3,4]")); MojTestErrCheck(err); err = check(db, _T("score"), _T("[1]")); MojTestErrCheck(err); err = check(db, _T("four years"), _T("[1]")); MojTestErrCheck(err); err = check(db, _T("Four Years"), _T("[1]")); MojTestErrCheck(err); err = check(db, _T("four decades"), _T("[]")); MojTestErrCheck(err); err = check(db, _T("fathers forth"), _T("[2,3]")); MojTestErrCheck(err); err = check(db, _T("four f"), _T("[1,3,4]")); MojTestErrCheck(err); err = check(db, _T("four f fo fou"), _T("[1,3,4]")); MojTestErrCheck(err); // bar and foo err = check(db, _T("f"), _T("[3]"), NULL, 2); MojTestErrCheck(err); // order by bar err = check(db, _T("f"), _T("[4,3,2,1]"), _T("bar")); MojTestErrCheck(err); err = check(db, _T("f"), _T("[1,2,3,4]"), _T("bar"), MojObject::Undefined, true); MojTestErrCheck(err); err = check(db, _T("f"), _T("[4,3,1,2]"), _T("foo")); MojTestErrCheck(err); // array value for bar MojObject array; err = array.push(1); MojErrCheck(err); err = array.push(2); MojErrCheck(err); err = check(db, _T("f"), _T("[4,3]"), _T("bar"), array); MojTestErrCheck(err); // limit MojDbQuery query; err = initQuery(query, _T("f")); MojTestErrCheck(err); query.limit(2); err = check(db, query, _T("[1,2]")); MojTestErrCheck(err); err = initQuery(query, _T("f"), _T("bar")); MojTestErrCheck(err); query.limit(2); err = check(db, query, _T("[4,3]")); MojTestErrCheck(err); // accent insensitivity err = check(db, _T("COTE"), _T("[5,6,7,8,9]")); MojTestErrCheck(err); // case-insensitive ordering err = check(db, _T("a"), _T("[12,13,11]"), _T("foo")); MojTestErrCheck(err); // multi-prop query.clear(); err = query.from(_T("SearchTest:1")); MojTestErrCheck(err); MojString val; err = val.assign(_T("test")); MojTestErrCheck(err); err = query.where(_T("multiprop"), MojDbQuery::OpSearch, val, MojDbCollationPrimary); MojTestErrCheck(err); err = check(db, query, _T("[14,15]")); MojTestErrCheck(err); return MojErrNone; }
MojErr MojDbServiceHandler::findImpl(MojServiceMessage* msg, MojObject& payload, MojDbReq& req, MojDbCursor& cursor, bool doCount) { MojAssert(msg); MojLogTrace(s_log); MojObject queryObj; MojErr err = payload.getRequired(MojDbServiceDefs::QueryKey, queryObj); MojErrCheck(err); bool doWatch = false; payload.get(MojDbServiceDefs::WatchKey, doWatch); MojDbQuery query; err = query.fromObject(queryObj); MojErrCheck(err); MojUInt32 limit = query.limit(); if (limit == MojDbQuery::LimitDefault){ query.limit(MaxQueryLimit); } else if (limit > MaxQueryLimit) { MojErrThrowMsg(MojErrDbInvalidQuery, _T("db: limit greater than %d not allowed"), MaxQueryLimit); } if (doWatch) { MojRefCountedPtr<Watcher> watcher(new Watcher(msg)); MojAllocCheck(watcher.get()); err = m_db.find(query, cursor, watcher->m_watchSlot, req); MojErrCheck(err); } else { err = m_db.find(query, cursor, req); MojErrCheck(err); } // append results MojObjectVisitor& writer = msg->writer(); err = writer.beginObject(); MojErrCheck(err); err = writer.boolProp(MojServiceMessage::ReturnValueKey, true); MojErrCheck(err); err = writer.propName(MojDbServiceDefs::ResultsKey); MojErrCheck(err); err = writer.beginArray(); MojErrCheck(err); err = cursor.visit(writer); MojErrCheck(err); err = writer.endArray(); MojErrCheck(err); // append next page MojDbQuery::Page page; err = cursor.nextPage(page); MojErrCheck(err); if (!page.empty()) { MojObject pageObj; err = page.toObject(pageObj); MojErrCheck(err); err = writer.objectProp(MojDbServiceDefs::NextKey, pageObj); MojErrCheck(err); } // append count if (doCount) { MojUInt32 count = 0; err = cursor.count(count); MojErrCheck(err); err = writer.intProp(MojDbServiceDefs::CountKey, (MojInt64) count); MojErrCheck(err); } err = writer.endObject(); MojErrCheck(err); if (doWatch) { // if this is a watched query, it cannot be part of a batch so it's safe to reply here err = msg->reply(); MojErrCheck(err); } // notifications can fire any time after the cursor is closed, // so don't close it until after sending the reply. err = cursor.close(); MojErrCheck(err); return MojErrNone; }
MojErr MojDbPerfUpdateTest::updateObjsViaMerge(MojDb& db, const MojChar* kindId, MojErr (MojDbPerfTest::*createFn) (MojObject&, MojUInt64)) { // register all the kinds MojTime time; MojErr err = putKinds(db, time); MojTestErrCheck(err); // put objects using createFn MojObject objs; err = putObjs(db, kindId, numInsertForPut, createFn, objs); MojTestErrCheck(err); MojObject midObj; bool found = objs.at(numInsertForPut/2, midObj); MojTestAssert(found); MojTime objTime; err = mergeObj(db, midObj, objTime); MojTestErrCheck(err); MojUInt64 mergeTime = objTime.microsecs(); err = MojPrintF("\n -------------------- \n"); MojTestErrCheck(err); err = MojPrintF(" merging single object - index %llu - of kind %s %llu times took: %llu microsecs\n", numInsertForPut/2, kindId, numMergeIterations, mergeTime); MojTestErrCheck(err); err = MojPrintF(" time per merge: %llu microsecs", (mergeTime) / (numMergeIterations)); MojTestErrCheck(err); err = MojPrintF("\n\n"); MojTestErrCheck(err); MojString buf; err = buf.format("Merge single object - index %llu - %llu times,%s,%llu,%llu,%llu,\n", numInsertForPut/2, numMergeIterations, kindId, mergeTime, mergeTime/numMergeIterations, mergeTime/(1*numMergeIterations)); MojTestErrCheck(err); err = fileWrite(file, buf); MojTestErrCheck(err); MojTime batchTime; MojObject::ArrayIterator beginArr; err = objs.arrayBegin(beginArr); MojErrCheck(err); err = batchMergeObj(db, beginArr, beginArr + (numInsertForPut / 10), batchTime); MojTestErrCheck(err); mergeTime = batchTime.microsecs(); err = MojPrintF(" merging batch - %llu objects - of kind %s %llu times took: %llu microsecs\n", numInsertForPut/10, kindId, numBatchMergeIterations, mergeTime); MojTestErrCheck(err); err = MojPrintF(" time per batch merge: %llu microsecs\n", (mergeTime) / (numBatchMergeIterations)); MojTestErrCheck(err); err = MojPrintF(" time per object: %llu microsecs", (mergeTime) / (numInsertForPut/10 * numBatchMergeIterations)); MojTestErrCheck(err); err = MojPrintF("\n\n"); MojTestErrCheck(err); err = buf.format("Batch merge %llu objects %llu times,%s,%llu,%llu,%llu,\n", numInsertForPut/10, numBatchMergeIterations, kindId, mergeTime, mergeTime/numBatchMergeIterations, mergeTime/(numInsertForPut/10*numBatchMergeIterations)); MojTestErrCheck(err); err = fileWrite(file, buf); MojTestErrCheck(err); MojTime mergeQueryTime; MojTestErrCheck(err); MojDbQuery query; err = query.from(kindId); MojTestErrCheck(err); query.limit(numInsertForPut/5); query.desc(true); MojObject queryObj; err = query.toObject(queryObj); MojTestErrCheck(err); MojString queryStr; err = queryObj.stringValue(queryStr); MojTestErrCheck(err); MojObject props; err = props.putString(_T("newKey"), _T("here's a new value")); MojTestErrCheck(err); MojUInt32 count; err = queryMergeObj(db, query, props, count, mergeQueryTime); MojTestErrCheck(err); mergeTime = mergeQueryTime.microsecs(); err = MojPrintF(" merging with query - %d objects - of kind %s %llu times took: %llu microsecs\n", count, kindId, numBatchMergeIterations, mergeTime); MojTestErrCheck(err); err = MojPrintF(" time per merge: %llu microsecs\n", (mergeTime) / (numBatchMergeIterations)); MojTestErrCheck(err); err = MojPrintF(" time per object: %llu microsecs", (mergeTime) / (count * numBatchMergeIterations)); MojTestErrCheck(err); err = MojPrintF("\n\n"); MojTestErrCheck(err); err = buf.format("Merge with query %s,,,,,\nMerge with query - %d objects - %llu times,%s,%llu,%llu,%llu,\n", queryStr.data(), count, numBatchMergeIterations, kindId, mergeTime, mergeTime/numBatchMergeIterations, mergeTime/(count*numBatchMergeIterations)); MojTestErrCheck(err); err = fileWrite(file, buf); MojTestErrCheck(err); return MojErrNone; }
MojErr MojDb::delKind(const MojObject& id, bool& foundOut, MojUInt32 flags, MojDbReqRef req) { MojLogTrace(s_log); foundOut = false; MojErr err = beginReq(req, true); MojErrCheck(err); // del kind obj MojString idStr; err = id.stringValue(idStr); MojErrCheck(err); MojDbKind *pk = NULL; // If Kinds has sub-kinds, we give an error err = m_kindEngine.getKind(idStr.data(), pk); MojErrCheck(err); if (pk->nsubkinds() > 0) { MojLogWarning(s_log, _T("delKind_error: %s has %d subkinds \n"), idStr.data(), pk->nsubkinds()); MojErrThrow(MojErrDbKindHasSubKinds); } //MojLogInfo(s_log, _T("delKind: %s \n"), idStr.data()); MojLogWarning(s_log, _T("delKind: %s \n"), idStr.data()); err = m_kindEngine.checkOwnerPermission(idStr, req); MojErrCheck(err); MojString dbId; err = formatKindId(idStr, dbId); MojErrCheck(err); MojObject deleted; bool found = false; MojDbAdminGuard adminGuard(req); err = delImpl(dbId, found, deleted, req, flags); MojErrCheck(err); if (found) { // del objects MojDbQuery query; err = query.from(idStr); MojErrCheck(err); err = query.includeDeleted(true); MojErrCheck(err); MojUInt32 count; req->fixmode(true); err = delImpl(query, count, req, flags | FlagPurge); MojErrCheck(err); // del associated permissions query.clear(); err = query.from(MojDbKindEngine::PermissionId); MojErrCheck(err); err = query.where(MojDbServiceDefs::ObjectKey, MojDbQuery::OpEq, idStr); MojErrCheck(err); req->fixmode(true); err = delImpl(query, count, req, flags); MojErrCheck(err); // del kind MojErr errAcc = m_kindEngine.delKind(idStr, req); MojErrAccumulate(err, errAcc); } err = commitKind(idStr, req, err); MojErrCheck(err); foundOut = found; return MojErrNone; }
void MojoDatabase::GetEmailSyncList(Signal::SlotRef slot, const MojObject& folderId, const MojInt64& rev, bool desc, MojDbQuery::Page& page, MojInt32 limit) { MojErr err; MojDbQuery query; err = query.from(PopEmailAdapter::POP_EMAIL_KIND);//EmailSchema::Kind::EMAIL); ErrorToException(err); // Select relevant fields err = query.select(PopEmailAdapter::ID); ErrorToException(err); err = query.select(PopEmailAdapter::SERVER_UID); ErrorToException(err); err = query.select(EmailSchema::TIMESTAMP); ErrorToException(err); err = query.where(EmailSchema::FOLDER_ID, MojDbQuery::OpEq, folderId); ErrorToException(err); if (rev > 0) { err = query.where(PopFolderAdapter::LAST_SYNC_REV, MojDbQuery::OpGreaterThan, rev); ErrorToException(err); } query.page(page); // Sort by timestamp descending err = query.order(EmailSchema::TIMESTAMP); ErrorToException(err); query.desc(desc); // Set limit if(limit > 0) { query.limit(limit); } slot.cancel(); // cancel existing slot in case we're in a callback err = m_dbClient.find(slot, query); ErrorToException(err); }
MojErr MojDb::delImpl(const MojDbQuery& quer, MojUInt32& countOut, MojDbReq& req, MojUInt32 flags) { MojLogTrace(s_log); countOut = 0; MojInt32 warns = 0; MojDbQuery newQuery = quer; MojUInt32 queryLimit = newQuery.limit(); if(newQuery.limit() == MojDbQuery::LimitDefault) newQuery.limit(AutoBatchSize); while(queryLimit > 0) { MojDbCursor cursor; MojErr err = findImpl(newQuery, cursor, NULL, req, OpDelete); MojErrCheck(err); MojAssert(cursor.txn()); MojUInt32 count = 0; MojUInt32 numberInBatch = 0; bool found = false; MojObject obj; for (;;) { MojDbStorageItem* item = NULL; err = cursor.get(item, found); // We simply skip ghost keys and continue; A warning is already written to the system log earlier if (err == MojErrInternalIndexOnFind) { warns++; numberInBatch++; continue; } MojErrCheck(err); if (!found) break; err = item->toObject(obj, m_kindEngine); MojErrCheck(err); const MojObject& id = item->id(); MojObject deleted; err = delObj(id, obj, item, deleted, req, flags); MojErrCheck(err); ++count; numberInBatch++; } if (warns > 0) MojLogInfo(s_log, _T("delquery index_warnings: %s, count: %d\n"), newQuery.from().data(), warns); countOut += count; err = cursor.close(); MojErrCheck(err); if(numberInBatch >= AutoBatchSize) // sing > - just in case something messed up { err = commitBatch(req); MojErrCheck(err); } if(count == 0) break; queryLimit -= newQuery.limit(); if(queryLimit > AutoBatchSize) newQuery.limit(AutoBatchSize); else newQuery.limit(queryLimit); } return MojErrNone; }
MojErr MojDbPermissionTest::checkPermissions(MojDb& db) { MojDbReq reqOwner(false); MojErr err = reqOwner.domain(_T("com.foo")); MojTestErrCheck(err); MojDbReq reqFooBar(false); err = reqFooBar.domain(_T("com.foo.bar")); MojTestErrCheck(err); MojDbReq reqFooBarBad(false); err = reqFooBarBad.domain(_T("com.foobar")); MojTestErrCheck(err); MojDbReq reqGranted(false); err = reqGranted.domain("com.granted"); MojTestErrCheck(err); MojDbReq reqEmpty(false); MojDbReq reqBad(false); err = reqBad.domain("com.bad"); MojTestErrCheck(err); MojObject objNoId; err = objNoId.putString(MojDb::KindKey, _T("PermissionTest:1")); MojTestErrCheck(err); MojObject obj2NoId; err = obj2NoId.putString(MojDb::KindKey, _T("PermissionTest2:1")); MojTestErrCheck(err); // put new obj MojObject obj1 = objNoId; err = db.put(obj1, MojDb::FlagNone, reqOwner); MojTestErrCheck(err); MojObject obj2 = objNoId; err = db.put(obj2, MojDb::FlagNone, reqGranted); MojTestErrCheck(err); MojObject obj3 = objNoId; err = db.put(obj3, MojDb::FlagNone, reqBad); MojTestErrExpected(err, MojErrDbPermissionDenied); obj3 = objNoId; err = db.put(obj3, MojDb::FlagNone, reqFooBarBad); MojTestErrExpected(err, MojErrDbPermissionDenied); obj3 = objNoId; err = db.put(obj3, MojDb::FlagNone, reqFooBar); MojTestErrCheck(err); err = db.put(obj2NoId, MojDb::FlagNone, reqEmpty); MojTestErrCheck(err); // put existing obj err = obj1.put(_T("foo"), 1); MojTestErrCheck(err); err = db.put(obj1, MojDb::FlagNone, reqOwner); MojTestErrCheck(err); err = obj2.put(_T("foo"), 2); MojTestErrCheck(err); err = db.put(obj2, MojDb::FlagNone, reqGranted); MojTestErrCheck(err); err = obj2.put(_T("foo"), 3); MojTestErrCheck(err); err = db.put(obj2, MojDb::FlagNone, reqBad); MojTestErrExpected(err, MojErrDbPermissionDenied); // find MojDbQuery query; err = query.from(_T("PermissionTest:1")); MojTestErrCheck(err); MojDbCursor cursor; err = db.find(query, cursor, reqOwner); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); err = db.find(query, cursor, reqGranted); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); err = db.find(query, cursor, reqBad); MojTestErrExpected(err, MojErrDbPermissionDenied); err = cursor.close(); MojTestErrCheck(err); // get MojObject id1; MojTestAssert(obj1.get(MojDb::IdKey, id1)); bool found = false; MojObject gotten; err = db.get(id1, gotten, found, reqOwner); MojTestErrCheck(err); err = db.get(id1, gotten, found, reqGranted); MojTestErrCheck(err); err = db.get(id1, gotten, found, reqBad); MojTestErrExpected(err, MojErrDbPermissionDenied); // del by id err = db.del(id1, found, MojDb::FlagNone, reqBad); MojTestErrExpected(err, MojErrDbPermissionDenied); MojTestAssert(!found); err = db.del(id1, found, MojDb::FlagNone, reqOwner); MojTestErrCheck(err); MojTestAssert(found); MojObject id2; MojTestAssert(obj2.get(MojDb::IdKey, id2)); err = db.del(id2, found, MojDb::FlagNone, reqGranted); MojTestErrCheck(err); MojTestAssert(found); // del query MojUInt32 count = 0; err = db.del(query, count, MojDb::FlagNone, reqOwner); MojTestErrCheck(err); err = db.del(query, count, MojDb::FlagNone, reqGranted); MojTestErrCheck(err); err = db.del(query, count, MojDb::FlagNone, reqBad); MojTestErrExpected(err, MojErrDbPermissionDenied); return MojErrNone; }
bool MojDbIndex::canAnswer(const MojDbQuery& query) const { LOG_TRACE("Entering function %s", __FUNCTION__); MojAssert(isOpen()); MojAssert(!m_propNames.empty()); // if this index is not ready yet, return false if (!m_ready) return false; // The goal here is to figure out whether the results for the query are contiguous in // the index. That will be the case if all the props referenced are contiguous in our // prop vector, any prop referenced by an inequality op comes at the end of the // range of referenced props, any unreferenced props in our vector are at the end, // and the order prop is either the last referenced prop or, if all ops are equality ops, // the prop following the last referenced prop. const MojDbQuery::WhereMap& map = query.where(); MojSize numQueryProps = map.size(); // if there are more props in the query than in our index, no can do if (numQueryProps > m_propNames.size()) return false; // skip the first prop if we have an incDel index and the query does not reference the del prop StringVec::ConstIterator propName = m_propNames.begin(); PropVec::ConstIterator prop = m_props.begin(); if (m_includeDeleted && !map.contains(MojDb::DelKey)) { ++propName; ++prop; } // if there are no props in query, but the order matches our first prop, we're good const MojString& orderProp = query.order(); if (numQueryProps == 0) { if (orderProp.empty()) { return isIdIndex(); } else { return *propName == orderProp; } } // check all remaining props for (; propName != m_propNames.end(); ++propName, ++prop) { --numQueryProps; // ensure that the current prop is referenced in the query (no gaps) MojDbQuery::WhereMap::ConstIterator mapIter = map.find(*propName); if (mapIter == map.end()) return false; // ensure that if it is an inequality op, it is at the end if (numQueryProps > 0 && mapIter->lowerOp() != MojDbQuery::OpEq) return false; // ensure that collation matches if (mapIter->collation() != (*prop)->collation()) return false; // hooray, we made it through all the props in the query without failing any tests. // there may still be unreferenced props at the end of the vector, but that's ok. if (numQueryProps == 0) { // make sure that we can satisfy the ordering. if there is no ordering, or we are // ordering on a referenced prop, we're golden if (!orderProp.empty() && !map.contains(orderProp)) { // ensure that the order prop is next in our vec StringVec::ConstIterator next = propName + 1; if (next == m_propNames.end() || *next != orderProp) return false; } // can do! return true; } } return false; }
MojErr MojDbWatchTest::pageTest(MojDb& db) { MojObject id; MojObject idFirst; MojObject idFourth; MojObject idLast; MojInt64 rev; for (int i = 100; i < 150; ++i) { MojErr err = put(db, 100, i, id, rev); MojTestErrCheck(err); if (i == 100) { idFirst = id; } else if (i == 103) { idFourth = id; } else if (i == 149) { idLast = id; } } MojDbQuery query; MojErr err = query.from(_T("WatchTest:1")); MojTestErrCheck(err); err = query.where(_T("foo"), MojDbQuery::OpGreaterThanEq, 100); MojTestErrCheck(err); query.limit(3); MojRefCountedPtr<TestWatcher> watcher(new TestWatcher); MojTestAssert(watcher.get()); MojDbCursor cursor; err = db.find(query, cursor, watcher->m_slot); MojTestErrCheck(err); bool found = false; MojUInt32 count = 0; do { MojObject obj; err = cursor.get(obj, found); MojTestErrCheck(err); if (found) ++count; } while (found); MojTestAssert(count == 3); MojDbQuery::Page page; err = cursor.nextPage(page); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); err = merge(db, idFourth, 53); MojTestErrCheck(err); MojTestAssert(watcher->m_count == 0); query.page(page); MojRefCountedPtr<TestWatcher> watcher2(new TestWatcher); MojTestAssert(watcher2.get()); err = db.find(query, cursor, watcher2->m_slot); MojTestErrCheck(err); found = false; count = 0; do { MojObject obj; err = cursor.get(obj, found); MojTestErrCheck(err); if (found) ++count; } while (found); MojTestAssert(count == 3); err = cursor.close(); MojTestErrCheck(err); err = db.del(idFirst, found); MojTestErrCheck(err); MojTestAssert(found); MojTestAssert(watcher->m_count == 1); MojTestAssert(watcher2->m_count == 0); err = db.del(idFourth, found); MojTestErrCheck(err); MojTestAssert(found); MojTestAssert(watcher->m_count == 1); MojTestAssert(watcher2->m_count == 1); // desc order query.page(MojDbQuery::Page()); query.desc(true); MojRefCountedPtr<TestWatcher> watcher3(new TestWatcher); MojTestAssert(watcher3.get()); err = db.find(query, cursor, watcher3->m_slot); MojTestErrCheck(err); found = false; count = 0; do { MojObject obj; err = cursor.get(obj, found); MojTestErrCheck(err); if (found) ++count; } while (found); MojTestAssert(count == 3); err = cursor.close(); MojTestErrCheck(err); err = merge(db, idLast, 53); MojTestErrCheck(err); MojTestAssert(watcher3->m_count == 1); MojRefCountedPtr<TestWatcher> watcher4(new TestWatcher); MojTestAssert(watcher4.get()); err = db.find(query, cursor, watcher4->m_slot); MojTestErrCheck(err); found = false; count = 0; do { MojObject obj; err = cursor.get(obj, found); MojTestErrCheck(err); if (found) ++count; } while (found); MojTestAssert(count == 3); err = cursor.close(); MojTestErrCheck(err); err = merge(db, idLast, 54); MojTestErrCheck(err); MojTestAssert(watcher4->m_count == 1); return MojErrNone; }
MojErr MojDbPurgeTest::delKindTest(MojDb& db) { // start from scratch - purge everything MojUInt32 count = 0; MojErr err = db.purge(count, 0); MojTestErrCheck(err); MojTestAssert(count > 0); // purge again, make sure nothing is left err = db.purge(count, 0); MojTestErrCheck(err); MojTestAssert(count == 0); // make sure at least 2 objects exist MojDbQuery q; err = q.from(_T("PurgeTest:1")); MojTestErrCheck(err); MojDbCursor cursor; err = db.find(q, cursor); MojTestErrCheck(err); MojUInt32 objCount; err = cursor.count(objCount); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); if (objCount <= 1) { for (int i = objCount; i < 2; i++) { MojObject obj; err = obj.fromJson(MojTestObjStr1); MojTestErrCheck(err); err = db.put(obj); MojTestErrCheck(err); objCount++; } } // delete half the objects q.limit(objCount / 2); MojUInt32 delCount; err = db.del(q, delCount); MojTestErrCheck(err); // delete the kind MojString idStr; err = idStr.assign(_T("PurgeTest:1")); MojTestErrCheck(err); MojObject id(idStr); bool found; err = db.delKind(id, found); MojTestErrCheck(err); MojTestAssert(found); /*MojUInt32 deletedObjCount; err = q.where(_T("_del"), MojDbQuery::OpEq, true); MojTestErrCheck(err); q.includeDeleted(true); err = db.find(q, cursor); MojTestErrCheck(err); err = cursor.count(deletedObjCount); MojTestErrCheck(err); err = cursor.close(); MojTestErrCheck(err); // now all the objects should be deleted MojTestAssert(deletedObjCount == objCount);*/ // purge now err = db.purge(count, 0); MojTestErrCheck(err); return MojErrNone; }
/* * Send one IMMessage that was in the outbox. See java code in TransportCallbackThread.sendImHelper() * Change the message status from pending to either successful or failed in the DB when done * Set timestamp on the message and save it. * * errors returned up to OutgoingIMHandler * * send payload format: * im.libpurple.palm //sendMessage string=“{"serviceName":"gmail","usernameTo":"*****@*****.**","messageText":"Test send from phone","username":"******"}” * luna://im.libpurple.palm/sendMessage '{"serviceName":"aol","usernameTo":"palm","messageText":"test send IM","username":"******"}' * * @param imMsg - imMessage that was read out of the DB with status pending and folder set to outbox */ MojErr SendOneMessageHandler::doSend(const MojObject imMsg) { MojErr err = MojErrNone; // body // text should be saved unescaped in the DB bool found = false; err = imMsg.get(MOJDB_MSG_TEXT, m_messageText, found); MojErrCheck(err); // userNameTo - use "to" address MojObject addrArray; // array found = imMsg.get(MOJDB_TO, addrArray); // now read the address out of the object MojObject toAddrObject; if (found) { found = addrArray.at(0, toAddrObject); err = toAddrObject.get(MOJDB_ADDRESS, m_usernameTo, found); MojErrCheck(err); } // user = "******" address MojObject fromAddrObject; found = imMsg.get(MOJDB_FROM, fromAddrObject); if (found) { err = fromAddrObject.get(MOJDB_ADDRESS, m_username, found); MojErrCheck(err); } // serviceName err = imMsg.get(MOJDB_SERVICENAME, m_serviceName, found); MojErrCheck(err); // get the id so we can update the status in the DB after sending err = imMsg.getRequired("_id", m_currentMsgdbId); MojErrCheck(err); // need to search buddy list to see if this user is on it. // first we have to figure out the accountId - query ImLoginState // construct our where clause - find by username and servicename MojDbQuery query; query.where("serviceName", MojDbQuery::OpEq, m_serviceName); query.where("username", MojDbQuery::OpEq, m_username); query.from(IM_LOGINSTATE_KIND); // call del // virtual MojErr del(Signal::SlotRef handler, const MojDbQuery& query, // MojUInt32 flags = MojDb::FlagNone); err = m_dbClient.find(this->m_findAccountIdSlot, query); if (err) { MojString error; MojErrToString(err, error); MojLogError(IMServiceApp::s_log, _T("doSend: dbClient.find() failed: error %d - %s"), err, error.data()); // tell the outgoing Command handler we are done failMessage(ERROR_SEND_GENERIC_ERROR); } return MojErrNone; }
MojErr MojDbRevTest::run() { MojDb db; MojErr err = db.open(MojDbTestDir); MojTestErrCheck(err); // put type MojObject obj; err = obj.fromJson(MojKindStr); MojTestErrCheck(err); // put obj err = db.putKind(obj); MojTestErrCheck(err); err = obj.fromJson(MojTestObjStr1); MojTestErrCheck(err); err = db.put(obj); MojTestErrCheck(err); MojObject id; err = obj.getRequired(MojDb::IdKey, id); MojTestErrCheck(err); // get obj and verify rev eq MojObject rev; err = obj.getRequired(MojDb::RevKey, rev); MojTestErrCheck(err); err = checkRevEq(db, id, rev); MojTestErrCheck(err); // put identical obj and verify rev not changed err = db.put(obj); MojTestErrCheck(err); err = checkRevEq(db, id, rev); MojTestErrCheck(err); // put with changed prop and verify rev gt err = obj.fromJson(MojTestObjStr2); MojTestErrCheck(err); err = obj.put(MojDb::IdKey, id); MojTestErrCheck(err); err = db.put(obj); // this put will fail because we haven't included a rev MojTestErrExpected(err, MojErrDbRevNotSpecified); bool found = false; err = db.del(id, found); // verify that we can put without a rev if the object is deleted MojTestErrCheck(err); MojTestAssert(found); err = db.put(obj); MojTestErrCheck(err); err = obj.getRequired(MojDb::RevKey, rev); MojTestErrCheck(err); err = obj.put(MojDb::RevKey, 1); MojTestErrCheck(err); err = db.put(obj); // this put will fail because the revision num is lower MojTestErrExpected(err, MojErrDbRevisionMismatch); err = obj.put(MojDb::RevKey, rev); MojTestErrCheck(err); err = db.put(obj); // now this should succeed MojTestErrCheck(err); // merge with unchanged prop and verify rev not changed err = obj.fromJson(MojTestObjStr3); MojTestErrCheck(err); err = obj.put(MojDb::IdKey, id); MojTestErrCheck(err); err = db.merge(obj); MojTestErrCheck(err); err = checkRevEq(db, id, rev); MojTestErrCheck(err); // merge with changed prop and verify rev gt err = obj.fromJson(MojTestObjStr4); MojTestErrCheck(err); err = obj.put(MojDb::IdKey, id); MojTestErrCheck(err); err = db.merge(obj); MojTestErrCheck(err); err = checkRevGt(db, id, rev); MojTestErrCheck(err); // query merge with unchanged prop and verify rev not changed MojDbQuery query; err = query.from("RevTest:1"); MojTestErrCheck(err); err = obj.fromJson(MojTestObjStr4); MojTestErrCheck(err); MojUInt32 count = 0; err = db.merge(query, obj, count); MojTestErrCheck(err); MojTestAssert(count == 1); err = checkRevEq(db, id, rev); MojTestErrCheck(err); // query merge with changed prop and verify rev gt err = obj.fromJson(MojTestObjStr3); MojTestErrCheck(err); err = db.merge(query, obj, count); MojTestErrCheck(err); MojTestAssert(count == 1); err = checkRevGt(db, id, rev); MojTestErrCheck(err); // del verify rev gt err = db.del(id, found); MojTestErrCheck(err); MojTestAssert(found); err = checkRevGt(db, id, rev); MojTestErrCheck(err); // del again and verify rev not changed err = db.del(id, found); MojTestErrCheck(err); MojTestAssert(found); err = checkRevEq(db, id, rev); MojTestErrCheck(err); // undel and verify rev gt err = obj.fromJson(MojTestObjStr5); MojTestErrCheck(err); err = obj.put(MojDb::IdKey, id); MojTestErrCheck(err); err = db.merge(obj); MojTestErrCheck(err); err = checkRevGt(db, id, rev); MojTestErrCheck(err); // query del and verify rev gt err = db.del(query, count); MojTestErrCheck(err); MojTestAssert(count == 1); err = checkRevGt(db, id, rev); MojTestErrCheck(err); err = db.close(); MojTestErrCheck(err); return MojErrNone; }