MojErr MojDbSearchCursor::loadObjects(const ObjectSet& ids) { MojInt32 warns = 0; for (ObjectSet::ConstIterator i = ids.begin(); i != ids.end(); ++i) { // get item by id MojObject obj; MojDbStorageItem* item = NULL; bool found = false; MojErr err = m_storageQuery->getById(*i, item, found); if (err == MojErrInternalIndexOnFind) { warns++; continue; } MojErrCheck(err); if (found) { // get object from item err = item->toObject(obj, *m_kindEngine); MojErrCheck(err); // filter results if (m_queryFilter.get() && !m_queryFilter->test(obj)) continue; // create object item MojRefCountedPtr<MojDbObjectItem> item(new MojDbObjectItem(obj)); MojAllocCheck(item.get()); // add to vec err = m_items.push(item); MojErrCheck(err); } } if (warns > 0) MojLogWarning(MojDb::s_log, _T("Search warnings: %d \n"), warns); return MojErrNone; }
MojErr MojDb::merge(const MojDbQuery& query, const MojObject& props, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req) { MojLogTrace(s_log); countOut = 0; MojErr err = beginReq(req); MojErrCheck(err); MojDbCursor cursor; err = findImpl(query, cursor, NULL, req, OpUpdate); MojErrCheck(err); MojAssert(cursor.txn()); MojUInt32 count = 0; MojUInt32 warns = 0; bool found = false; MojObject prev; for (;;) { // get prev rev from cursor MojDbStorageItem* prevItem = NULL; err = cursor.get(prevItem, found); if (err == MojErrInternalIndexOnFind) { warns++; continue; } MojErrCheck(err); if (!found) break; err = prevItem->toObject(prev, m_kindEngine); MojErrCheck(err); // merge obj into prev MojObject merged; err = mergeInto(merged, props, prev); MojErrCheck(err); // and update the db const MojObject& id = prevItem->id(); err = putObj(id, merged, &prev, prevItem, req, OpUpdate); MojErrCheck(err); ++count; } if (warns > 0) MojLogWarning(s_log, _T("Merge index_warnings: %s; count: %d\n"), query.from().data(), warns); err = cursor.close(); MojErrCheck(err); err = req->end(); MojErrCheck(err); countOut = count; return MojErrNone; }
MojErr MojDbSearchCursor::load() { LOG_TRACE("Entering function %s", __FUNCTION__); // pull unique ids from index ObjectSet ids; MojErr err = loadIds(ids); MojErrCheck(err); // load objects into memory err = loadObjects(ids); MojErrCheck(err); // sort results if (!m_orderProp.empty()) { err = sort(); MojErrCheck(err); } // distinct if (!m_distinct.empty()) { distinct(); } // reverse for desc if (m_query.desc()) { err = m_items.reverse(); MojErrCheck(err); } // next page if (!m_page.empty()) { err = setPagePosition(); MojErrCheck(err); } else { // set begin/last position. m_pos = m_items.begin(); if (m_limit >= m_items.size()) { m_limitPos = m_items.end(); } else { // if item size is bigger than limit, set next page. m_limitPos = m_items.begin() + m_limit; MojDbStorageItem* nextItem = m_limitPos->get(); const MojObject nextId = nextItem->id(); m_page.fromObject(nextId); } } // set remainder count m_count = m_items.end() - m_pos; return MojErrNone; }
/*********************************************************************** * setPagePosition * * Move cursor position to page id provided in query. * 1. Find item "_id" which is equal to page id iteratively. * 2. If found, set begin/last position and next page. ***********************************************************************/ MojErr MojDbSearchCursor::setPagePosition() { LOG_TRACE("Entering function %s", __FUNCTION__); MojErr err; MojDbStorageItem* item; m_pos = m_items.begin(); ItemVec::ConstIterator last = m_items.end(); m_limitPos = last; // retrieve page id from query MojObject pageKey; err = m_page.toObject(pageKey); MojErrCheck(err); while (m_pos != last) { // retrieve id from query item = m_pos->get(); const MojObject id = item->id(); // If match, set begin/last position and next page if(pageKey.compare(id) == 0) { if (m_limit >= (last-m_pos)) { m_limitPos = m_items.end(); m_page.clear(); } else { m_limitPos = m_pos + m_limit; // set next page MojDbStorageItem* nextItem = m_limitPos->get(); const MojObject nextId = nextItem->id(); m_page.fromObject(nextId); // print log for nextId MojString strOut; err = nextId.toJson(strOut); MojErrCheck(err); LOG_DEBUG("[db_mojodb] nextId : %s \n", strOut.data()); } break; } ++m_pos; } return MojErrNone; }
MojErr MojDbSearchCursor::loadObjects(const ObjectSet& ids) { LOG_TRACE("Entering function %s", __FUNCTION__); MojInt32 warns = 0; for (ObjectSet::ConstIterator i = ids.begin(); i != ids.end(); ++i) { // get item by id MojObject obj; MojDbStorageItem* item = NULL; bool found = false; MojErr err = m_storageQuery->getById(*i, item, found); if (err == MojErrInternalIndexOnFind) { warns++; continue; } MojErrCheck(err); if (found) { // get object from item err = item->toObject(obj, *m_kindEngine); MojErrCheck(err); // filter results if (m_queryFilter.get() && !MojBoolResult(m_queryFilter->test, obj)) continue; // create object item MojRefCountedPtr<MojDbObjectItem> item(new MojDbObjectItem(obj)); MojAllocCheck(item.get()); // add to vec err = m_items.push(item); MojErrCheck(err); } } if (warns > 0) LOG_DEBUG("[db_mojodb] Search warnings: %d \n", warns); return MojErrNone; }
MojErr MojDb::delImpl(const MojDbQuery& quer, MojUInt32& countOut, MojDbReq& req, MojUInt32 flags) { MojLogTrace(s_log); countOut = 0; MojInt32 warns = 0; MojDbQuery newQuery = quer; MojUInt32 queryLimit = newQuery.limit(); if(newQuery.limit() == MojDbQuery::LimitDefault) newQuery.limit(AutoBatchSize); while(queryLimit > 0) { MojDbCursor cursor; MojErr err = findImpl(newQuery, cursor, NULL, req, OpDelete); MojErrCheck(err); MojAssert(cursor.txn()); MojUInt32 count = 0; MojUInt32 numberInBatch = 0; bool found = false; MojObject obj; for (;;) { MojDbStorageItem* item = NULL; err = cursor.get(item, found); // We simply skip ghost keys and continue; A warning is already written to the system log earlier if (err == MojErrInternalIndexOnFind) { warns++; numberInBatch++; continue; } MojErrCheck(err); if (!found) break; err = item->toObject(obj, m_kindEngine); MojErrCheck(err); const MojObject& id = item->id(); MojObject deleted; err = delObj(id, obj, item, deleted, req, flags); MojErrCheck(err); ++count; numberInBatch++; } if (warns > 0) MojLogInfo(s_log, _T("delquery index_warnings: %s, count: %d\n"), newQuery.from().data(), warns); countOut += count; err = cursor.close(); MojErrCheck(err); if(numberInBatch >= AutoBatchSize) // sing > - just in case something messed up { err = commitBatch(req); MojErrCheck(err); } if(count == 0) break; queryLimit -= newQuery.limit(); if(queryLimit > AutoBatchSize) newQuery.limit(AutoBatchSize); else newQuery.limit(queryLimit); } return MojErrNone; }
MojErr MojDbKind::stats(MojObject& objOut, MojSize& usageOut, MojDbReq& req, bool verify) { MojLogTrace(s_log); #if defined(TESTDBKIND) MojLogInfo(s_log, _T("Subkinds for - %s ; count = %d\n"), m_id.data(), m_subs.size()); int n = 0; for (KindVec::ConstIterator i = m_subs.begin(); i != m_subs.end(); ++i) { MojLogInfo(s_log, _T("SubKind %d: %s"), n++, (*i)->id().data()); } MojLogInfo(s_log, _T("Supers for - %s ; count = %d\n"), m_id.data(), m_supers.size()); n = 0; for (KindVec::ConstIterator i = m_supers.begin(); i != m_supers.end(); ++i) { MojLogInfo(s_log, _T("Super %d: %s"), n++, (*i)->id().data()); } #endif // analyze objects MojDbQuery query; MojErr err = query.from(m_id); MojErrCheck(err); err = query.includeDeleted(true); MojErrCheck(err); MojDbCursor cursor; err = m_kindEngine->find(query, cursor, NULL, req, OpRead); MojLogInfo(s_log, _T("KindStats start: %s ; Indexes = %zu; Using Index: %s; \n"), m_id.data(), m_indexes.size(), cursor.m_dbIndex->name().data()); MojErrCheck(err); MojSize count = 0; MojSize size = 0; MojSize delCount = 0; MojSize delSize = 0; MojSize warnings = 0; for (;;) { MojDbStorageItem* item = NULL; bool found = false; cursor.verifymode(true); err = cursor.get(item, found); if (err == MojErrInternalIndexOnFind) { warnings++; continue; } if (err != MojErrNone) // for all other errors break and dump current stats break; if (!found) break; MojObject obj; err = item->toObject(obj, *m_kindEngine, true); if (err != MojErrNone) break; bool deleted = false; if (obj.get(MojDb::DelKey, deleted) && deleted) { delSize += item->size(); delCount++; } else { size += item->size(); count++; } } MojLogInfo(s_log, _T("KindStats Summary: %s : Count: %zu; delCount: %zu; warnings: %zu \n"), m_id.data(), count, delCount, warnings); usageOut += size + delSize; MojObject info; err = info.put(SizeKey, (MojInt64) size); MojErrCheck(err); err = info.put(CountKey, (MojInt64) count); MojErrCheck(err); if (delCount > 0) { err = info.put(DelSizeKey, (MojInt64) delSize); MojErrCheck(err); err = info.put(DelCountKey, (MojInt64) delCount); MojErrCheck(err); } if (warnings > 0) { err = info.put(WarnKey, (MojInt64) warnings); MojErrCheck(err); } err = objOut.put(ObjectsKey, info); MojErrCheck(err); // and indexes MojObject indexes; for (IndexVec::ConstIterator i = m_indexes.begin(); i != m_indexes.end(); ++i) { MojObject indexInfo; err = (*i)->stats(indexInfo, usageOut, req); MojErrCheck(err); if (verify) { MojDbIndex *pi = i->get(); MojErr err2 = verifyIndex(pi, indexInfo, req); MojErrCheck(err2); } err = indexes.put((*i)->name(), indexInfo); MojErrCheck(err); } err = objOut.put(IndexesKey, indexes); MojErrCheck(err); return MojErrNone; }
MojErr MojDbKind::verifyIndex(MojDbIndex *pIndex, MojObject &iinfo, MojDbReq& req) { // Goes throudh each index entry and verifies that it points to a valid object // For debugging purposes as stats for indexes does not access the target objects // Index->stats function does not have enough context to find the object // db/stats usage '{"verify":true,"kind":"xyz"}' - each optional MojDbQuery query; MojErr err = query.from(m_id); MojErrCheck(err); err = query.includeDeleted(true); MojErrCheck(err); MojDbCursor cursor; query.m_forceIndex = pIndex; // Important: Otherwise, it will pick the default index cursor.verifymode(true); // to get the errors err = m_kindEngine->find(query, cursor, NULL, req, OpRead); MojLogInfo(s_log, _T("Kind_verifyIndex: Kind: %s; Index: %s; idIndex: %zX; size: %zu; CursorIndex: %s \n"), m_name.data(), pIndex->name().data(), pIndex->idIndex(), pIndex->size(), cursor.m_dbIndex->name().data()); MojErrCheck(err); MojSize count = 0; MojSize delCount = 0; MojSize warnCount = 0; char s[1024]; for (;;) { MojDbStorageItem* item = NULL; bool found = false; err = cursor.get(item, found); if (err == MojErrInternalIndexOnFind) { warnCount++; MojDbIsamQuery *iquery = (MojDbIsamQuery *)cursor.m_storageQuery.get(); MojErr err2 = MojByteArrayToHex(iquery->m_keyData, iquery->m_keySize, s); MojErrCheck(err2); MojChar *ids = (iquery->m_keySize > 18) ? (MojChar *)(iquery->m_keyData + iquery->m_keySize - 17) : NULL; MojLogInfo(s_log, _T("VerifyIndex Warning: %s; KeySize: %zu; %s ;id: %s \n"), cursor.m_dbIndex->name().data(), iquery->m_keySize, s, ids); continue; } MojErrCheck(err); if (!found) break; MojObject obj; err = item->toObject(obj, *m_kindEngine, true); MojErrCheck(err); bool deleted = false; if (obj.get(MojDb::DelKey, deleted) && deleted) { delCount++; } else { count++; } } MojLogInfo(s_log, _T("Kind_verifyIndex Counts: Kind: %s; Index: %s; count: %zu; delcount: %zu; warnings: %zu \n"), m_name.data(), pIndex->name().data(), count, delCount, warnCount); err = iinfo.put(VerifyCountKey, (MojInt64)count); MojErrCheck(err); err = iinfo.put(VerifyWarnCountKey, (MojInt64) warnCount); MojErrCheck(err); err = iinfo.put(VerifyDelCountKey, (MojInt64) delCount); MojErrCheck(err); return MojErrNone; }