MojErr MojDbSearchCursor::loadObjects(const ObjectSet& ids)
{
MojInt32 warns = 0;
for (ObjectSet::ConstIterator i = ids.begin(); i != ids.end(); ++i) {
// get item by id
MojObject obj;
MojDbStorageItem* item = NULL;
bool found = false;
MojErr err = m_storageQuery->getById(*i, item, found);
if (err == MojErrInternalIndexOnFind) {
warns++;
continue;
}
MojErrCheck(err);
if (found) {
// get object from item
err = item->toObject(obj, *m_kindEngine);
MojErrCheck(err);
// filter results
if (m_queryFilter.get() && !m_queryFilter->test(obj))
continue;
// create object item
MojRefCountedPtr<MojDbObjectItem> item(new MojDbObjectItem(obj));
MojAllocCheck(item.get());
// add to vec
err = m_items.push(item);
MojErrCheck(err);
}
}
if (warns > 0)
MojLogWarning(MojDb::s_log, _T("Search warnings: %d \n"), warns);
return MojErrNone;
}
MojErr MojDb::merge(const MojDbQuery& query, const MojObject& props, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
MojErr err = beginReq(req);
MojErrCheck(err);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpUpdate);
MojErrCheck(err);
MojAssert(cursor.txn());
MojUInt32 count = 0;
MojUInt32 warns = 0;
bool found = false;
MojObject prev;
for (;;) {
// get prev rev from cursor
MojDbStorageItem* prevItem = NULL;
err = cursor.get(prevItem, found);
if (err == MojErrInternalIndexOnFind) {
warns++;
continue;
}
MojErrCheck(err);
if (!found)
break;
err = prevItem->toObject(prev, m_kindEngine);
MojErrCheck(err);
// merge obj into prev
MojObject merged;
err = mergeInto(merged, props, prev);
MojErrCheck(err);
// and update the db
const MojObject& id = prevItem->id();
err = putObj(id, merged, &prev, prevItem, req, OpUpdate);
MojErrCheck(err);
++count;
}
if (warns > 0)
MojLogWarning(s_log, _T("Merge index_warnings: %s; count: %d\n"), query.from().data(), warns);
err = cursor.close();
MojErrCheck(err);
err = req->end();
MojErrCheck(err);
countOut = count;
return MojErrNone;
}
MojErr MojDbSearchCursor::load()
{
LOG_TRACE("Entering function %s", __FUNCTION__);
// pull unique ids from index
ObjectSet ids;
MojErr err = loadIds(ids);
MojErrCheck(err);
// load objects into memory
err = loadObjects(ids);
MojErrCheck(err);
// sort results
if (!m_orderProp.empty()) {
err = sort();
MojErrCheck(err);
}
// distinct
if (!m_distinct.empty()) {
distinct();
}
// reverse for desc
if (m_query.desc()) {
err = m_items.reverse();
MojErrCheck(err);
}
// next page
if (!m_page.empty()) {
err = setPagePosition();
MojErrCheck(err);
} else {
// set begin/last position.
m_pos = m_items.begin();
if (m_limit >= m_items.size()) {
m_limitPos = m_items.end();
} else {
// if item size is bigger than limit, set next page.
m_limitPos = m_items.begin() + m_limit;
MojDbStorageItem* nextItem = m_limitPos->get();
const MojObject nextId = nextItem->id();
m_page.fromObject(nextId);
}
}
// set remainder count
m_count = m_items.end() - m_pos;
return MojErrNone;
}
/***********************************************************************
* setPagePosition
*
* Move cursor position to page id provided in query.
* 1. Find item "_id" which is equal to page id iteratively.
* 2. If found, set begin/last position and next page.
***********************************************************************/
MojErr MojDbSearchCursor::setPagePosition()
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojErr err;
MojDbStorageItem* item;
m_pos = m_items.begin();
ItemVec::ConstIterator last = m_items.end();
m_limitPos = last;
// retrieve page id from query
MojObject pageKey;
err = m_page.toObject(pageKey);
MojErrCheck(err);
while (m_pos != last) {
// retrieve id from query
item = m_pos->get();
const MojObject id = item->id();
// If match, set begin/last position and next page
if(pageKey.compare(id) == 0) {
if (m_limit >= (last-m_pos)) {
m_limitPos = m_items.end();
m_page.clear();
} else {
m_limitPos = m_pos + m_limit;
// set next page
MojDbStorageItem* nextItem = m_limitPos->get();
const MojObject nextId = nextItem->id();
m_page.fromObject(nextId);
// print log for nextId
MojString strOut;
err = nextId.toJson(strOut);
MojErrCheck(err);
LOG_DEBUG("[db_mojodb] nextId : %s \n", strOut.data());
}
break;
}
++m_pos;
}
return MojErrNone;
}
MojErr MojDbSearchCursor::loadObjects(const ObjectSet& ids)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojInt32 warns = 0;
for (ObjectSet::ConstIterator i = ids.begin(); i != ids.end(); ++i) {
// get item by id
MojObject obj;
MojDbStorageItem* item = NULL;
bool found = false;
MojErr err = m_storageQuery->getById(*i, item, found);
if (err == MojErrInternalIndexOnFind) {
warns++;
continue;
}
MojErrCheck(err);
if (found) {
// get object from item
err = item->toObject(obj, *m_kindEngine);
MojErrCheck(err);
// filter results
if (m_queryFilter.get() && !MojBoolResult(m_queryFilter->test, obj))
continue;
// create object item
MojRefCountedPtr<MojDbObjectItem> item(new MojDbObjectItem(obj));
MojAllocCheck(item.get());
// add to vec
err = m_items.push(item);
MojErrCheck(err);
}
}
if (warns > 0)
LOG_DEBUG("[db_mojodb] Search warnings: %d \n", warns);
return MojErrNone;
}
MojErr MojDb::delImpl(const MojDbQuery& quer, MojUInt32& countOut, MojDbReq& req, MojUInt32 flags)
{
MojLogTrace(s_log);
countOut = 0;
MojInt32 warns = 0;
MojDbQuery newQuery = quer;
MojUInt32 queryLimit = newQuery.limit();
if(newQuery.limit() == MojDbQuery::LimitDefault)
newQuery.limit(AutoBatchSize);
while(queryLimit > 0)
{
MojDbCursor cursor;
MojErr err = findImpl(newQuery, cursor, NULL, req, OpDelete);
MojErrCheck(err);
MojAssert(cursor.txn());
MojUInt32 count = 0;
MojUInt32 numberInBatch = 0;
bool found = false;
MojObject obj;
for (;;) {
MojDbStorageItem* item = NULL;
err = cursor.get(item, found);
// We simply skip ghost keys and continue; A warning is already written to the system log earlier
if (err == MojErrInternalIndexOnFind) {
warns++;
numberInBatch++;
continue;
}
MojErrCheck(err);
if (!found)
break;
err = item->toObject(obj, m_kindEngine);
MojErrCheck(err);
const MojObject& id = item->id();
MojObject deleted;
err = delObj(id, obj, item, deleted, req, flags);
MojErrCheck(err);
++count;
numberInBatch++;
}
if (warns > 0)
MojLogInfo(s_log, _T("delquery index_warnings: %s, count: %d\n"), newQuery.from().data(), warns);
countOut += count;
err = cursor.close();
MojErrCheck(err);
if(numberInBatch >= AutoBatchSize) // sing > - just in case something messed up
{
err = commitBatch(req);
MojErrCheck(err);
}
if(count == 0)
break;
queryLimit -= newQuery.limit();
if(queryLimit > AutoBatchSize)
newQuery.limit(AutoBatchSize);
else
newQuery.limit(queryLimit);
}
return MojErrNone;
}
MojErr MojDbKind::stats(MojObject& objOut, MojSize& usageOut, MojDbReq& req, bool verify)
{
MojLogTrace(s_log);
#if defined(TESTDBKIND)
MojLogInfo(s_log, _T("Subkinds for - %s ; count = %d\n"), m_id.data(), m_subs.size());
int n = 0;
for (KindVec::ConstIterator i = m_subs.begin(); i != m_subs.end(); ++i) {
MojLogInfo(s_log, _T("SubKind %d: %s"), n++, (*i)->id().data());
}
MojLogInfo(s_log, _T("Supers for - %s ; count = %d\n"), m_id.data(), m_supers.size());
n = 0;
for (KindVec::ConstIterator i = m_supers.begin(); i != m_supers.end(); ++i) {
MojLogInfo(s_log, _T("Super %d: %s"), n++, (*i)->id().data());
}
#endif
// analyze objects
MojDbQuery query;
MojErr err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
MojLogInfo(s_log, _T("KindStats start: %s ; Indexes = %zu; Using Index: %s; \n"),
m_id.data(), m_indexes.size(), cursor.m_dbIndex->name().data());
MojErrCheck(err);
MojSize count = 0;
MojSize size = 0;
MojSize delCount = 0;
MojSize delSize = 0;
MojSize warnings = 0;
for (;;) {
MojDbStorageItem* item = NULL;
bool found = false;
cursor.verifymode(true);
err = cursor.get(item, found);
if (err == MojErrInternalIndexOnFind) {
warnings++;
continue;
}
if (err != MojErrNone) // for all other errors break and dump current stats
break;
if (!found)
break;
MojObject obj;
err = item->toObject(obj, *m_kindEngine, true);
if (err != MojErrNone)
break;
bool deleted = false;
if (obj.get(MojDb::DelKey, deleted) && deleted) {
delSize += item->size();
delCount++;
} else {
size += item->size();
count++;
}
}
MojLogInfo(s_log, _T("KindStats Summary: %s : Count: %zu; delCount: %zu; warnings: %zu \n"), m_id.data(), count, delCount, warnings);
usageOut += size + delSize;
MojObject info;
err = info.put(SizeKey, (MojInt64) size);
MojErrCheck(err);
err = info.put(CountKey, (MojInt64) count);
MojErrCheck(err);
if (delCount > 0) {
err = info.put(DelSizeKey, (MojInt64) delSize);
MojErrCheck(err);
err = info.put(DelCountKey, (MojInt64) delCount);
MojErrCheck(err);
}
if (warnings > 0) {
err = info.put(WarnKey, (MojInt64) warnings);
MojErrCheck(err);
}
err = objOut.put(ObjectsKey, info);
MojErrCheck(err);
// and indexes
MojObject indexes;
for (IndexVec::ConstIterator i = m_indexes.begin(); i != m_indexes.end(); ++i) {
MojObject indexInfo;
err = (*i)->stats(indexInfo, usageOut, req);
MojErrCheck(err);
if (verify) {
MojDbIndex *pi = i->get();
MojErr err2 = verifyIndex(pi, indexInfo, req);
MojErrCheck(err2);
}
err = indexes.put((*i)->name(), indexInfo);
MojErrCheck(err);
}
err = objOut.put(IndexesKey, indexes);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbKind::verifyIndex(MojDbIndex *pIndex, MojObject &iinfo, MojDbReq& req)
{
// Goes throudh each index entry and verifies that it points to a valid object
// For debugging purposes as stats for indexes does not access the target objects
// Index->stats function does not have enough context to find the object
// db/stats usage '{"verify":true,"kind":"xyz"}' - each optional
MojDbQuery query;
MojErr err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
query.m_forceIndex = pIndex; // Important: Otherwise, it will pick the default index
cursor.verifymode(true); // to get the errors
err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
MojLogInfo(s_log, _T("Kind_verifyIndex: Kind: %s; Index: %s; idIndex: %zX; size: %zu; CursorIndex: %s \n"), m_name.data(),
pIndex->name().data(), pIndex->idIndex(), pIndex->size(), cursor.m_dbIndex->name().data());
MojErrCheck(err);
MojSize count = 0;
MojSize delCount = 0;
MojSize warnCount = 0;
char s[1024];
for (;;) {
MojDbStorageItem* item = NULL;
bool found = false;
err = cursor.get(item, found);
if (err == MojErrInternalIndexOnFind) {
warnCount++;
MojDbIsamQuery *iquery = (MojDbIsamQuery *)cursor.m_storageQuery.get();
MojErr err2 = MojByteArrayToHex(iquery->m_keyData, iquery->m_keySize, s);
MojErrCheck(err2);
MojChar *ids = (iquery->m_keySize > 18) ? (MojChar *)(iquery->m_keyData + iquery->m_keySize - 17) : NULL;
MojLogInfo(s_log, _T("VerifyIndex Warning: %s; KeySize: %zu; %s ;id: %s \n"),
cursor.m_dbIndex->name().data(), iquery->m_keySize, s, ids);
continue;
}
MojErrCheck(err);
if (!found)
break;
MojObject obj;
err = item->toObject(obj, *m_kindEngine, true);
MojErrCheck(err);
bool deleted = false;
if (obj.get(MojDb::DelKey, deleted) && deleted) {
delCount++;
} else {
count++;
}
}
MojLogInfo(s_log, _T("Kind_verifyIndex Counts: Kind: %s; Index: %s; count: %zu; delcount: %zu; warnings: %zu \n"), m_name.data(),
pIndex->name().data(), count, delCount, warnCount);
err = iinfo.put(VerifyCountKey, (MojInt64)count);
MojErrCheck(err);
err = iinfo.put(VerifyWarnCountKey, (MojInt64) warnCount);
MojErrCheck(err);
err = iinfo.put(VerifyDelCountKey, (MojInt64) delCount);
MojErrCheck(err);
return MojErrNone;
}