Beispiel #1
0
MojErr MojDbSearchCursor::loadObjects(const ObjectSet& ids)
{
    MojInt32 warns = 0;
    for (ObjectSet::ConstIterator i = ids.begin(); i != ids.end(); ++i) {
        // get item by id
        MojObject obj;
        MojDbStorageItem* item = NULL;
        bool found = false;
        MojErr err = m_storageQuery->getById(*i, item, found);
        if (err == MojErrInternalIndexOnFind) {
            warns++;
            continue;
        }

        MojErrCheck(err);
        if (found) {
            // get object from item
            err = item->toObject(obj, *m_kindEngine);
            MojErrCheck(err);
            // filter results
            if (m_queryFilter.get() && !m_queryFilter->test(obj))
                continue;
            // create object item
            MojRefCountedPtr<MojDbObjectItem> item(new MojDbObjectItem(obj));
            MojAllocCheck(item.get());
            // add to vec
            err = m_items.push(item);
            MojErrCheck(err);
        }
    }
    if (warns > 0)
        MojLogWarning(MojDb::s_log, _T("Search warnings: %d \n"), warns);
    return MojErrNone;
}
Beispiel #2
0
MojErr MojDb::merge(const MojDbQuery& query, const MojObject& props, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req)
{
	MojLogTrace(s_log);

	countOut = 0;
	MojErr err = beginReq(req);
	MojErrCheck(err);

	MojDbCursor cursor;
	err = findImpl(query, cursor, NULL, req, OpUpdate);
	MojErrCheck(err);
	MojAssert(cursor.txn());

	MojUInt32 count = 0;
	MojUInt32 warns = 0;
	bool found = false;
	MojObject prev;
	for (;;) {
		// get prev rev from cursor
		MojDbStorageItem* prevItem = NULL;
		err = cursor.get(prevItem, found);
		if (err == MojErrInternalIndexOnFind) {
			warns++;
			continue;
		}
		MojErrCheck(err);
		if (!found)
			break;
		err = prevItem->toObject(prev, m_kindEngine);
		MojErrCheck(err);
		// merge obj into prev
		MojObject merged;
		err = mergeInto(merged, props, prev);
		MojErrCheck(err);
		// and update the db
		const MojObject& id = prevItem->id();
		err = putObj(id, merged, &prev, prevItem, req, OpUpdate);
		MojErrCheck(err);
		++count;
	}
	if (warns > 0)
		MojLogWarning(s_log, _T("Merge index_warnings: %s; count: %d\n"), query.from().data(), warns);
	err = cursor.close();
	MojErrCheck(err);
	err = req->end();
	MojErrCheck(err);

	countOut = count;

	return MojErrNone;
}
Beispiel #3
0
MojErr MojDbSearchCursor::loadObjects(const ObjectSet& ids)
{
    LOG_TRACE("Entering function %s", __FUNCTION__);

    MojInt32 warns = 0;
    for (ObjectSet::ConstIterator i = ids.begin(); i != ids.end(); ++i) {
        // get item by id
        MojObject obj;
        MojDbStorageItem* item = NULL;
        bool found = false;
        MojErr err = m_storageQuery->getById(*i, item, found);
        if (err == MojErrInternalIndexOnFind) {
            warns++;
            continue;
        }

        MojErrCheck(err);
        if (found) {
            // get object from item
            err = item->toObject(obj, *m_kindEngine);
            MojErrCheck(err);
            // filter results
            if (m_queryFilter.get() && !MojBoolResult(m_queryFilter->test, obj))
                continue;
            // create object item
            MojRefCountedPtr<MojDbObjectItem> item(new MojDbObjectItem(obj));
            MojAllocCheck(item.get());
            // add to vec
            err = m_items.push(item);
            MojErrCheck(err);
        }
    }
    if (warns > 0)
        LOG_DEBUG("[db_mojodb] Search warnings: %d \n", warns);
    return MojErrNone;
}
Beispiel #4
0
MojErr MojDb::delImpl(const MojDbQuery& quer, MojUInt32& countOut, MojDbReq& req, MojUInt32 flags)
{
	MojLogTrace(s_log);

	countOut = 0;
	MojInt32 warns = 0;
    MojDbQuery newQuery = quer;
    MojUInt32 queryLimit = newQuery.limit();

    if(newQuery.limit() == MojDbQuery::LimitDefault)
        newQuery.limit(AutoBatchSize);

    while(queryLimit > 0)
    {
        MojDbCursor cursor;
        MojErr err = findImpl(newQuery, cursor, NULL, req, OpDelete);

        MojErrCheck(err);
        MojAssert(cursor.txn());

        MojUInt32 count = 0;
        MojUInt32 numberInBatch = 0;

        bool found = false;

        MojObject obj;
        for (;;) {
            MojDbStorageItem* item = NULL;
            err = cursor.get(item, found);
            // We simply skip ghost keys and continue; A warning is already written to the system log earlier
            if (err == MojErrInternalIndexOnFind) {
                warns++;
                numberInBatch++;
                continue;
            }
            MojErrCheck(err);
            if (!found)
                break;
            err = item->toObject(obj, m_kindEngine);
            MojErrCheck(err);
            const MojObject& id = item->id();
            MojObject deleted;
            err = delObj(id, obj, item, deleted, req, flags);
            MojErrCheck(err);
            ++count;
            numberInBatch++;
        }

        if (warns > 0)
            MojLogInfo(s_log, _T("delquery index_warnings: %s, count: %d\n"), newQuery.from().data(), warns);
        countOut += count;

        err = cursor.close();
        MojErrCheck(err);

        if(numberInBatch >= AutoBatchSize) // sing > - just in case something messed up
        {
            err = commitBatch(req);
            MojErrCheck(err);
        }

        if(count == 0)
            break;

        queryLimit -= newQuery.limit();
        if(queryLimit > AutoBatchSize)
            newQuery.limit(AutoBatchSize);
        else
            newQuery.limit(queryLimit);
    }

    return MojErrNone;
}
Beispiel #5
0
MojErr MojDbKind::stats(MojObject& objOut, MojSize& usageOut, MojDbReq& req, bool verify)
{
	MojLogTrace(s_log);

#if defined(TESTDBKIND)
	MojLogInfo(s_log, _T("Subkinds for - %s ; count = %d\n"), m_id.data(), m_subs.size());
	int n = 0;
	for (KindVec::ConstIterator i = m_subs.begin(); i != m_subs.end(); ++i) {
		MojLogInfo(s_log, _T("SubKind %d: %s"), n++, (*i)->id().data());
	}
	MojLogInfo(s_log, _T("Supers for - %s ; count = %d\n"), m_id.data(), m_supers.size());
	n = 0;
	for (KindVec::ConstIterator i = m_supers.begin(); i != m_supers.end(); ++i) {
		MojLogInfo(s_log, _T("Super %d: %s"), n++, (*i)->id().data());
	}
#endif 
	// analyze objects
	MojDbQuery query;
	MojErr err = query.from(m_id);
	MojErrCheck(err);
	err = query.includeDeleted(true);
	MojErrCheck(err);
	MojDbCursor cursor;
	err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
	MojLogInfo(s_log, _T("KindStats start: %s ; Indexes = %zu; Using Index: %s; \n"), 
				m_id.data(), m_indexes.size(), cursor.m_dbIndex->name().data());

	MojErrCheck(err);
	MojSize count = 0;
	MojSize size = 0;
	MojSize delCount = 0;
	MojSize delSize = 0;
	MojSize warnings = 0;
	for (;;) {
		MojDbStorageItem* item = NULL;
		bool found = false;
		cursor.verifymode(true);
		err = cursor.get(item, found);
		if (err == MojErrInternalIndexOnFind) {
			warnings++;
			continue;
		}
		if (err != MojErrNone)		// for all other errors break and dump current stats 
			break;	
		if (!found)
			break;
		MojObject obj;
		err = item->toObject(obj, *m_kindEngine, true);
		if (err != MojErrNone)
			break;
		bool deleted = false;
		if (obj.get(MojDb::DelKey, deleted) && deleted) {
			delSize += item->size();
			delCount++;
		} else {
			size += item->size();
			count++;
		}
	}

	MojLogInfo(s_log, _T("KindStats Summary: %s : Count: %zu; delCount: %zu; warnings: %zu \n"), m_id.data(), count, delCount, warnings);
	
	usageOut += size + delSize;
	MojObject info;
	err = info.put(SizeKey, (MojInt64) size);
	MojErrCheck(err);
	err = info.put(CountKey, (MojInt64) count);
	MojErrCheck(err);
	if (delCount > 0) {
		err = info.put(DelSizeKey, (MojInt64) delSize);
		MojErrCheck(err);
		err = info.put(DelCountKey, (MojInt64) delCount);
		MojErrCheck(err);
	}
	if (warnings > 0) {
		err = info.put(WarnKey, (MojInt64) warnings);
		MojErrCheck(err);
	}
	err = objOut.put(ObjectsKey, info);
	MojErrCheck(err);

	// and indexes
	MojObject indexes;
	
	for (IndexVec::ConstIterator i = m_indexes.begin(); i != m_indexes.end(); ++i) {
		MojObject indexInfo;
		err = (*i)->stats(indexInfo, usageOut, req);
		MojErrCheck(err);

		if (verify) {
			MojDbIndex *pi = i->get();
			MojErr err2 = verifyIndex(pi, indexInfo, req);
			MojErrCheck(err2);
		}

		err = indexes.put((*i)->name(), indexInfo);
		MojErrCheck(err);
	}
	err = objOut.put(IndexesKey, indexes);
	MojErrCheck(err);

	return MojErrNone;
}
Beispiel #6
0
MojErr MojDbKind::verifyIndex(MojDbIndex *pIndex, MojObject &iinfo, MojDbReq& req)
{
	// Goes throudh each index entry and verifies that it points to a valid object
	// For debugging purposes as stats for indexes does not access the target objects
	// Index->stats function does not have enough context to find the object
	// db/stats usage '{"verify":true,"kind":"xyz"}' - each optional
	
	MojDbQuery query;
	MojErr err = query.from(m_id);
	MojErrCheck(err);
	err = query.includeDeleted(true);
	MojErrCheck(err);
	MojDbCursor cursor;
	query.m_forceIndex = pIndex;		// Important: Otherwise, it will pick the default index
	cursor.verifymode(true);		// to get the errors
	err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
	MojLogInfo(s_log, _T("Kind_verifyIndex: Kind: %s; Index: %s; idIndex: %zX; size: %zu; CursorIndex: %s \n"), m_name.data(), 
					pIndex->name().data(), pIndex->idIndex(), pIndex->size(), cursor.m_dbIndex->name().data());
	MojErrCheck(err);
	MojSize count = 0;
	MojSize delCount = 0;
	MojSize warnCount = 0;
	char s[1024];
	for (;;) {
		MojDbStorageItem* item = NULL;
		bool found = false;
		err = cursor.get(item, found);
		if (err == MojErrInternalIndexOnFind) {
			warnCount++;
			MojDbIsamQuery *iquery = (MojDbIsamQuery *)cursor.m_storageQuery.get();
			MojErr err2 =  MojByteArrayToHex(iquery->m_keyData, iquery->m_keySize, s); 
			MojErrCheck(err2);
			MojChar *ids = (iquery->m_keySize > 18) ? (MojChar *)(iquery->m_keyData + iquery->m_keySize - 17) : NULL; 
			MojLogInfo(s_log, _T("VerifyIndex Warning: %s; KeySize: %zu; %s ;id: %s \n"), 
					cursor.m_dbIndex->name().data(), iquery->m_keySize, s, ids);
			continue;
		}
		MojErrCheck(err);
		if (!found)
			break;
		MojObject obj;
		err = item->toObject(obj, *m_kindEngine, true);
		MojErrCheck(err);
		bool deleted = false;
		if (obj.get(MojDb::DelKey, deleted) && deleted) {
			delCount++;
		} else {
			count++;
		}
	}

	MojLogInfo(s_log, _T("Kind_verifyIndex Counts: Kind: %s; Index: %s; count: %zu; delcount: %zu; warnings: %zu \n"), m_name.data(), 
					pIndex->name().data(), count, delCount, warnCount);
	
	err = iinfo.put(VerifyCountKey, (MojInt64)count);
	MojErrCheck(err);
	err = iinfo.put(VerifyWarnCountKey, (MojInt64) warnCount);
	MojErrCheck(err);
	err = iinfo.put(VerifyDelCountKey, (MojInt64) delCount);
	MojErrCheck(err);

	return MojErrNone;
}