MojErr MojDbLevelIndex::del(const MojDbKey& key, MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(txn);
MojAssert(isOpen());
MojDbLevelItem keyItem;
keyItem.fromBytesNoCopy(key.data(), key.size());
bool found = false;
MojErr err = m_db->del(keyItem, found, txn);
#ifdef MOJ_DEBUG
char s[1024];
size_t size1 = keyItem.size();
MojErr err2 = MojByteArrayToHex(keyItem.data(), size1, s);
MojErrCheck(err2);
if (size1 > 16) // if the object-id is in key
strncat(s, (char *)(keyItem.data()) + (size1 - 17), 16);
LOG_DEBUG("[db_ldb] ldbindexdel: %s; keylen: %zu, key: %s ; err = %d\n", m_db->m_name.data(), size1, s, err);
if (!found)
LOG_WARNING(MSGID_LEVEL_DB_WARNING, 1,
PMLOGKS("index", s),
"ldbindexdel_warn: not found: %s \n", s);
#endif
MojErrCheck(err);
if (!found) {
//MojErrThrow(MojErrDbInconsistentIndex); // fix this to work around to deal with out of sync indexes
MojErrThrow(MojErrInternalIndexOnDel);
}
return MojErrNone;
}
MojErr MojDbSandwichDatabase::put(MojDbSandwichItem& key, MojDbSandwichItem& val, MojDbStorageTxn* txn, bool updateIdQuota)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(m_db.Valid() );
MojAssert( !txn || dynamic_cast<MojDbSandwichEnvTxn *> (txn) );
MojErr err;
if (txn)
{
MojInt64 quotaOffset = val.size();
if (updateIdQuota)
quotaOffset += key.size();
err = txn->offsetQuota(quotaOffset);
MojErrCheck(err);
}
MojDbSandwichEnvTxn * leveldb_txn = static_cast<MojDbSandwichEnvTxn *> (txn);
leveldb::Status s;
if(leveldb_txn)
{
s = leveldb_txn->ref(impl()).Put(*key.impl(), *val.impl());
}
else
s = m_db.Put(*key.impl(), *val.impl());
#if defined(MOJ_DEBUG)
char str_buf[1024];
size_t size1 = key.size();
size_t size2 = val.size();
MojErr err2 = MojByteArrayToHex(key.data(), size1, str_buf);
MojErrCheck(err2);
if (size1 > 16) // if the object-id is in key
strncat(str_buf, (char *)(key.data()) + (size1 - 17), 16);
LOG_DEBUG("[db_ldb] ldb put: %s; keylen: %zu, key: %s ; vallen = %zu; err = %s\n",
this->m_name.data(), size1, str_buf, size2, s.ToString().c_str());
#endif
/*
if(leveldb_txn)
;//MojLdbErrCheck(batch->status(), _T("db->put"));
else
*/
MojLdbErrCheck(s, _T("db->put"));
postUpdate(txn, key.size() + val.size());
return MojErrNone;
}
MojErr MojDbIsamQuery::getImpl(MojDbStorageItem*& itemOut, bool& foundOut, bool getItem)
{
itemOut = NULL;
MojUInt32 group = 0;
MojErr err = getKey(group, foundOut);
MojErrCheck(err);
if (foundOut && getItem) {
err = getVal(itemOut, foundOut);
if (err == MojErrInternalIndexOnFind) {
#if defined (MOJ_DEBUG)
char s[1024];
char *s2 = NULL;
MojErr err2 = MojByteArrayToHex(m_keyData, m_keySize, s);
MojErrCheck(err2);
if (m_keySize > 17)
s2 = ((char *)m_keyData) + m_keySize - 17;
MojSize idIndex = m_plan->idIndex();
const MojChar * from = m_plan->query().from().data();
MojLogInfo(MojDb::s_log, _T("isamquery_warnindex: from: %s; indexid: %zu; group: %d; KeySize: %zu; %s ;id: %s \n"),
from, idIndex, (int)group, m_keySize, s, (s2?s2:"NULL"));
#endif
}
MojErrCheck(err);
}
if (foundOut) {
//If distinct query is, We need to check that field is duplicated or not
//In case of duplication, count will not incremented,
//and set "itemOut" to NULL for getting next DB result.
if(!m_distinct.empty() && itemOut)
{
bool distincted = false;
err = distinct(itemOut, distincted);
MojErrCheck(err);
if(!distincted)
incrementCount();
else
itemOut = NULL;
}
else
incrementCount();
}
return MojErrNone;
}
MojErr MojDbLevelQuery::getById(const MojObject& id, MojDbStorageItem*& itemOut, bool& foundOut)
{
itemOut = NULL;
foundOut = false;
MojDbLevelItem* item = NULL;
MojErr err = MojErrNone;
if (m_db) {
// retrun val from primary db
MojDbLevelItem primaryKey;
err = primaryKey.fromObject(id);
MojErrCheck(err);
err = m_db->get(primaryKey, m_txn, false, m_primaryVal, foundOut);
MojErrCheck(err);
if (!foundOut) {
char s[1024];
int size = primaryKey.size();
(void) MojByteArrayToHex(primaryKey.data(), size, s);
MojLogInfo(MojDbLevelEngine::s_log, _T("bdbq_byId_warnindex: KeySize: %d; %s ;id: %s \n"),
size, s, primaryKey.data()+1);
//MojErrThrow(MojErrDbInconsistentIndex);
MojErrThrow(MojErrInternalIndexOnFind);
}
item = &m_primaryVal;
} else {
// return val from cursor
item = &m_val;
}
item->id(id);
// check for exclusions
bool exclude = false;
err = checkExclude(item, exclude);
MojErrCheck(err);
if (!exclude) {
itemOut = item;
}
return MojErrNone;
}
MojErr MojDbLevelDatabase::del(MojDbLevelItem& key, bool& foundOut, MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(m_db);
MojAssert( !txn || dynamic_cast<MojDbLevelAbstractTxn *> (txn) );
foundOut = false;
MojErr err = txn->offsetQuota(-(MojInt64) key.size());
MojErrCheck(err);
MojDbLevelAbstractTxn * leveldb_txn = static_cast<MojDbLevelAbstractTxn *> (txn);
leveldb::Status st;
if(leveldb_txn)
{
leveldb_txn->tableTxn(impl()).Delete(*key.impl());
}
else
st = m_db->Delete(MojDbLevelEngine::getWriteOptions(), *key.impl());
#if defined(MOJ_DEBUG)
char str_buf[1024]; // big enough for any key
size_t size = key.size();
MojErr err2 = MojByteArrayToHex(key.data(), size, str_buf);
MojErrCheck(err2);
if (size > 16) // if the object-id is in key
strncat(str_buf, (char *)(key.data()) + (size - 17), 16);
LOG_DEBUG("[db_ldb] ldbdel: %s; keylen: %zu, key= %s; err = %d \n", this->m_name.data(), size, str_buf, !st.ok());
#endif
if (st.IsNotFound() == false) {
MojLdbErrCheck(st, _T("db->del"));
foundOut = true;
}
postUpdate(txn, key.size());
return MojErrNone;
}
MojErr MojDbIndex::insertKeys(const KeySet& keys, MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
int count = 0;
for (KeySet::ConstIterator i = keys.begin(); i != keys.end(); ++i) {
MojErr err = m_index->insert(*i, txn);
#if defined(MOJ_DEBUG_LOGGING)
char s[1024];
size_t size = (*i).size();
MojErr err2 = MojByteArrayToHex((*i).data(), size, s);
MojErrCheck(err2);
if (size > 16) // if the object-id is in key
strncat(s, (char *)((*i).data()) + (size - 17), 16);
LOG_DEBUG("[db_mojodb] insertKey %d for: %s; key= %s ; err= %d\n", count+1, this->m_name.data(), s, err);
#endif
MojErrCheck(err);
count ++;
}
return MojErrNone;
}
MojErr MojDbIndex::insertKeys(const KeySet& keys, MojDbStorageTxn* txn)
{
MojLogTrace(s_log);
int count = 0;
for (KeySet::ConstIterator i = keys.begin(); i != keys.end(); ++i) {
MojErr err = m_index->insert(*i, txn);
#if defined(MOJ_DEBUG_LOGGING)
char s[1024];
size_t size = (*i).size();
MojErr err2 = MojByteArrayToHex((*i).data(), size, s);
MojErrCheck(err2);
if (size > 16) // if the object-id is in key
strncat(s, (char *)((*i).data()) + (size - 17), 16);
MojLogInfo(s_log, _T("insertKey %d for: %s; key= %s ; err= %d\n"), count+1, this->m_name.data(), s, err);
#endif
MojErrCheck(err);
count ++;
}
return MojErrNone;
}
MojErr MojDbIndex::delKeys(const KeySet& keys, MojDbStorageTxn* txn, bool forcedel)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
int count = 0;
for (KeySet::ConstIterator i = keys.begin(); i != keys.end(); ++i) {
MojErr err = m_index->del(*i, txn);
#if defined(MOJ_DEBUG_LOGGING)
char s[1024];
char *s2 = NULL;
if (m_kind)
s2 = (char *)(m_kind->id().data());
size_t size = (*i).size();
MojErr err2 = MojByteArrayToHex((*i).data(), size, s);
MojErrCheck(err2);
if (size > 16) // if the object-id is in key
strncat(s, (char *)((*i).data()) + (size - 17), 16);
LOG_DEBUG("[db_mojodb] delKey %d for: %s - %s; key= %s ; err= %d\n", count+1, s2, this->m_name.data(), s, err);
#endif
// This has some potential risk
if (err == MojErrInternalIndexOnDel) {
m_delMisses++;
#if defined(MOJ_DEBUG_LOGGING)
LOG_DEBUG("[db_mojodb] delKey %d for: %s - %s; key= %s; err = %d \n", count+1, s2, this->m_name.data(), s, err);
#endif
if (forcedel)
err = MojErrNone;
}
MojErrCheck(err);
count++;
}
return MojErrNone;
}
MojErr MojDbLevelIndex::insert(const MojDbKey& key, MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(txn);
MojDbLevelItem keyItem;
keyItem.fromBytesNoCopy(key.data(), key.size());
MojDbLevelItem valItem; // empty item? not clear why do we need to insert it
MojErr err = m_db->put(keyItem, valItem, txn, true);
#ifdef MOJ_DEBUG
char s[1024];
size_t size1 = keyItem.size();
size_t size2 = valItem.size();
MojErr err2 = MojByteArrayToHex(keyItem.data(), size1, s);
MojErrCheck(err2);
if (size1 > 16) // if the object-id is in key
strncat(s, (char *)(keyItem.data()) + (size1 - 17), 16);
LOG_DEBUG("[db_ldb] ldbindexinsert: %s; keylen: %zu, key: %s ; vallen = %zu; err = %d\n",
m_db->m_name.data(), size1, s, size2, err);
#endif
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbKind::verifyIndex(MojDbIndex *pIndex, MojObject &iinfo, MojDbReq& req)
{
// Goes throudh each index entry and verifies that it points to a valid object
// For debugging purposes as stats for indexes does not access the target objects
// Index->stats function does not have enough context to find the object
// db/stats usage '{"verify":true,"kind":"xyz"}' - each optional
MojDbQuery query;
MojErr err = query.from(m_id);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojDbCursor cursor;
query.m_forceIndex = pIndex; // Important: Otherwise, it will pick the default index
cursor.verifymode(true); // to get the errors
err = m_kindEngine->find(query, cursor, NULL, req, OpRead);
MojLogInfo(s_log, _T("Kind_verifyIndex: Kind: %s; Index: %s; idIndex: %zX; size: %zu; CursorIndex: %s \n"), m_name.data(),
pIndex->name().data(), pIndex->idIndex(), pIndex->size(), cursor.m_dbIndex->name().data());
MojErrCheck(err);
MojSize count = 0;
MojSize delCount = 0;
MojSize warnCount = 0;
char s[1024];
for (;;) {
MojDbStorageItem* item = NULL;
bool found = false;
err = cursor.get(item, found);
if (err == MojErrInternalIndexOnFind) {
warnCount++;
MojDbIsamQuery *iquery = (MojDbIsamQuery *)cursor.m_storageQuery.get();
MojErr err2 = MojByteArrayToHex(iquery->m_keyData, iquery->m_keySize, s);
MojErrCheck(err2);
MojChar *ids = (iquery->m_keySize > 18) ? (MojChar *)(iquery->m_keyData + iquery->m_keySize - 17) : NULL;
MojLogInfo(s_log, _T("VerifyIndex Warning: %s; KeySize: %zu; %s ;id: %s \n"),
cursor.m_dbIndex->name().data(), iquery->m_keySize, s, ids);
continue;
}
MojErrCheck(err);
if (!found)
break;
MojObject obj;
err = item->toObject(obj, *m_kindEngine, true);
MojErrCheck(err);
bool deleted = false;
if (obj.get(MojDb::DelKey, deleted) && deleted) {
delCount++;
} else {
count++;
}
}
MojLogInfo(s_log, _T("Kind_verifyIndex Counts: Kind: %s; Index: %s; count: %zu; delcount: %zu; warnings: %zu \n"), m_name.data(),
pIndex->name().data(), count, delCount, warnCount);
err = iinfo.put(VerifyCountKey, (MojInt64)count);
MojErrCheck(err);
err = iinfo.put(VerifyWarnCountKey, (MojInt64) warnCount);
MojErrCheck(err);
err = iinfo.put(VerifyDelCountKey, (MojInt64) delCount);
MojErrCheck(err);
return MojErrNone;
}
