MojErr MojDbLevelIndex::del(const MojDbKey& key, MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(txn);
MojAssert(isOpen());
MojDbLevelItem keyItem;
keyItem.fromBytesNoCopy(key.data(), key.size());
bool found = false;
MojErr err = m_db->del(keyItem, found, txn);
#ifdef MOJ_DEBUG
char s[1024];
size_t size1 = keyItem.size();
MojErr err2 = MojByteArrayToHex(keyItem.data(), size1, s);
MojErrCheck(err2);
if (size1 > 16) // if the object-id is in key
strncat(s, (char *)(keyItem.data()) + (size1 - 17), 16);
LOG_DEBUG("[db_ldb] ldbindexdel: %s; keylen: %zu, key: %s ; err = %d\n", m_db->m_name.data(), size1, s, err);
if (!found)
LOG_WARNING(MSGID_LEVEL_DB_WARNING, 1,
PMLOGKS("index", s),
"ldbindexdel_warn: not found: %s \n", s);
#endif
MojErrCheck(err);
if (!found) {
//MojErrThrow(MojErrDbInconsistentIndex); // fix this to work around to deal with out of sync indexes
MojErrThrow(MojErrInternalIndexOnDel);
}
return MojErrNone;
}
MojErr MojDbIndexTest::assertContainsText(TestIndex& ti, MojObject id, const MojChar* str)
{
MojString strObj;
MojErr err = strObj.assign(str);
MojErrCheck(err);
MojRefCountedPtr<MojDbTextCollator> collator(new MojDbTextCollator);
MojAllocCheck(collator.get());
err = collator->init(_T("en_US"), MojDbCollationPrimary);
MojTestErrCheck(err);
MojDbKey key;
err = collator->sortKey(strObj, key);
MojTestErrCheck(err);
MojObjectWriter writer;
err = id.visit(writer);
MojTestErrCheck(err);
const MojByte* idData = NULL;
MojSize idSize = 0;
err = writer.buf().data(idData, idSize);
MojTestErrCheck(err);
err = key.byteVec().append(idData, idData + idSize);
MojTestErrCheck(err);
err = assertContains(ti, id, key);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDbPropExtractor::handleVal(const MojObject& val, KeySet& valsOut, MojSize idx) const
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(idx < m_prop.size());
MojErr err = MojErrNone;
if (idx == m_prop.size() - 1) {
// if we're at the end of the prop path, use this object as the value
if (m_tokenizer.get() && val.type() == MojObject::TypeString) {
MojString text;
err = val.stringValue(text);
MojErrCheck(err);
if (m_tokenizer.get()) {
err = m_tokenizer->tokenize(text, m_collator.get(), valsOut);
MojErrCheck(err);
}
} else {
MojDbKey key;
err = key.assign(val, m_collator.get());
MojErrCheck(err);
err = valsOut.put(key);
MojErrCheck(err);
}
} else {
// otherwise, keep recursing
err = valsImpl(val, valsOut, idx + 1);
MojErrCheck(err);
}
return MojErrNone;
}
MojErr MojDbPropExtractor::fromObjectImpl(const MojObject& obj, const MojDbPropExtractor& defaultConfig, const MojChar* locale)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
// super
MojErr err = MojDbExtractor::fromObject(obj, locale);
MojErrCheck(err);
// default value
m_default = defaultConfig.m_default;
MojObject defaultVal;
if (obj.get(DefaultKey, defaultVal)) {
MojObject::Type type = defaultVal.type();
MojDbKey key;
if (type == MojObject::TypeArray) {
// if the value is an array, act on its elements rather than the array object itself
MojObject::ConstArrayIterator end = defaultVal.arrayEnd();
for (MojObject::ConstArrayIterator j = defaultVal.arrayBegin(); j != end; ++j) {
err = key.assign(*j);
MojErrCheck(err);
err = m_default.put(key);
MojErrCheck(err);
}
} else {
err = key.assign(defaultVal);
MojErrCheck(err);
err = m_default.put(key);
MojErrCheck(err);
}
}
// tokenizer
m_tokenizer = defaultConfig.m_tokenizer;
MojString tokenize;
bool found = false;
err = obj.get(TokenizeKey, tokenize, found);
MojErrCheck(err);
if (found) {
if (tokenize == AllKey || tokenize == DefaultKey) {
m_tokenizer.reset(new MojDbTextTokenizer);
MojAllocCheck(m_tokenizer.get());
err = m_tokenizer->init(locale);
MojErrCheck(err);
} else {
MojErrThrow(MojErrDbInvalidTokenization);
}
}
// collator
if (m_collation == MojDbCollationInvalid)
m_collation = defaultConfig.m_collation;
err = updateLocale(locale);
MojErrCheck(err);
// set prop
err = prop(m_name);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbIndex::open(MojDbStorageIndex* index, const MojObject& id, MojDbReq& req, bool created)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(!isOpen() && !m_props.empty());
MojAssert(index);
// we don't want to take built-in props into account when sorting indexes,
m_sortKey = m_propNames;
m_id = id;
MojDbKey idKey;
MojErr err = idKey.assign(id);
MojErrCheck(err);
err = m_idSet.put(idKey);
MojErrCheck(err);
err = addBuiltinProps();
MojErrCheck(err);
if (created && !isIdIndex()) {
// if this index was just created, we need to re-index before committing the transaction
MojDbStorageTxn* txn = req.txn();
txn->notifyPreCommit(m_preCommitSlot);
txn->notifyPostCommit(m_postCommitSlot);
} else {
// otherwise it's ready
m_ready = true;
}
// and we're open
m_index.reset(index);
m_collection = m_index.get();
return MojErrNone;
}
MojErr MojDbTextTokenizerTest::check(const MojChar* text, const MojChar* tokens)
{
// tokenize string
MojString textStr;
MojErr err = textStr.assign(text);
MojTestErrCheck(err);
MojSet<MojDbKey> set;
MojRefCountedPtr<MojDbTextTokenizer> tokenizer(new MojDbTextTokenizer);
MojAllocCheck(tokenizer.get());
err = tokenizer->init(_T("en_US"));
MojTestErrCheck(err);
err = tokenizer->tokenize(textStr, NULL, set);
MojTestErrCheck(err);
// check that tokens match
MojObject obj;
err = obj.fromJson(tokens);
MojTestErrCheck(err);
MojSize objSize = obj.size();
MojSize setSize = set.size();
MojTestAssert(objSize == setSize);
for (MojObject::ConstArrayIterator i = obj.arrayBegin(); i != obj.arrayEnd(); ++i) {
MojDbKey key;
err = key.assign(*i);
MojTestErrCheck(err);
MojTestAssert(set.contains(key));
}
return MojErrNone;
}
MojErr MojDbIndexTest::assertContains(TestIndex& ti, MojObject id, MojObject key)
{
MojObjectWriter writer;
MojErr err = key.visit(writer);
MojTestErrCheck(err);
err = id.visit(writer);
MojTestErrCheck(err);
MojDbKey compoundKey;
err = compoundKey.assign(writer.buf());
MojTestErrCheck(err);
err = assertContains(ti, id, compoundKey);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDbWatcher::activate(const MojDbKey& limitKey)
{
MojAssert(m_index);
MojThreadGuard guard(m_mutex);
m_state = StateActive;
bool fired = !m_fireKey.empty();
bool inRange = fired &&
(limitKey.empty() ||
(m_desc && m_fireKey >= limitKey) ||
(!m_desc && m_fireKey <= limitKey));
if (inRange) {
// we were fired before activation, so if the maxKey our cursor returned
// is >= the minKey with which we were fired, go ahead and do the fire
MojErr err = fireImpl();
MojErrCheck(err);
} else {
// keep limit so we can reject fires for larger keys
m_limitKey = limitKey;
}
MojLogDebug(MojDb::s_log, _T("Watcher_activate: state= %d; fired = %d; inrange = %d; index name = %s; domain = %s\n"), (int)m_state,
(int)fired, (int)inRange, ((m_index) ? m_index->name().data(): NULL), ((m_domain) ? m_domain.data(): NULL));
return MojErrNone;
}
MojErr MojDbTextTokenizer::tokenize(const MojString& text, MojDbTextCollator* collator, KeySet& keysOut) const
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(m_ubrk.get());
// convert to UChar from str
MojDbTextUtils::UnicodeVec unicodeStr;
MojErr err = MojDbTextUtils::strToUnicode(text, unicodeStr);
MojErrCheck(err);
// clone break iterator and set text
MojByte buf[U_BRK_SAFECLONE_BUFFERSIZE];
UErrorCode status = U_ZERO_ERROR;
MojInt32 size = sizeof(buf);
IterPtr ubrk(ubrk_safeClone(m_ubrk.get(), buf, &size, &status));
MojUnicodeErrCheck(status);
MojAssert(ubrk.get());
ubrk_setText(ubrk.get(), unicodeStr.begin(), (MojInt32) unicodeStr.size(), &status);
MojUnicodeErrCheck(status);
MojInt32 tokBegin = -1;
MojInt32 pos = ubrk_first(ubrk.get());
while (pos != UBRK_DONE) {
UWordBreak status = (UWordBreak) ubrk_getRuleStatus(ubrk.get());
if (status != UBRK_WORD_NONE) {
MojAssert(tokBegin != -1);
MojDbKey key;
const UChar* tokChars = unicodeStr.begin() + tokBegin;
MojSize tokSize = (MojSize) (pos - tokBegin);
if (collator) {
err = collator->sortKey(tokChars, tokSize, key);
MojErrCheck(err);
} else {
MojString tok;
err = MojDbTextUtils::unicodeToStr(tokChars, tokSize, tok);
MojErrCheck(err);
err = key.assign(tok);
MojErrCheck(err);
}
err = keysOut.put(key);
MojErrCheck(err);
}
tokBegin = pos;
pos = ubrk_next(ubrk.get());
}
return MojErrNone;
}
MojErr MojDbLevelIndex::drop(MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojDbLevelCursor cursor;
MojErr err = cursor.open(m_db.get(), txn, 0);
MojErrCheck(err);
MojDbKey prefix;
err = prefix.assign(m_id);
MojErrCheck(err);
err = cursor.delPrefix(prefix);
MojErrCheck(err);
err = cursor.close();
MojErrCheck(err);
return err;
}
MojErr MojDbLevelIndex::stats(MojDbStorageTxn* txn, MojSize& countOut, MojSize& sizeOut)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojDbLevelCursor cursor;
MojErr err = cursor.open(m_db.get(), txn, 0);
MojErrCheck(err);
MojDbKey prefix;
err = prefix.assign(m_id);
MojErrCheck(err);
err = cursor.statsPrefix(prefix, countOut, sizeOut);
MojErrCheck(err);
err = cursor.close();
MojErrCheck(err);
return err;
}
MojErr MojDbIndexTest::assertContains(TestIndex& ti, MojObject id, const MojChar* json)
{
MojObject array;
MojErr err = array.fromJson(json);
MojTestErrCheck(err);
MojObjectWriter(writer);
MojObject val;
MojSize idx = 0;
while (array.at(idx++, val)) {
err = val.visit(writer);
MojTestErrCheck(err);
}
err = id.visit(writer);
MojTestErrCheck(err);
MojDbKey key;
err = key.assign(writer.buf());
MojTestErrCheck(err);
err = assertContains(ti, id, key);
MojTestErrCheck(err);
return MojErrNone;
}
MojErr MojDbIndexTest::assertContains(TestIndex& ti, MojObject id, const MojDbKey& key)
{
MojDbKey prefixedKey(key);
MojErr err = prefixedKey.byteVec().insert(0, 1, MojObjectWriter::MarkerZeroIntValue);
MojErrCheck(err);
if (ti.m_incDel) {
MojDbKey::ByteVec vec = prefixedKey.byteVec();
MojErr err = vec.insert(1, 1, MojObjectWriter::MarkerTrueValue);
MojTestErrCheck(err);
MojDbKey keyTrue;
err = keyTrue.assign(vec.begin(), vec.size());
MojTestErrCheck(err);
err = vec.setAt(1, MojObjectWriter::MarkerFalseValue);
MojTestErrCheck(err);
MojDbKey keyFalse;
err = keyFalse.assign(vec.begin(), vec.size());
MojTestErrCheck(err);
MojTestAssert(ti.m_set.contains(keyTrue) || ti.m_set.contains(keyFalse));
} else {
MojTestAssert(ti.m_set.contains(prefixedKey));
}
return MojErrNone;
}
MojErr MojDbQueryPlan::pushVal(MojDbKeyBuilder& builder, const MojObject& val, MojDbTextCollator* collator)
{
MojErr err = MojErrNone;
MojDbKey key;
MojDbKeyBuilder::KeySet keys;
if (val.type() == MojObject::TypeArray) {
MojObject::ConstArrayIterator end = val.arrayEnd();
for (MojObject::ConstArrayIterator i = val.arrayBegin(); i != end; ++i) {
err = key.assign(*i, collator);
MojErrCheck(err);
err = keys.put(key);
MojErrCheck(err);
}
} else {
err = key.assign(val, collator);
MojErrCheck(err);
err = keys.put(key);
MojErrCheck(err);
}
err = builder.push(keys);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbLevelIndex::insert(const MojDbKey& key, MojDbStorageTxn* txn)
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(txn);
MojDbLevelItem keyItem;
keyItem.fromBytesNoCopy(key.data(), key.size());
MojDbLevelItem valItem; // empty item? not clear why do we need to insert it
MojErr err = m_db->put(keyItem, valItem, txn, true);
#ifdef MOJ_DEBUG
char s[1024];
size_t size1 = keyItem.size();
size_t size2 = valItem.size();
MojErr err2 = MojByteArrayToHex(keyItem.data(), size1, s);
MojErrCheck(err2);
if (size1 > 16) // if the object-id is in key
strncat(s, (char *)(keyItem.data()) + (size1 - 17), 16);
LOG_DEBUG("[db_ldb] ldbindexinsert: %s; keylen: %zu, key: %s ; vallen = %zu; err = %d\n",
m_db->m_name.data(), size1, s, size2, err);
#endif
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbTextCollator::sortKey(const UChar* chars, MojSize size, MojDbKey& keyOut) const
{
LOG_TRACE("Entering function %s", __FUNCTION__);
MojErr err = MojErrNone;
MojObjectWriter writer;
if (size == 0) {
err = writer.stringValue(_T(""), 0);
MojErrCheck(err);
} else {
// get sort key
MojInt32 destCapacity = 0;
MojInt32 destLength = 0;
MojDbKey::ByteVec vec;
err = vec.resize(size * 3);
MojErrCheck(err);
do {
MojByte* dest = NULL;
err = vec.begin(dest);
MojErrCheck(err);
destCapacity = (MojInt32) vec.size();
destLength = ucol_getSortKey(m_ucol, chars, (MojInt32) size, dest, destCapacity);
if (destLength == 0) {
MojErrThrow(MojErrDbUnicode);
}
err = vec.resize(destLength);
MojErrCheck(err);
} while (destLength > destCapacity);
// write it
MojAssert(vec.size() >= 1 && vec.back() == _T('\0'));
err = writer.stringValue((const MojChar*) vec.begin(), vec.size() - 1);
MojErrCheck(err);
}
err = keyOut.assign(writer.buf());
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDbQueryPlan::rangesFromKeys(MojDbKey lowerKey, MojDbKey upperKey, MojDbKey prefix, MojUInt32 index,
const MojDbQuery::WhereClause* clause)
{
MojErr err = MojErrNone;
MojUInt32 group = 0;
MojDbQuery::CompOp lowerOp = MojDbQuery::OpEq;
MojDbQuery::CompOp upperOp = MojDbQuery::OpNone;
if (clause) {
lowerOp = clause->lowerOp();
upperOp = clause->upperOp();
}
// set up upper bound
switch (upperOp) {
case MojDbQuery::OpNone:
MojAssert(lowerOp != MojDbQuery::OpNone);
upperKey = prefix;
// no break. fall through to OpLessThanEq case
case MojDbQuery::OpLessThanEq:
// match while less-than ++upperKey
err = upperKey.increment();
MojErrCheck(err);
break;
default:
MojAssert(upperOp == MojDbQuery::OpLessThan);
break;
}
// set up lower bound
switch (lowerOp) {
case MojDbQuery::OpNone:
// seek to prefix and match while less than upperKey
MojAssert(upperOp != MojDbQuery::OpNone);
err = addRange(lowerKey, upperKey);
MojErrCheck(err);
break;
case MojDbQuery::OpSearch:
group = index % m_groupCount;
// no break. fall through to OpPrefix case
case MojDbQuery::OpPrefix:
// remove null terminator
MojAssert(!prefix.empty());
if (prefix.byteVec().back() == 0) {
err = prefix.byteVec().pop();
MojErrCheck(err);
}
// no break. fall through to OpEq case
case MojDbQuery::OpEq:
// seek to lowerKey and match while less than ++prefix
err = prefix.increment();
MojErrCheck(err);
err = addRange(lowerKey, prefix, group);
MojErrCheck(err);
break;
case MojDbQuery::OpNotEq:
// seek to prefix and match while less than lowerKey
err = addRange(prefix, lowerKey);
MojErrCheck(err);
// seek to ++lowerKey, and match while less than ++prefix
err = lowerKey.increment();
MojErrCheck(err);
err = prefix.increment();
MojErrCheck(err);
err = addRange(lowerKey, prefix);
MojErrCheck(err);
break;
case MojDbQuery::OpGreaterThan:
// seek to ++lowerKey and match while less than upperKey
err = lowerKey.increment();
MojErrCheck(err);
// no break. fall through to OpGreaterThanEq case
case MojDbQuery::OpGreaterThanEq:
// seek to lowerKey and match while less than upperKey
err = addRange(lowerKey, upperKey);
MojErrCheck(err);
break;
default:
MojAssertNotReached();
break;
}
return MojErrNone;
}
