MojErr MojDb::updateLocale(const MojChar* locale, MojDbReqRef req)
{
MojAssert(locale);
MojLogTrace(s_log);
MojErr err = beginReq(req, true);
MojErrCheck(err);
MojString oldLocale;
err = getLocale(oldLocale, req);
MojErrCheck(err);
MojString newLocale;
err = newLocale.assign(locale);
MojErrCheck(err);
MojErr updateErr = err = updateLocaleImpl(oldLocale, newLocale, req);
MojErrCatchAll(err) {
err = req->abort();
MojErrCheck(err);
err = m_kindEngine.close();
MojErrCheck(err);
MojDbReq openReq;
err = beginReq(openReq, true);
MojErrCheck(err);
err = m_kindEngine.open(this, openReq);
MojErrCheck(err);
err = openReq.end();
MojErrCheck(err);
MojErrThrow(updateErr);
}
return MojErrNone;
}
MojErr MojDb::del(const MojObject* idsBegin, const MojObject* idsEnd, MojUInt32& countOut, MojObject& arrOut, MojUInt32 flags, MojDbReqRef req)
{
MojAssert(idsBegin || idsBegin == idsEnd);
MojAssert(idsEnd >= idsBegin);
MojLogTrace(s_log);
countOut = 0;
MojErr err = beginReq(req);
MojErrCheck(err);
// do the dels
MojUInt32 count= 0;
for (const MojObject* i = idsBegin; i != idsEnd; ++i) {
MojObject foundObj;
bool found = false;
err = delImpl(*i, found, foundObj, req, flags);
MojErrCheck(err);
if (found) {
count++;
err = arrOut.push(foundObj);
MojErrCheck(err);
}
}
// commit txn
err = req->end();
MojErrCheck(err);
countOut = count;
return MojErrNone;
}
MojErr MojDb::watch(const MojDbQuery& query, MojDbCursor& cursor, WatchSignal::SlotRef watchHandler, bool& firedOut, MojDbReqRef req)
{
MojLogTrace(s_log);
firedOut = false;
MojErr err = beginReq(req);
MojErrCheck(err);
MojRefCountedPtr<MojDbWatcher> watcher(new MojDbWatcher(watchHandler));
MojAllocCheck(watcher.get());
MojDbQuery limitedQuery = query;
limitedQuery.limit(1);
err = findImpl(limitedQuery, cursor, watcher.get(), req, OpRead);
MojErrCheck(err);
MojDbStorageItem* item = NULL;
bool found = false;
cursor.verifymode(false);
err = cursor.get(item, found);
MojErrCheck(err);
if (found) {
const MojDbKey& key = cursor.storageQuery()->endKey();
err = watcher->fire(key);
MojErrCheck(err);
firedOut = true;
}
err = req->end(false);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::quotaStats(MojObject& objOut, MojDbReqRef req)
{
MojLogTrace(s_log);
MojErr err = beginReq(req);
MojErrCheck(err);
err = m_quotaEngine.stats(objOut, req);
MojErrCheck(err);
err = req->end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::stats(MojObject& objOut, MojDbReqRef req, bool verify, MojString *pKind)
{
MojLogTrace(s_log);
MojErr err = beginReq(req);
MojErrCheck(err);
err = m_kindEngine.stats(objOut, req, verify, pKind);
MojErrCheck(err);
err = req->end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::merge(const MojDbQuery& query, const MojObject& props, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
MojErr err = beginReq(req);
MojErrCheck(err);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpUpdate);
MojErrCheck(err);
MojAssert(cursor.txn());
MojUInt32 count = 0;
MojUInt32 warns = 0;
bool found = false;
MojObject prev;
for (;;) {
// get prev rev from cursor
MojDbStorageItem* prevItem = NULL;
err = cursor.get(prevItem, found);
if (err == MojErrInternalIndexOnFind) {
warns++;
continue;
}
MojErrCheck(err);
if (!found)
break;
err = prevItem->toObject(prev, m_kindEngine);
MojErrCheck(err);
// merge obj into prev
MojObject merged;
err = mergeInto(merged, props, prev);
MojErrCheck(err);
// and update the db
const MojObject& id = prevItem->id();
err = putObj(id, merged, &prev, prevItem, req, OpUpdate);
MojErrCheck(err);
++count;
}
if (warns > 0)
MojLogWarning(s_log, _T("Merge index_warnings: %s; count: %d\n"), query.from().data(), warns);
err = cursor.close();
MojErrCheck(err);
err = req->end();
MojErrCheck(err);
countOut = count;
return MojErrNone;
}
MojErr MojDb::purgeStatus(MojObject& revOut, MojDbReqRef req)
{
MojLogTrace(s_log);
revOut = -1;
MojErr err = beginReq(req);
MojErrCheck(err);
err = getState(LastPurgedRevKey, revOut, req);
MojErrCheck(err);
err = req->end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::find(const MojDbQuery& query, MojDbCursor& cursor, MojDbReqRef req)
{
MojLogTrace(s_log);
MojErr err = beginReq(req);
MojErrCheck(err);
err = findImpl(query, cursor, NULL, req, OpRead);
MojErrCheck(err);
err = req->end(false);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::del(const MojDbQuery& query, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
MojErr err = beginReq(req);
MojErrCheck(err);
err = delImpl(query, countOut, req, flags);
MojErrCheck(err);
err = req->end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::find(const MojDbQuery& query, MojDbCursor& cursor, WatchSignal::SlotRef watchHandler, MojDbReqRef req)
{
MojLogTrace(s_log);
MojErr err = beginReq(req);
MojErrCheck(err);
MojRefCountedPtr<MojDbWatcher> watcher(new MojDbWatcher(watchHandler));
MojAllocCheck(watcher.get());
err = findImpl(query, cursor, watcher.get(), req, OpRead);
MojErrCheck(err);
err = req->end(false);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::put(MojObject* begin, const MojObject* end, MojUInt32 flags, MojDbReqRef req)
{
MojAssert(begin || begin == end);
MojAssert(end >= begin);
MojLogTrace(s_log);
MojErr err = beginReq(req);
MojErrCheck(err);
for (MojObject* i = begin; i != end; ++i) {
err = putImpl(*i, flags, req);
MojErrCheck(err);
}
err = req->end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::putConfig(MojObject* begin, const MojObject* end, MojDbReq& req, MojDbPutHandler& handler)
{
MojAssert(begin || begin == end);
MojAssert(end >= begin);
MojLogTrace(s_log);
MojErr err = beginReq(req, true);
MojErrCheck(err);
for (MojObject* i = begin; i != end; ++i) {
err = handler.put(*i, req);
MojErrCheck(err);
}
err = req.end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::get(const MojObject* idsBegin, const MojObject* idsEnd, MojObjectVisitor& visitor, MojDbReqRef req)
{
MojAssert(idsBegin || idsBegin == idsEnd);
MojAssert(idsEnd >= idsBegin);
MojLogTrace(s_log);
MojErr err = beginReq(req);
MojErrCheck(err);
// do the gets
for (const MojObject* i = idsBegin; i != idsEnd; ++i) {
err = getImpl(*i, visitor, OpRead, req);
MojErrCheck(err);
}
// commit txn
err = req->end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::putKind(MojObject& obj, MojUInt32 flags, MojDbReqRef req)
{
MojErr err = beginReq(req, true);
MojErrCheck(err);
// get id
MojString id;
err = obj.getRequired(MojDbServiceDefs::IdKey, id);
MojErrCheck(err);
MojLogInfo(s_log, _T("putKind: %s \n"), id.data());
MojLogWarning(s_log, _T("putKind: %s \n"), id.data());
// set _kind and _id
err = obj.putString(KindKey, MojDbKindEngine::KindKindId);
MojErrCheck(err);
MojString dbId;
err = formatKindId(id, dbId);
MojErrCheck(err);
err = obj.putString(IdKey, dbId);
MojErrCheck(err);
// put the object
MojDbAdminGuard guard(req);
err = putImpl(obj, flags | FlagForce, req);
MojErrCheck(err);
guard.unset();
// attempt the putKind
MojErr errAcc = m_kindEngine.putKind(obj, req);
MojErrAccumulate(err, errAcc);
err = commitKind(id, req, err);
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::delKind(const MojObject& id, bool& foundOut, MojUInt32 flags, MojDbReqRef req)
{
MojLogTrace(s_log);
foundOut = false;
MojErr err = beginReq(req, true);
MojErrCheck(err);
// del kind obj
MojString idStr;
err = id.stringValue(idStr);
MojErrCheck(err);
MojDbKind *pk = NULL;
// If Kinds has sub-kinds, we give an error
err = m_kindEngine.getKind(idStr.data(), pk);
MojErrCheck(err);
if (pk->nsubkinds() > 0) {
MojLogWarning(s_log, _T("delKind_error: %s has %d subkinds \n"), idStr.data(), pk->nsubkinds());
MojErrThrow(MojErrDbKindHasSubKinds);
}
//MojLogInfo(s_log, _T("delKind: %s \n"), idStr.data());
MojLogWarning(s_log, _T("delKind: %s \n"), idStr.data());
err = m_kindEngine.checkOwnerPermission(idStr, req);
MojErrCheck(err);
MojString dbId;
err = formatKindId(idStr, dbId);
MojErrCheck(err);
MojObject deleted;
bool found = false;
MojDbAdminGuard adminGuard(req);
err = delImpl(dbId, found, deleted, req, flags);
MojErrCheck(err);
if (found) {
// del objects
MojDbQuery query;
err = query.from(idStr);
MojErrCheck(err);
err = query.includeDeleted(true);
MojErrCheck(err);
MojUInt32 count;
req->fixmode(true);
err = delImpl(query, count, req, flags | FlagPurge);
MojErrCheck(err);
// del associated permissions
query.clear();
err = query.from(MojDbKindEngine::PermissionId);
MojErrCheck(err);
err = query.where(MojDbServiceDefs::ObjectKey, MojDbQuery::OpEq, idStr);
MojErrCheck(err);
req->fixmode(true);
err = delImpl(query, count, req, flags);
MojErrCheck(err);
// del kind
MojErr errAcc = m_kindEngine.delKind(idStr, req);
MojErrAccumulate(err, errAcc);
}
err = commitKind(idStr, req, err);
MojErrCheck(err);
foundOut = found;
return MojErrNone;
}
MojErr MojDb::dump(const MojChar* path, MojUInt32& countOut, bool incDel, MojDbReqRef req, bool backup,
MojUInt32 maxBytes, const MojObject* incrementalKey, MojObject* backupResponse)
{
MojAssert(path);
MojLogTrace(s_log);
MojErr err = beginReq(req);
MojErrCheck(err);
if (!req->admin()) {
MojLogError(s_log, _T("access denied: '%s' cannot dump db to path: '%s'"), req->domain().data(), path);
MojErrThrow(MojErrDbAccessDenied);
}
MojFile file;
err = file.open(path, MOJ_O_WRONLY | MOJ_O_CREAT | MOJ_O_TRUNC, MOJ_S_IRUSR | MOJ_S_IWUSR);
MojErrCheck(err);
// write out kinds first, then existing objects, then deleted objects
MojSize bytesWritten = 0;
MojSize totalwarns = 0;
MojSize newwarns = 0;
MojDbQuery objQuery;
MojVector<MojObject> kindVec;
MojObject revParam = -1;
MojObject delRevParam = -1;
// if we were given an incremental key, pull out the revs now
if (incrementalKey) {
incrementalKey->get(MojDbServiceDefs::RevKey, revParam);
incrementalKey->get(MojDbServiceDefs::DeletedRevKey, delRevParam);
}
err = m_kindEngine.getKinds(kindVec);
MojErrCheck(err);
// write kinds - if incremental, only write the kinds that have changed since the respective revs
MojString countStr;
for (MojVector<MojObject>::ConstIterator i = kindVec.begin(); i != kindVec.end(); ++i) {
if (backup) {
bool backupKind = false;
i->get(MojDbKind::SyncKey, backupKind);
if (!backupKind)
continue;
MojString id;
err = i->getRequired(MojDbServiceDefs::IdKey, id);
MojErrCheck(err);
MojDbQuery countQuery;
err = countQuery.from(id);
MojErrCheck(err);
MojDbCursor cursor;
err = find(countQuery, cursor, req);
MojErrCheck(err);
MojUInt32 count = 0;
err = cursor.count(count);
MojErrCheck(err);
if (count > 0) {
if (i != kindVec.begin()) {
err = countStr.appendFormat(_T(", "));
MojErrCheck(err);
}
err = countStr.appendFormat("%s=%u", id.data(), count);
MojErrCheck(err);
}
}
bool deleted = false;
i->get(DelKey, deleted);
MojObject kindRev;
err = i->getRequired(RevKey, kindRev);
MojErrCheck(err);
if ((deleted && kindRev > delRevParam) || (!deleted && kindRev > revParam)) {
err = dumpObj(file, (*i), bytesWritten, maxBytes);
MojErrCheck(err);
countOut++;
}
}
// dump all the non-deleted objects
err = dumpImpl(file, backup, false, revParam, delRevParam, true, countOut, req, backupResponse, MojDbServiceDefs::RevKey, bytesWritten, newwarns, maxBytes);
MojErrCheck(err);
totalwarns += newwarns;
// If we're supposed to include deleted objects, dump the deleted objects now.
// There's a chance that we may have run out of space in our backup. If that's the case,
// we don't want to try to dump deleted objects - we can detect this by looking for the HasMoreKey
if (incDel && backupResponse && !backupResponse->contains(MojDbServiceDefs::HasMoreKey)) {
err = dumpImpl(file, backup, true, revParam, delRevParam, false, countOut, req, backupResponse, MojDbServiceDefs::DeletedRevKey, bytesWritten, newwarns, maxBytes);
MojErrCheck(err);
}
totalwarns += newwarns;
// Add the Full and Version keys
if (backup && backupResponse) {
bool incremental = (incrementalKey != NULL);
err = backupResponse->putBool(MojDbServiceDefs::FullKey, !incremental);
MojErrCheck(err);
err = backupResponse->put(MojDbServiceDefs::VersionKey, DatabaseVersion);
MojErrCheck(err);
err = backupResponse->put(MojDbServiceDefs::WarningsKey, (MojInt32)totalwarns);
MojErrCheck(err);
MojString description;
err = description.format(_T("incremental=%u"), countOut);
MojErrCheck(err);
if (!countStr.empty()) {
err = description.appendFormat(_T(", %s"), countStr.data());
MojErrCheck(err);
}
err = backupResponse->put(_T("description"), description);
MojErrCheck(err);
}
err = req->end();
MojErrCheck(err);
return MojErrNone;
}
MojErr MojDb::purge(MojUInt32& countOut, MojInt64 numDays, MojDbReqRef req)
{
MojLogTrace(s_log);
countOut = 0;
if (numDays <= -1) {
numDays = m_purgeWindow;
}
MojErr err = beginReq(req);
MojErrCheck(err);
MojLogDebug(s_log, _T("purging objects deleted more than %lld days ago..."), numDays);
MojTime time;
err = MojGetCurrentTime(time);
MojErrCheck(err);
// store the revision number to current timestamp mapping
MojObject revTimeMapping;
MojInt64 rev;
err = nextId(rev);
MojErrCheck(err);
err = revTimeMapping.put(RevNumKey, rev);
MojErrCheck(err);
err = revTimeMapping.put(TimestampKey, time.microsecs());
MojErrCheck(err);
err = revTimeMapping.putString(KindKey, MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
err = putImpl(revTimeMapping, MojDb::FlagNone, req);
MojErrCheck(err);
// find the revision number for numDays prior to now
MojInt64 purgeTime = time.microsecs() - (MojTime::UnitsPerDay * numDays);
MojDbQuery query;
err = query.from(MojDbKindEngine::RevTimestampId);
MojErrCheck(err);
query.limit(1);
err = query.where(TimestampKey, MojDbQuery::OpLessThanEq, purgeTime);
MojErrCheck(err);
err = query.order(TimestampKey);
MojErrCheck(err);
query.desc(true);
MojDbCursor cursor;
err = findImpl(query, cursor, NULL, req, OpDelete);
MojErrCheck(err);
bool found = false;
MojObject obj;
err = cursor.get(obj, found);
MojErrCheck(err);
err = cursor.close();
MojErrCheck(err);
MojUInt32 batchCount = 0;
MojUInt32 totalCount = 0;
while ((found))
{
// Do it in AutoBatchSize batches
batchCount = 0;
req->fixmode(true); // purge even if index mis-matches
err = purgeImpl(obj, batchCount, req);
MojLogDebug(s_log, _T("purge batch processed: batch: %d; total: %d; err = %d\n"),
batchCount, (totalCount + batchCount), err);
MojErrCheck(err);
totalCount += batchCount;
countOut = totalCount;
if (batchCount < AutoBatchSize) // last batch
break;
err = commitBatch(req);
MojErrCheck(err);
continue;
}
// end request
err = req->end();
MojErrCheck(err);
MojLogDebug(s_log, _T("purged %d objects"), countOut);
return MojErrNone;
}
MojErr MojDb::load(const MojChar* path, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req)
{
MojAssert(path);
MojLogTrace(s_log);
MojErr err = beginReq(req, true);
MojErrCheck(err);
MojFile file;
err = file.open(path, MOJ_O_RDONLY);
MojErrCheck(err);
MojJsonParser parser;
parser.begin();
MojSize bytesRead = 0;
MojObjectBuilder visitor;
int total_mutexes, mutexes_free, mutexes_used, mutexes_used_highwater, mutex_regionsize;
m_objDb->mutexStats(&total_mutexes, &mutexes_free, &mutexes_used, &mutexes_used_highwater, &mutex_regionsize);
MojLogDebug(s_log, _T("Starting load of %s, total_mutexes: %d, mutexes_free: %d, mutexes_used: %d, mutexes_used_highwater: %d, &mutex_regionsize: %d\n"),
path, total_mutexes, mutexes_free, mutexes_used, mutexes_used_highwater, mutex_regionsize);
int orig_mutexes_used = mutexes_used;
struct timeval startTime = {0,0}, stopTime = {0,0};
gettimeofday(&startTime, NULL);
int total_transaction_time = 0;
int total = 0;
int transactions = 0;
do {
MojChar buf[MojFile::MojFileBufSize];
err = file.read(buf, sizeof(buf), bytesRead);
MojErrCheck(err);
const MojChar* parseEnd = buf;
while (parseEnd < (buf + bytesRead)) {
err = parser.parseChunk(visitor, parseEnd, bytesRead - (parseEnd - buf), parseEnd);
MojErrCheck(err);
if (parser.finished()) {
//store the object
err = loadImpl(visitor.object(), flags, req);
MojErrCheck(err);
countOut++;
parser.begin();
visitor.reset();
total++;
if ((total % 10) == 0) {
// For debugging mutex consumption during load operations, we periodically retrieve the mutex stats.
m_objDb->mutexStats(&total_mutexes, &mutexes_free, &mutexes_used, &mutexes_used_highwater, &mutex_regionsize);
MojLogDebug(s_log, _T("Loading %s record %d, total_mutexes: %d, mutexes_free: %d, mutexes_used: %d, mutexes_used_highwater: %d, &mutex_regionsize: %d\n"),
path, total, total_mutexes, mutexes_free, mutexes_used, mutexes_used_highwater, mutex_regionsize);
}
// If a loadStepSize is configured, then break up the load into separate transactions.
// This is intended to prevent run-away mutex consumption in some particular scenarios.
// The transactions do not reverse or prevent mutex consumption, but seem to reduce the
// growth and eventually cause it to level off.
if ((m_loadStepSize > 0) && ((total % m_loadStepSize) == 0)) {
// Close and reopen transaction, to prevent a very large transaction from building up.
MojLogDebug(s_log, _T("Loading %s record %d, closing and reopening transaction.\n"),
path, total);
struct timeval transactionStartTime = {0,0}, transactionStopTime = {0,0};
gettimeofday(&transactionStartTime, NULL);
err = req->end();
MojErrCheck(err);
err = req->endBatch();
MojErrCheck(err);
req->beginBatch(); // beginBatch() invocation for first transaction happened in MojDbServiceHandlerBase::invokeImpl
err = beginReq(req, true);
MojErrCheck(err);
gettimeofday(&transactionStopTime, NULL);
long int elapsedTransactionTimeMS = (transactionStopTime.tv_sec - transactionStartTime.tv_sec) * 1000 +
(transactionStopTime.tv_usec - transactionStartTime.tv_usec) / 1000;
total_transaction_time += (int)elapsedTransactionTimeMS;
transactions++;
}
}
}
} while (bytesRead > 0);
err = parser.end(visitor);
MojErrCheck(err);
if (parser.finished()) {
err = loadImpl(visitor.object(), flags, req);
MojErrCheck(err);
countOut++;
} else if (bytesRead > 0) {
MojErrThrow(MojErrJsonParseEof);
}
err = req->end();
MojErrCheck(err);
gettimeofday(&stopTime, NULL);
long int elapsedTimeMS = (stopTime.tv_sec - startTime.tv_sec) * 1000 +
(stopTime.tv_usec - startTime.tv_usec) / 1000;
m_objDb->mutexStats(&total_mutexes, &mutexes_free, &mutexes_used, &mutexes_used_highwater, &mutex_regionsize);
MojLogDebug(s_log, _T("Finished load of %s, total_mutexes: %d, mutexes_free: %d, mutexes_used: %d, mutexes_used_highwater: %d, &mutex_regionsize: %d\n"),
path, total_mutexes, mutexes_free, mutexes_used, mutexes_used_highwater, mutex_regionsize);
MojLogDebug(s_log, _T("Loaded %s with %d records in %ldms (%dms of that for %d extra transactions), consuming %d mutexes, afterwards %d are available out of %d\n"),
path, total, elapsedTimeMS, total_transaction_time, transactions, mutexes_used - orig_mutexes_used, mutexes_free, total_mutexes);
return MojErrNone;
}
