MojErr MojDb::updateLocale(const MojChar* locale, MojDbReqRef req) { MojAssert(locale); MojLogTrace(s_log); MojErr err = beginReq(req, true); MojErrCheck(err); MojString oldLocale; err = getLocale(oldLocale, req); MojErrCheck(err); MojString newLocale; err = newLocale.assign(locale); MojErrCheck(err); MojErr updateErr = err = updateLocaleImpl(oldLocale, newLocale, req); MojErrCatchAll(err) { err = req->abort(); MojErrCheck(err); err = m_kindEngine.close(); MojErrCheck(err); MojDbReq openReq; err = beginReq(openReq, true); MojErrCheck(err); err = m_kindEngine.open(this, openReq); MojErrCheck(err); err = openReq.end(); MojErrCheck(err); MojErrThrow(updateErr); } return MojErrNone; }
MojErr MojDb::del(const MojObject* idsBegin, const MojObject* idsEnd, MojUInt32& countOut, MojObject& arrOut, MojUInt32 flags, MojDbReqRef req) { MojAssert(idsBegin || idsBegin == idsEnd); MojAssert(idsEnd >= idsBegin); MojLogTrace(s_log); countOut = 0; MojErr err = beginReq(req); MojErrCheck(err); // do the dels MojUInt32 count= 0; for (const MojObject* i = idsBegin; i != idsEnd; ++i) { MojObject foundObj; bool found = false; err = delImpl(*i, found, foundObj, req, flags); MojErrCheck(err); if (found) { count++; err = arrOut.push(foundObj); MojErrCheck(err); } } // commit txn err = req->end(); MojErrCheck(err); countOut = count; return MojErrNone; }
MojErr MojDb::watch(const MojDbQuery& query, MojDbCursor& cursor, WatchSignal::SlotRef watchHandler, bool& firedOut, MojDbReqRef req) { MojLogTrace(s_log); firedOut = false; MojErr err = beginReq(req); MojErrCheck(err); MojRefCountedPtr<MojDbWatcher> watcher(new MojDbWatcher(watchHandler)); MojAllocCheck(watcher.get()); MojDbQuery limitedQuery = query; limitedQuery.limit(1); err = findImpl(limitedQuery, cursor, watcher.get(), req, OpRead); MojErrCheck(err); MojDbStorageItem* item = NULL; bool found = false; cursor.verifymode(false); err = cursor.get(item, found); MojErrCheck(err); if (found) { const MojDbKey& key = cursor.storageQuery()->endKey(); err = watcher->fire(key); MojErrCheck(err); firedOut = true; } err = req->end(false); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::quotaStats(MojObject& objOut, MojDbReqRef req) { MojLogTrace(s_log); MojErr err = beginReq(req); MojErrCheck(err); err = m_quotaEngine.stats(objOut, req); MojErrCheck(err); err = req->end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::stats(MojObject& objOut, MojDbReqRef req, bool verify, MojString *pKind) { MojLogTrace(s_log); MojErr err = beginReq(req); MojErrCheck(err); err = m_kindEngine.stats(objOut, req, verify, pKind); MojErrCheck(err); err = req->end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::merge(const MojDbQuery& query, const MojObject& props, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req) { MojLogTrace(s_log); countOut = 0; MojErr err = beginReq(req); MojErrCheck(err); MojDbCursor cursor; err = findImpl(query, cursor, NULL, req, OpUpdate); MojErrCheck(err); MojAssert(cursor.txn()); MojUInt32 count = 0; MojUInt32 warns = 0; bool found = false; MojObject prev; for (;;) { // get prev rev from cursor MojDbStorageItem* prevItem = NULL; err = cursor.get(prevItem, found); if (err == MojErrInternalIndexOnFind) { warns++; continue; } MojErrCheck(err); if (!found) break; err = prevItem->toObject(prev, m_kindEngine); MojErrCheck(err); // merge obj into prev MojObject merged; err = mergeInto(merged, props, prev); MojErrCheck(err); // and update the db const MojObject& id = prevItem->id(); err = putObj(id, merged, &prev, prevItem, req, OpUpdate); MojErrCheck(err); ++count; } if (warns > 0) MojLogWarning(s_log, _T("Merge index_warnings: %s; count: %d\n"), query.from().data(), warns); err = cursor.close(); MojErrCheck(err); err = req->end(); MojErrCheck(err); countOut = count; return MojErrNone; }
MojErr MojDb::purgeStatus(MojObject& revOut, MojDbReqRef req) { MojLogTrace(s_log); revOut = -1; MojErr err = beginReq(req); MojErrCheck(err); err = getState(LastPurgedRevKey, revOut, req); MojErrCheck(err); err = req->end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::find(const MojDbQuery& query, MojDbCursor& cursor, MojDbReqRef req) { MojLogTrace(s_log); MojErr err = beginReq(req); MojErrCheck(err); err = findImpl(query, cursor, NULL, req, OpRead); MojErrCheck(err); err = req->end(false); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::del(const MojDbQuery& query, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req) { MojLogTrace(s_log); countOut = 0; MojErr err = beginReq(req); MojErrCheck(err); err = delImpl(query, countOut, req, flags); MojErrCheck(err); err = req->end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::find(const MojDbQuery& query, MojDbCursor& cursor, WatchSignal::SlotRef watchHandler, MojDbReqRef req) { MojLogTrace(s_log); MojErr err = beginReq(req); MojErrCheck(err); MojRefCountedPtr<MojDbWatcher> watcher(new MojDbWatcher(watchHandler)); MojAllocCheck(watcher.get()); err = findImpl(query, cursor, watcher.get(), req, OpRead); MojErrCheck(err); err = req->end(false); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::put(MojObject* begin, const MojObject* end, MojUInt32 flags, MojDbReqRef req) { MojAssert(begin || begin == end); MojAssert(end >= begin); MojLogTrace(s_log); MojErr err = beginReq(req); MojErrCheck(err); for (MojObject* i = begin; i != end; ++i) { err = putImpl(*i, flags, req); MojErrCheck(err); } err = req->end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::putConfig(MojObject* begin, const MojObject* end, MojDbReq& req, MojDbPutHandler& handler) { MojAssert(begin || begin == end); MojAssert(end >= begin); MojLogTrace(s_log); MojErr err = beginReq(req, true); MojErrCheck(err); for (MojObject* i = begin; i != end; ++i) { err = handler.put(*i, req); MojErrCheck(err); } err = req.end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::get(const MojObject* idsBegin, const MojObject* idsEnd, MojObjectVisitor& visitor, MojDbReqRef req) { MojAssert(idsBegin || idsBegin == idsEnd); MojAssert(idsEnd >= idsBegin); MojLogTrace(s_log); MojErr err = beginReq(req); MojErrCheck(err); // do the gets for (const MojObject* i = idsBegin; i != idsEnd; ++i) { err = getImpl(*i, visitor, OpRead, req); MojErrCheck(err); } // commit txn err = req->end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::putKind(MojObject& obj, MojUInt32 flags, MojDbReqRef req) { MojErr err = beginReq(req, true); MojErrCheck(err); // get id MojString id; err = obj.getRequired(MojDbServiceDefs::IdKey, id); MojErrCheck(err); MojLogInfo(s_log, _T("putKind: %s \n"), id.data()); MojLogWarning(s_log, _T("putKind: %s \n"), id.data()); // set _kind and _id err = obj.putString(KindKey, MojDbKindEngine::KindKindId); MojErrCheck(err); MojString dbId; err = formatKindId(id, dbId); MojErrCheck(err); err = obj.putString(IdKey, dbId); MojErrCheck(err); // put the object MojDbAdminGuard guard(req); err = putImpl(obj, flags | FlagForce, req); MojErrCheck(err); guard.unset(); // attempt the putKind MojErr errAcc = m_kindEngine.putKind(obj, req); MojErrAccumulate(err, errAcc); err = commitKind(id, req, err); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::delKind(const MojObject& id, bool& foundOut, MojUInt32 flags, MojDbReqRef req) { MojLogTrace(s_log); foundOut = false; MojErr err = beginReq(req, true); MojErrCheck(err); // del kind obj MojString idStr; err = id.stringValue(idStr); MojErrCheck(err); MojDbKind *pk = NULL; // If Kinds has sub-kinds, we give an error err = m_kindEngine.getKind(idStr.data(), pk); MojErrCheck(err); if (pk->nsubkinds() > 0) { MojLogWarning(s_log, _T("delKind_error: %s has %d subkinds \n"), idStr.data(), pk->nsubkinds()); MojErrThrow(MojErrDbKindHasSubKinds); } //MojLogInfo(s_log, _T("delKind: %s \n"), idStr.data()); MojLogWarning(s_log, _T("delKind: %s \n"), idStr.data()); err = m_kindEngine.checkOwnerPermission(idStr, req); MojErrCheck(err); MojString dbId; err = formatKindId(idStr, dbId); MojErrCheck(err); MojObject deleted; bool found = false; MojDbAdminGuard adminGuard(req); err = delImpl(dbId, found, deleted, req, flags); MojErrCheck(err); if (found) { // del objects MojDbQuery query; err = query.from(idStr); MojErrCheck(err); err = query.includeDeleted(true); MojErrCheck(err); MojUInt32 count; req->fixmode(true); err = delImpl(query, count, req, flags | FlagPurge); MojErrCheck(err); // del associated permissions query.clear(); err = query.from(MojDbKindEngine::PermissionId); MojErrCheck(err); err = query.where(MojDbServiceDefs::ObjectKey, MojDbQuery::OpEq, idStr); MojErrCheck(err); req->fixmode(true); err = delImpl(query, count, req, flags); MojErrCheck(err); // del kind MojErr errAcc = m_kindEngine.delKind(idStr, req); MojErrAccumulate(err, errAcc); } err = commitKind(idStr, req, err); MojErrCheck(err); foundOut = found; return MojErrNone; }
MojErr MojDb::dump(const MojChar* path, MojUInt32& countOut, bool incDel, MojDbReqRef req, bool backup, MojUInt32 maxBytes, const MojObject* incrementalKey, MojObject* backupResponse) { MojAssert(path); MojLogTrace(s_log); MojErr err = beginReq(req); MojErrCheck(err); if (!req->admin()) { MojLogError(s_log, _T("access denied: '%s' cannot dump db to path: '%s'"), req->domain().data(), path); MojErrThrow(MojErrDbAccessDenied); } MojFile file; err = file.open(path, MOJ_O_WRONLY | MOJ_O_CREAT | MOJ_O_TRUNC, MOJ_S_IRUSR | MOJ_S_IWUSR); MojErrCheck(err); // write out kinds first, then existing objects, then deleted objects MojSize bytesWritten = 0; MojSize totalwarns = 0; MojSize newwarns = 0; MojDbQuery objQuery; MojVector<MojObject> kindVec; MojObject revParam = -1; MojObject delRevParam = -1; // if we were given an incremental key, pull out the revs now if (incrementalKey) { incrementalKey->get(MojDbServiceDefs::RevKey, revParam); incrementalKey->get(MojDbServiceDefs::DeletedRevKey, delRevParam); } err = m_kindEngine.getKinds(kindVec); MojErrCheck(err); // write kinds - if incremental, only write the kinds that have changed since the respective revs MojString countStr; for (MojVector<MojObject>::ConstIterator i = kindVec.begin(); i != kindVec.end(); ++i) { if (backup) { bool backupKind = false; i->get(MojDbKind::SyncKey, backupKind); if (!backupKind) continue; MojString id; err = i->getRequired(MojDbServiceDefs::IdKey, id); MojErrCheck(err); MojDbQuery countQuery; err = countQuery.from(id); MojErrCheck(err); MojDbCursor cursor; err = find(countQuery, cursor, req); MojErrCheck(err); MojUInt32 count = 0; err = cursor.count(count); MojErrCheck(err); if (count > 0) { if (i != kindVec.begin()) { err = countStr.appendFormat(_T(", ")); MojErrCheck(err); } err = countStr.appendFormat("%s=%u", id.data(), count); MojErrCheck(err); } } bool deleted = false; i->get(DelKey, deleted); MojObject kindRev; err = i->getRequired(RevKey, kindRev); MojErrCheck(err); if ((deleted && kindRev > delRevParam) || (!deleted && kindRev > revParam)) { err = dumpObj(file, (*i), bytesWritten, maxBytes); MojErrCheck(err); countOut++; } } // dump all the non-deleted objects err = dumpImpl(file, backup, false, revParam, delRevParam, true, countOut, req, backupResponse, MojDbServiceDefs::RevKey, bytesWritten, newwarns, maxBytes); MojErrCheck(err); totalwarns += newwarns; // If we're supposed to include deleted objects, dump the deleted objects now. // There's a chance that we may have run out of space in our backup. If that's the case, // we don't want to try to dump deleted objects - we can detect this by looking for the HasMoreKey if (incDel && backupResponse && !backupResponse->contains(MojDbServiceDefs::HasMoreKey)) { err = dumpImpl(file, backup, true, revParam, delRevParam, false, countOut, req, backupResponse, MojDbServiceDefs::DeletedRevKey, bytesWritten, newwarns, maxBytes); MojErrCheck(err); } totalwarns += newwarns; // Add the Full and Version keys if (backup && backupResponse) { bool incremental = (incrementalKey != NULL); err = backupResponse->putBool(MojDbServiceDefs::FullKey, !incremental); MojErrCheck(err); err = backupResponse->put(MojDbServiceDefs::VersionKey, DatabaseVersion); MojErrCheck(err); err = backupResponse->put(MojDbServiceDefs::WarningsKey, (MojInt32)totalwarns); MojErrCheck(err); MojString description; err = description.format(_T("incremental=%u"), countOut); MojErrCheck(err); if (!countStr.empty()) { err = description.appendFormat(_T(", %s"), countStr.data()); MojErrCheck(err); } err = backupResponse->put(_T("description"), description); MojErrCheck(err); } err = req->end(); MojErrCheck(err); return MojErrNone; }
MojErr MojDb::purge(MojUInt32& countOut, MojInt64 numDays, MojDbReqRef req) { MojLogTrace(s_log); countOut = 0; if (numDays <= -1) { numDays = m_purgeWindow; } MojErr err = beginReq(req); MojErrCheck(err); MojLogDebug(s_log, _T("purging objects deleted more than %lld days ago..."), numDays); MojTime time; err = MojGetCurrentTime(time); MojErrCheck(err); // store the revision number to current timestamp mapping MojObject revTimeMapping; MojInt64 rev; err = nextId(rev); MojErrCheck(err); err = revTimeMapping.put(RevNumKey, rev); MojErrCheck(err); err = revTimeMapping.put(TimestampKey, time.microsecs()); MojErrCheck(err); err = revTimeMapping.putString(KindKey, MojDbKindEngine::RevTimestampId); MojErrCheck(err); err = putImpl(revTimeMapping, MojDb::FlagNone, req); MojErrCheck(err); // find the revision number for numDays prior to now MojInt64 purgeTime = time.microsecs() - (MojTime::UnitsPerDay * numDays); MojDbQuery query; err = query.from(MojDbKindEngine::RevTimestampId); MojErrCheck(err); query.limit(1); err = query.where(TimestampKey, MojDbQuery::OpLessThanEq, purgeTime); MojErrCheck(err); err = query.order(TimestampKey); MojErrCheck(err); query.desc(true); MojDbCursor cursor; err = findImpl(query, cursor, NULL, req, OpDelete); MojErrCheck(err); bool found = false; MojObject obj; err = cursor.get(obj, found); MojErrCheck(err); err = cursor.close(); MojErrCheck(err); MojUInt32 batchCount = 0; MojUInt32 totalCount = 0; while ((found)) { // Do it in AutoBatchSize batches batchCount = 0; req->fixmode(true); // purge even if index mis-matches err = purgeImpl(obj, batchCount, req); MojLogDebug(s_log, _T("purge batch processed: batch: %d; total: %d; err = %d\n"), batchCount, (totalCount + batchCount), err); MojErrCheck(err); totalCount += batchCount; countOut = totalCount; if (batchCount < AutoBatchSize) // last batch break; err = commitBatch(req); MojErrCheck(err); continue; } // end request err = req->end(); MojErrCheck(err); MojLogDebug(s_log, _T("purged %d objects"), countOut); return MojErrNone; }
MojErr MojDb::load(const MojChar* path, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req) { MojAssert(path); MojLogTrace(s_log); MojErr err = beginReq(req, true); MojErrCheck(err); MojFile file; err = file.open(path, MOJ_O_RDONLY); MojErrCheck(err); MojJsonParser parser; parser.begin(); MojSize bytesRead = 0; MojObjectBuilder visitor; int total_mutexes, mutexes_free, mutexes_used, mutexes_used_highwater, mutex_regionsize; m_objDb->mutexStats(&total_mutexes, &mutexes_free, &mutexes_used, &mutexes_used_highwater, &mutex_regionsize); MojLogDebug(s_log, _T("Starting load of %s, total_mutexes: %d, mutexes_free: %d, mutexes_used: %d, mutexes_used_highwater: %d, &mutex_regionsize: %d\n"), path, total_mutexes, mutexes_free, mutexes_used, mutexes_used_highwater, mutex_regionsize); int orig_mutexes_used = mutexes_used; struct timeval startTime = {0,0}, stopTime = {0,0}; gettimeofday(&startTime, NULL); int total_transaction_time = 0; int total = 0; int transactions = 0; do { MojChar buf[MojFile::MojFileBufSize]; err = file.read(buf, sizeof(buf), bytesRead); MojErrCheck(err); const MojChar* parseEnd = buf; while (parseEnd < (buf + bytesRead)) { err = parser.parseChunk(visitor, parseEnd, bytesRead - (parseEnd - buf), parseEnd); MojErrCheck(err); if (parser.finished()) { //store the object err = loadImpl(visitor.object(), flags, req); MojErrCheck(err); countOut++; parser.begin(); visitor.reset(); total++; if ((total % 10) == 0) { // For debugging mutex consumption during load operations, we periodically retrieve the mutex stats. m_objDb->mutexStats(&total_mutexes, &mutexes_free, &mutexes_used, &mutexes_used_highwater, &mutex_regionsize); MojLogDebug(s_log, _T("Loading %s record %d, total_mutexes: %d, mutexes_free: %d, mutexes_used: %d, mutexes_used_highwater: %d, &mutex_regionsize: %d\n"), path, total, total_mutexes, mutexes_free, mutexes_used, mutexes_used_highwater, mutex_regionsize); } // If a loadStepSize is configured, then break up the load into separate transactions. // This is intended to prevent run-away mutex consumption in some particular scenarios. // The transactions do not reverse or prevent mutex consumption, but seem to reduce the // growth and eventually cause it to level off. if ((m_loadStepSize > 0) && ((total % m_loadStepSize) == 0)) { // Close and reopen transaction, to prevent a very large transaction from building up. MojLogDebug(s_log, _T("Loading %s record %d, closing and reopening transaction.\n"), path, total); struct timeval transactionStartTime = {0,0}, transactionStopTime = {0,0}; gettimeofday(&transactionStartTime, NULL); err = req->end(); MojErrCheck(err); err = req->endBatch(); MojErrCheck(err); req->beginBatch(); // beginBatch() invocation for first transaction happened in MojDbServiceHandlerBase::invokeImpl err = beginReq(req, true); MojErrCheck(err); gettimeofday(&transactionStopTime, NULL); long int elapsedTransactionTimeMS = (transactionStopTime.tv_sec - transactionStartTime.tv_sec) * 1000 + (transactionStopTime.tv_usec - transactionStartTime.tv_usec) / 1000; total_transaction_time += (int)elapsedTransactionTimeMS; transactions++; } } } } while (bytesRead > 0); err = parser.end(visitor); MojErrCheck(err); if (parser.finished()) { err = loadImpl(visitor.object(), flags, req); MojErrCheck(err); countOut++; } else if (bytesRead > 0) { MojErrThrow(MojErrJsonParseEof); } err = req->end(); MojErrCheck(err); gettimeofday(&stopTime, NULL); long int elapsedTimeMS = (stopTime.tv_sec - startTime.tv_sec) * 1000 + (stopTime.tv_usec - startTime.tv_usec) / 1000; m_objDb->mutexStats(&total_mutexes, &mutexes_free, &mutexes_used, &mutexes_used_highwater, &mutex_regionsize); MojLogDebug(s_log, _T("Finished load of %s, total_mutexes: %d, mutexes_free: %d, mutexes_used: %d, mutexes_used_highwater: %d, &mutex_regionsize: %d\n"), path, total_mutexes, mutexes_free, mutexes_used, mutexes_used_highwater, mutex_regionsize); MojLogDebug(s_log, _T("Loaded %s with %d records in %ldms (%dms of that for %d extra transactions), consuming %d mutexes, afterwards %d are available out of %d\n"), path, total, elapsedTimeMS, total_transaction_time, transactions, mutexes_used - orig_mutexes_used, mutexes_free, total_mutexes); return MojErrNone; }