// NOTE: This function may be called at any time after // registerShutdownTask is called below. It must not depend on the // prior execution of mongo initializers or the existence of threads. static void cleanupTask() { { auto serviceContext = getGlobalServiceContext(); Client::initThreadIfNotAlready(); Client& client = cc(); ServiceContext::UniqueOperationContext uniqueTxn; OperationContext* txn = client.getOperationContext(); if (!txn) { uniqueTxn = client.makeOperationContext(); txn = uniqueTxn.get(); } if (serviceContext) serviceContext->setKillAllOperations(); if (auto cursorManager = Grid::get(txn)->getCursorManager()) { cursorManager->shutdown(); } if (auto pool = Grid::get(txn)->getExecutorPool()) { pool->shutdownAndJoin(); } if (auto catalog = Grid::get(txn)->catalogClient(txn)) { catalog->shutDown(txn); } } audit::logShutdown(Client::getCurrent()); }
void TaskRunner::_runTasks() { // We initialize cc() because ServiceContextMongoD::_newOpCtx() expects cc() to be equal to the // client used to create the operation context. Client* client = &cc(); if (AuthorizationManager::get(client->getServiceContext())->isAuthEnabled()) { AuthorizationSession::get(client)->grantInternalAuthorization(client); } ServiceContext::UniqueOperationContext opCtx; while (Task task = _waitForNextTask()) { if (!opCtx) { opCtx = client->makeOperationContext(); } NextAction nextAction = runSingleTask(task, opCtx.get(), Status::OK()); if (nextAction != NextAction::kKeepOperationContext) { opCtx.reset(); } if (nextAction == NextAction::kCancel) { break; } // Release thread back to pool after disposing if no scheduled tasks in queue. if (nextAction == NextAction::kDisposeOperationContext || nextAction == NextAction::kInvalid) { stdx::lock_guard<stdx::mutex> lk(_mutex); if (_tasks.empty()) { _finishRunTasks_inlock(); return; } } } opCtx.reset(); std::list<Task> tasks; UniqueLock lk{_mutex}; auto cancelTasks = [&]() { tasks.swap(_tasks); lk.unlock(); // Cancel remaining tasks with a CallbackCanceled status. for (auto&& task : tasks) { runSingleTask(std::move(task), nullptr, Status(ErrorCodes::CallbackCanceled, "this task has been canceled by a previously invoked task")); } tasks.clear(); }; cancelTasks(); lk.lock(); _finishRunTasks_inlock(); cancelTasks(); }
void TaskRunner::_runTasks() { Client* client = nullptr; ServiceContext::UniqueOperationContext txn; while (Task task = _waitForNextTask()) { if (!txn) { if (!client) { // We initialize cc() because ServiceContextMongoD::_newOpCtx() expects cc() // to be equal to the client used to create the operation context. Client::initThreadIfNotAlready(); client = &cc(); if (getGlobalAuthorizationManager()->isAuthEnabled()) { AuthorizationSession::get(client)->grantInternalAuthorization(); } } txn = client->makeOperationContext(); } NextAction nextAction = runSingleTask(task, txn.get(), Status::OK()); if (nextAction != NextAction::kKeepOperationContext) { txn.reset(); } if (nextAction == NextAction::kCancel) { break; } // Release thread back to pool after disposing if no scheduled tasks in queue. if (nextAction == NextAction::kDisposeOperationContext || nextAction == NextAction::kInvalid) { stdx::lock_guard<stdx::mutex> lk(_mutex); if (_tasks.empty()) { _finishRunTasks_inlock(); return; } } } txn.reset(); std::list<Task> tasks; { stdx::lock_guard<stdx::mutex> lk(_mutex); tasks.swap(_tasks); } // Cancel remaining tasks with a CallbackCanceled status. for (auto task : tasks) { runSingleTask(task, nullptr, Status(ErrorCodes::CallbackCanceled, "this task has been canceled by a previously invoked task")); } stdx::lock_guard<stdx::mutex> lk(_mutex); _finishRunTasks_inlock(); }
void DatabasesCloner::_onEachDBCloneFinish(const Status& status, const std::string& name) { UniqueLock lk(_mutex); if (!status.isOK()) { warning() << "database '" << name << "' (" << (_stats.databasesCloned + 1) << " of " << _databaseCloners.size() << ") clone failed due to " << status.toString(); _fail_inlock(&lk, status); return; } if (StringData(name).equalCaseInsensitive("admin")) { LOG(1) << "Finished the 'admin' db, now calling isAdminDbValid."; // Do special checks for the admin database because of auth. collections. auto adminStatus = Status(ErrorCodes::NotYetInitialized, ""); { // TODO: Move isAdminDbValid() out of the collection/database cloner code paths. OperationContext* opCtx = cc().getOperationContext(); ServiceContext::UniqueOperationContext opCtxPtr; if (!opCtx) { opCtxPtr = cc().makeOperationContext(); opCtx = opCtxPtr.get(); } adminStatus = _storage->isAdminDbValid(opCtx); } if (!adminStatus.isOK()) { LOG(1) << "Validation failed on 'admin' db due to " << adminStatus; _fail_inlock(&lk, adminStatus); return; } } _stats.databasesCloned++; if (_stats.databasesCloned == _databaseCloners.size()) { _succeed_inlock(&lk); return; } // Start next database cloner. auto&& dbCloner = _databaseCloners[_stats.databasesCloned]; auto startStatus = dbCloner->startup(); if (!startStatus.isOK()) { warning() << "failed to schedule database '" << name << "' (" << (_stats.databasesCloned + 1) << " of " << _databaseCloners.size() << ") due to " << startStatus.toString(); _fail_inlock(&lk, startStatus); return; } }
void run() { const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext(); OperationContext& opCtx = *opCtxPtr; const NamespaceString nss("unittests.matchertests"); AutoGetCollectionForReadCommand ctx(&opCtx, nss); const CollatorInterface* collator = nullptr; const boost::intrusive_ptr<ExpressionContext> expCtx( new ExpressionContext(opCtxPtr.get(), collator)); M m(BSON("$where" << "function(){ return this.a == 1; }"), expCtx, ExtensionsCallbackReal(&opCtx, &nss), MatchExpressionParser::AllowedFeatures::kJavascript); ASSERT(m.matches(BSON("a" << 1))); ASSERT(!m.matches(BSON("a" << 2))); }
// NOTE: This function may be called at any time after // registerShutdownTask is called below. It must not depend on the // prior execution of mongo initializers or the existence of threads. static void cleanupTask() { { Client& client = cc(); ServiceContext::UniqueOperationContext uniqueTxn; OperationContext* txn = client.getOperationContext(); if (!txn) { uniqueTxn = client.makeOperationContext(); txn = uniqueTxn.get(); } auto cursorManager = grid.getCursorManager(); cursorManager->shutdown(); grid.shardRegistry()->shutdown(); grid.catalogManager(txn)->shutDown(txn); } audit::logShutdown(ClientBasic::getCurrent()); }
// NOTE: This function may be called at any time after // registerShutdownTask is called below. It must not depend on the // prior execution of mongo initializers or the existence of threads. static void cleanupTask() { { Client::initThreadIfNotAlready(); Client& client = cc(); ServiceContext::UniqueOperationContext uniqueTxn; OperationContext* txn = client.getOperationContext(); if (!txn) { uniqueTxn = client.makeOperationContext(); txn = uniqueTxn.get(); } auto cursorManager = grid.getCursorManager(); cursorManager->shutdown(); grid.getExecutorPool()->shutdownAndJoin(); grid.catalogClient(txn)->shutDown(txn); } audit::logShutdown(Client::getCurrent()); }
/** * The main durability thread loop. There is a single instance of this function running. */ static void durThread(ClockSource* cs, int64_t serverStartMs) { Client::initThread("durability"); log() << "Durability thread started"; bool samePartition = true; try { const std::string dbpathDir = boost::filesystem::path(storageGlobalParams.dbpath).string(); samePartition = onSamePartition(getJournalDir().string(), dbpathDir); } catch (...) { } // Spawn the journal writer thread JournalWriter journalWriter(&commitNotify, &applyToDataFilesNotify, NumAsyncJournalWrites); journalWriter.start(); // Used as an estimate of how much / how fast to remap uint64_t commitCounter(0); uint64_t estimatedPrivateMapSize(0); uint64_t remapLastTimestamp(0); while (shutdownRequested.loadRelaxed() == 0) { unsigned ms = storageGlobalParams.journalCommitIntervalMs.load(); if (ms == 0) { ms = samePartition ? 100 : 30; } // +1 so it never goes down to zero const int64_t oneThird = (ms / 3) + 1; // Reset the stats based on the reset interval if (stats.curr()->getCurrentDurationMillis() > DurStatsResetIntervalMillis) { stats.reset(); } try { stdx::unique_lock<stdx::mutex> lock(flushMutex); for (unsigned i = 0; i <= 2; i++) { if (stdx::cv_status::no_timeout == flushRequested.wait_for(lock, Milliseconds(oneThird).toSystemDuration())) { // Someone forced a flush break; } if (commitNotify.nWaiting()) { // One or more getLastError j:true is pending break; } if (commitJob.bytes() > UncommittedBytesLimit / 2) { // The number of written bytes is growing break; } } // The commit logic itself LOG(4) << "groupCommit begin"; Timer t; const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext(); OperationContext& opCtx = *opCtxPtr; AutoAcquireFlushLockForMMAPV1Commit autoFlushLock(opCtx.lockState()); // We need to snapshot the commitNumber after the flush lock has been obtained, // because at this point we know that we have a stable snapshot of the data. const CommitNotifier::When commitNumber(commitNotify.now()); LOG(4) << "Processing commit number " << commitNumber; if (!commitJob.hasWritten()) { // We do not need the journal lock anymore. Free it here, for the really // unlikely possibility that the writeBuffer command below blocks. autoFlushLock.release(); // getlasterror request could have came after the data was already committed. // No need to call committingReset though, because we have not done any // writes (hasWritten == false). JournalWriter::Buffer* const buffer = journalWriter.newBuffer(); buffer->setNoop(); buffer->journalListenerToken = getJournalListener()->getToken(); journalWriter.writeBuffer(buffer, commitNumber); } else { // This copies all the in-memory changes into the journal writer's buffer. JournalWriter::Buffer* const buffer = journalWriter.newBuffer(); PREPLOGBUFFER(buffer->getHeader(), buffer->getBuilder(), cs, serverStartMs); estimatedPrivateMapSize += commitJob.bytes(); commitCounter++; // Now that the write intents have been copied to the buffer, the commit job is // free to be reused. We need to reset the commit job's contents while under // the S flush lock, because otherwise someone might have done a write and this // would wipe out their changes without ever being committed. commitJob.committingReset(); double systemMemoryPressurePercentage = ProcessInfo::getSystemMemoryPressurePercentage(); // Now that the in-memory modifications have been collected, we can potentially // release the flush lock if remap is not necessary. // When we remap due to memory pressure, we look at two criteria // 1. If the amount of 4k pages touched exceeds 512 MB, // a reasonable estimate of memory pressure on Linux. // 2. Check if the amount of free memory on the machine is running low, // since #1 is underestimates the memory pressure on Windows since // commits in 64MB chunks. const bool shouldRemap = (estimatedPrivateMapSize >= UncommittedBytesLimit) || (systemMemoryPressurePercentage > 0.0) || (commitCounter % NumCommitsBeforeRemap == 0) || (mmapv1GlobalOptions.journalOptions & MMAPV1Options::JournalAlwaysRemap); double remapFraction = 0.0; if (shouldRemap) { // We want to remap all private views about every 2 seconds. There could be // ~1000 views so we do a little each pass. There will be copy on write // faults after remapping, so doing a little bit at a time will avoid big // load spikes when the pages are touched. // // TODO: Instead of the time-based logic above, consider using ProcessInfo // and watching for getResidentSize to drop, which is more precise. remapFraction = (curTimeMicros64() - remapLastTimestamp) / 2000000.0; if (mmapv1GlobalOptions.journalOptions & MMAPV1Options::JournalAlwaysRemap) { remapFraction = 1; } else { // We don't want to get close to the UncommittedBytesLimit const double remapMemFraction = estimatedPrivateMapSize / ((double)UncommittedBytesLimit); remapFraction = std::max(remapMemFraction, remapFraction); remapFraction = std::max(systemMemoryPressurePercentage, remapFraction); } } else { LOG(4) << "Early release flush lock"; // We will not be doing a remap so drop the flush lock. That way we will be // doing the journal I/O outside of lock, so other threads can proceed. invariant(!shouldRemap); autoFlushLock.release(); } buffer->journalListenerToken = getJournalListener()->getToken(); // Request async I/O to the journal. This may block. journalWriter.writeBuffer(buffer, commitNumber); // Data has now been written to the shared view. If remap was requested, we // would still be holding the S flush lock here, so just upgrade it and // perform the remap. if (shouldRemap) { // Need to wait for the previously scheduled journal writes to complete // before any remap is attempted. journalWriter.flush(); journalWriter.assertIdle(); // Upgrading the journal lock to flush stops all activity on the system, // because we will be remapping memory and we don't want readers to be // accessing it. Technically this step could be avoided on systems, which // support atomic remap. autoFlushLock.upgradeFlushLockToExclusive(); remapPrivateView(opCtxPtr.get(), remapFraction); autoFlushLock.release(); // Reset the private map estimate outside of the lock estimatedPrivateMapSize = 0; remapLastTimestamp = curTimeMicros64(); stats.curr()->_commitsInWriteLock++; stats.curr()->_commitsInWriteLockMicros += t.micros(); } } stats.curr()->_commits++; stats.curr()->_commitsMicros += t.micros(); LOG(4) << "groupCommit end"; } catch (DBException& e) { severe() << "dbexception in durThread causing immediate shutdown: " << redact(e); invariant(false); } catch (std::ios_base::failure& e) { severe() << "ios_base exception in durThread causing immediate shutdown: " << redact(e.what()); invariant(false); } catch (std::bad_alloc& e) { severe() << "bad_alloc exception in durThread causing immediate shutdown: " << redact(e.what()); invariant(false); } catch (std::exception& e) { severe() << "exception in durThread causing immediate shutdown: " << redact(e.what()); invariant(false); } catch (...) { severe() << "unhandled exception in durThread causing immediate shutdown"; invariant(false); } } // Stops the journal thread and ensures everything was written invariant(!commitJob.hasWritten()); journalWriter.flush(); journalWriter.shutdown(); log() << "Durability thread stopped"; }