bool BackgroundJob::wait(unsigned msTimeOut) {
verify(!_selfDelete); // you cannot call wait on a self-deleting job
const auto deadline = Date_t::now() + Milliseconds(msTimeOut);
stdx::unique_lock<stdx::mutex> l(_status->mutex);
while (_status->state != Done) {
if (msTimeOut) {
if (stdx::cv_status::timeout ==
_status->done.wait_until(l, deadline.toSystemTimePoint()))
return false;
} else {
_status->done.wait(l);
}
}
return true;
}
void WiredTigerOplogManager::_oplogJournalThreadLoop(WiredTigerSessionCache* sessionCache,
WiredTigerRecordStore* oplogRecordStore,
const bool updateOldestTimestamp) noexcept {
Client::initThread("WTOplogJournalThread");
// This thread updates the oplog read timestamp, the timestamp used to read from the oplog with
// forward cursors. The timestamp is used to hide oplog entries that might be committed but
// have uncommitted entries ahead of them.
while (true) {
stdx::unique_lock<stdx::mutex> lk(_oplogVisibilityStateMutex);
{
MONGO_IDLE_THREAD_BLOCK;
_opsWaitingForJournalCV.wait(lk,
[&] { return _shuttingDown || _opsWaitingForJournal; });
// If we're not shutting down and nobody is actively waiting for the oplog to become
// durable, delay journaling a bit to reduce the sync rate.
auto journalDelay = Milliseconds(storageGlobalParams.journalCommitIntervalMs.load());
if (journalDelay == Milliseconds(0)) {
journalDelay = Milliseconds(WiredTigerKVEngine::kDefaultJournalDelayMillis);
}
auto now = Date_t::now();
auto deadline = now + journalDelay;
auto shouldSyncOpsWaitingForJournal = [&] {
return _shuttingDown || _opsWaitingForVisibility ||
oplogRecordStore->haveCappedWaiters();
};
// Eventually it would be more optimal to merge this with the normal journal flushing
// and block for either oplog tailers or operations waiting for oplog visibility. For
// now this loop will poll once a millisecond up to the journalDelay to see if we have
// any waiters yet. This reduces sync-related I/O on the primary when secondaries are
// lagged, but will avoid significant delays in confirming majority writes on replica
// sets with infrequent writes.
// Callers of waitForAllEarlierOplogWritesToBeVisible() like causally consistent reads
// will preempt this delay.
while (now < deadline &&
!_opsWaitingForJournalCV.wait_until(
lk, now.toSystemTimePoint(), shouldSyncOpsWaitingForJournal)) {
now += Milliseconds(1);
}
}
while (!_shuttingDown && MONGO_FAIL_POINT(WTPausePrimaryOplogDurabilityLoop)) {
lk.unlock();
sleepmillis(10);
lk.lock();
}
if (_shuttingDown) {
log() << "oplog journal thread loop shutting down";
return;
}
invariant(_opsWaitingForJournal);
_opsWaitingForJournal = false;
lk.unlock();
const uint64_t newTimestamp = fetchAllCommittedValue(sessionCache->conn());
// The newTimestamp may actually go backward during secondary batch application,
// where we commit data file changes separately from oplog changes, so ignore
// a non-incrementing timestamp.
if (newTimestamp <= _oplogReadTimestamp.load()) {
LOG(2) << "no new oplog entries were made visible: " << newTimestamp;
continue;
}
// In order to avoid oplog holes after an unclean shutdown, we must ensure this proposed
// oplog read timestamp's documents are durable before publishing that timestamp.
sessionCache->waitUntilDurable(/*forceCheckpoint=*/false, false);
lk.lock();
// Publish the new timestamp value. Avoid going backward.
auto oldTimestamp = getOplogReadTimestamp();
if (newTimestamp > oldTimestamp) {
_setOplogReadTimestamp(lk, newTimestamp);
}
lk.unlock();
if (updateOldestTimestamp) {
const bool force = false;
sessionCache->getKVEngine()->setOldestTimestamp(Timestamp(newTimestamp), force);
}
// Wake up any await_data cursors and tell them more data might be visible now.
oplogRecordStore->notifyCappedWaitersIfNeeded();
}
}
void ThreadPool::_consumeTasks() {
stdx::unique_lock<stdx::mutex> lk(_mutex);
while (_state == running) {
if (_pendingTasks.empty()) {
if (_threads.size() > _options.minThreads) {
// Since there are more than minThreads threads, this thread may be eligible for
// retirement. If it isn't now, it may be later, so it must put a time limit on how
// long it waits on _workAvailable.
const auto now = Date_t::now();
const auto nextThreadRetirementDate =
_lastFullUtilizationDate + _options.maxIdleThreadAge;
if (now >= nextThreadRetirementDate) {
_lastFullUtilizationDate = now;
LOG(1) << "Reaping this thread; next thread reaped no earlier than "
<< _lastFullUtilizationDate + _options.maxIdleThreadAge;
break;
}
LOG(3) << "Not reaping because the earliest retirement date is "
<< nextThreadRetirementDate;
_workAvailable.wait_until(lk, nextThreadRetirementDate.toSystemTimePoint());
} else {
// Since the number of threads is not more than minThreads, this thread is not
// eligible for retirement. It is OK to sleep until _workAvailable is signaled,
// because any new threads that put the number of total threads above minThreads
// would be eligible for retirement once they had no work left to do.
LOG(3) << "waiting for work; I am one of " << _threads.size() << " thread(s);"
<< " the minimum number of threads is " << _options.minThreads;
_workAvailable.wait(lk);
}
continue;
}
_doOneTask(&lk);
}
// We still hold the lock, but this thread is retiring. If the whole pool is shutting down, this
// thread lends a hand in draining the work pool and returns so it can be joined. Otherwise, it
// falls through to the detach code, below.
if (_state == joinRequired || _state == joining) {
// Drain the leftover pending tasks.
while (!_pendingTasks.empty()) {
_doOneTask(&lk);
}
--_numIdleThreads;
return;
}
--_numIdleThreads;
if (_state != running) {
severe() << "State of pool " << _options.poolName << " is " << static_cast<int32_t>(_state)
<< ", but expected " << static_cast<int32_t>(running);
fassertFailedNoTrace(28701);
}
// This thread is ending because it was idle for too long. Find self in _threads, remove self
// from _threads, detach self.
for (size_t i = 0; i < _threads.size(); ++i) {
auto& t = _threads[i];
if (t.get_id() != stdx::this_thread::get_id()) {
continue;
}
t.detach();
t.swap(_threads.back());
_threads.pop_back();
return;
}
severe().stream() << "Could not find this thread, with id " << stdx::this_thread::get_id()
<< " in pool " << _options.poolName;
fassertFailedNoTrace(28703);
}
void FTDCController::doLoop() {
try {
// Update config
{
stdx::lock_guard<stdx::mutex> lock(_mutex);
_config = _configTemp;
}
Client::initThread("ftdc");
Client* client = &cc();
auto swMgr = FTDCFileManager::create(&_config, _path, &_rotateCollectors, client);
_mgr = uassertStatusOK(std::move(swMgr));
while (true) {
// Compute the next interval to run regardless of how we were woken up
// Skipping an interval due to a race condition with a config signal is harmless.
auto now = getGlobalServiceContext()->getClockSource()->now();
// Get next time to run at
auto next_time = FTDCUtil::roundTime(now, _config.period);
// Wait for the next run or signal to shutdown
{
stdx::unique_lock<stdx::mutex> lock(_mutex);
// We ignore spurious wakeups by just doing an iteration of the loop
auto status = _condvar.wait_until(lock, next_time.toSystemTimePoint());
// Are we done running?
if (_state == State::kStopRequested) {
break;
}
// if we hit a timeout on the condvar, we need to do another collection
// if we were signalled, then we have a config update only or were asked to stop
if (status == stdx::cv_status::no_timeout) {
// Update the current configuration settings
_config = _configTemp;
continue;
}
}
// TODO: consider only running this thread if we are enabled
// for now, we just keep an idle thread as it is simplier
if (_config.enabled) {
BSONObj obj = _periodicCollectors.collect(client);
Status s = _mgr->writeSampleAndRotateIfNeeded(client, obj);
uassertStatusOK(s);
}
}
} catch (...) {
warning() << "Uncaught exception in '" << exceptionToStatus()
<< "' in full-time diagnostic data capture subsystem. Shutting down the "
"full-time diagnostic data capture subsystem.";
}
}
