void RangeDeleter::doWork() { _env->initThread(); while (!inShutdown() && !stopRequested()) { string errMsg; boost::scoped_ptr<OperationContext> txn(getGlobalServiceContext()->newOpCtx()); RangeDeleteEntry* nextTask = NULL; { boost::unique_lock<boost::mutex> sl(_queueMutex); while (_taskQueue.empty()) { _taskQueueNotEmptyCV.timed_wait( sl, duration::milliseconds(kNotEmptyTimeoutMillis)); if (stopRequested()) { log() << "stopping range deleter worker" << endl; return; } if (_taskQueue.empty()) { // Try to check if some deletes are ready and move them to the // ready queue. TaskList::iterator iter = _notReadyQueue.begin(); while (iter != _notReadyQueue.end()) { RangeDeleteEntry* entry = *iter; set<CursorId> cursorsNow; { if (entry->options.waitForOpenCursors) { _env->getCursorIds(txn.get(), entry->options.range.ns, &cursorsNow); } } set<CursorId> cursorsLeft; std::set_intersection(entry->cursorsToWait.begin(), entry->cursorsToWait.end(), cursorsNow.begin(), cursorsNow.end(), std::inserter(cursorsLeft, cursorsLeft.end())); entry->cursorsToWait.swap(cursorsLeft); if (entry->cursorsToWait.empty()) { (*iter)->stats.queueEndTS = jsTime(); _taskQueue.push_back(*iter); _taskQueueNotEmptyCV.notify_one(); iter = _notReadyQueue.erase(iter); } else { logCursorsWaiting(entry); ++iter; } } } } if (stopRequested()) { log() << "stopping range deleter worker" << endl; return; } nextTask = _taskQueue.front(); _taskQueue.pop_front(); _deletesInProgress++; } { nextTask->stats.deleteStartTS = jsTime(); bool delResult = _env->deleteRange(txn.get(), *nextTask, &nextTask->stats.deletedDocCount, &errMsg); nextTask->stats.deleteEndTS = jsTime(); if (delResult) { nextTask->stats.waitForReplStartTS = jsTime(); if (!_waitForMajority(txn.get(), &errMsg)) { warning() << "Error encountered while waiting for replication: " << errMsg; } nextTask->stats.waitForReplEndTS = jsTime(); } else { warning() << "Error encountered while trying to delete range: " << errMsg << endl; } } { boost::lock_guard<boost::mutex> sl(_queueMutex); NSMinMax setEntry(nextTask->options.range.ns, nextTask->options.range.minKey, nextTask->options.range.maxKey); deletePtrElement(&_deleteSet, &setEntry); _deletesInProgress--; if (nextTask->notifyDone) { nextTask->notifyDone->notifyOne(); } } recordDelStats(new DeleteJobStats(nextTask->stats)); delete nextTask; nextTask = NULL; } }
void RangeDeleter::doWork() { _env->initThread(); while (!inShutdown() && !stopRequested()) { string errMsg; RangeDeleteEntry* nextTask = NULL; { scoped_lock sl(_queueMutex); while (_taskQueue.empty()) { _taskQueueNotEmptyCV.timed_wait( sl.boost(), duration::milliseconds(NotEmptyTimeoutMillis)); if (stopRequested()) { log() << "stopping range deleter worker" << endl; return; } if (_taskQueue.empty()) { // Try to check if some deletes are ready and move them to the // ready queue. TaskList::iterator iter = _notReadyQueue.begin(); while (iter != _notReadyQueue.end()) { RangeDeleteEntry* entry = *iter; set<CursorId> cursorsNow; { boost::scoped_ptr<OperationContext> txn(getGlobalEnvironment()->newOpCtx()); _env->getCursorIds(txn.get(), entry->ns, &cursorsNow); } set<CursorId> cursorsLeft; std::set_intersection(entry->cursorsToWait.begin(), entry->cursorsToWait.end(), cursorsNow.begin(), cursorsNow.end(), std::inserter(cursorsLeft, cursorsLeft.end())); entry->cursorsToWait.swap(cursorsLeft); if (entry->cursorsToWait.empty()) { (*iter)->stats.queueEndTS = jsTime(); _taskQueue.push_back(*iter); _taskQueueNotEmptyCV.notify_one(); iter = _notReadyQueue.erase(iter); } else { const unsigned long long int elapsedMillis = entry->stats.queueStartTS.millis - curTimeMillis64(); if ( elapsedMillis > LogCursorsThresholdMillis && entry->timeSinceLastLog.millis > LogCursorsIntervalMillis) { entry->timeSinceLastLog = jsTime(); logCursorsWaiting(entry->ns, entry->min, entry->max, elapsedMillis, entry->cursorsToWait); } ++iter; } } } } if (stopRequested()) { log() << "stopping range deleter worker" << endl; return; } nextTask = _taskQueue.front(); _taskQueue.pop_front(); _deletesInProgress++; } { boost::scoped_ptr<OperationContext> txn(getGlobalEnvironment()->newOpCtx()); ReplTime lastOp; nextTask->stats.deleteStartTS = jsTime(); bool delResult = _env->deleteRange(txn.get(), *nextTask, &nextTask->stats.deletedDocCount, &lastOp, &errMsg); nextTask->stats.deleteEndTS = jsTime(); if (delResult) { nextTask->stats.waitForReplStartTS = jsTime(); if (!_env->waitForReplication(lastOp, DelWriteConcern, WaitForReplTimeoutSecs, &errMsg)) { warning() << "Error encountered while waiting for replication: " << errMsg << endl; } nextTask->stats.waitForReplEndTS = jsTime(); } else { warning() << "Error encountered while trying to delete range: " << errMsg << endl; } } { scoped_lock sl(_queueMutex); NSMinMax setEntry(nextTask->ns, nextTask->min, nextTask->max); deletePtrElement(&_deleteSet, &setEntry); _deletesInProgress--; if (nextTask->notifyDone) { nextTask->notifyDone->notifyOne(); } } recordDelStats(new DeleteJobStats(nextTask->stats)); delete nextTask; nextTask = NULL; } }
bool RangeDeleter::deleteNow(OperationContext* txn, const RangeDeleterOptions& options, string* errMsg) { if (stopRequested()) { *errMsg = "deleter is already stopped."; return false; } string dummy; if (errMsg == NULL) errMsg = &dummy; const string& ns(options.range.ns); const BSONObj& min(options.range.minKey); const BSONObj& max(options.range.maxKey); NSMinMax deleteRange(ns, min, max); { boost::lock_guard<boost::mutex> sl(_queueMutex); if (!canEnqueue_inlock(ns, min, max, errMsg)) { return false; } _deleteSet.insert(&deleteRange); // Note: count for pending deletes is an integral part of the shutdown story. // Therefore, to simplify things, there is no "pending" state for deletes in // deleteNow, the state transition is simply inProgress -> done. _deletesInProgress++; } set<CursorId> cursorsToWait; if (options.waitForOpenCursors) { _env->getCursorIds(txn, ns, &cursorsToWait); } long long checkIntervalMillis = 5; RangeDeleteEntry taskDetails(options); taskDetails.stats.queueStartTS = jsTime(); for (; !cursorsToWait.empty(); sleepmillis(checkIntervalMillis)) { logCursorsWaiting(&taskDetails); set<CursorId> cursorsNow; _env->getCursorIds(txn, ns, &cursorsNow); set<CursorId> cursorsLeft; std::set_intersection(cursorsToWait.begin(), cursorsToWait.end(), cursorsNow.begin(), cursorsNow.end(), std::inserter(cursorsLeft, cursorsLeft.end())); cursorsToWait.swap(cursorsLeft); if (stopRequested()) { *errMsg = "deleter was stopped."; boost::lock_guard<boost::mutex> sl(_queueMutex); _deleteSet.erase(&deleteRange); _deletesInProgress--; if (_deletesInProgress == 0) { _nothingInProgressCV.notify_one(); } return false; } if (checkIntervalMillis < kMaxCursorCheckIntervalMillis) { checkIntervalMillis *= 2; } } taskDetails.stats.queueEndTS = jsTime(); taskDetails.stats.deleteStartTS = jsTime(); bool result = _env->deleteRange(txn, taskDetails, &taskDetails.stats.deletedDocCount, errMsg); taskDetails.stats.deleteEndTS = jsTime(); if (result) { taskDetails.stats.waitForReplStartTS = jsTime(); result = _waitForMajority(txn, errMsg); taskDetails.stats.waitForReplEndTS = jsTime(); } { boost::lock_guard<boost::mutex> sl(_queueMutex); _deleteSet.erase(&deleteRange); _deletesInProgress--; if (_deletesInProgress == 0) { _nothingInProgressCV.notify_one(); } } recordDelStats(new DeleteJobStats(taskDetails.stats)); return result; }
bool RangeDeleter::deleteNow(OperationContext* txn, const std::string& ns, const BSONObj& min, const BSONObj& max, const BSONObj& shardKeyPattern, bool secondaryThrottle, string* errMsg) { if (stopRequested()) { *errMsg = "deleter is already stopped."; return false; } string dummy; if (errMsg == NULL) errMsg = &dummy; NSMinMax deleteRange(ns, min, max); { scoped_lock sl(_queueMutex); if (!canEnqueue_inlock(ns, min, max, errMsg)) { return false; } _deleteSet.insert(&deleteRange); // Note: count for pending deletes is an integral part of the shutdown story. // Therefore, to simplify things, there is no "pending" state for deletes in // deleteNow, the state transition is simply inProgress -> done. _deletesInProgress++; } set<CursorId> cursorsToWait; _env->getCursorIds(txn, ns, &cursorsToWait); long long checkIntervalMillis = 5; if (!cursorsToWait.empty()) { log() << "rangeDeleter waiting for " << cursorsToWait.size() << " cursors in " << ns << " to finish" << endl; } RangeDeleteEntry taskDetails(ns, min, max, shardKeyPattern, secondaryThrottle); taskDetails.stats.queueStartTS = jsTime(); Date_t timeSinceLastLog; for (; !cursorsToWait.empty(); sleepmillis(checkIntervalMillis)) { const unsigned long long timeNow = curTimeMillis64(); const unsigned long long elapsedTimeMillis = timeNow - taskDetails.stats.queueStartTS.millis; const unsigned long long lastLogMillis = timeNow - timeSinceLastLog.millis; if (elapsedTimeMillis > LogCursorsThresholdMillis && lastLogMillis > LogCursorsIntervalMillis) { timeSinceLastLog = jsTime(); logCursorsWaiting(ns, min, max, elapsedTimeMillis, cursorsToWait); } set<CursorId> cursorsNow; _env->getCursorIds(txn, ns, &cursorsNow); set<CursorId> cursorsLeft; std::set_intersection(cursorsToWait.begin(), cursorsToWait.end(), cursorsNow.begin(), cursorsNow.end(), std::inserter(cursorsLeft, cursorsLeft.end())); cursorsToWait.swap(cursorsLeft); if (stopRequested()) { *errMsg = "deleter was stopped."; scoped_lock sl(_queueMutex); _deleteSet.erase(&deleteRange); _deletesInProgress--; if (_deletesInProgress == 0) { _nothingInProgressCV.notify_one(); } return false; } if (checkIntervalMillis < MaxCurorCheckIntervalMillis) { checkIntervalMillis *= 2; } } taskDetails.stats.queueEndTS = jsTime(); ReplTime lastOp; taskDetails.stats.deleteStartTS = jsTime(); bool result = _env->deleteRange(txn, taskDetails, &taskDetails.stats.deletedDocCount, &lastOp, errMsg); taskDetails.stats.deleteEndTS = jsTime(); if (result) { taskDetails.stats.waitForReplStartTS = jsTime(); result = _env->waitForReplication(lastOp, DelWriteConcern, WaitForReplTimeoutSecs, errMsg); taskDetails.stats.waitForReplEndTS = jsTime(); } { scoped_lock sl(_queueMutex); _deleteSet.erase(&deleteRange); _deletesInProgress--; if (_deletesInProgress == 0) { _nothingInProgressCV.notify_one(); } } recordDelStats(new DeleteJobStats(taskDetails.stats)); return result; }