void RangeDeleter::doWork() {
_env->initThread();
while (!inShutdown() && !stopRequested()) {
string errMsg;
boost::scoped_ptr<OperationContext> txn(getGlobalServiceContext()->newOpCtx());
RangeDeleteEntry* nextTask = NULL;
{
boost::unique_lock<boost::mutex> sl(_queueMutex);
while (_taskQueue.empty()) {
_taskQueueNotEmptyCV.timed_wait(
sl, duration::milliseconds(kNotEmptyTimeoutMillis));
if (stopRequested()) {
log() << "stopping range deleter worker" << endl;
return;
}
if (_taskQueue.empty()) {
// Try to check if some deletes are ready and move them to the
// ready queue.
TaskList::iterator iter = _notReadyQueue.begin();
while (iter != _notReadyQueue.end()) {
RangeDeleteEntry* entry = *iter;
set<CursorId> cursorsNow;
{
if (entry->options.waitForOpenCursors) {
_env->getCursorIds(txn.get(),
entry->options.range.ns,
&cursorsNow);
}
}
set<CursorId> cursorsLeft;
std::set_intersection(entry->cursorsToWait.begin(),
entry->cursorsToWait.end(),
cursorsNow.begin(),
cursorsNow.end(),
std::inserter(cursorsLeft,
cursorsLeft.end()));
entry->cursorsToWait.swap(cursorsLeft);
if (entry->cursorsToWait.empty()) {
(*iter)->stats.queueEndTS = jsTime();
_taskQueue.push_back(*iter);
_taskQueueNotEmptyCV.notify_one();
iter = _notReadyQueue.erase(iter);
}
else {
logCursorsWaiting(entry);
++iter;
}
}
}
}
if (stopRequested()) {
log() << "stopping range deleter worker" << endl;
return;
}
nextTask = _taskQueue.front();
_taskQueue.pop_front();
_deletesInProgress++;
}
{
nextTask->stats.deleteStartTS = jsTime();
bool delResult = _env->deleteRange(txn.get(),
*nextTask,
&nextTask->stats.deletedDocCount,
&errMsg);
nextTask->stats.deleteEndTS = jsTime();
if (delResult) {
nextTask->stats.waitForReplStartTS = jsTime();
if (!_waitForMajority(txn.get(), &errMsg)) {
warning() << "Error encountered while waiting for replication: " << errMsg;
}
nextTask->stats.waitForReplEndTS = jsTime();
}
else {
warning() << "Error encountered while trying to delete range: "
<< errMsg << endl;
}
}
{
boost::lock_guard<boost::mutex> sl(_queueMutex);
NSMinMax setEntry(nextTask->options.range.ns,
nextTask->options.range.minKey,
nextTask->options.range.maxKey);
deletePtrElement(&_deleteSet, &setEntry);
_deletesInProgress--;
if (nextTask->notifyDone) {
nextTask->notifyDone->notifyOne();
}
}
recordDelStats(new DeleteJobStats(nextTask->stats));
delete nextTask;
nextTask = NULL;
}
}
void RangeDeleter::doWork() {
_env->initThread();
while (!inShutdown() && !stopRequested()) {
string errMsg;
RangeDeleteEntry* nextTask = NULL;
{
scoped_lock sl(_queueMutex);
while (_taskQueue.empty()) {
_taskQueueNotEmptyCV.timed_wait(
sl.boost(), duration::milliseconds(NotEmptyTimeoutMillis));
if (stopRequested()) {
log() << "stopping range deleter worker" << endl;
return;
}
if (_taskQueue.empty()) {
// Try to check if some deletes are ready and move them to the
// ready queue.
TaskList::iterator iter = _notReadyQueue.begin();
while (iter != _notReadyQueue.end()) {
RangeDeleteEntry* entry = *iter;
set<CursorId> cursorsNow;
{
boost::scoped_ptr<OperationContext> txn(getGlobalEnvironment()->newOpCtx());
_env->getCursorIds(txn.get(), entry->ns, &cursorsNow);
}
set<CursorId> cursorsLeft;
std::set_intersection(entry->cursorsToWait.begin(),
entry->cursorsToWait.end(),
cursorsNow.begin(),
cursorsNow.end(),
std::inserter(cursorsLeft,
cursorsLeft.end()));
entry->cursorsToWait.swap(cursorsLeft);
if (entry->cursorsToWait.empty()) {
(*iter)->stats.queueEndTS = jsTime();
_taskQueue.push_back(*iter);
_taskQueueNotEmptyCV.notify_one();
iter = _notReadyQueue.erase(iter);
}
else {
const unsigned long long int elapsedMillis =
entry->stats.queueStartTS.millis - curTimeMillis64();
if ( elapsedMillis > LogCursorsThresholdMillis &&
entry->timeSinceLastLog.millis > LogCursorsIntervalMillis) {
entry->timeSinceLastLog = jsTime();
logCursorsWaiting(entry->ns,
entry->min,
entry->max,
elapsedMillis,
entry->cursorsToWait);
}
++iter;
}
}
}
}
if (stopRequested()) {
log() << "stopping range deleter worker" << endl;
return;
}
nextTask = _taskQueue.front();
_taskQueue.pop_front();
_deletesInProgress++;
}
{
boost::scoped_ptr<OperationContext> txn(getGlobalEnvironment()->newOpCtx());
ReplTime lastOp;
nextTask->stats.deleteStartTS = jsTime();
bool delResult = _env->deleteRange(txn.get(),
*nextTask,
&nextTask->stats.deletedDocCount,
&lastOp,
&errMsg);
nextTask->stats.deleteEndTS = jsTime();
if (delResult) {
nextTask->stats.waitForReplStartTS = jsTime();
if (!_env->waitForReplication(lastOp,
DelWriteConcern,
WaitForReplTimeoutSecs,
&errMsg)) {
warning() << "Error encountered while waiting for replication: "
<< errMsg << endl;
}
nextTask->stats.waitForReplEndTS = jsTime();
}
else {
warning() << "Error encountered while trying to delete range: "
<< errMsg << endl;
}
}
{
scoped_lock sl(_queueMutex);
NSMinMax setEntry(nextTask->ns, nextTask->min, nextTask->max);
deletePtrElement(&_deleteSet, &setEntry);
_deletesInProgress--;
if (nextTask->notifyDone) {
nextTask->notifyDone->notifyOne();
}
}
recordDelStats(new DeleteJobStats(nextTask->stats));
delete nextTask;
nextTask = NULL;
}
}
bool RangeDeleter::deleteNow(OperationContext* txn,
const RangeDeleterOptions& options,
string* errMsg) {
if (stopRequested()) {
*errMsg = "deleter is already stopped.";
return false;
}
string dummy;
if (errMsg == NULL) errMsg = &dummy;
const string& ns(options.range.ns);
const BSONObj& min(options.range.minKey);
const BSONObj& max(options.range.maxKey);
NSMinMax deleteRange(ns, min, max);
{
boost::lock_guard<boost::mutex> sl(_queueMutex);
if (!canEnqueue_inlock(ns, min, max, errMsg)) {
return false;
}
_deleteSet.insert(&deleteRange);
// Note: count for pending deletes is an integral part of the shutdown story.
// Therefore, to simplify things, there is no "pending" state for deletes in
// deleteNow, the state transition is simply inProgress -> done.
_deletesInProgress++;
}
set<CursorId> cursorsToWait;
if (options.waitForOpenCursors) {
_env->getCursorIds(txn, ns, &cursorsToWait);
}
long long checkIntervalMillis = 5;
RangeDeleteEntry taskDetails(options);
taskDetails.stats.queueStartTS = jsTime();
for (; !cursorsToWait.empty(); sleepmillis(checkIntervalMillis)) {
logCursorsWaiting(&taskDetails);
set<CursorId> cursorsNow;
_env->getCursorIds(txn, ns, &cursorsNow);
set<CursorId> cursorsLeft;
std::set_intersection(cursorsToWait.begin(),
cursorsToWait.end(),
cursorsNow.begin(),
cursorsNow.end(),
std::inserter(cursorsLeft, cursorsLeft.end()));
cursorsToWait.swap(cursorsLeft);
if (stopRequested()) {
*errMsg = "deleter was stopped.";
boost::lock_guard<boost::mutex> sl(_queueMutex);
_deleteSet.erase(&deleteRange);
_deletesInProgress--;
if (_deletesInProgress == 0) {
_nothingInProgressCV.notify_one();
}
return false;
}
if (checkIntervalMillis < kMaxCursorCheckIntervalMillis) {
checkIntervalMillis *= 2;
}
}
taskDetails.stats.queueEndTS = jsTime();
taskDetails.stats.deleteStartTS = jsTime();
bool result = _env->deleteRange(txn,
taskDetails,
&taskDetails.stats.deletedDocCount,
errMsg);
taskDetails.stats.deleteEndTS = jsTime();
if (result) {
taskDetails.stats.waitForReplStartTS = jsTime();
result = _waitForMajority(txn, errMsg);
taskDetails.stats.waitForReplEndTS = jsTime();
}
{
boost::lock_guard<boost::mutex> sl(_queueMutex);
_deleteSet.erase(&deleteRange);
_deletesInProgress--;
if (_deletesInProgress == 0) {
_nothingInProgressCV.notify_one();
}
}
recordDelStats(new DeleteJobStats(taskDetails.stats));
return result;
}
bool RangeDeleter::deleteNow(OperationContext* txn,
const std::string& ns,
const BSONObj& min,
const BSONObj& max,
const BSONObj& shardKeyPattern,
bool secondaryThrottle,
string* errMsg) {
if (stopRequested()) {
*errMsg = "deleter is already stopped.";
return false;
}
string dummy;
if (errMsg == NULL) errMsg = &dummy;
NSMinMax deleteRange(ns, min, max);
{
scoped_lock sl(_queueMutex);
if (!canEnqueue_inlock(ns, min, max, errMsg)) {
return false;
}
_deleteSet.insert(&deleteRange);
// Note: count for pending deletes is an integral part of the shutdown story.
// Therefore, to simplify things, there is no "pending" state for deletes in
// deleteNow, the state transition is simply inProgress -> done.
_deletesInProgress++;
}
set<CursorId> cursorsToWait;
_env->getCursorIds(txn, ns, &cursorsToWait);
long long checkIntervalMillis = 5;
if (!cursorsToWait.empty()) {
log() << "rangeDeleter waiting for " << cursorsToWait.size()
<< " cursors in " << ns << " to finish" << endl;
}
RangeDeleteEntry taskDetails(ns, min, max, shardKeyPattern, secondaryThrottle);
taskDetails.stats.queueStartTS = jsTime();
Date_t timeSinceLastLog;
for (; !cursorsToWait.empty(); sleepmillis(checkIntervalMillis)) {
const unsigned long long timeNow = curTimeMillis64();
const unsigned long long elapsedTimeMillis =
timeNow - taskDetails.stats.queueStartTS.millis;
const unsigned long long lastLogMillis = timeNow - timeSinceLastLog.millis;
if (elapsedTimeMillis > LogCursorsThresholdMillis &&
lastLogMillis > LogCursorsIntervalMillis) {
timeSinceLastLog = jsTime();
logCursorsWaiting(ns, min, max, elapsedTimeMillis, cursorsToWait);
}
set<CursorId> cursorsNow;
_env->getCursorIds(txn, ns, &cursorsNow);
set<CursorId> cursorsLeft;
std::set_intersection(cursorsToWait.begin(),
cursorsToWait.end(),
cursorsNow.begin(),
cursorsNow.end(),
std::inserter(cursorsLeft, cursorsLeft.end()));
cursorsToWait.swap(cursorsLeft);
if (stopRequested()) {
*errMsg = "deleter was stopped.";
scoped_lock sl(_queueMutex);
_deleteSet.erase(&deleteRange);
_deletesInProgress--;
if (_deletesInProgress == 0) {
_nothingInProgressCV.notify_one();
}
return false;
}
if (checkIntervalMillis < MaxCurorCheckIntervalMillis) {
checkIntervalMillis *= 2;
}
}
taskDetails.stats.queueEndTS = jsTime();
ReplTime lastOp;
taskDetails.stats.deleteStartTS = jsTime();
bool result = _env->deleteRange(txn,
taskDetails,
&taskDetails.stats.deletedDocCount,
&lastOp,
errMsg);
taskDetails.stats.deleteEndTS = jsTime();
if (result) {
taskDetails.stats.waitForReplStartTS = jsTime();
result = _env->waitForReplication(lastOp,
DelWriteConcern,
WaitForReplTimeoutSecs,
errMsg);
taskDetails.stats.waitForReplEndTS = jsTime();
}
{
scoped_lock sl(_queueMutex);
_deleteSet.erase(&deleteRange);
_deletesInProgress--;
if (_deletesInProgress == 0) {
_nothingInProgressCV.notify_one();
}
}
recordDelStats(new DeleteJobStats(taskDetails.stats));
return result;
}
