void BackgroundSync::applyOpsFromOplog() { GTID lastLiveGTID; GTID lastUnappliedGTID; while (1) { try { BSONObj curr; { boost::unique_lock<boost::mutex> lck(_mutex); // wait until we know an item has been produced while (_deque.size() == 0 && !_applierShouldExit) { _queueDone.notify_all(); _queueCond.wait(lck); } if (_deque.size() == 0 && _applierShouldExit) { return; } curr = _deque.front(); } GTID currEntry = getGTIDFromOplogEntry(curr); theReplSet->gtidManager->noteApplyingGTID(currEntry); // we must do applyTransactionFromOplog in a loop // because once we have called noteApplyingGTID, we must // continue until we are successful in applying the transaction. for (uint32_t numTries = 0; numTries <= 100; numTries++) { try { numTries++; TimerHolder timer(&applyBatchStats); applyTransactionFromOplog(curr); opsAppliedStats.increment(); break; } catch (std::exception &e) { log() << "exception during applying transaction from oplog: " << e.what() << endl; log() << "oplog entry: " << curr.str() << endl; if (numTries == 100) { // something is really wrong if we fail 100 times, let's abort ::abort(); } sleepsecs(1); } } LOG(3) << "applied " << curr.toString(false, true) << endl; theReplSet->gtidManager->noteGTIDApplied(currEntry); { boost::unique_lock<boost::mutex> lck(_mutex); dassert(_deque.size() > 0); _deque.pop_front(); bufferCountGauge.increment(-1); bufferSizeGauge.increment(-curr.objsize()); // this is a flow control mechanism, with bad numbers // hard coded for now just to get something going. // If the opSync thread notices that we have over 20000 // transactions in the queue, it waits until we get below // 10000. This is where we signal that we have gotten there // Once we have spilling of transactions working, this // logic will need to be redone if (_deque.size() == 10000) { _queueCond.notify_all(); } } } catch (DBException& e) { sethbmsg(str::stream() << "db exception in producer on applier thread: " << e.toString()); sleepsecs(2); } catch (std::exception& e2) { sethbmsg(str::stream() << "exception in producer on applier thread: " << e2.what()); sleepsecs(2); } } }