TEST_F(SyncTailTest, MultiApplyAssignsOperationsToWriterThreadsBasedOnNamespaceHash) { // This test relies on implementation details of how multiApply uses hashing to distribute ops // to threads. It is possible for this test to fail, even if the implementation of multiApply is // correct. If it fails, consider adjusting the namespace names (to adjust the hash values) or // the number of threads in the pool. NamespaceString nss1("test.t0"); NamespaceString nss2("test.t1"); OldThreadPool writerPool(3); stdx::mutex mutex; std::vector<MultiApplier::Operations> operationsApplied; auto applyOperationFn = [&mutex, &operationsApplied]( MultiApplier::OperationPtrs* operationsForWriterThreadToApply) { stdx::lock_guard<stdx::mutex> lock(mutex); operationsApplied.emplace_back(); for (auto&& opPtr : *operationsForWriterThreadToApply) { operationsApplied.back().push_back(*opPtr); } }; auto op1 = makeInsertDocumentOplogEntry({Timestamp(Seconds(1), 0), 1LL}, nss1, BSON("x" << 1)); auto op2 = makeInsertDocumentOplogEntry({Timestamp(Seconds(2), 0), 1LL}, nss2, BSON("x" << 2)); NamespaceString nssForInsert; std::vector<BSONObj> operationsWrittenToOplog; _storageInterface->insertDocumentsFn = [&mutex, &nssForInsert, &operationsWrittenToOplog]( OperationContext* txn, const NamespaceString& nss, const std::vector<BSONObj>& docs) { stdx::lock_guard<stdx::mutex> lock(mutex); nssForInsert = nss; operationsWrittenToOplog = docs; return Status::OK(); }; auto lastOpTime = unittest::assertGet(multiApply(_txn.get(), &writerPool, {op1, op2}, applyOperationFn)); ASSERT_EQUALS(op2.getOpTime(), lastOpTime); // Each writer thread should be given exactly one operation to apply. std::vector<OpTime> seen; { stdx::lock_guard<stdx::mutex> lock(mutex); ASSERT_EQUALS(operationsApplied.size(), 2U); for (auto&& operationsAppliedByThread : operationsApplied) { ASSERT_EQUALS(1U, operationsAppliedByThread.size()); const auto& oplogEntry = operationsAppliedByThread.front(); ASSERT_TRUE(std::find(seen.cbegin(), seen.cend(), oplogEntry.getOpTime()) == seen.cend()); ASSERT_TRUE(oplogEntry == op1 || oplogEntry == op2); seen.push_back(oplogEntry.getOpTime()); } } // Check ops in oplog. stdx::lock_guard<stdx::mutex> lock(mutex); ASSERT_EQUALS(2U, operationsWrittenToOplog.size()); ASSERT_EQUALS(NamespaceString(rsOplogName), nssForInsert); ASSERT_EQUALS(op1.raw, operationsWrittenToOplog[0]); ASSERT_EQUALS(op2.raw, operationsWrittenToOplog[1]); }
TEST_F(SyncTailTest, MultiApplyAssignsOperationsToWriterThreadsBasedOnNamespaceHash) { NamespaceString nss1("test.t0"); NamespaceString nss2("test.t1"); OldThreadPool writerPool(2); // Ensure that namespaces are hashed to different threads in pool. ASSERT_EQUALS(0U, StringMapTraits::hash(nss1.ns()) % writerPool.getNumThreads()); ASSERT_EQUALS(1U, StringMapTraits::hash(nss2.ns()) % writerPool.getNumThreads()); stdx::mutex mutex; std::vector<MultiApplier::Operations> operationsApplied; auto applyOperationFn = [&mutex, &operationsApplied]( MultiApplier::OperationPtrs* operationsForWriterThreadToApply) { stdx::lock_guard<stdx::mutex> lock(mutex); operationsApplied.emplace_back(); for (auto&& opPtr : *operationsForWriterThreadToApply) { operationsApplied.back().push_back(*opPtr); } }; auto op1 = makeInsertDocumentOplogEntry({Timestamp(Seconds(1), 0), 1LL}, nss1, BSON("x" << 1)); auto op2 = makeInsertDocumentOplogEntry({Timestamp(Seconds(2), 0), 1LL}, nss2, BSON("x" << 2)); NamespaceString nssForInsert; std::vector<BSONObj> operationsWrittenToOplog; _storageInterface->insertDocumentsFn = [&mutex, &nssForInsert, &operationsWrittenToOplog]( OperationContext* txn, const NamespaceString& nss, const std::vector<BSONObj>& docs) { stdx::lock_guard<stdx::mutex> lock(mutex); nssForInsert = nss; operationsWrittenToOplog = docs; return Status::OK(); }; auto lastOpTime = unittest::assertGet(multiApply(_txn.get(), &writerPool, {op1, op2}, applyOperationFn)); ASSERT_EQUALS(op2.getOpTime(), lastOpTime); // Each writer thread should be given exactly one operation to apply. std::vector<OpTime> seen; { stdx::lock_guard<stdx::mutex> lock(mutex); ASSERT_EQUALS(writerPool.getNumThreads(), operationsApplied.size()); for (auto&& operationsAppliedByThread : operationsApplied) { ASSERT_EQUALS(1U, operationsAppliedByThread.size()); const auto& oplogEntry = operationsAppliedByThread.front(); ASSERT_TRUE(std::find(seen.cbegin(), seen.cend(), oplogEntry.getOpTime()) == seen.cend()); ASSERT_TRUE(oplogEntry == op1 || oplogEntry == op2); seen.push_back(oplogEntry.getOpTime()); } } // Check ops in oplog. stdx::lock_guard<stdx::mutex> lock(mutex); ASSERT_EQUALS(2U, operationsWrittenToOplog.size()); ASSERT_EQUALS(NamespaceString(rsOplogName), nssForInsert); ASSERT_EQUALS(op1.raw, operationsWrittenToOplog[0]); ASSERT_EQUALS(op2.raw, operationsWrittenToOplog[1]); }
bool _testOplogEntryIsForCappedCollection(OperationContext* txn, const NamespaceString& nss, const CollectionOptions& options) { auto writerPool = SyncTail::makeWriterPool(); MultiApplier::Operations operationsApplied; auto applyOperationFn = [&operationsApplied](MultiApplier::OperationPtrs* operationsToApply) { for (auto&& opPtr : *operationsToApply) { operationsApplied.push_back(*opPtr); } }; createCollection(txn, nss, options); auto op = makeInsertDocumentOplogEntry({Timestamp(Seconds(1), 0), 1LL}, nss, BSON("a" << 1)); ASSERT_FALSE(op.isForCappedCollection); auto lastOpTime = unittest::assertGet(multiApply(txn, writerPool.get(), {op}, applyOperationFn)); ASSERT_EQUALS(op.getOpTime(), lastOpTime); ASSERT_EQUALS(1U, operationsApplied.size()); const auto& opApplied = operationsApplied.front(); ASSERT_EQUALS(op, opApplied); // "isForCappedCollection" is not parsed from raw oplog entry document. return opApplied.isForCappedCollection; }
TEST_F(SyncTailTest, MultiSyncApplyFallsBackOnApplyingInsertsIndividuallyWhenGroupedInsertFails) { int seconds = 0; auto makeOp = [&seconds](const NamespaceString& nss) { return makeInsertDocumentOplogEntry( {Timestamp(Seconds(seconds), 0), 1LL}, nss, BSON("_id" << seconds++)); }; NamespaceString nss("test." + _agent.getSuiteName() + "_" + _agent.getTestName() + "_1"); auto createOp = makeCreateCollectionOplogEntry({Timestamp(Seconds(seconds++), 0), 1LL}, nss); // Generate operations to apply: // {create}, {insert_1}, {insert_2}, .. {insert_(limit)}, {insert_(limit+1)} std::size_t limit = 64; MultiApplier::Operations insertOps; for (std::size_t i = 0; i < limit + 1; ++i) { insertOps.push_back(makeOp(nss)); } MultiApplier::Operations operationsToApply; operationsToApply.push_back(createOp); std::copy(insertOps.begin(), insertOps.end(), std::back_inserter(operationsToApply)); std::size_t numFailedGroupedInserts = 0; MultiApplier::Operations operationsApplied; auto syncApply = [&numFailedGroupedInserts, &operationsApplied](OperationContext*, const BSONObj& op, bool) -> Status { // Reject grouped insert operations. if (op["o"].type() == BSONType::Array) { numFailedGroupedInserts++; return {ErrorCodes::OperationFailed, "grouped inserts not supported"}; } operationsApplied.push_back(OplogEntry(op)); return Status::OK(); }; MultiApplier::OperationPtrs ops; for (auto&& op : operationsToApply) { ops.push_back(&op); } ASSERT_OK(multiSyncApply_noAbort(_txn.get(), &ops, syncApply)); // On failing to apply the grouped insert operation, multiSyncApply should apply the operations // as given in "operationsToApply": // {create}, {insert_1}, {insert_2}, .. {insert_(limit)}, {insert_(limit+1)} ASSERT_EQUALS(limit + 2, operationsApplied.size()); ASSERT_EQUALS(createOp, operationsApplied[0]); for (std::size_t i = 0; i < limit + 1; ++i) { const auto& insertOp = insertOps[i]; ASSERT_EQUALS(insertOp, operationsApplied[i + 1]); } // Ensure that multiSyncApply does not attempt to group remaining operations in first failed // grouped insert operation. ASSERT_EQUALS(1U, numFailedGroupedInserts); }
TEST_F(SyncTailTest, MultiInitialSyncApplySkipsDocumentOnNamespaceNotFound) { BSONObj emptyDoc; SyncTailWithLocalDocumentFetcher syncTail(emptyDoc); NamespaceString nss("local." + _agent.getSuiteName() + "_" + _agent.getTestName()); NamespaceString badNss("local." + _agent.getSuiteName() + "_" + _agent.getTestName() + "bad"); auto doc1 = BSON("_id" << 1); auto doc2 = BSON("_id" << 2); auto doc3 = BSON("_id" << 3); auto op0 = makeCreateCollectionOplogEntry({Timestamp(Seconds(1), 0), 1LL}, nss); auto op1 = makeInsertDocumentOplogEntry({Timestamp(Seconds(2), 0), 1LL}, nss, doc1); auto op2 = makeInsertDocumentOplogEntry({Timestamp(Seconds(3), 0), 1LL}, badNss, doc2); auto op3 = makeInsertDocumentOplogEntry({Timestamp(Seconds(4), 0), 1LL}, nss, doc3); MultiApplier::OperationPtrs ops = {&op0, &op1, &op2, &op3}; ASSERT_OK(multiInitialSyncApply_noAbort(_txn.get(), &ops, &syncTail)); OplogInterfaceLocal collectionReader(_txn.get(), nss.ns()); auto iter = collectionReader.makeIterator(); ASSERT_EQUALS(doc3, unittest::assertGet(iter->next()).first); ASSERT_EQUALS(doc1, unittest::assertGet(iter->next()).first); ASSERT_EQUALS(ErrorCodes::CollectionIsEmpty, iter->next().getStatus()); }
TEST_F(SyncTailTest, MultiSyncApplyUsesLimitWhenGroupingInsertOperation) { int seconds = 0; auto makeOp = [&seconds](const NamespaceString& nss) { return makeInsertDocumentOplogEntry( {Timestamp(Seconds(seconds), 0), 1LL}, nss, BSON("_id" << seconds++)); }; NamespaceString nss("test." + _agent.getSuiteName() + "_" + _agent.getTestName() + "_1"); auto createOp = makeCreateCollectionOplogEntry({Timestamp(Seconds(seconds++), 0), 1LL}, nss); // Generate operations to apply: // {create}, {insert_1}, {insert_2}, .. {insert_(limit)}, {insert_(limit+1)} std::size_t limit = 64; MultiApplier::Operations insertOps; for (std::size_t i = 0; i < limit + 1; ++i) { insertOps.push_back(makeOp(nss)); } MultiApplier::Operations operationsToApply; operationsToApply.push_back(createOp); std::copy(insertOps.begin(), insertOps.end(), std::back_inserter(operationsToApply)); MultiApplier::Operations operationsApplied; auto syncApply = [&operationsApplied](OperationContext*, const BSONObj& op, bool) { operationsApplied.push_back(OplogEntry(op)); return Status::OK(); }; MultiApplier::OperationPtrs ops; for (auto&& op : operationsToApply) { ops.push_back(&op); } ASSERT_OK(multiSyncApply_noAbort(_txn.get(), &ops, syncApply)); // multiSyncApply should combine operations as follows: // {create}, {grouped_insert}, {insert_(limit+1)} ASSERT_EQUALS(3U, operationsApplied.size()); ASSERT_EQUALS(createOp, operationsApplied[0]); const auto& groupedInsertOp = operationsApplied[1]; ASSERT_EQUALS(insertOps.front().getOpTime(), groupedInsertOp.getOpTime()); ASSERT_EQUALS(insertOps.front().ns, groupedInsertOp.ns); ASSERT_EQUALS(BSONType::Array, groupedInsertOp.o.type()); auto groupedInsertDocuments = groupedInsertOp.o.Array(); ASSERT_EQUALS(limit, groupedInsertDocuments.size()); for (std::size_t i = 0; i < limit; ++i) { const auto& insertOp = insertOps[i]; ASSERT_EQUALS(insertOp.o.Obj(), groupedInsertDocuments[i].Obj()); } // (limit + 1)-th insert operations should not be included in group of first (limit) inserts. ASSERT_EQUALS(insertOps.back(), operationsApplied[2]); }
TEST_F(SyncTailTest, MultiSyncApplyGroupsInsertOperationByNamespaceBeforeApplying) { int seconds = 0; auto makeOp = [&seconds](const NamespaceString& nss) { return makeInsertDocumentOplogEntry( {Timestamp(Seconds(seconds), 0), 1LL}, nss, BSON("_id" << seconds++)); }; NamespaceString nss1("test." + _agent.getSuiteName() + "_" + _agent.getTestName() + "_1"); NamespaceString nss2("test." + _agent.getSuiteName() + "_" + _agent.getTestName() + "_2"); auto createOp1 = makeCreateCollectionOplogEntry({Timestamp(Seconds(seconds++), 0), 1LL}, nss1); auto createOp2 = makeCreateCollectionOplogEntry({Timestamp(Seconds(seconds++), 0), 1LL}, nss2); auto insertOp1a = makeOp(nss1); auto insertOp1b = makeOp(nss1); auto insertOp2a = makeOp(nss2); auto insertOp2b = makeOp(nss2); MultiApplier::Operations operationsApplied; auto syncApply = [&operationsApplied](OperationContext*, const BSONObj& op, bool) { operationsApplied.push_back(OplogEntry(op)); return Status::OK(); }; MultiApplier::OperationPtrs ops = { &createOp1, &createOp2, &insertOp1a, &insertOp2a, &insertOp1b, &insertOp2b}; ASSERT_OK(multiSyncApply_noAbort(_txn.get(), &ops, syncApply)); ASSERT_EQUALS(4U, operationsApplied.size()); ASSERT_EQUALS(createOp1, operationsApplied[0]); ASSERT_EQUALS(createOp2, operationsApplied[1]); // Check grouped insert operations in namespace "nss1". ASSERT_EQUALS(insertOp1a.getOpTime(), operationsApplied[2].getOpTime()); ASSERT_EQUALS(insertOp1a.ns, operationsApplied[2].ns); ASSERT_EQUALS(BSONType::Array, operationsApplied[2].o.type()); auto group1 = operationsApplied[2].o.Array(); ASSERT_EQUALS(2U, group1.size()); ASSERT_EQUALS(insertOp1a.o.Obj(), group1[0].Obj()); ASSERT_EQUALS(insertOp1b.o.Obj(), group1[1].Obj()); // Check grouped insert operations in namespace "nss2". ASSERT_EQUALS(insertOp2a.getOpTime(), operationsApplied[3].getOpTime()); ASSERT_EQUALS(insertOp2a.ns, operationsApplied[3].ns); ASSERT_EQUALS(BSONType::Array, operationsApplied[3].o.type()); auto group2 = operationsApplied[3].o.Array(); ASSERT_EQUALS(2U, group2.size()); ASSERT_EQUALS(insertOp2a.o.Obj(), group2[0].Obj()); ASSERT_EQUALS(insertOp2b.o.Obj(), group2[1].Obj()); }