TEST(ParticipantList, ReceiveParticipantListMissingParticipantThatAlreadyVotedCommitThrows) {
ParticipantList participantList;
participantList.recordVoteCommit(ShardId("shard0000"), dummyTimestamp);
ASSERT_THROWS_CODE(participantList.recordFullList({ShardId("shard0001")}),
AssertionException,
ErrorCodes::InternalError);
}
TEST(ParticipantList, ReceiveConflictingParticipantListsFirstListIsSupersetOfSecondThrows) {
ParticipantList participantList;
participantList.recordFullList({ShardId("shard0000"), ShardId("shard0001")});
ASSERT_THROWS_CODE(participantList.recordFullList({ShardId("shard0000")}),
AssertionException,
ErrorCodes::InternalError);
}
TEST(ParticipantList, ReceiveVoteCommitFromParticipantNotInListThrows) {
ParticipantList participantList;
participantList.recordFullList({ShardId("shard0000")});
ASSERT_THROWS_CODE(participantList.recordVoteCommit(ShardId("shard0001"), dummyTimestamp),
AssertionException,
ErrorCodes::InternalError);
}
TEST(ParticipantList, ParticipantChangesPrepareTimestampThrows) {
ParticipantList participantList;
participantList.recordVoteCommit(ShardId("shard0000"), Timestamp::min());
ASSERT_THROWS_CODE(participantList.recordVoteCommit(ShardId("shard0000"), Timestamp::max()),
AssertionException,
ErrorCodes::InternalError);
}
TEST(ParticipantList, ReceiveConflictingParticipantListsNoOverlapThrows) {
ParticipantList participantList;
participantList.recordFullList({ShardId("shard0000"), ShardId("shard0001")});
ASSERT_THROWS_CODE(participantList.recordFullList({ShardId("shard0002"), ShardId("shard0003")}),
AssertionException,
ErrorCodes::InternalError);
}
TEST(ParticipantList, ParticipantChangesVoteFromCommitToAbortThrows) {
ParticipantList participantList;
participantList.recordVoteCommit(ShardId("shard0000"), dummyTimestamp);
ASSERT_THROWS_CODE(participantList.recordVoteAbort(ShardId("shard0000")),
AssertionException,
ErrorCodes::InternalError);
}
std::vector<AsyncRequestsSender::Response> TransactionRouter::abortTransaction(
OperationContext* opCtx, bool isImplicit) {
// The router has yet to send any commands to a remote shard for this transaction.
// Return the same error that would have been returned by a shard.
uassert(ErrorCodes::NoSuchTransaction,
"no known command has been sent by this router for this transaction",
!_participants.empty());
auto abortCmd = BSON("abortTransaction" << 1);
std::vector<AsyncRequestsSender::Request> abortRequests;
for (const auto& participantEntry : _participants) {
abortRequests.emplace_back(ShardId(participantEntry.first), abortCmd);
}
// Implicit aborts log earlier.
if (!isImplicit) {
LOG(0) << _txnIdToString() << " Aborting transaction on " << _participants.size()
<< " shard(s)";
}
return gatherResponses(opCtx,
NamespaceString::kAdminDb,
ReadPreferenceSetting{ReadPreference::PrimaryOnly},
Shard::RetryPolicy::kIdempotent,
abortRequests);
}
void CollectionShardingState::onDeleteOp(OperationContext* txn,
const CollectionShardingState::DeleteState& deleteState) {
dassert(txn->lockState()->isCollectionLockedForMode(_nss.ns(), MODE_IX));
if (txn->writesAreReplicated() && serverGlobalParams.clusterRole == ClusterRole::ShardServer &&
_nss == NamespaceString::kConfigCollectionNamespace) {
if (auto idElem = deleteState.idDoc["_id"]) {
uassert(40070,
"cannot delete shardIdentity document while in --shardsvr mode",
idElem.str() != ShardIdentityType::IdName);
}
}
// For backwards compatibility, cancel a pending asynchronous addShard task created on the
// primary config as a result of a 3.2 mongos doing addShard for the shard with id
// deletedDocId.
if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer &&
_nss == ShardType::ConfigNS) {
BSONElement idElement = deleteState.idDoc["_id"];
invariant(!idElement.eoo());
auto shardIdStr = idElement.valuestrsafe();
txn->recoveryUnit()->registerChange(
new RemoveShardLogOpHandler(txn, ShardId(std::move(shardIdStr))));
}
checkShardVersionOrThrow(txn);
if (_sourceMgr && deleteState.isMigrating) {
_sourceMgr->getCloner()->onDeleteOp(txn, deleteState.idDoc);
}
}
std::vector<ShardId> TransactionRouter::_getPendingParticipants() const {
std::vector<ShardId> pendingParticipants;
for (const auto& participant : _participants) {
if (participant.second.stmtIdCreatedAt == _latestStmtId) {
pendingParticipants.emplace_back(ShardId(participant.first));
}
}
return pendingParticipants;
}
StatusWith<SetShardVersionRequest> SetShardVersionRequest::parseFromBSON(const BSONObj& cmdObj) {
SetShardVersionRequest request;
{
std::string shardName;
Status status = bsonExtractStringField(cmdObj, kShardName, &shardName);
request._shardName = ShardId(shardName);
if (!status.isOK())
return status;
}
{
std::string shardCS;
Status status = bsonExtractStringField(cmdObj, kShardConnectionString, &shardCS);
if (!status.isOK())
return status;
auto shardCSStatus = ConnectionString::parse(shardCS);
if (!shardCSStatus.isOK())
return shardCSStatus.getStatus();
request._shardCS = std::move(shardCSStatus.getValue());
}
{
Status status = bsonExtractBooleanFieldWithDefault(cmdObj, kInit, false, &request._init);
if (!status.isOK())
return status;
}
{
Status status = bsonExtractBooleanFieldWithDefault(
cmdObj, kAuthoritative, false, &request._isAuthoritative);
if (!status.isOK())
return status;
}
{
Status status = bsonExtractBooleanFieldWithDefault(
cmdObj, kNoConnectionVersioning, false, &request._noConnectionVersioning);
if (!status.isOK())
return status;
}
if (request.isInit()) {
return request;
}
// Only initialize the version information if this is not an "init" request
{
std::string ns;
Status status = bsonExtractStringField(cmdObj, kCmdName, &ns);
if (!status.isOK())
return status;
NamespaceString nss(ns);
if (!nss.isValid()) {
return {ErrorCodes::InvalidNamespace,
str::stream() << ns << " is not a valid namespace"};
}
request._nss = std::move(nss);
}
{
auto versionStatus = ChunkVersion::parseFromBSONForSetShardVersion(cmdObj);
if (!versionStatus.isOK())
return versionStatus.getStatus();
request._version = versionStatus.getValue();
}
return request;
}
void CollectionCloner::_establishCollectionCursorsCallback(const RemoteCommandCallbackArgs& rcbd,
EstablishCursorsCommand cursorCommand) {
if (_state == State::kShuttingDown) {
Status shuttingDownStatus{ErrorCodes::CallbackCanceled, "Cloner shutting down."};
_finishCallback(shuttingDownStatus);
return;
}
auto response = rcbd.response;
if (!response.isOK()) {
_finishCallback(response.status);
return;
}
Status commandStatus = getStatusFromCommandResult(response.data);
if (commandStatus == ErrorCodes::NamespaceNotFound) {
_finishCallback(Status::OK());
return;
}
if (!commandStatus.isOK()) {
_finishCallback(commandStatus.withContext(
str::stream() << "Error querying collection '" << _sourceNss.ns() << "'"));
return;
}
std::vector<CursorResponse> cursorResponses;
Status parseResponseStatus =
_parseCursorResponse(response.data, &cursorResponses, cursorCommand);
if (!parseResponseStatus.isOK()) {
_finishCallback(parseResponseStatus);
return;
}
LOG(1) << "Collection cloner running with " << cursorResponses.size()
<< " cursors established.";
// Initialize the 'AsyncResultsMerger'(ARM).
std::vector<ClusterClientCursorParams::RemoteCursor> remoteCursors;
for (auto&& cursorResponse : cursorResponses) {
// A placeholder 'ShardId' is used until the ARM is made less sharding specific.
remoteCursors.emplace_back(
ShardId("CollectionClonerSyncSource"), _source, std::move(cursorResponse));
}
// An empty list of authenticated users is passed into the cluster parameters
// as user information is not used in the ARM in context of collection cloning.
_clusterClientCursorParams =
stdx::make_unique<ClusterClientCursorParams>(_sourceNss, UserNameIterator());
_clusterClientCursorParams->remotes = std::move(remoteCursors);
if (_collectionCloningBatchSize > 0)
_clusterClientCursorParams->batchSize = _collectionCloningBatchSize;
Client::initThreadIfNotAlready();
_arm = stdx::make_unique<AsyncResultsMerger>(
cc().getOperationContext(), _executor, _clusterClientCursorParams.get());
// This completion guard invokes _finishCallback on destruction.
auto cancelRemainingWorkInLock = [this]() { _cancelRemainingWork_inlock(); };
auto finishCallbackFn = [this](const Status& status) { _finishCallback(status); };
auto onCompletionGuard =
std::make_shared<OnCompletionGuard>(cancelRemainingWorkInLock, finishCallbackFn);
// Lock guard must be declared after completion guard. If there is an error in this function
// that will cause the destructor of the completion guard to run, the destructor must be run
// outside the mutex. This is a necessary condition to invoke _finishCallback.
stdx::lock_guard<stdx::mutex> lock(_mutex);
Status scheduleStatus = _scheduleNextARMResultsCallback(onCompletionGuard);
_arm->detachFromOperationContext();
if (!scheduleStatus.isOK()) {
onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, scheduleStatus);
return;
}
}
TEST(ParticipantList, ReceiveSameParticipantListMultipleTimesSucceeds) {
ParticipantList participantList;
participantList.recordFullList({ShardId("shard0000"), ShardId("shard0001")});
participantList.recordFullList({ShardId("shard0000"), ShardId("shard0001")});
participantList.recordFullList({ShardId("shard0000"), ShardId("shard0001")});
}
TEST(ParticipantList, ParticipantResendsVoteCommitSucceeds) {
ParticipantList participantList;
participantList.recordVoteCommit(ShardId("shard0000"), dummyTimestamp);
participantList.recordVoteCommit(ShardId("shard0000"), dummyTimestamp);
}
TEST(ParticipantList, ParticipantResendsVoteAbortSucceeds) {
ParticipantList participantList;
participantList.recordVoteAbort(ShardId("shard0001"));
participantList.recordVoteAbort(ShardId("shard0001"));
}
boost::intrusive_ptr<DocumentSource> DocumentSourceMergeCursors::createFromBson(
BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx) {
if (elem.type() == BSONType::Object) {
// This is the modern serialization format. We de-serialize using the IDL.
auto ownedObj = elem.embeddedObject().getOwned();
auto armParams =
AsyncResultsMergerParams::parse(IDLParserErrorContext(kStageName), ownedObj);
return new DocumentSourceMergeCursors(
Grid::get(expCtx->opCtx)->getExecutorPool()->getArbitraryExecutor(),
std::move(armParams),
expCtx,
std::move(ownedObj));
}
// This is the old serialization format which can still be generated by mongos processes
// older than 4.0.
// TODO SERVER-34009 Remove support for this format.
uassert(17026,
"$mergeCursors stage expected either an array or an object as argument",
elem.type() == BSONType::Array);
const auto serializedRemotes = elem.Array();
uassert(50729,
"$mergeCursors stage expected array with at least one entry",
serializedRemotes.size() > 0);
boost::optional<NamespaceString> nss;
std::vector<RemoteCursor> remotes;
for (auto&& cursor : serializedRemotes) {
BSONElement nsElem;
BSONElement hostElem;
BSONElement idElem;
uassert(17027,
"$mergeCursors stage requires each cursor in array to be an object",
cursor.type() == BSONType::Object);
for (auto&& cursorElem : cursor.Obj()) {
const auto fieldName = cursorElem.fieldNameStringData();
if (fieldName == "ns"_sd) {
nsElem = cursorElem;
} else if (fieldName == "host"_sd) {
hostElem = cursorElem;
} else if (fieldName == "id"_sd) {
idElem = cursorElem;
} else {
uasserted(50730,
str::stream() << "Unrecognized option " << fieldName
<< " within cursor provided to $mergeCursors: "
<< cursor);
}
}
uassert(
50731,
"$mergeCursors stage requires \'ns\' field with type string for each cursor in array",
nsElem.type() == BSONType::String);
// We require each cursor to have the same namespace. This isn't a fundamental limit of the
// system, but needs to be true due to the implementation of AsyncResultsMerger, which
// tracks one namespace for all cursors.
uassert(50720,
"$mergeCursors requires each cursor to have the same namespace",
!nss || nss->ns() == nsElem.valueStringData());
nss = NamespaceString(nsElem.String());
uassert(
50721,
"$mergeCursors stage requires \'host\' field with type string for each cursor in array",
hostElem.type() == BSONType::String);
auto host = uassertStatusOK(HostAndPort::parse(hostElem.valueStringData()));
uassert(50722,
"$mergeCursors stage requires \'id\' field with type long for each cursor in array",
idElem.type() == BSONType::NumberLong);
auto cursorId = idElem.Long();
// We are assuming that none of the cursors have been iterated at all, and so will not have
// any data in the initial batch.
// TODO SERVER-33323 We use a fake shard id because the AsyncResultsMerger won't use it for
// anything, and finding the real one is non-trivial.
RemoteCursor remoteCursor;
remoteCursor.setShardId(ShardId("fakeShardIdForMergeCursors"));
remoteCursor.setHostAndPort(std::move(host));
std::vector<BSONObj> emptyBatch;
remoteCursor.setCursorResponse(CursorResponse{*nss, cursorId, emptyBatch});
remotes.push_back(std::move(remoteCursor));
}
invariant(nss); // We know there is at least one cursor in 'serializedRemotes', and we require
// each cursor to have a 'ns' field.
AsyncResultsMergerParams params;
params.setRemotes(std::move(remotes));
params.setNss(*nss);
return new DocumentSourceMergeCursors(
Grid::get(expCtx->opCtx)->getExecutorPool()->getArbitraryExecutor(),
std::move(params),
expCtx,
elem.embeddedObject().getOwned());
}
