BSONObj CommandHelpers::appendMajorityWriteConcern(const BSONObj& cmdObj) {
WriteConcernOptions newWC = kMajorityWriteConcern;
if (cmdObj.hasField(kWriteConcernField)) {
auto wc = cmdObj.getField(kWriteConcernField);
// The command has a writeConcern field and it's majority, so we can
// return it as-is.
if (wc["w"].ok() && wc["w"].str() == "majority") {
return cmdObj;
}
if (wc["wtimeout"].ok()) {
// They set a timeout, but aren't using majority WC. We want to use their
// timeout along with majority WC.
newWC = WriteConcernOptions(WriteConcernOptions::kMajority,
WriteConcernOptions::SyncMode::UNSET,
wc["wtimeout"].Number());
}
}
// Append all original fields except the writeConcern field to the new command.
BSONObjBuilder cmdObjWithWriteConcern;
for (const auto& elem : cmdObj) {
const auto name = elem.fieldNameStringData();
if (name != "writeConcern" && !cmdObjWithWriteConcern.hasField(name)) {
cmdObjWithWriteConcern.append(elem);
}
}
// Finally, add the new write concern.
cmdObjWithWriteConcern.append(kWriteConcernField, newWC.toBSON());
return cmdObjWithWriteConcern.obj();
}
void CommandHelpers::appendCommandWCStatus(BSONObjBuilder& result,
const Status& awaitReplicationStatus,
const WriteConcernResult& wcResult) {
if (!awaitReplicationStatus.isOK() && !result.hasField("writeConcernError")) {
WriteConcernErrorDetail wcError;
wcError.setStatus(awaitReplicationStatus);
if (wcResult.wTimedOut) {
wcError.setErrInfo(BSON("wtimeout" << true));
}
result.append("writeConcernError", wcError.toBSON());
}
}
BSONObj DocumentSourceMatch::getObjectForMatch(const Document& input,
const std::set<std::string>& fields) {
BSONObjBuilder matchObject;
for (auto&& field : fields) {
// getNestedField does not handle dotted paths correctly, so instead of retrieving the
// entire path, we just extract the first element of the path.
FieldPath path(field);
auto prefix = path.getFieldName(0);
if (!matchObject.hasField(prefix)) {
// Avoid adding the same prefix twice.
input.getField(prefix).addToBsonObj(&matchObject, prefix);
}
}
return matchObject.obj();
}
virtual bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int options,
string& errmsg, BSONObjBuilder& result,
bool fromRepl = false ) {
// --- parse
NamespaceString ns( dbname, cmdObj[name].String() );
Status status = userAllowedWriteNS( ns );
if ( !status.isOK() )
return appendCommandStatus( result, status );
if ( cmdObj["indexes"].type() != Array ) {
errmsg = "indexes has to be an array";
result.append( "cmdObj", cmdObj );
return false;
}
std::vector<BSONObj> specs;
{
BSONObjIterator i( cmdObj["indexes"].Obj() );
while ( i.more() ) {
BSONElement e = i.next();
if ( e.type() != Object ) {
errmsg = "everything in indexes has to be an Object";
result.append( "cmdObj", cmdObj );
return false;
}
specs.push_back( e.Obj() );
}
}
if ( specs.size() == 0 ) {
errmsg = "no indexes to add";
return false;
}
// check specs
for ( size_t i = 0; i < specs.size(); i++ ) {
BSONObj spec = specs[i];
if ( spec["ns"].eoo() ) {
spec = _addNsToSpec( ns, spec );
specs[i] = spec;
}
if ( spec["ns"].type() != String ) {
errmsg = "spec has no ns";
result.append( "spec", spec );
return false;
}
if ( ns != spec["ns"].String() ) {
errmsg = "namespace mismatch";
result.append( "spec", spec );
return false;
}
}
{
// We first take a read lock to see if we need to do anything
// as many calls are ensureIndex (and hence no-ops), this is good so its a shared
// lock for common calls. We only take write lock if needed.
// Note: createIndexes command does not currently respect shard versioning.
Client::ReadContext readContext( ns,
storageGlobalParams.dbpath,
false /* doVersion */ );
const Collection* collection = readContext.ctx().db()->getCollection( ns.ns() );
if ( collection ) {
for ( size_t i = 0; i < specs.size(); i++ ) {
BSONObj spec = specs[i];
StatusWith<BSONObj> statusWithSpec =
collection->getIndexCatalog()->prepareSpecForCreate( spec );
status = statusWithSpec.getStatus();
if ( status.code() == ErrorCodes::IndexAlreadyExists ) {
specs.erase( specs.begin() + i );
i--;
continue;
}
if ( !status.isOK() )
return appendCommandStatus( result, status );
}
if ( specs.size() == 0 ) {
result.append( "numIndexesBefore",
collection->getIndexCatalog()->numIndexesTotal() );
result.append( "note", "all indexes already exist" );
return true;
}
// need to create index
}
}
// now we know we have to create index(es)
// Note: createIndexes command does not currently respect shard versioning.
Client::WriteContext writeContext( ns.ns(),
storageGlobalParams.dbpath,
false /* doVersion */ );
Database* db = writeContext.ctx().db();
Collection* collection = db->getCollection( txn, ns.ns() );
result.appendBool( "createdCollectionAutomatically", collection == NULL );
if ( !collection ) {
collection = db->createCollection( txn, ns.ns() );
invariant( collection );
}
result.append( "numIndexesBefore", collection->getIndexCatalog()->numIndexesTotal() );
for ( size_t i = 0; i < specs.size(); i++ ) {
BSONObj spec = specs[i];
if ( spec["unique"].trueValue() ) {
status = checkUniqueIndexConstraints( ns.ns(), spec["key"].Obj() );
if ( !status.isOK() ) {
appendCommandStatus( result, status );
return false;
}
}
status = collection->getIndexCatalog()->createIndex(txn, spec, true);
if ( status.code() == ErrorCodes::IndexAlreadyExists ) {
if ( !result.hasField( "note" ) )
result.append( "note", "index already exists" );
continue;
}
if ( !status.isOK() ) {
appendCommandStatus( result, status );
return false;
}
if ( !fromRepl ) {
std::string systemIndexes = ns.getSystemIndexesCollection();
repl::logOp(txn, "i", systemIndexes.c_str(), spec);
}
}
result.append( "numIndexesAfter", collection->getIndexCatalog()->numIndexesTotal() );
return true;
}
Status dropCollection(OperationContext* opCtx,
const NamespaceString& collectionName,
BSONObjBuilder& result,
const repl::OpTime& dropOpTime,
DropCollectionSystemCollectionMode systemCollectionMode) {
if (!serverGlobalParams.quiet.load()) {
log() << "CMD: drop " << collectionName;
}
return writeConflictRetry(opCtx, "drop", collectionName.ns(), [&] {
AutoGetDb autoDb(opCtx, collectionName.db(), MODE_X);
Database* const db = autoDb.getDb();
Collection* coll = db ? db->getCollection(opCtx, collectionName) : nullptr;
auto view =
db && !coll ? db->getViewCatalog()->lookup(opCtx, collectionName.ns()) : nullptr;
if (MONGO_FAIL_POINT(hangDuringDropCollection)) {
log() << "hangDuringDropCollection fail point enabled. Blocking until fail point is "
"disabled.";
MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangDuringDropCollection);
}
if (!db || (!coll && !view)) {
return Status(ErrorCodes::NamespaceNotFound, "ns not found");
}
const bool shardVersionCheck = true;
OldClientContext context(opCtx, collectionName.ns(), shardVersionCheck);
bool userInitiatedWritesAndNotPrimary = opCtx->writesAreReplicated() &&
!repl::ReplicationCoordinator::get(opCtx)->canAcceptWritesFor(opCtx, collectionName);
if (userInitiatedWritesAndNotPrimary) {
return Status(ErrorCodes::NotMaster,
str::stream() << "Not primary while dropping collection "
<< collectionName);
}
WriteUnitOfWork wunit(opCtx);
if (!result.hasField("ns")) {
result.append("ns", collectionName.ns());
}
if (coll) {
invariant(!view);
int numIndexes = coll->getIndexCatalog()->numIndexesTotal(opCtx);
BackgroundOperation::assertNoBgOpInProgForNs(collectionName.ns());
Status s = systemCollectionMode ==
DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops
? db->dropCollection(opCtx, collectionName.ns(), dropOpTime)
: db->dropCollectionEvenIfSystem(opCtx, collectionName, dropOpTime);
if (!s.isOK()) {
return s;
}
result.append("nIndexesWas", numIndexes);
} else {
invariant(view);
Status status = db->dropView(opCtx, collectionName.ns());
if (!status.isOK()) {
return status;
}
}
wunit.commit();
return Status::OK();
});
}
void WriteBackListener::run() {
int secsToSleep = 0;
scoped_ptr<ChunkVersion> lastNeededVersion;
int lastNeededCount = 0;
bool needsToReloadShardInfo = false;
while ( ! inShutdown() ) {
if ( ! Shard::isAShardNode( _addr ) ) {
LOG(1) << _addr << " is not a shard node" << endl;
sleepsecs( 60 );
continue;
}
try {
if (needsToReloadShardInfo) {
// It's possible this shard was removed
Shard::reloadShardInfo();
needsToReloadShardInfo = false;
}
scoped_ptr<ScopedDbConnection> conn(
ScopedDbConnection::getInternalScopedDbConnection( _addr ) );
BSONObj result;
{
BSONObjBuilder cmd;
cmd.appendOID( "writebacklisten" , &serverID ); // Command will block for data
if ( ! conn->get()->runCommand( "admin" , cmd.obj() , result ) ) {
result = result.getOwned();
log() << "writebacklisten command failed! " << result << endl;
conn->done();
continue;
}
}
conn->done();
LOG(1) << "writebacklisten result: " << result << endl;
BSONObj data = result.getObjectField( "data" );
if ( data.getBoolField( "writeBack" ) ) {
string ns = data["ns"].valuestrsafe();
ConnectionIdent cid( "" , 0 );
OID wid;
if ( data["connectionId"].isNumber() && data["id"].type() == jstOID ) {
string s = "";
if ( data["instanceIdent"].type() == String )
s = data["instanceIdent"].String();
cid = ConnectionIdent( s , data["connectionId"].numberLong() );
wid = data["id"].OID();
}
else {
warning() << "mongos/mongod version mismatch (1.7.5 is the split)" << endl;
}
int len; // not used, but needed for next call
Message msg( (void*)data["msg"].binData( len ) , false );
massert( 10427 , "invalid writeback message" , msg.header()->valid() );
DBConfigPtr db = grid.getDBConfig( ns );
ChunkVersion needVersion = ChunkVersion::fromBSON( data, "version" );
//
// TODO: Refactor the sharded strategy to correctly handle all sharding state changes itself,
// we can't rely on WBL to do this for us b/c anything could reset our state in-between.
// We should always reload here for efficiency when possible, but staleness is also caught in the
// loop below.
//
ChunkManagerPtr manager;
ShardPtr primary;
db->getChunkManagerOrPrimary( ns, manager, primary );
ChunkVersion currVersion;
if( manager ) currVersion = manager->getVersion();
LOG(1) << "connectionId: " << cid << " writebackId: " << wid << " needVersion : " << needVersion.toString()
<< " mine : " << currVersion.toString() << endl;
LOG(1) << msg.toString() << endl;
//
// We should reload only if we need to update our version to be compatible *and* we
// haven't already done so. This avoids lots of reloading when we remove/add a sharded collection
//
bool alreadyReloaded = lastNeededVersion &&
lastNeededVersion->isEquivalentTo( needVersion );
if( alreadyReloaded ){
LOG(1) << "wbl already reloaded config information for version "
<< needVersion << ", at version " << currVersion << endl;
}
else if( lastNeededVersion ) {
log() << "new version change detected to " << needVersion.toString()
<< ", " << lastNeededCount << " writebacks processed at "
<< lastNeededVersion->toString() << endl;
lastNeededCount = 0;
}
//
// Set our lastNeededVersion for next time
//
lastNeededVersion.reset( new ChunkVersion( needVersion ) );
lastNeededCount++;
//
// Determine if we should reload, if so, reload
//
bool shouldReload = ! needVersion.isWriteCompatibleWith( currVersion ) &&
! alreadyReloaded;
if( shouldReload && currVersion.isSet()
&& needVersion.isSet()
&& currVersion.hasCompatibleEpoch( needVersion ) )
{
//
// If we disagree about versions only, reload the chunk manager
//
db->getChunkManagerIfExists( ns, true );
}
else if( shouldReload ){
//
// If we disagree about anything else, reload the full db
//
warning() << "reloading config data for " << db->getName() << ", "
<< "wanted version " << needVersion.toString()
<< " but currently have version " << currVersion.toString() << endl;
db->reload();
}
// do request and then call getLastError
// we have to call getLastError so we can return the right fields to the user if they decide to call getLastError
BSONObj gle;
int attempts = 0;
while ( true ) {
attempts++;
try {
Request r( msg , 0 );
r.init();
r.d().reservedField() |= Reserved_FromWriteback;
ClientInfo * ci = r.getClientInfo();
if (!noauth) {
ci->getAuthorizationManager()->grantInternalAuthorization(
"_writebackListener");
}
ci->noAutoSplit();
r.process( attempts );
ci->newRequest(); // this so we flip prev and cur shards
BSONObjBuilder b;
string errmsg;
if ( ! ci->getLastError( "admin",
BSON( "getLastError" << 1 ),
b,
errmsg,
true ) )
{
b.appendBool( "commandFailed" , true );
if( ! b.hasField( "errmsg" ) ){
b.append( "errmsg", errmsg );
gle = b.obj();
}
else if( errmsg.size() > 0 ){
// Rebuild GLE object with errmsg
// TODO: Make this less clumsy by improving GLE interface
gle = b.obj();
if( gle["errmsg"].type() == String ){
BSONObj gleNoErrmsg =
gle.filterFieldsUndotted( BSON( "errmsg" << 1 ),
false );
BSONObjBuilder bb;
bb.appendElements( gleNoErrmsg );
bb.append( "errmsg", gle["errmsg"].String() +
" ::and:: " +
errmsg );
gle = bb.obj().getOwned();
}
}
}
else{
gle = b.obj();
}
if ( gle["code"].numberInt() == 9517 ) {
log() << "new version change detected, "
<< lastNeededCount << " writebacks processed previously" << endl;
lastNeededVersion.reset();
lastNeededCount = 1;
log() << "writeback failed because of stale config, retrying attempts: " << attempts << endl;
LOG(1) << "writeback error : " << gle << endl;
//
// Bringing this in line with the similar retry logic elsewhere
//
// TODO: Reloading the chunk manager may not help if we dropped a
// collection, but we don't actually have that info in the writeback
// error
//
if( attempts <= 2 ){
db->getChunkManagerIfExists( ns, true );
}
else{
versionManager.forceRemoteCheckShardVersionCB( ns );
sleepsecs( attempts - 1 );
}
uassert( 15884, str::stream()
<< "Could not reload chunk manager after "
<< attempts << " attempts.", attempts <= 4 );
continue;
}
ci->clearSinceLastGetError();
}
catch ( DBException& e ) {
error() << "error processing writeback: " << e << endl;
BSONObjBuilder b;
e.getInfo().append( b, "err", "code" );
gle = b.obj();
}
break;
}
{
scoped_lock lk( _seenWritebacksLock );
WBStatus& s = _seenWritebacks[cid];
s.id = wid;
s.gle = gle;
}
}
else if ( result["noop"].trueValue() ) {
// no-op
}
else {
log() << "unknown writeBack result: " << result << endl;
}
secsToSleep = 0;
continue;
}
catch ( std::exception& e ) {
// Attention! Do not call any method that would throw an exception
// (or assert) in this block.
if ( inShutdown() ) {
// we're shutting down, so just clean up
return;
}
log() << "WriteBackListener exception : " << e.what() << endl;
needsToReloadShardInfo = true;
}
catch ( ... ) {
log() << "WriteBackListener uncaught exception!" << endl;
}
secsToSleep++;
sleepsecs(secsToSleep);
if ( secsToSleep > 10 )
secsToSleep = 0;
}
log() << "WriteBackListener exiting : address no longer in cluster " << _addr;
}