static bool checkShardVersion(OperationContext* txn,
ShardingState* shardingState,
const BatchedCommandRequest& request,
WriteOpResult* result) {
const NamespaceString nss( request.getTargetingNS() );
txn->lockState()->assertWriteLocked( nss.ns() );
ChunkVersion requestShardVersion =
request.isMetadataSet() && request.getMetadata()->isShardVersionSet() ?
request.getMetadata()->getShardVersion() : ChunkVersion::IGNORED();
if ( shardingState->enabled() ) {
CollectionMetadataPtr metadata = shardingState->getCollectionMetadata( nss.ns() );
if ( !ChunkVersion::isIgnoredVersion( requestShardVersion ) ) {
ChunkVersion shardVersion =
metadata ? metadata->getShardVersion() : ChunkVersion::UNSHARDED();
if ( !requestShardVersion.isWriteCompatibleWith( shardVersion ) ) {
result->setError(new WriteErrorDetail);
buildStaleError(requestShardVersion, shardVersion, result->getError());
return false;
}
}
}
return true;
}
static bool checkShardVersion( ShardingState* shardingState,
const BatchedCommandRequest& request,
WriteErrorDetail** error ) {
const NamespaceString nss( request.getTargetingNS() );
Lock::assertWriteLocked( nss.ns() );
ChunkVersion requestShardVersion =
request.isMetadataSet() && request.getMetadata()->isShardVersionSet() ?
request.getMetadata()->getShardVersion() : ChunkVersion::IGNORED();
if ( shardingState->enabled() ) {
CollectionMetadataPtr metadata = shardingState->getCollectionMetadata( nss.ns() );
if ( !ChunkVersion::isIgnoredVersion( requestShardVersion ) ) {
ChunkVersion shardVersion =
metadata ? metadata->getShardVersion() : ChunkVersion::UNSHARDED();
if ( !requestShardVersion.isWriteCompatibleWith( shardVersion ) ) {
*error = new WriteErrorDetail;
buildStaleError( requestShardVersion, shardVersion, *error );
return false;
}
}
}
return true;
}
bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) {
// Steps
// 1. check basic config
// 2. extract params from command
// 3. fast check
// 4. slow check (LOCKS)
// step 1
lastError.disableForCommand();
ShardedConnectionInfo* info = ShardedConnectionInfo::get( true );
// make sure we have the mongos id for writebacks
if ( ! checkMongosID( info , cmdObj["serverID"] , errmsg ) )
return false;
bool authoritative = cmdObj.getBoolField( "authoritative" );
// check config server is ok or enable sharding
if ( ! checkConfigOrInit( cmdObj["configdb"].valuestrsafe() , authoritative , errmsg , result ) )
return false;
// check shard name/hosts are correct
if ( cmdObj["shard"].type() == String ) {
shardingState.gotShardName( cmdObj["shard"].String() );
}
// Handle initial shard connection
if( cmdObj["version"].eoo() && cmdObj["init"].trueValue() ){
result.append( "initialized", true );
// Send back wire version to let mongos know what protocol we can speak
result.append( "minWireVersion", minWireVersion );
result.append( "maxWireVersion", maxWireVersion );
return true;
}
// we can run on a slave up to here
if ( ! isMaster( "admin" ) ) {
result.append( "errmsg" , "not master" );
result.append( "note" , "from post init in setShardVersion" );
return false;
}
// step 2
string ns = cmdObj["setShardVersion"].valuestrsafe();
if ( ns.size() == 0 ) {
errmsg = "need to specify namespace";
return false;
}
if( ! ChunkVersion::canParseBSON( cmdObj, "version" ) ){
errmsg = "need to specify version";
return false;
}
const ChunkVersion version = ChunkVersion::fromBSON( cmdObj, "version" );
// step 3
const ChunkVersion oldVersion = info->getVersion(ns);
const ChunkVersion globalVersion = shardingState.getVersion(ns);
oldVersion.addToBSON( result, "oldVersion" );
if ( globalVersion.isSet() && version.isSet() ) {
// this means there is no reset going on an either side
// so its safe to make some assumptions
if ( version.isWriteCompatibleWith( globalVersion ) ) {
// mongos and mongod agree!
if ( ! oldVersion.isWriteCompatibleWith( version ) ) {
if ( oldVersion < globalVersion &&
oldVersion.hasCompatibleEpoch(globalVersion) )
{
info->setVersion( ns , version );
}
else if ( authoritative ) {
// this means there was a drop and our version is reset
info->setVersion( ns , version );
}
else {
result.append( "ns" , ns );
result.appendBool( "need_authoritative" , true );
errmsg = "verifying drop on '" + ns + "'";
return false;
}
}
return true;
}
}
// step 4
// this is because of a weird segfault I saw and I can't see why this should ever be set
massert( 13647 , str::stream() << "context should be empty here, is: " << cc().getContext()->ns() , cc().getContext() == 0 );
if ( oldVersion.isSet() && ! globalVersion.isSet() ) {
// this had been reset
info->setVersion( ns , ChunkVersion( 0, OID() ) );
}
if ( ! version.isSet() && ! globalVersion.isSet() ) {
// this connection is cleaning itself
info->setVersion( ns , ChunkVersion( 0, OID() ) );
return true;
}
// Cases below all either return OR fall-through to remote metadata reload.
if ( version.isSet() || !globalVersion.isSet() ) {
// Not Dropping
// TODO: Refactor all of this
if ( version < oldVersion && version.hasCompatibleEpoch( oldVersion ) ) {
errmsg = "this connection already had a newer version of collection '" + ns + "'";
result.append( "ns" , ns );
version.addToBSON( result, "newVersion" );
globalVersion.addToBSON( result, "globalVersion" );
return false;
}
// TODO: Refactor all of this
if ( version < globalVersion && version.hasCompatibleEpoch( globalVersion ) ) {
while ( shardingState.inCriticalMigrateSection() ) {
log() << "waiting till out of critical section" << endl;
shardingState.waitTillNotInCriticalSection( 10 );
}
errmsg = "shard global version for collection is higher than trying to set to '" + ns + "'";
result.append( "ns" , ns );
version.addToBSON( result, "version" );
globalVersion.addToBSON( result, "globalVersion" );
result.appendBool( "reloadConfig" , true );
return false;
}
if ( ! globalVersion.isSet() && ! authoritative ) {
// Needed b/c when the last chunk is moved off a shard, the version gets reset to zero, which
// should require a reload.
while ( shardingState.inCriticalMigrateSection() ) {
log() << "waiting till out of critical section" << endl;
shardingState.waitTillNotInCriticalSection( 10 );
}
// need authoritative for first look
result.append( "ns" , ns );
result.appendBool( "need_authoritative" , true );
errmsg = "first time for collection '" + ns + "'";
return false;
}
// Fall through to metadata reload below
}
else {
// Dropping
if ( ! authoritative ) {
result.appendBool( "need_authoritative" , true );
result.append( "ns" , ns );
globalVersion.addToBSON( result, "globalVersion" );
errmsg = "dropping needs to be authoritative";
return false;
}
// Fall through to metadata reload below
}
ChunkVersion currVersion;
Status status = shardingState.refreshMetadataIfNeeded( ns, version, &currVersion );
if (!status.isOK()) {
// The reload itself was interrupted or confused here
errmsg = str::stream() << "could not refresh metadata for " << ns
<< " with requested shard version " << version.toString()
<< ", stored shard version is " << currVersion.toString()
<< causedBy( status.reason() );
warning() << errmsg << endl;
result.append( "ns" , ns );
version.addToBSON( result, "version" );
currVersion.addToBSON( result, "globalVersion" );
result.appendBool( "reloadConfig", true );
return false;
}
else if ( !version.isWriteCompatibleWith( currVersion ) ) {
// We reloaded a version that doesn't match the version mongos was trying to
// set.
errmsg = str::stream() << "requested shard version differs from"
<< " config shard version for " << ns
<< ", requested version is " << version.toString()
<< " but found version " << currVersion.toString();
OCCASIONALLY warning() << errmsg << endl;
// WARNING: the exact fields below are important for compatibility with mongos
// version reload.
result.append( "ns" , ns );
currVersion.addToBSON( result, "globalVersion" );
// If this was a reset of a collection or the last chunk moved out, inform mongos to
// do a full reload.
if (currVersion.epoch() != version.epoch() || !currVersion.isSet() ) {
result.appendBool( "reloadConfig", true );
// Zero-version also needed to trigger full mongos reload, sadly
// TODO: Make this saner, and less impactful (full reload on last chunk is bad)
ChunkVersion( 0, 0, OID() ).addToBSON( result, "version" );
// For debugging
version.addToBSON( result, "origVersion" );
}
else {
version.addToBSON( result, "version" );
}
return false;
}
info->setVersion( ns , version );
return true;
}
void WriteBackListener::run() {
int secsToSleep = 0;
scoped_ptr<ChunkVersion> lastNeededVersion;
int lastNeededCount = 0;
bool needsToReloadShardInfo = false;
while ( ! inShutdown() ) {
if ( ! Shard::isAShardNode( _addr ) ) {
LOG(1) << _addr << " is not a shard node" << endl;
sleepsecs( 60 );
continue;
}
try {
if (needsToReloadShardInfo) {
// It's possible this shard was removed
Shard::reloadShardInfo();
needsToReloadShardInfo = false;
}
scoped_ptr<ScopedDbConnection> conn(
ScopedDbConnection::getInternalScopedDbConnection( _addr ) );
BSONObj result;
{
BSONObjBuilder cmd;
cmd.appendOID( "writebacklisten" , &serverID ); // Command will block for data
if ( ! conn->get()->runCommand( "admin" , cmd.obj() , result ) ) {
result = result.getOwned();
log() << "writebacklisten command failed! " << result << endl;
conn->done();
continue;
}
}
conn->done();
LOG(1) << "writebacklisten result: " << result << endl;
BSONObj data = result.getObjectField( "data" );
if ( data.getBoolField( "writeBack" ) ) {
string ns = data["ns"].valuestrsafe();
ConnectionIdent cid( "" , 0 );
OID wid;
if ( data["connectionId"].isNumber() && data["id"].type() == jstOID ) {
string s = "";
if ( data["instanceIdent"].type() == String )
s = data["instanceIdent"].String();
cid = ConnectionIdent( s , data["connectionId"].numberLong() );
wid = data["id"].OID();
}
else {
warning() << "mongos/mongod version mismatch (1.7.5 is the split)" << endl;
}
int len; // not used, but needed for next call
Message msg( (void*)data["msg"].binData( len ) , false );
massert( 10427 , "invalid writeback message" , msg.header()->valid() );
DBConfigPtr db = grid.getDBConfig( ns );
ChunkVersion needVersion = ChunkVersion::fromBSON( data, "version" );
//
// TODO: Refactor the sharded strategy to correctly handle all sharding state changes itself,
// we can't rely on WBL to do this for us b/c anything could reset our state in-between.
// We should always reload here for efficiency when possible, but staleness is also caught in the
// loop below.
//
ChunkManagerPtr manager;
ShardPtr primary;
db->getChunkManagerOrPrimary( ns, manager, primary );
ChunkVersion currVersion;
if( manager ) currVersion = manager->getVersion();
LOG(1) << "connectionId: " << cid << " writebackId: " << wid << " needVersion : " << needVersion.toString()
<< " mine : " << currVersion.toString() << endl;
LOG(1) << msg.toString() << endl;
//
// We should reload only if we need to update our version to be compatible *and* we
// haven't already done so. This avoids lots of reloading when we remove/add a sharded collection
//
bool alreadyReloaded = lastNeededVersion &&
lastNeededVersion->isEquivalentTo( needVersion );
if( alreadyReloaded ){
LOG(1) << "wbl already reloaded config information for version "
<< needVersion << ", at version " << currVersion << endl;
}
else if( lastNeededVersion ) {
log() << "new version change detected to " << needVersion.toString()
<< ", " << lastNeededCount << " writebacks processed at "
<< lastNeededVersion->toString() << endl;
lastNeededCount = 0;
}
//
// Set our lastNeededVersion for next time
//
lastNeededVersion.reset( new ChunkVersion( needVersion ) );
lastNeededCount++;
//
// Determine if we should reload, if so, reload
//
bool shouldReload = ! needVersion.isWriteCompatibleWith( currVersion ) &&
! alreadyReloaded;
if( shouldReload && currVersion.isSet()
&& needVersion.isSet()
&& currVersion.hasCompatibleEpoch( needVersion ) )
{
//
// If we disagree about versions only, reload the chunk manager
//
db->getChunkManagerIfExists( ns, true );
}
else if( shouldReload ){
//
// If we disagree about anything else, reload the full db
//
warning() << "reloading config data for " << db->getName() << ", "
<< "wanted version " << needVersion.toString()
<< " but currently have version " << currVersion.toString() << endl;
db->reload();
}
// do request and then call getLastError
// we have to call getLastError so we can return the right fields to the user if they decide to call getLastError
BSONObj gle;
int attempts = 0;
while ( true ) {
attempts++;
try {
Request r( msg , 0 );
r.init();
r.d().reservedField() |= Reserved_FromWriteback;
ClientInfo * ci = r.getClientInfo();
if (!noauth) {
ci->getAuthorizationManager()->grantInternalAuthorization(
"_writebackListener");
}
ci->noAutoSplit();
r.process( attempts );
ci->newRequest(); // this so we flip prev and cur shards
BSONObjBuilder b;
string errmsg;
if ( ! ci->getLastError( "admin",
BSON( "getLastError" << 1 ),
b,
errmsg,
true ) )
{
b.appendBool( "commandFailed" , true );
if( ! b.hasField( "errmsg" ) ){
b.append( "errmsg", errmsg );
gle = b.obj();
}
else if( errmsg.size() > 0 ){
// Rebuild GLE object with errmsg
// TODO: Make this less clumsy by improving GLE interface
gle = b.obj();
if( gle["errmsg"].type() == String ){
BSONObj gleNoErrmsg =
gle.filterFieldsUndotted( BSON( "errmsg" << 1 ),
false );
BSONObjBuilder bb;
bb.appendElements( gleNoErrmsg );
bb.append( "errmsg", gle["errmsg"].String() +
" ::and:: " +
errmsg );
gle = bb.obj().getOwned();
}
}
}
else{
gle = b.obj();
}
if ( gle["code"].numberInt() == 9517 ) {
log() << "new version change detected, "
<< lastNeededCount << " writebacks processed previously" << endl;
lastNeededVersion.reset();
lastNeededCount = 1;
log() << "writeback failed because of stale config, retrying attempts: " << attempts << endl;
LOG(1) << "writeback error : " << gle << endl;
//
// Bringing this in line with the similar retry logic elsewhere
//
// TODO: Reloading the chunk manager may not help if we dropped a
// collection, but we don't actually have that info in the writeback
// error
//
if( attempts <= 2 ){
db->getChunkManagerIfExists( ns, true );
}
else{
versionManager.forceRemoteCheckShardVersionCB( ns );
sleepsecs( attempts - 1 );
}
uassert( 15884, str::stream()
<< "Could not reload chunk manager after "
<< attempts << " attempts.", attempts <= 4 );
continue;
}
ci->clearSinceLastGetError();
}
catch ( DBException& e ) {
error() << "error processing writeback: " << e << endl;
BSONObjBuilder b;
e.getInfo().append( b, "err", "code" );
gle = b.obj();
}
break;
}
{
scoped_lock lk( _seenWritebacksLock );
WBStatus& s = _seenWritebacks[cid];
s.id = wid;
s.gle = gle;
}
}
else if ( result["noop"].trueValue() ) {
// no-op
}
else {
log() << "unknown writeBack result: " << result << endl;
}
secsToSleep = 0;
continue;
}
catch ( std::exception& e ) {
// Attention! Do not call any method that would throw an exception
// (or assert) in this block.
if ( inShutdown() ) {
// we're shutting down, so just clean up
return;
}
log() << "WriteBackListener exception : " << e.what() << endl;
needsToReloadShardInfo = true;
}
catch ( ... ) {
log() << "WriteBackListener uncaught exception!" << endl;
}
secsToSleep++;
sleepsecs(secsToSleep);
if ( secsToSleep > 10 )
secsToSleep = 0;
}
log() << "WriteBackListener exiting : address no longer in cluster " << _addr;
}
/**
* @ return true if not in sharded mode
or if version for this client is ok
*/
bool shardVersionOk( const string& ns , string& errmsg, ChunkVersion& received, ChunkVersion& wanted ) {
if ( ! shardingState.enabled() )
return true;
if ( ! isMasterNs( ns.c_str() ) ) {
// right now connections to secondaries aren't versioned at all
return true;
}
ShardedConnectionInfo* info = ShardedConnectionInfo::get( false );
if ( ! info ) {
// this means the client has nothing sharded
// so this allows direct connections to do whatever they want
// which i think is the correct behavior
return true;
}
if ( info->inForceVersionOkMode() ) {
return true;
}
// TODO : all collections at some point, be sharded or not, will have a version
// (and a CollectionMetadata)
received = info->getVersion( ns );
wanted = shardingState.getVersion( ns );
if( received.isWriteCompatibleWith( wanted ) ) return true;
//
// Figure out exactly why not compatible, send appropriate error message
// The versions themselves are returned in the error, so not needed in messages here
//
// Check epoch first, to send more meaningful message, since other parameters probably
// won't match either
if( ! wanted.hasCompatibleEpoch( received ) ){
errmsg = str::stream() << "version epoch mismatch detected for " << ns << ", "
<< "the collection may have been dropped and recreated";
return false;
}
if( ! wanted.isSet() && received.isSet() ){
errmsg = str::stream() << "this shard no longer contains chunks for " << ns << ", "
<< "the collection may have been dropped";
return false;
}
if( wanted.isSet() && ! received.isSet() ){
errmsg = str::stream() << "this shard contains versioned chunks for " << ns << ", "
<< "but no version set in request";
return false;
}
if( wanted.majorVersion() != received.majorVersion() ){
//
// Could be > or < - wanted is > if this is the source of a migration,
// wanted < if this is the target of a migration
//
errmsg = str::stream() << "version mismatch detected for " << ns << ", "
<< "stored major version " << wanted.majorVersion()
<< " does not match received " << received.majorVersion();
return false;
}
// Those are all the reasons the versions can mismatch
verify( false );
return false;
}