int ConfigDiffTracker<ValType, ShardType>::calculateConfigDiff(
const std::vector<ChunkType>& chunks) {
_assertAttached();
// Apply the chunk changes to the ranges and versions
//
// Overall idea here is to work in two steps :
// 1. For all the new chunks we find, increment the maximum version per-shard and
// per-collection, and remove any conflicting chunks from the ranges.
// 2. For all the new chunks we're interested in (all of them for mongos, just chunks on
// the shard for mongod) add them to the ranges.
std::vector<ChunkType> newTracked;
// Store epoch now so it doesn't change when we change max
OID currEpoch = _maxVersion->epoch();
_validDiffs = 0;
for (const ChunkType& chunk : chunks) {
ChunkVersion chunkVersion =
ChunkVersion::fromBSON(chunk.toBSON(), ChunkType::DEPRECATED_lastmod());
if (!chunkVersion.isSet() || !chunkVersion.hasEqualEpoch(currEpoch)) {
warning() << "got invalid chunk version " << chunkVersion << " in document "
<< chunk.toString() << " when trying to load differing chunks at version "
<< ChunkVersion(
_maxVersion->majorVersion(), _maxVersion->minorVersion(), currEpoch);
// Don't keep loading, since we know we'll be broken here
return -1;
}
_validDiffs++;
// Get max changed version and chunk version
if (chunkVersion > *_maxVersion) {
*_maxVersion = chunkVersion;
}
// Chunk version changes
ShardType shard = shardFor(chunk.getShard());
typename MaxChunkVersionMap::const_iterator shardVersionIt = _maxShardVersions->find(shard);
if (shardVersionIt == _maxShardVersions->end() || shardVersionIt->second < chunkVersion) {
(*_maxShardVersions)[shard] = chunkVersion;
}
// See if we need to remove any chunks we are currently tracking because of this
// chunk's changes
removeOverlapping(chunk.getMin(), chunk.getMax());
// Figure out which of the new chunks we need to track
// Important - we need to actually own this doc, in case the cursor decides to getMore
// or unbuffer.
if (isTracked(chunk)) {
newTracked.push_back(chunk);
}
}
LOG(3) << "found " << _validDiffs << " new chunks for collection " << _ns << " (tracking "
<< newTracked.size() << "), new version is " << *_maxVersion;
for (const ChunkType& chunk : newTracked) {
// Invariant enforced by sharding - it's possible to read inconsistent state due to
// getMore and yielding, so we want to detect it as early as possible.
//
// TODO: This checks for overlap, we also should check for holes here iff we're
// tracking all chunks.
if (isOverlapping(chunk.getMin(), chunk.getMax())) {
return -1;
}
_currMap->insert(rangeFor(chunk));
}
return _validDiffs;
}
int ConfigDiffTracker<ValType,ShardType>::
calculateConfigDiff( DBClientCursorInterface& diffCursor )
{
MONGO_LOG_DEFAULT_COMPONENT_LOCAL(::mongo::logger::LogComponent::kSharding);
verifyAttached();
// Apply the chunk changes to the ranges and versions
//
// Overall idea here is to work in two steps :
// 1. For all the new chunks we find, increment the maximum version per-shard and
// per-collection, and remove any conflicting chunks from the ranges
// 2. For all the new chunks we're interested in (all of them for mongos, just chunks on the
// shard for mongod) add them to the ranges
//
vector<BSONObj> newTracked;
// Store epoch now so it doesn't change when we change max
OID currEpoch = _maxVersion->epoch();
_validDiffs = 0;
while( diffCursor.more() ){
BSONObj diffChunkDoc = diffCursor.next();
ChunkVersion chunkVersion = ChunkVersion::fromBSON(diffChunkDoc, ChunkType::DEPRECATED_lastmod());
if( diffChunkDoc[ChunkType::min()].type() != Object ||
diffChunkDoc[ChunkType::max()].type() != Object ||
diffChunkDoc[ChunkType::shard()].type() != String )
{
warning() << "got invalid chunk document " << diffChunkDoc
<< " when trying to load differing chunks" << endl;
continue;
}
if( ! chunkVersion.isSet() || ! chunkVersion.hasEqualEpoch( currEpoch ) ){
warning() << "got invalid chunk version " << chunkVersion << " in document " << diffChunkDoc
<< " when trying to load differing chunks at version "
<< ChunkVersion( _maxVersion->majorVersion(),
_maxVersion->minorVersion(),
currEpoch ) << endl;
// Don't keep loading, since we know we'll be broken here
return -1;
}
_validDiffs++;
// Get max changed version and chunk version
if( chunkVersion > *_maxVersion ) *_maxVersion = chunkVersion;
// Chunk version changes
ShardType shard = shardFor( diffChunkDoc[ChunkType::shard()].String() );
typename map<ShardType, ChunkVersion>::iterator shardVersionIt = _maxShardVersions->find( shard );
if( shardVersionIt == _maxShardVersions->end() || shardVersionIt->second < chunkVersion ){
(*_maxShardVersions)[ shard ] = chunkVersion;
}
// See if we need to remove any chunks we are currently tracking b/c of this chunk's changes
removeOverlapping(diffChunkDoc[ChunkType::min()].Obj(),
diffChunkDoc[ChunkType::max()].Obj());
// Figure out which of the new chunks we need to track
// Important - we need to actually own this doc, in case the cursor decides to getMore or unbuffer
if( isTracked( diffChunkDoc ) ) newTracked.push_back( diffChunkDoc.getOwned() );
}
LOG(3) << "found " << _validDiffs
<< " new chunks for collection " << _ns
<< " (tracking " << newTracked.size()
<< "), new version is " << *_maxVersion
<< endl;
for( vector<BSONObj>::iterator it = newTracked.begin(); it != newTracked.end(); it++ ){
BSONObj chunkDoc = *it;
// Important - we need to make sure we actually own the min and max here
BSONObj min = chunkDoc[ChunkType::min()].Obj().getOwned();
BSONObj max = chunkDoc[ChunkType::max()].Obj().getOwned();
// Invariant enforced by sharding
// It's possible to read inconsistent state b/c of getMore() and yielding, so we want
// to detect as early as possible.
// TODO: This checks for overlap, we also should check for holes here iff we're tracking
// all chunks
if( isOverlapping( min, max ) ) return -1;
_currMap->insert( rangeFor( chunkDoc, min, max ) );
}
return _validDiffs;
}