void try_merge_buddy_recursive( ChunkMap::iterator cmit ) {
// compute address of buddy
intptr_t address = cmit->second.address;
intptr_t buddy_address = (address ^ cmit->second.size);
DVLOG(5) << cmit->second << " buddy address " << (void *) buddy_address;
// does it exist?
ChunkMap::iterator buddy_iterator = chunks_.find( buddy_address );
if( buddy_iterator != chunks_.end() &&
buddy_iterator->second.size == cmit->second.size &&
buddy_iterator->second.in_use == false ) {
DVLOG(5) << "buddy found! address " << (void *) address << " buddy address " << (void *) buddy_address;
// remove the higher-addressed chunk
ChunkMap::iterator higher_iterator = address < buddy_address ? buddy_iterator : cmit;
remove_from_free_list( higher_iterator );
chunks_.erase( higher_iterator );
// keep the the lower-addressed chunk in the map:
// update its size and move it to the right free list
ChunkMap::iterator lower_iterator = address < buddy_address ? cmit : buddy_iterator;
remove_from_free_list( lower_iterator ); // should these be swapped? I think so.
lower_iterator->second.size *= 2;
add_to_free_list( lower_iterator );
// see if we have more to merge
try_merge_buddy_recursive( lower_iterator );
}
}
void ChunkRangeManager::assertValid() const {
if (_ranges.empty())
return;
try {
// No Nulls
for (ChunkRangeMap::const_iterator it = _ranges.begin(), end = _ranges.end(); it != end;
++it) {
verify(it->second);
}
// Check endpoints
verify(allOfType(MinKey, _ranges.begin()->second->getMin()));
verify(allOfType(MaxKey, boost::prior(_ranges.end())->second->getMax()));
// Make sure there are no gaps or overlaps
for (ChunkRangeMap::const_iterator it = boost::next(_ranges.begin()), end = _ranges.end();
it != end;
++it) {
ChunkRangeMap::const_iterator last = boost::prior(it);
verify(it->second->getMin() == last->second->getMax());
}
// Check Map keys
for (ChunkRangeMap::const_iterator it = _ranges.begin(), end = _ranges.end(); it != end;
++it) {
verify(it->first == it->second->getMax());
}
// Make sure we match the original chunks
const ChunkMap chunks = _ranges.begin()->second->getManager()->_chunkMap;
for (ChunkMap::const_iterator i = chunks.begin(); i != chunks.end(); ++i) {
const ChunkPtr chunk = i->second;
ChunkRangeMap::const_iterator min = _ranges.upper_bound(chunk->getMin());
ChunkRangeMap::const_iterator max = _ranges.lower_bound(chunk->getMax());
verify(min != _ranges.end());
verify(max != _ranges.end());
verify(min == max);
verify(min->second->getShardId() == chunk->getShardId());
verify(min->second->containsKey(chunk->getMin()));
verify(min->second->containsKey(chunk->getMax()) ||
(min->second->getMax() == chunk->getMax()));
}
} catch (...) {
error() << "\t invalid ChunkRangeMap! printing ranges:";
for (ChunkRangeMap::const_iterator it = _ranges.begin(), end = _ranges.end(); it != end;
++it) {
log() << it->first << ": " << it->second->toString();
}
throw;
}
}
ChunkMap::iterator add_to_chunk_map( const AllocatorChunk& ac ) {
ChunkMap::iterator it;
bool inserted = false;
boost::tie( it, inserted ) = chunks_.insert( std::make_pair( ac.address, ac ) );
assert( inserted );
return it;
}
void ChunkRangeManager::reloadAll(const ChunkMap& chunks) {
_ranges.clear();
_insertRange(chunks.begin(), chunks.end());
DEV assertValid();
}
bool ChunkManager::_load(ChunkMap& chunkMap,
set<ShardId>& shardIds,
ShardVersionMap* shardVersions,
const ChunkManager* oldManager) {
// Reset the max version, but not the epoch, when we aren't loading from the oldManager
_version = ChunkVersion(0, 0, _version.epoch());
// If we have a previous version of the ChunkManager to work from, use that info to reduce
// our config query
if (oldManager && oldManager->getVersion().isSet()) {
// Get the old max version
_version = oldManager->getVersion();
// Load a copy of the old versions
*shardVersions = oldManager->_shardVersions;
// Load a copy of the chunk map, replacing the chunk manager with our own
const ChunkMap& oldChunkMap = oldManager->getChunkMap();
// Could be v.expensive
// TODO: If chunks were immutable and didn't reference the manager, we could do more
// interesting things here
for (const auto& oldChunkMapEntry : oldChunkMap) {
shared_ptr<Chunk> oldC = oldChunkMapEntry.second;
shared_ptr<Chunk> newC(new Chunk(
this, oldC->getMin(), oldC->getMax(), oldC->getShardId(), oldC->getLastmod()));
newC->setBytesWritten(oldC->getBytesWritten());
chunkMap.insert(make_pair(oldC->getMax(), newC));
}
LOG(2) << "loading chunk manager for collection " << _ns
<< " using old chunk manager w/ version " << _version.toString() << " and "
<< oldChunkMap.size() << " chunks";
}
// Attach a diff tracker for the versioned chunk data
CMConfigDiffTracker differ(this);
differ.attach(_ns, chunkMap, _version, *shardVersions);
// Diff tracker should *always* find at least one chunk if collection exists
// Get the diff query required
auto diffQuery = differ.configDiffQuery();
std::vector<ChunkType> chunks;
uassertStatusOK(
grid.catalogManager()->getChunks(diffQuery.query, diffQuery.sort, boost::none, &chunks));
int diffsApplied = differ.calculateConfigDiff(chunks);
if (diffsApplied > 0) {
LOG(2) << "loaded " << diffsApplied << " chunks into new chunk manager for " << _ns
<< " with version " << _version;
// Add all existing shards we find to the shards set
for (ShardVersionMap::iterator it = shardVersions->begin(); it != shardVersions->end();) {
shared_ptr<Shard> shard = grid.shardRegistry()->getShard(it->first);
if (shard) {
shardIds.insert(it->first);
++it;
} else {
shardVersions->erase(it++);
}
}
return true;
} else if (diffsApplied == 0) {
// No chunks were found for the ns
warning() << "no chunks found when reloading " << _ns << ", previous version was "
<< _version;
// Set all our data to empty
chunkMap.clear();
shardVersions->clear();
_version = ChunkVersion(0, 0, OID());
return true;
} else { // diffsApplied < 0
bool allInconsistent = (differ.numValidDiffs() == 0);
if (allInconsistent) {
// All versions are different, this can be normal
warning() << "major change in chunk information found when reloading " << _ns
<< ", previous version was " << _version;
} else {
// Inconsistent load halfway through (due to yielding cursor during load)
// should be rare
warning() << "inconsistent chunks found when reloading " << _ns
<< ", previous version was " << _version << ", this should be rare";
}
// Set all our data to empty to be extra safe
chunkMap.clear();
shardVersions->clear();
_version = ChunkVersion(0, 0, OID());
return allInconsistent;
}
}
