Exemplo n.º 1
0
  void try_merge_buddy_recursive( ChunkMap::iterator cmit ) {
    // compute address of buddy
    intptr_t address = cmit->second.address;
    intptr_t buddy_address = (address ^ cmit->second.size);
    DVLOG(5) << cmit->second << " buddy address " << (void *) buddy_address;

    // does it exist?
    ChunkMap::iterator buddy_iterator = chunks_.find( buddy_address );
    if( buddy_iterator != chunks_.end() && 
        buddy_iterator->second.size == cmit->second.size &&
        buddy_iterator->second.in_use == false ) {
      DVLOG(5) << "buddy found! address " << (void *) address << " buddy address " << (void *) buddy_address;

      // remove the higher-addressed chunk
      ChunkMap::iterator higher_iterator = address < buddy_address ? buddy_iterator : cmit;
      remove_from_free_list( higher_iterator );
      chunks_.erase( higher_iterator );

      // keep the the lower-addressed chunk in the map:
      // update its size and move it to the right free list
      ChunkMap::iterator lower_iterator = address < buddy_address ? cmit : buddy_iterator;
      remove_from_free_list( lower_iterator ); // should these be swapped? I think so.
      lower_iterator->second.size *= 2;
      add_to_free_list( lower_iterator );

      // see if we have more to merge
      try_merge_buddy_recursive( lower_iterator );
    }
  }
Exemplo n.º 2
0
void ChunkRangeManager::assertValid() const {
    if (_ranges.empty())
        return;

    try {
        // No Nulls
        for (ChunkRangeMap::const_iterator it = _ranges.begin(), end = _ranges.end(); it != end;
             ++it) {
            verify(it->second);
        }

        // Check endpoints
        verify(allOfType(MinKey, _ranges.begin()->second->getMin()));
        verify(allOfType(MaxKey, boost::prior(_ranges.end())->second->getMax()));

        // Make sure there are no gaps or overlaps
        for (ChunkRangeMap::const_iterator it = boost::next(_ranges.begin()), end = _ranges.end();
             it != end;
             ++it) {
            ChunkRangeMap::const_iterator last = boost::prior(it);
            verify(it->second->getMin() == last->second->getMax());
        }

        // Check Map keys
        for (ChunkRangeMap::const_iterator it = _ranges.begin(), end = _ranges.end(); it != end;
             ++it) {
            verify(it->first == it->second->getMax());
        }

        // Make sure we match the original chunks
        const ChunkMap chunks = _ranges.begin()->second->getManager()->_chunkMap;
        for (ChunkMap::const_iterator i = chunks.begin(); i != chunks.end(); ++i) {
            const ChunkPtr chunk = i->second;

            ChunkRangeMap::const_iterator min = _ranges.upper_bound(chunk->getMin());
            ChunkRangeMap::const_iterator max = _ranges.lower_bound(chunk->getMax());

            verify(min != _ranges.end());
            verify(max != _ranges.end());
            verify(min == max);
            verify(min->second->getShardId() == chunk->getShardId());
            verify(min->second->containsKey(chunk->getMin()));
            verify(min->second->containsKey(chunk->getMax()) ||
                   (min->second->getMax() == chunk->getMax()));
        }

    } catch (...) {
        error() << "\t invalid ChunkRangeMap! printing ranges:";

        for (ChunkRangeMap::const_iterator it = _ranges.begin(), end = _ranges.end(); it != end;
             ++it) {
            log() << it->first << ": " << it->second->toString();
        }

        throw;
    }
}
Exemplo n.º 3
0
 ChunkMap::iterator add_to_chunk_map( const AllocatorChunk& ac ) {
   ChunkMap::iterator it;
   bool inserted = false;
   boost::tie( it, inserted ) = chunks_.insert( std::make_pair( ac.address, ac ) );
   assert( inserted );
   return it;
 }
Exemplo n.º 4
0
void ChunkRangeManager::reloadAll(const ChunkMap& chunks) {
    _ranges.clear();
    _insertRange(chunks.begin(), chunks.end());

    DEV assertValid();
}
Exemplo n.º 5
0
bool ChunkManager::_load(ChunkMap& chunkMap,
                         set<ShardId>& shardIds,
                         ShardVersionMap* shardVersions,
                         const ChunkManager* oldManager) {
    // Reset the max version, but not the epoch, when we aren't loading from the oldManager
    _version = ChunkVersion(0, 0, _version.epoch());

    // If we have a previous version of the ChunkManager to work from, use that info to reduce
    // our config query
    if (oldManager && oldManager->getVersion().isSet()) {
        // Get the old max version
        _version = oldManager->getVersion();

        // Load a copy of the old versions
        *shardVersions = oldManager->_shardVersions;

        // Load a copy of the chunk map, replacing the chunk manager with our own
        const ChunkMap& oldChunkMap = oldManager->getChunkMap();

        // Could be v.expensive
        // TODO: If chunks were immutable and didn't reference the manager, we could do more
        // interesting things here
        for (const auto& oldChunkMapEntry : oldChunkMap) {
            shared_ptr<Chunk> oldC = oldChunkMapEntry.second;
            shared_ptr<Chunk> newC(new Chunk(
                this, oldC->getMin(), oldC->getMax(), oldC->getShardId(), oldC->getLastmod()));

            newC->setBytesWritten(oldC->getBytesWritten());

            chunkMap.insert(make_pair(oldC->getMax(), newC));
        }

        LOG(2) << "loading chunk manager for collection " << _ns
               << " using old chunk manager w/ version " << _version.toString() << " and "
               << oldChunkMap.size() << " chunks";
    }

    // Attach a diff tracker for the versioned chunk data
    CMConfigDiffTracker differ(this);
    differ.attach(_ns, chunkMap, _version, *shardVersions);

    // Diff tracker should *always* find at least one chunk if collection exists
    // Get the diff query required
    auto diffQuery = differ.configDiffQuery();

    std::vector<ChunkType> chunks;
    uassertStatusOK(
        grid.catalogManager()->getChunks(diffQuery.query, diffQuery.sort, boost::none, &chunks));

    int diffsApplied = differ.calculateConfigDiff(chunks);
    if (diffsApplied > 0) {
        LOG(2) << "loaded " << diffsApplied << " chunks into new chunk manager for " << _ns
               << " with version " << _version;

        // Add all existing shards we find to the shards set
        for (ShardVersionMap::iterator it = shardVersions->begin(); it != shardVersions->end();) {
            shared_ptr<Shard> shard = grid.shardRegistry()->getShard(it->first);
            if (shard) {
                shardIds.insert(it->first);
                ++it;
            } else {
                shardVersions->erase(it++);
            }
        }

        return true;
    } else if (diffsApplied == 0) {
        // No chunks were found for the ns
        warning() << "no chunks found when reloading " << _ns << ", previous version was "
                  << _version;

        // Set all our data to empty
        chunkMap.clear();
        shardVersions->clear();

        _version = ChunkVersion(0, 0, OID());

        return true;
    } else {  // diffsApplied < 0

        bool allInconsistent = (differ.numValidDiffs() == 0);
        if (allInconsistent) {
            // All versions are different, this can be normal
            warning() << "major change in chunk information found when reloading " << _ns
                      << ", previous version was " << _version;
        } else {
            // Inconsistent load halfway through (due to yielding cursor during load)
            // should be rare
            warning() << "inconsistent chunks found when reloading " << _ns
                      << ", previous version was " << _version << ", this should be rare";
        }

        // Set all our data to empty to be extra safe
        chunkMap.clear();
        shardVersions->clear();

        _version = ChunkVersion(0, 0, OID());

        return allInconsistent;
    }
}