BSONObj ClientCursor::extractKey( const KeyPattern& usingKeyPattern ) const {
KeyPattern currentIndex( _c->indexKeyPattern() );
if ( usingKeyPattern.isCoveredBy( currentIndex ) && ! currentIndex.isSpecial() ){
BSONObj currKey = _c->currKey();
BSONObj prettyKey = currKey.replaceFieldNames( currentIndex.toBSON() );
return usingKeyPattern.extractSingleKey( prettyKey );
}
return usingKeyPattern.extractSingleKey( _c->current() );
}
bool CollectionShardingState::_shouldSplitChunk(OperationContext* opCtx,
const ShardKeyPattern& shardKeyPattern,
const Chunk& chunk) {
const auto balancerConfig = Grid::get(opCtx)->getBalancerConfiguration();
invariant(balancerConfig);
const KeyPattern keyPattern = shardKeyPattern.getKeyPattern();
const bool minIsInf = (0 == keyPattern.globalMin().woCompare(chunk.getMin()));
const bool maxIsInf = (0 == keyPattern.globalMax().woCompare(chunk.getMax()));
return chunk.shouldSplit(balancerConfig->getMaxChunkSizeBytes(), minIsInf, maxIsInf);
}
ShardCollectionType::ShardCollectionType(const NamespaceString& uuid,
const NamespaceString& nss,
const OID& epoch,
const KeyPattern& keyPattern,
const BSONObj& defaultCollation,
const bool& unique)
: _uuid(uuid),
_nss(nss),
_epoch(epoch),
_keyPattern(keyPattern.toBSON()),
_defaultCollation(defaultCollation.getOwned()),
_unique(unique) {}
ShardKeyPattern::ShardKeyPattern(const KeyPattern& keyPattern)
: _keyPatternPaths(parseShardKeyPattern(keyPattern.toBSON())),
_keyPattern(_keyPatternPaths.empty() ? KeyPattern(BSONObj()) : keyPattern) {}
Status MigrationChunkClonerSourceLegacy::_storeCurrentLocs(OperationContext* txn) {
ScopedTransaction scopedXact(txn, MODE_IS);
AutoGetCollection autoColl(txn, _args.getNss(), MODE_IS);
Collection* const collection = autoColl.getCollection();
if (!collection) {
return {ErrorCodes::NamespaceNotFound,
str::stream() << "Collection " << _args.getNss().ns() << " does not exist."};
}
// Allow multiKey based on the invariant that shard keys must be single-valued. Therefore, any
// multi-key index prefixed by shard key cannot be multikey over the shard key fields.
IndexDescriptor* idx =
collection->getIndexCatalog()->findShardKeyPrefixedIndex(txn,
_shardKeyPattern.toBSON(),
false); // requireSingleKey
if (!idx) {
return {ErrorCodes::IndexNotFound,
str::stream() << "can't find index with prefix " << _shardKeyPattern.toBSON()
<< " in storeCurrentLocs for "
<< _args.getNss().ns()};
}
// Install the stage, which will listen for notifications on the collection
{
stdx::lock_guard<stdx::mutex> sl(_mutex);
invariant(!_deleteNotifyExec);
// Takes ownership of 'ws' and 'dns'.
auto statusWithPlanExecutor =
PlanExecutor::make(txn,
stdx::make_unique<WorkingSet>(),
stdx::make_unique<DeleteNotificationStage>(this, txn),
collection,
PlanExecutor::YIELD_MANUAL);
invariant(statusWithPlanExecutor.isOK());
_deleteNotifyExec = std::move(statusWithPlanExecutor.getValue());
_deleteNotifyExec->registerExec(collection);
}
// Assume both min and max non-empty, append MinKey's to make them fit chosen index
const KeyPattern kp(idx->keyPattern());
BSONObj min = Helpers::toKeyFormat(kp.extendRangeBound(_args.getMinKey(), false));
BSONObj max = Helpers::toKeyFormat(kp.extendRangeBound(_args.getMaxKey(), false));
std::unique_ptr<PlanExecutor> exec(InternalPlanner::indexScan(txn,
collection,
idx,
min,
max,
false, // endKeyInclusive
PlanExecutor::YIELD_MANUAL));
// We can afford to yield here because any change to the base data that we might miss is already
// being queued and will migrate in the 'transferMods' stage.
exec->setYieldPolicy(PlanExecutor::YIELD_AUTO, collection);
// Use the average object size to estimate how many objects a full chunk would carry do that
// while traversing the chunk's range using the sharding index, below there's a fair amount of
// slack before we determine a chunk is too large because object sizes will vary.
unsigned long long maxRecsWhenFull;
long long avgRecSize;
const long long totalRecs = collection->numRecords(txn);
if (totalRecs > 0) {
avgRecSize = collection->dataSize(txn) / totalRecs;
maxRecsWhenFull = _args.getMaxChunkSizeBytes() / avgRecSize;
maxRecsWhenFull = std::min((unsigned long long)(Chunk::MaxObjectPerChunk + 1),
130 * maxRecsWhenFull / 100 /* slack */);
} else {
avgRecSize = 0;
maxRecsWhenFull = Chunk::MaxObjectPerChunk + 1;
}
// Do a full traversal of the chunk and don't stop even if we think it is a large chunk we want
// the number of records to better report, in that case.
bool isLargeChunk = false;
unsigned long long recCount = 0;
BSONObj obj;
RecordId recordId;
PlanExecutor::ExecState state;
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, &recordId))) {
if (!isLargeChunk) {
stdx::lock_guard<stdx::mutex> lk(_mutex);
_cloneLocs.insert(recordId);
}
if (++recCount > maxRecsWhenFull) {
isLargeChunk = true;
// Continue on despite knowing that it will fail, just to get the correct value for
// recCount
}
}
if (PlanExecutor::DEAD == state || PlanExecutor::FAILURE == state) {
return {ErrorCodes::InternalError,
str::stream() << "Executor error while scanning for documents belonging to chunk: "
<< WorkingSetCommon::toStatusString(obj)};
}
exec.reset();
if (isLargeChunk) {
return {
ErrorCodes::ChunkTooBig,
str::stream() << "Cannot move chunk: the maximum number of documents for a chunk is "
<< maxRecsWhenFull
<< ", the maximum chunk size is "
<< _args.getMaxChunkSizeBytes()
<< ", average document size is "
<< avgRecSize
<< ". Found "
<< recCount
<< " documents in chunk "
<< " ns: "
<< _args.getNss().ns()
<< " "
<< _args.getMinKey()
<< " -> "
<< _args.getMaxKey()};
}
_averageObjectSizeForCloneLocs = static_cast<uint64_t>(collection->averageObjectSize(txn) + 12);
return Status::OK();
}
bool ShardKeyPattern::isUniqueIndexCompatible( const KeyPattern& uniqueIndexPattern ) const {
return mongo::isUniqueIndexCompatible( pattern.toBSON(), uniqueIndexPattern.toBSON() );
}
StatusWith<int> CollectionRangeDeleter::_doDeletion(OperationContext* opCtx,
Collection* collection,
BSONObj const& keyPattern,
ChunkRange const& range,
int maxToDelete) {
invariant(collection != nullptr);
invariant(!isEmpty());
auto const& nss = collection->ns();
// The IndexChunk has a keyPattern that may apply to more than one index - we need to
// select the index and get the full index keyPattern here.
auto catalog = collection->getIndexCatalog();
const IndexDescriptor* idx = catalog->findShardKeyPrefixedIndex(opCtx, keyPattern, false);
if (!idx) {
std::string msg = str::stream() << "Unable to find shard key index for "
<< keyPattern.toString() << " in " << nss.ns();
LOG(0) << msg;
return {ErrorCodes::InternalError, msg};
}
// Extend bounds to match the index we found
const KeyPattern indexKeyPattern(idx->keyPattern());
const auto extend = [&](const auto& key) {
return Helpers::toKeyFormat(indexKeyPattern.extendRangeBound(key, false));
};
const auto min = extend(range.getMin());
const auto max = extend(range.getMax());
LOG(1) << "begin removal of " << min << " to " << max << " in " << nss.ns();
const auto indexName = idx->indexName();
const IndexDescriptor* descriptor =
collection->getIndexCatalog()->findIndexByName(opCtx, indexName);
if (!descriptor) {
std::string msg = str::stream() << "shard key index with name " << indexName << " on '"
<< nss.ns() << "' was dropped";
LOG(0) << msg;
return {ErrorCodes::InternalError, msg};
}
auto deleteStageParams = std::make_unique<DeleteStageParams>();
deleteStageParams->fromMigrate = true;
deleteStageParams->isMulti = true;
deleteStageParams->returnDeleted = true;
if (serverGlobalParams.moveParanoia) {
deleteStageParams->removeSaver =
std::make_unique<RemoveSaver>("moveChunk", nss.ns(), "cleaning");
}
auto exec = InternalPlanner::deleteWithIndexScan(opCtx,
collection,
std::move(deleteStageParams),
descriptor,
min,
max,
BoundInclusion::kIncludeStartKeyOnly,
PlanExecutor::YIELD_MANUAL,
InternalPlanner::FORWARD);
PlanYieldPolicy planYieldPolicy(exec.get(), PlanExecutor::YIELD_MANUAL);
int numDeleted = 0;
do {
BSONObj deletedObj;
PlanExecutor::ExecState state = exec->getNext(&deletedObj, nullptr);
if (state == PlanExecutor::IS_EOF) {
break;
}
if (state == PlanExecutor::FAILURE) {
warning() << PlanExecutor::statestr(state) << " - cursor error while trying to delete "
<< redact(min) << " to " << redact(max) << " in " << nss
<< ": FAILURE, stats: " << Explain::getWinningPlanStats(exec.get());
break;
}
invariant(PlanExecutor::ADVANCED == state);
ShardingStatistics::get(opCtx).countDocsDeletedOnDonor.addAndFetch(1);
} while (++numDeleted < maxToDelete);
return numDeleted;
}
void ShardCollectionType::setKeyPattern(const KeyPattern& keyPattern) {
invariant(!keyPattern.toBSON().isEmpty());
_keyPattern = keyPattern;
}
StatusWith<int> CollectionRangeDeleter::_doDeletion(OperationContext* opCtx,
Collection* collection,
BSONObj const& keyPattern,
ChunkRange const& range,
int maxToDelete) {
invariant(collection != nullptr);
invariant(!isEmpty());
auto const& nss = collection->ns();
// The IndexChunk has a keyPattern that may apply to more than one index - we need to
// select the index and get the full index keyPattern here.
auto catalog = collection->getIndexCatalog();
const IndexDescriptor* idx = catalog->findShardKeyPrefixedIndex(opCtx, keyPattern, false);
if (!idx) {
std::string msg = str::stream() << "Unable to find shard key index for "
<< keyPattern.toString() << " in " << nss.ns();
LOG(0) << msg;
return {ErrorCodes::InternalError, msg};
}
// Extend bounds to match the index we found
const KeyPattern indexKeyPattern(idx->keyPattern());
const auto extend = [&](const auto& key) {
return Helpers::toKeyFormat(indexKeyPattern.extendRangeBound(key, false));
};
const auto min = extend(range.getMin());
const auto max = extend(range.getMax());
LOG(1) << "begin removal of " << min << " to " << max << " in " << nss.ns();
const auto indexName = idx->indexName();
IndexDescriptor* descriptor = collection->getIndexCatalog()->findIndexByName(opCtx, indexName);
if (!descriptor) {
std::string msg = str::stream() << "shard key index with name " << indexName << " on '"
<< nss.ns() << "' was dropped";
LOG(0) << msg;
return {ErrorCodes::InternalError, msg};
}
boost::optional<Helpers::RemoveSaver> saver;
if (serverGlobalParams.moveParanoia) {
saver.emplace("moveChunk", nss.ns(), "cleaning");
}
auto halfOpen = BoundInclusion::kIncludeStartKeyOnly;
auto manual = PlanExecutor::YIELD_MANUAL;
auto forward = InternalPlanner::FORWARD;
auto fetch = InternalPlanner::IXSCAN_FETCH;
auto exec = InternalPlanner::indexScan(
opCtx, collection, descriptor, min, max, halfOpen, manual, forward, fetch);
int numDeleted = 0;
do {
RecordId rloc;
BSONObj obj;
PlanExecutor::ExecState state = exec->getNext(&obj, &rloc);
if (state == PlanExecutor::IS_EOF) {
break;
}
if (state == PlanExecutor::FAILURE || state == PlanExecutor::DEAD) {
warning() << PlanExecutor::statestr(state) << " - cursor error while trying to delete "
<< redact(min) << " to " << redact(max) << " in " << nss << ": "
<< redact(WorkingSetCommon::toStatusString(obj))
<< ", stats: " << Explain::getWinningPlanStats(exec.get());
break;
}
invariant(PlanExecutor::ADVANCED == state);
exec->saveState();
writeConflictRetry(opCtx, "delete range", nss.ns(), [&] {
WriteUnitOfWork wuow(opCtx);
if (saver) {
uassertStatusOK(saver->goingToDelete(obj));
}
collection->deleteDocument(opCtx, kUninitializedStmtId, rloc, nullptr, true);
wuow.commit();
});
try {
exec->restoreState();
} catch (const DBException& ex) {
warning() << "error restoring cursor state while trying to delete " << redact(min)
<< " to " << redact(max) << " in " << nss
<< ", stats: " << Explain::getWinningPlanStats(exec.get()) << ": "
<< redact(ex.toStatus());
break;
}
ShardingStatistics::get(opCtx).countDocsDeletedOnDonor.addAndFetch(1);
} while (++numDeleted < maxToDelete);
return numDeleted;
}