IndexBounds ChunkManager::getIndexBoundsForQuery(const BSONObj& key,
const CanonicalQuery& canonicalQuery) {
// $text is not allowed in planning since we don't have text index on mongos.
//
// TODO: Treat $text query as a no-op in planning. So with shard key {a: 1},
// the query { a: 2, $text: { ... } } will only target to {a: 2}.
if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::TEXT)) {
IndexBounds bounds;
IndexBoundsBuilder::allValuesBounds(key, &bounds); // [minKey, maxKey]
return bounds;
}
// Consider shard key as an index
string accessMethod = IndexNames::findPluginName(key);
dassert(accessMethod == IndexNames::BTREE || accessMethod == IndexNames::HASHED);
// Use query framework to generate index bounds
QueryPlannerParams plannerParams;
// Must use "shard key" index
plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN;
IndexEntry indexEntry(key,
accessMethod,
false /* multiKey */,
false /* sparse */,
false /* unique */,
"shardkey",
NULL /* filterExpr */,
BSONObj());
plannerParams.indices.push_back(indexEntry);
OwnedPointerVector<QuerySolution> solutions;
Status status = QueryPlanner::plan(canonicalQuery, plannerParams, &solutions.mutableVector());
uassert(status.code(), status.reason(), status.isOK());
IndexBounds bounds;
for (vector<QuerySolution*>::const_iterator it = solutions.begin();
bounds.size() == 0 && it != solutions.end();
it++) {
// Try next solution if we failed to generate index bounds, i.e. bounds.size() == 0
bounds = collapseQuerySolution((*it)->root.get());
}
if (bounds.size() == 0) {
// We cannot plan the query without collection scan, so target to all shards.
IndexBoundsBuilder::allValuesBounds(key, &bounds); // [minKey, maxKey]
}
return bounds;
}
BSONObj buildMergeLogEntry( const OwnedPointerVector<ChunkType>& chunksToMerge,
const ChunkVersion& currShardVersion,
const ChunkVersion& newMergedVersion ) {
BSONObjBuilder logDetailB;
BSONArrayBuilder mergedB( logDetailB.subarrayStart( "merged" ) );
for ( OwnedPointerVector<ChunkType>::const_iterator it = chunksToMerge.begin();
it != chunksToMerge.end(); ++it ) {
ChunkType* chunkToMerge = *it;
mergedB.append( chunkToMerge->toBSON() );
}
mergedB.done();
currShardVersion.addToBSON( logDetailB, "prevShardVersion" );
newMergedVersion.addToBSON( logDetailB, "mergedVersion" );
return logDetailB.obj();
}
BSONObj generateSection(OperationContext* txn, const BSONElement& configElement) const {
RangeDeleter* deleter = getDeleter();
if (!deleter) {
return BSONObj();
}
BSONObjBuilder result;
OwnedPointerVector<DeleteJobStats> statsList;
deleter->getStatsHistory(&statsList.mutableVector());
BSONArrayBuilder oldStatsBuilder;
for (OwnedPointerVector<DeleteJobStats>::const_iterator it = statsList.begin();
it != statsList.end();
++it) {
BSONObjBuilder entryBuilder;
entryBuilder.append("deletedDocs", (*it)->deletedDocCount);
if ((*it)->queueEndTS > Date_t()) {
entryBuilder.append("queueStart", (*it)->queueStartTS);
entryBuilder.append("queueEnd", (*it)->queueEndTS);
}
if ((*it)->deleteEndTS > Date_t()) {
entryBuilder.append("deleteStart", (*it)->deleteStartTS);
entryBuilder.append("deleteEnd", (*it)->deleteEndTS);
if ((*it)->waitForReplEndTS > Date_t()) {
entryBuilder.append("waitForReplStart", (*it)->waitForReplStartTS);
entryBuilder.append("waitForReplEnd", (*it)->waitForReplEndTS);
}
}
oldStatsBuilder.append(entryBuilder.obj());
}
result.append("lastDeleteStats", oldStatsBuilder.arr());
return result.obj();
}
// static
Status ListFilters::list(const QuerySettings& querySettings, BSONObjBuilder* bob) {
invariant(bob);
// Format of BSON result:
//
// {
// hints: [
// {
// query: <query>,
// sort: <sort>,
// projection: <projection>,
// indexes: [<index1>, <index2>, <index3>, ...]
// }
// }
BSONArrayBuilder hintsBuilder(bob->subarrayStart("filters"));
OwnedPointerVector<AllowedIndexEntry> entries;
entries.mutableVector() = querySettings.getAllAllowedIndices();
for (vector<AllowedIndexEntry*>::const_iterator i = entries.begin();
i != entries.end(); ++i) {
AllowedIndexEntry* entry = *i;
invariant(entry);
BSONObjBuilder hintBob(hintsBuilder.subobjStart());
hintBob.append("query", entry->query);
hintBob.append("sort", entry->sort);
hintBob.append("projection", entry->projection);
BSONArrayBuilder indexesBuilder(hintBob.subarrayStart("indexes"));
for (vector<BSONObj>::const_iterator j = entry->indexKeyPatterns.begin();
j != entry->indexKeyPatterns.end(); ++j) {
const BSONObj& index = *j;
indexesBuilder.append(index);
}
indexesBuilder.doneFast();
}
hintsBuilder.doneFast();
return Status::OK();
}
BSONObj buildApplyOpsCmd( const OwnedPointerVector<ChunkType>& chunksToMerge,
const ChunkVersion& currShardVersion,
const ChunkVersion& newMergedVersion ) {
BSONObjBuilder applyOpsCmdB;
BSONArrayBuilder updatesB( applyOpsCmdB.subarrayStart( "applyOps" ) );
// The chunk we'll be "expanding" is the first chunk
const ChunkType* chunkToMerge = *chunksToMerge.begin();
// Fill in details not tracked by metadata
ChunkType mergedChunk;
chunkToMerge->cloneTo( &mergedChunk );
mergedChunk.setName( Chunk::genID( chunkToMerge->getNS(), chunkToMerge->getMin() ) );
mergedChunk.setMax( ( *chunksToMerge.vector().rbegin() )->getMax() );
mergedChunk.setVersion( newMergedVersion );
updatesB.append( buildOpMergeChunk( mergedChunk ) );
// Don't remove chunk we're expanding
OwnedPointerVector<ChunkType>::const_iterator it = chunksToMerge.begin();
for ( ++it; it != chunksToMerge.end(); ++it ) {
ChunkType* chunkToMerge = *it;
chunkToMerge->setName( Chunk::genID( chunkToMerge->getNS(), chunkToMerge->getMin() ) );
updatesB.append( buildOpRemoveChunk( *chunkToMerge ) );
}
updatesB.done();
applyOpsCmdB.append( "preCondition",
buildOpPrecond( chunkToMerge->getNS(),
chunkToMerge->getShard(),
currShardVersion ) );
return applyOpsCmdB.obj();
}
bool mergeChunks( OperationContext* txn,
const NamespaceString& nss,
const BSONObj& minKey,
const BSONObj& maxKey,
const OID& epoch,
string* errMsg ) {
//
// Get sharding state up-to-date
//
ConnectionString configLoc = ConnectionString::parse( shardingState.getConfigServer(),
*errMsg );
if ( !configLoc.isValid() ){
warning() << *errMsg << endl;
return false;
}
//
// Get the distributed lock
//
ScopedDistributedLock collLock( configLoc, nss.ns() );
collLock.setLockMessage( stream() << "merging chunks in " << nss.ns() << " from "
<< minKey << " to " << maxKey );
Status acquisitionStatus = collLock.tryAcquire();
if (!acquisitionStatus.isOK()) {
*errMsg = stream() << "could not acquire collection lock for " << nss.ns()
<< " to merge chunks in [" << minKey << "," << maxKey << ")"
<< causedBy(acquisitionStatus);
warning() << *errMsg << endl;
return false;
}
//
// We now have the collection lock, refresh metadata to latest version and sanity check
//
ChunkVersion shardVersion;
Status status = shardingState.refreshMetadataNow(txn, nss.ns(), &shardVersion);
if ( !status.isOK() ) {
*errMsg = str::stream() << "could not merge chunks, failed to refresh metadata for "
<< nss.ns() << causedBy( status.reason() );
warning() << *errMsg << endl;
return false;
}
if ( epoch.isSet() && shardVersion.epoch() != epoch ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " has changed" << " since merge was sent" << "(sent epoch : "
<< epoch.toString()
<< ", current epoch : " << shardVersion.epoch().toString() << ")";
warning() << *errMsg << endl;
return false;
}
CollectionMetadataPtr metadata = shardingState.getCollectionMetadata( nss.ns() );
if ( !metadata || metadata->getKeyPattern().isEmpty() ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " is not sharded";
warning() << *errMsg << endl;
return false;
}
dassert( metadata->getShardVersion().equals( shardVersion ) );
if ( !metadata->isValidKey( minKey ) || !metadata->isValidKey( maxKey ) ) {
*errMsg = stream() << "could not merge chunks, the range "
<< rangeToString( minKey, maxKey ) << " is not valid"
<< " for collection " << nss.ns() << " with key pattern "
<< metadata->getKeyPattern();
warning() << *errMsg << endl;
return false;
}
//
// Get merged chunk information
//
ChunkVersion mergeVersion = metadata->getCollVersion();
mergeVersion.incMinor();
OwnedPointerVector<ChunkType> chunksToMerge;
ChunkType itChunk;
itChunk.setMin( minKey );
itChunk.setMax( minKey );
itChunk.setNS( nss.ns() );
itChunk.setShard( shardingState.getShardName() );
while ( itChunk.getMax().woCompare( maxKey ) < 0 &&
metadata->getNextChunk( itChunk.getMax(), &itChunk ) ) {
auto_ptr<ChunkType> saved( new ChunkType );
itChunk.cloneTo( saved.get() );
chunksToMerge.mutableVector().push_back( saved.release() );
}
if ( chunksToMerge.empty() ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range starting at " << minKey
<< " and ending at " << maxKey
<< " does not belong to shard " << shardingState.getShardName();
warning() << *errMsg << endl;
return false;
}
//
// Validate the range starts and ends at chunks and has no holes, error if not valid
//
BSONObj firstDocMin = ( *chunksToMerge.begin() )->getMin();
BSONObj firstDocMax = ( *chunksToMerge.begin() )->getMax();
// minKey is inclusive
bool minKeyInRange = rangeContains( firstDocMin, firstDocMax, minKey );
if ( !minKeyInRange ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range starting at " << minKey
<< " does not belong to shard " << shardingState.getShardName();
warning() << *errMsg << endl;
return false;
}
BSONObj lastDocMin = ( *chunksToMerge.rbegin() )->getMin();
BSONObj lastDocMax = ( *chunksToMerge.rbegin() )->getMax();
// maxKey is exclusive
bool maxKeyInRange = lastDocMin.woCompare( maxKey ) < 0 &&
lastDocMax.woCompare( maxKey ) >= 0;
if ( !maxKeyInRange ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " range ending at " << maxKey
<< " does not belong to shard " << shardingState.getShardName();
warning() << *errMsg << endl;
return false;
}
bool validRangeStartKey = firstDocMin.woCompare( minKey ) == 0;
bool validRangeEndKey = lastDocMax.woCompare( maxKey ) == 0;
if ( !validRangeStartKey || !validRangeEndKey ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " does not contain a chunk "
<< ( !validRangeStartKey ? "starting at " + minKey.toString() : "" )
<< ( !validRangeStartKey && !validRangeEndKey ? " or " : "" )
<< ( !validRangeEndKey ? "ending at " + maxKey.toString() : "" );
warning() << *errMsg << endl;
return false;
}
if ( chunksToMerge.size() == 1 ) {
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " already contains chunk for " << rangeToString( minKey, maxKey );
warning() << *errMsg << endl;
return false;
}
bool holeInRange = false;
// Look for hole in range
ChunkType* prevChunk = *chunksToMerge.begin();
ChunkType* nextChunk = NULL;
for ( OwnedPointerVector<ChunkType>::const_iterator it = chunksToMerge.begin();
it != chunksToMerge.end(); ++it ) {
if ( it == chunksToMerge.begin() ) continue;
nextChunk = *it;
if ( prevChunk->getMax().woCompare( nextChunk->getMin() ) != 0 ) {
holeInRange = true;
break;
}
prevChunk = nextChunk;
}
if ( holeInRange ) {
dassert( NULL != nextChunk );
*errMsg = stream() << "could not merge chunks, collection " << nss.ns()
<< " has a hole in the range " << rangeToString( minKey, maxKey )
<< " at " << rangeToString( prevChunk->getMax(),
nextChunk->getMin() );
warning() << *errMsg << endl;
return false;
}
//
// Run apply ops command
//
BSONObj applyOpsCmd = buildApplyOpsCmd( chunksToMerge,
shardVersion,
mergeVersion );
bool ok;
BSONObj result;
try {
ScopedDbConnection conn( configLoc, 30.0 );
ok = conn->runCommand( "config", applyOpsCmd, result );
if ( !ok ) *errMsg = result.toString();
conn.done();
}
catch( const DBException& ex ) {
ok = false;
*errMsg = ex.toString();
}
if ( !ok ) {
*errMsg = stream() << "could not merge chunks for " << nss.ns()
<< ", writing to config failed" << causedBy( errMsg );
warning() << *errMsg << endl;
return false;
}
//
// Install merged chunk metadata
//
{
Lock::DBLock writeLk(txn->lockState(), nss.db(), newlm::MODE_X);
shardingState.mergeChunks(txn, nss.ns(), minKey, maxKey, mergeVersion);
}
//
// Log change
//
BSONObj mergeLogEntry = buildMergeLogEntry( chunksToMerge,
shardVersion,
mergeVersion );
configServer.logChange( "merge", nss.ns(), mergeLogEntry );
return true;
}
// static
Status ClearFilters::clear(QuerySettings* querySettings, PlanCache* planCache,
const std::string& ns, const BSONObj& cmdObj) {
invariant(querySettings);
// According to the specification, the planCacheClearFilters command runs in two modes:
// - clear all hints; or
// - clear hints for single query shape when a query shape is described in the
// command arguments.
if (cmdObj.hasField("query")) {
CanonicalQuery* cqRaw;
Status status = PlanCacheCommand::canonicalize(ns, cmdObj, &cqRaw);
if (!status.isOK()) {
return status;
}
scoped_ptr<CanonicalQuery> cq(cqRaw);
querySettings->removeAllowedIndices(*cq);
// Remove entry from plan cache
planCache->remove(*cq);
return Status::OK();
}
// If query is not provided, make sure sort and projection are not in arguments.
// We do not want to clear the entire cache inadvertently when the user
// forgot to provide a value for "query".
if (cmdObj.hasField("sort") || cmdObj.hasField("projection")) {
return Status(ErrorCodes::BadValue, "sort or projection provided without query");
}
// Get entries from query settings. We need to remove corresponding entries from the plan
// cache shortly.
OwnedPointerVector<AllowedIndexEntry> entries;
entries.mutableVector() = querySettings->getAllAllowedIndices();
// OK to proceed with clearing entire cache.
querySettings->clearAllowedIndices();
const NamespaceString nss(ns);
const WhereCallbackReal whereCallback(nss.db());
// Remove corresponding entries from plan cache.
// Admin hints affect the planning process directly. If there were
// plans generated as a result of applying index filter, these need to be
// invalidated. This allows the planner to re-populate the plan cache with
// non-filtered indexed solutions next time the query is run.
// Resolve plan cache key from (query, sort, projection) in query settings entry.
// Concurrency note: There's no harm in removing plan cache entries one at at time.
// Only way that PlanCache::remove() can fail is when the query shape has been removed from
// the cache by some other means (re-index, collection info reset, ...). This is OK since
// that's the intended effect of calling the remove() function with the key from the hint entry.
for (vector<AllowedIndexEntry*>::const_iterator i = entries.begin();
i != entries.end(); ++i) {
AllowedIndexEntry* entry = *i;
invariant(entry);
// Create canonical query.
CanonicalQuery* cqRaw;
Status result = CanonicalQuery::canonicalize(
ns, entry->query, entry->sort, entry->projection, &cqRaw, whereCallback);
invariant(result.isOK());
scoped_ptr<CanonicalQuery> cq(cqRaw);
// Remove plan cache entry.
planCache->remove(*cq);
}
return Status::OK();
}
// static
Status ClearFilters::clear(OperationContext* txn,
QuerySettings* querySettings,
PlanCache* planCache,
const std::string& ns,
const BSONObj& cmdObj) {
invariant(querySettings);
// According to the specification, the planCacheClearFilters command runs in two modes:
// - clear all hints; or
// - clear hints for single query shape when a query shape is described in the
// command arguments.
if (cmdObj.hasField("query")) {
auto statusWithCQ = PlanCacheCommand::canonicalize(txn, ns, cmdObj);
if (!statusWithCQ.isOK()) {
return statusWithCQ.getStatus();
}
unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());
querySettings->removeAllowedIndices(planCache->computeKey(*cq));
// Remove entry from plan cache
planCache->remove(*cq);
LOG(0) << "Removed index filter on " << ns << " " << cq->toStringShort();
return Status::OK();
}
// If query is not provided, make sure sort and projection are not in arguments.
// We do not want to clear the entire cache inadvertently when the user
// forgot to provide a value for "query".
if (cmdObj.hasField("sort") || cmdObj.hasField("projection")) {
return Status(ErrorCodes::BadValue, "sort or projection provided without query");
}
// Get entries from query settings. We need to remove corresponding entries from the plan
// cache shortly.
OwnedPointerVector<AllowedIndexEntry> entries;
entries.mutableVector() = querySettings->getAllAllowedIndices();
// OK to proceed with clearing entire cache.
querySettings->clearAllowedIndices();
const NamespaceString nss(ns);
const ExtensionsCallbackReal extensionsCallback(txn, &nss);
// Remove corresponding entries from plan cache.
// Admin hints affect the planning process directly. If there were
// plans generated as a result of applying index filter, these need to be
// invalidated. This allows the planner to re-populate the plan cache with
// non-filtered indexed solutions next time the query is run.
// Resolve plan cache key from (query, sort, projection) in query settings entry.
// Concurrency note: There's no harm in removing plan cache entries one at at time.
// Only way that PlanCache::remove() can fail is when the query shape has been removed from
// the cache by some other means (re-index, collection info reset, ...). This is OK since
// that's the intended effect of calling the remove() function with the key from the hint entry.
for (vector<AllowedIndexEntry*>::const_iterator i = entries.begin(); i != entries.end(); ++i) {
AllowedIndexEntry* entry = *i;
invariant(entry);
// Create canonical query.
auto qr = stdx::make_unique<QueryRequest>(nss);
qr->setFilter(entry->query);
qr->setSort(entry->sort);
qr->setProj(entry->projection);
auto statusWithCQ = CanonicalQuery::canonicalize(txn, std::move(qr), extensionsCallback);
invariantOK(statusWithCQ.getStatus());
std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());
// Remove plan cache entry.
planCache->remove(*cq);
}
LOG(0) << "Removed all index filters for collection: " << ns;
return Status::OK();
}
