IndexBounds ChunkManager::getIndexBoundsForQuery(const BSONObj& key, const CanonicalQuery& canonicalQuery) { // $text is not allowed in planning since we don't have text index on mongos. // // TODO: Treat $text query as a no-op in planning. So with shard key {a: 1}, // the query { a: 2, $text: { ... } } will only target to {a: 2}. if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::TEXT)) { IndexBounds bounds; IndexBoundsBuilder::allValuesBounds(key, &bounds); // [minKey, maxKey] return bounds; } // Consider shard key as an index string accessMethod = IndexNames::findPluginName(key); dassert(accessMethod == IndexNames::BTREE || accessMethod == IndexNames::HASHED); // Use query framework to generate index bounds QueryPlannerParams plannerParams; // Must use "shard key" index plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN; IndexEntry indexEntry(key, accessMethod, false /* multiKey */, false /* sparse */, false /* unique */, "shardkey", NULL /* filterExpr */, BSONObj()); plannerParams.indices.push_back(indexEntry); OwnedPointerVector<QuerySolution> solutions; Status status = QueryPlanner::plan(canonicalQuery, plannerParams, &solutions.mutableVector()); uassert(status.code(), status.reason(), status.isOK()); IndexBounds bounds; for (vector<QuerySolution*>::const_iterator it = solutions.begin(); bounds.size() == 0 && it != solutions.end(); it++) { // Try next solution if we failed to generate index bounds, i.e. bounds.size() == 0 bounds = collapseQuerySolution((*it)->root.get()); } if (bounds.size() == 0) { // We cannot plan the query without collection scan, so target to all shards. IndexBoundsBuilder::allValuesBounds(key, &bounds); // [minKey, maxKey] } return bounds; }
BSONObj buildMergeLogEntry( const OwnedPointerVector<ChunkType>& chunksToMerge, const ChunkVersion& currShardVersion, const ChunkVersion& newMergedVersion ) { BSONObjBuilder logDetailB; BSONArrayBuilder mergedB( logDetailB.subarrayStart( "merged" ) ); for ( OwnedPointerVector<ChunkType>::const_iterator it = chunksToMerge.begin(); it != chunksToMerge.end(); ++it ) { ChunkType* chunkToMerge = *it; mergedB.append( chunkToMerge->toBSON() ); } mergedB.done(); currShardVersion.addToBSON( logDetailB, "prevShardVersion" ); newMergedVersion.addToBSON( logDetailB, "mergedVersion" ); return logDetailB.obj(); }
BSONObj generateSection(OperationContext* txn, const BSONElement& configElement) const { RangeDeleter* deleter = getDeleter(); if (!deleter) { return BSONObj(); } BSONObjBuilder result; OwnedPointerVector<DeleteJobStats> statsList; deleter->getStatsHistory(&statsList.mutableVector()); BSONArrayBuilder oldStatsBuilder; for (OwnedPointerVector<DeleteJobStats>::const_iterator it = statsList.begin(); it != statsList.end(); ++it) { BSONObjBuilder entryBuilder; entryBuilder.append("deletedDocs", (*it)->deletedDocCount); if ((*it)->queueEndTS > Date_t()) { entryBuilder.append("queueStart", (*it)->queueStartTS); entryBuilder.append("queueEnd", (*it)->queueEndTS); } if ((*it)->deleteEndTS > Date_t()) { entryBuilder.append("deleteStart", (*it)->deleteStartTS); entryBuilder.append("deleteEnd", (*it)->deleteEndTS); if ((*it)->waitForReplEndTS > Date_t()) { entryBuilder.append("waitForReplStart", (*it)->waitForReplStartTS); entryBuilder.append("waitForReplEnd", (*it)->waitForReplEndTS); } } oldStatsBuilder.append(entryBuilder.obj()); } result.append("lastDeleteStats", oldStatsBuilder.arr()); return result.obj(); }
// static Status ListFilters::list(const QuerySettings& querySettings, BSONObjBuilder* bob) { invariant(bob); // Format of BSON result: // // { // hints: [ // { // query: <query>, // sort: <sort>, // projection: <projection>, // indexes: [<index1>, <index2>, <index3>, ...] // } // } BSONArrayBuilder hintsBuilder(bob->subarrayStart("filters")); OwnedPointerVector<AllowedIndexEntry> entries; entries.mutableVector() = querySettings.getAllAllowedIndices(); for (vector<AllowedIndexEntry*>::const_iterator i = entries.begin(); i != entries.end(); ++i) { AllowedIndexEntry* entry = *i; invariant(entry); BSONObjBuilder hintBob(hintsBuilder.subobjStart()); hintBob.append("query", entry->query); hintBob.append("sort", entry->sort); hintBob.append("projection", entry->projection); BSONArrayBuilder indexesBuilder(hintBob.subarrayStart("indexes")); for (vector<BSONObj>::const_iterator j = entry->indexKeyPatterns.begin(); j != entry->indexKeyPatterns.end(); ++j) { const BSONObj& index = *j; indexesBuilder.append(index); } indexesBuilder.doneFast(); } hintsBuilder.doneFast(); return Status::OK(); }
BSONObj buildApplyOpsCmd( const OwnedPointerVector<ChunkType>& chunksToMerge, const ChunkVersion& currShardVersion, const ChunkVersion& newMergedVersion ) { BSONObjBuilder applyOpsCmdB; BSONArrayBuilder updatesB( applyOpsCmdB.subarrayStart( "applyOps" ) ); // The chunk we'll be "expanding" is the first chunk const ChunkType* chunkToMerge = *chunksToMerge.begin(); // Fill in details not tracked by metadata ChunkType mergedChunk; chunkToMerge->cloneTo( &mergedChunk ); mergedChunk.setName( Chunk::genID( chunkToMerge->getNS(), chunkToMerge->getMin() ) ); mergedChunk.setMax( ( *chunksToMerge.vector().rbegin() )->getMax() ); mergedChunk.setVersion( newMergedVersion ); updatesB.append( buildOpMergeChunk( mergedChunk ) ); // Don't remove chunk we're expanding OwnedPointerVector<ChunkType>::const_iterator it = chunksToMerge.begin(); for ( ++it; it != chunksToMerge.end(); ++it ) { ChunkType* chunkToMerge = *it; chunkToMerge->setName( Chunk::genID( chunkToMerge->getNS(), chunkToMerge->getMin() ) ); updatesB.append( buildOpRemoveChunk( *chunkToMerge ) ); } updatesB.done(); applyOpsCmdB.append( "preCondition", buildOpPrecond( chunkToMerge->getNS(), chunkToMerge->getShard(), currShardVersion ) ); return applyOpsCmdB.obj(); }
bool mergeChunks( OperationContext* txn, const NamespaceString& nss, const BSONObj& minKey, const BSONObj& maxKey, const OID& epoch, string* errMsg ) { // // Get sharding state up-to-date // ConnectionString configLoc = ConnectionString::parse( shardingState.getConfigServer(), *errMsg ); if ( !configLoc.isValid() ){ warning() << *errMsg << endl; return false; } // // Get the distributed lock // ScopedDistributedLock collLock( configLoc, nss.ns() ); collLock.setLockMessage( stream() << "merging chunks in " << nss.ns() << " from " << minKey << " to " << maxKey ); Status acquisitionStatus = collLock.tryAcquire(); if (!acquisitionStatus.isOK()) { *errMsg = stream() << "could not acquire collection lock for " << nss.ns() << " to merge chunks in [" << minKey << "," << maxKey << ")" << causedBy(acquisitionStatus); warning() << *errMsg << endl; return false; } // // We now have the collection lock, refresh metadata to latest version and sanity check // ChunkVersion shardVersion; Status status = shardingState.refreshMetadataNow(txn, nss.ns(), &shardVersion); if ( !status.isOK() ) { *errMsg = str::stream() << "could not merge chunks, failed to refresh metadata for " << nss.ns() << causedBy( status.reason() ); warning() << *errMsg << endl; return false; } if ( epoch.isSet() && shardVersion.epoch() != epoch ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " has changed" << " since merge was sent" << "(sent epoch : " << epoch.toString() << ", current epoch : " << shardVersion.epoch().toString() << ")"; warning() << *errMsg << endl; return false; } CollectionMetadataPtr metadata = shardingState.getCollectionMetadata( nss.ns() ); if ( !metadata || metadata->getKeyPattern().isEmpty() ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " is not sharded"; warning() << *errMsg << endl; return false; } dassert( metadata->getShardVersion().equals( shardVersion ) ); if ( !metadata->isValidKey( minKey ) || !metadata->isValidKey( maxKey ) ) { *errMsg = stream() << "could not merge chunks, the range " << rangeToString( minKey, maxKey ) << " is not valid" << " for collection " << nss.ns() << " with key pattern " << metadata->getKeyPattern(); warning() << *errMsg << endl; return false; } // // Get merged chunk information // ChunkVersion mergeVersion = metadata->getCollVersion(); mergeVersion.incMinor(); OwnedPointerVector<ChunkType> chunksToMerge; ChunkType itChunk; itChunk.setMin( minKey ); itChunk.setMax( minKey ); itChunk.setNS( nss.ns() ); itChunk.setShard( shardingState.getShardName() ); while ( itChunk.getMax().woCompare( maxKey ) < 0 && metadata->getNextChunk( itChunk.getMax(), &itChunk ) ) { auto_ptr<ChunkType> saved( new ChunkType ); itChunk.cloneTo( saved.get() ); chunksToMerge.mutableVector().push_back( saved.release() ); } if ( chunksToMerge.empty() ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " range starting at " << minKey << " and ending at " << maxKey << " does not belong to shard " << shardingState.getShardName(); warning() << *errMsg << endl; return false; } // // Validate the range starts and ends at chunks and has no holes, error if not valid // BSONObj firstDocMin = ( *chunksToMerge.begin() )->getMin(); BSONObj firstDocMax = ( *chunksToMerge.begin() )->getMax(); // minKey is inclusive bool minKeyInRange = rangeContains( firstDocMin, firstDocMax, minKey ); if ( !minKeyInRange ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " range starting at " << minKey << " does not belong to shard " << shardingState.getShardName(); warning() << *errMsg << endl; return false; } BSONObj lastDocMin = ( *chunksToMerge.rbegin() )->getMin(); BSONObj lastDocMax = ( *chunksToMerge.rbegin() )->getMax(); // maxKey is exclusive bool maxKeyInRange = lastDocMin.woCompare( maxKey ) < 0 && lastDocMax.woCompare( maxKey ) >= 0; if ( !maxKeyInRange ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " range ending at " << maxKey << " does not belong to shard " << shardingState.getShardName(); warning() << *errMsg << endl; return false; } bool validRangeStartKey = firstDocMin.woCompare( minKey ) == 0; bool validRangeEndKey = lastDocMax.woCompare( maxKey ) == 0; if ( !validRangeStartKey || !validRangeEndKey ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " does not contain a chunk " << ( !validRangeStartKey ? "starting at " + minKey.toString() : "" ) << ( !validRangeStartKey && !validRangeEndKey ? " or " : "" ) << ( !validRangeEndKey ? "ending at " + maxKey.toString() : "" ); warning() << *errMsg << endl; return false; } if ( chunksToMerge.size() == 1 ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " already contains chunk for " << rangeToString( minKey, maxKey ); warning() << *errMsg << endl; return false; } bool holeInRange = false; // Look for hole in range ChunkType* prevChunk = *chunksToMerge.begin(); ChunkType* nextChunk = NULL; for ( OwnedPointerVector<ChunkType>::const_iterator it = chunksToMerge.begin(); it != chunksToMerge.end(); ++it ) { if ( it == chunksToMerge.begin() ) continue; nextChunk = *it; if ( prevChunk->getMax().woCompare( nextChunk->getMin() ) != 0 ) { holeInRange = true; break; } prevChunk = nextChunk; } if ( holeInRange ) { dassert( NULL != nextChunk ); *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " has a hole in the range " << rangeToString( minKey, maxKey ) << " at " << rangeToString( prevChunk->getMax(), nextChunk->getMin() ); warning() << *errMsg << endl; return false; } // // Run apply ops command // BSONObj applyOpsCmd = buildApplyOpsCmd( chunksToMerge, shardVersion, mergeVersion ); bool ok; BSONObj result; try { ScopedDbConnection conn( configLoc, 30.0 ); ok = conn->runCommand( "config", applyOpsCmd, result ); if ( !ok ) *errMsg = result.toString(); conn.done(); } catch( const DBException& ex ) { ok = false; *errMsg = ex.toString(); } if ( !ok ) { *errMsg = stream() << "could not merge chunks for " << nss.ns() << ", writing to config failed" << causedBy( errMsg ); warning() << *errMsg << endl; return false; } // // Install merged chunk metadata // { Lock::DBLock writeLk(txn->lockState(), nss.db(), newlm::MODE_X); shardingState.mergeChunks(txn, nss.ns(), minKey, maxKey, mergeVersion); } // // Log change // BSONObj mergeLogEntry = buildMergeLogEntry( chunksToMerge, shardVersion, mergeVersion ); configServer.logChange( "merge", nss.ns(), mergeLogEntry ); return true; }
// static Status ClearFilters::clear(QuerySettings* querySettings, PlanCache* planCache, const std::string& ns, const BSONObj& cmdObj) { invariant(querySettings); // According to the specification, the planCacheClearFilters command runs in two modes: // - clear all hints; or // - clear hints for single query shape when a query shape is described in the // command arguments. if (cmdObj.hasField("query")) { CanonicalQuery* cqRaw; Status status = PlanCacheCommand::canonicalize(ns, cmdObj, &cqRaw); if (!status.isOK()) { return status; } scoped_ptr<CanonicalQuery> cq(cqRaw); querySettings->removeAllowedIndices(*cq); // Remove entry from plan cache planCache->remove(*cq); return Status::OK(); } // If query is not provided, make sure sort and projection are not in arguments. // We do not want to clear the entire cache inadvertently when the user // forgot to provide a value for "query". if (cmdObj.hasField("sort") || cmdObj.hasField("projection")) { return Status(ErrorCodes::BadValue, "sort or projection provided without query"); } // Get entries from query settings. We need to remove corresponding entries from the plan // cache shortly. OwnedPointerVector<AllowedIndexEntry> entries; entries.mutableVector() = querySettings->getAllAllowedIndices(); // OK to proceed with clearing entire cache. querySettings->clearAllowedIndices(); const NamespaceString nss(ns); const WhereCallbackReal whereCallback(nss.db()); // Remove corresponding entries from plan cache. // Admin hints affect the planning process directly. If there were // plans generated as a result of applying index filter, these need to be // invalidated. This allows the planner to re-populate the plan cache with // non-filtered indexed solutions next time the query is run. // Resolve plan cache key from (query, sort, projection) in query settings entry. // Concurrency note: There's no harm in removing plan cache entries one at at time. // Only way that PlanCache::remove() can fail is when the query shape has been removed from // the cache by some other means (re-index, collection info reset, ...). This is OK since // that's the intended effect of calling the remove() function with the key from the hint entry. for (vector<AllowedIndexEntry*>::const_iterator i = entries.begin(); i != entries.end(); ++i) { AllowedIndexEntry* entry = *i; invariant(entry); // Create canonical query. CanonicalQuery* cqRaw; Status result = CanonicalQuery::canonicalize( ns, entry->query, entry->sort, entry->projection, &cqRaw, whereCallback); invariant(result.isOK()); scoped_ptr<CanonicalQuery> cq(cqRaw); // Remove plan cache entry. planCache->remove(*cq); } return Status::OK(); }
// static Status ClearFilters::clear(OperationContext* txn, QuerySettings* querySettings, PlanCache* planCache, const std::string& ns, const BSONObj& cmdObj) { invariant(querySettings); // According to the specification, the planCacheClearFilters command runs in two modes: // - clear all hints; or // - clear hints for single query shape when a query shape is described in the // command arguments. if (cmdObj.hasField("query")) { auto statusWithCQ = PlanCacheCommand::canonicalize(txn, ns, cmdObj); if (!statusWithCQ.isOK()) { return statusWithCQ.getStatus(); } unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue()); querySettings->removeAllowedIndices(planCache->computeKey(*cq)); // Remove entry from plan cache planCache->remove(*cq); LOG(0) << "Removed index filter on " << ns << " " << cq->toStringShort(); return Status::OK(); } // If query is not provided, make sure sort and projection are not in arguments. // We do not want to clear the entire cache inadvertently when the user // forgot to provide a value for "query". if (cmdObj.hasField("sort") || cmdObj.hasField("projection")) { return Status(ErrorCodes::BadValue, "sort or projection provided without query"); } // Get entries from query settings. We need to remove corresponding entries from the plan // cache shortly. OwnedPointerVector<AllowedIndexEntry> entries; entries.mutableVector() = querySettings->getAllAllowedIndices(); // OK to proceed with clearing entire cache. querySettings->clearAllowedIndices(); const NamespaceString nss(ns); const ExtensionsCallbackReal extensionsCallback(txn, &nss); // Remove corresponding entries from plan cache. // Admin hints affect the planning process directly. If there were // plans generated as a result of applying index filter, these need to be // invalidated. This allows the planner to re-populate the plan cache with // non-filtered indexed solutions next time the query is run. // Resolve plan cache key from (query, sort, projection) in query settings entry. // Concurrency note: There's no harm in removing plan cache entries one at at time. // Only way that PlanCache::remove() can fail is when the query shape has been removed from // the cache by some other means (re-index, collection info reset, ...). This is OK since // that's the intended effect of calling the remove() function with the key from the hint entry. for (vector<AllowedIndexEntry*>::const_iterator i = entries.begin(); i != entries.end(); ++i) { AllowedIndexEntry* entry = *i; invariant(entry); // Create canonical query. auto qr = stdx::make_unique<QueryRequest>(nss); qr->setFilter(entry->query); qr->setSort(entry->sort); qr->setProj(entry->projection); auto statusWithCQ = CanonicalQuery::canonicalize(txn, std::move(qr), extensionsCallback); invariantOK(statusWithCQ.getStatus()); std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue()); // Remove plan cache entry. planCache->remove(*cq); } LOG(0) << "Removed all index filters for collection: " << ns; return Status::OK(); }