BSONObj ShardKeyPattern::normalizeShardKey(const BSONObj& shardKey) const {
// Shard keys are always of the form: { 'nested.path' : value, 'nested.path2' : value }
// and in the same order as the key pattern
if (!isValid())
return BSONObj();
// We want to return an empty key if users pass us something that's not a shard key
if (shardKey.nFields() > _keyPattern.toBSON().nFields())
return BSONObj();
BSONObjBuilder keyBuilder;
BSONObjIterator patternIt(_keyPattern.toBSON());
while (patternIt.more()) {
BSONElement patternEl = patternIt.next();
BSONElement keyEl = shardKey[patternEl.fieldNameStringData()];
if (!isShardKeyElement(keyEl, true))
return BSONObj();
keyBuilder.appendAs(keyEl, patternEl.fieldName());
}
dassert(isShardKey(keyBuilder.asTempObj()));
return keyBuilder.obj();
}
BSONObj //
ShardKeyPattern::extractShardKeyFromMatchable(const MatchableDocument& matchable) const {
if (!isValid())
return BSONObj();
BSONObjBuilder keyBuilder;
BSONObjIterator patternIt(_keyPattern.toBSON());
while (patternIt.more()) {
BSONElement patternEl = patternIt.next();
BSONElement matchEl =
extractKeyElementFromMatchable(matchable, patternEl.fieldNameStringData());
if (!isShardKeyElement(matchEl, true))
return BSONObj();
if (isHashedPatternEl(patternEl)) {
keyBuilder.append(
patternEl.fieldName(),
BSONElementHasher::hash64(matchEl, BSONElementHasher::DEFAULT_HASH_SEED));
} else {
// NOTE: The matched element may *not* have the same field name as the path -
// index keys don't contain field names, for example
keyBuilder.appendAs(matchEl, patternEl.fieldName());
}
}
dassert(isShardKey(keyBuilder.asTempObj()));
return keyBuilder.obj();
}
void waitForWriteConcern(OperationContext* opCtx,
const CommandInvocation* invocation,
const repl::OpTime& lastOpBeforeRun,
BSONObjBuilder& commandResponseBuilder) const override {
auto lastOpAfterRun = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp();
// Ensures that if we tried to do a write, we wait for write concern, even if that write was
// a noop.
if ((lastOpAfterRun == lastOpBeforeRun) &&
GlobalLockAcquisitionTracker::get(opCtx).getGlobalExclusiveLockTaken()) {
repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx);
lastOpAfterRun = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp();
}
WriteConcernResult res;
auto waitForWCStatus =
mongo::waitForWriteConcern(opCtx, lastOpAfterRun, opCtx->getWriteConcern(), &res);
CommandHelpers::appendCommandWCStatus(commandResponseBuilder, waitForWCStatus, res);
// SERVER-22421: This code is to ensure error response backwards compatibility with the
// user management commands. This can be removed in 3.6.
if (!waitForWCStatus.isOK() && invocation->definition()->isUserManagementCommand()) {
BSONObj temp = commandResponseBuilder.asTempObj().copy();
commandResponseBuilder.resetToEmpty();
CommandHelpers::appendCommandStatus(commandResponseBuilder, waitForWCStatus);
commandResponseBuilder.appendElementsUnique(temp);
}
}
bool Command::appendCommandStatus(BSONObjBuilder& result, const Status& status) {
appendCommandStatus(result, status.isOK(), status.reason());
BSONObj tmp = result.asTempObj();
if (!status.isOK() && !tmp.hasField("code")) {
result.append("code", status.code());
}
return status.isOK();
}
BSONObj TransactionRouter::Participant::attachTxnFieldsIfNeeded(
BSONObj cmd, bool isFirstStatementInThisParticipant) const {
bool hasStartTxn = false;
bool hasAutoCommit = false;
bool hasTxnNum = false;
BSONObjIterator iter(cmd);
while (iter.more()) {
auto elem = iter.next();
if (OperationSessionInfoFromClient::kStartTransactionFieldName ==
elem.fieldNameStringData()) {
hasStartTxn = true;
} else if (OperationSessionInfoFromClient::kAutocommitFieldName ==
elem.fieldNameStringData()) {
hasAutoCommit = true;
} else if (OperationSessionInfo::kTxnNumberFieldName == elem.fieldNameStringData()) {
hasTxnNum = true;
}
}
// TODO: SERVER-37045 assert when attaching startTransaction to killCursors command.
// The first command sent to a participant must start a transaction, unless it is a transaction
// command, which don't support the options that start transactions, i.e. startTransaction and
// readConcern. Otherwise the command must not have a read concern.
bool mustStartTransaction = isFirstStatementInThisParticipant && !isTransactionCommand(cmd);
if (!mustStartTransaction) {
dassert(!cmd.hasField(repl::ReadConcernArgs::kReadConcernFieldName));
}
BSONObjBuilder newCmd = mustStartTransaction
? appendFieldsForStartTransaction(std::move(cmd),
sharedOptions.readConcernArgs,
sharedOptions.atClusterTime,
!hasStartTxn)
: BSONObjBuilder(std::move(cmd));
if (isCoordinator) {
newCmd.append(kCoordinatorField, true);
}
if (!hasAutoCommit) {
newCmd.append(OperationSessionInfoFromClient::kAutocommitFieldName, false);
}
if (!hasTxnNum) {
newCmd.append(OperationSessionInfo::kTxnNumberFieldName, sharedOptions.txnNumber);
} else {
auto osi =
OperationSessionInfoFromClient::parse("OperationSessionInfo"_sd, newCmd.asTempObj());
invariant(sharedOptions.txnNumber == *osi.getTxnNumber());
}
return newCmd.obj();
}
bool CommandHelpers::extractOrAppendOk(BSONObjBuilder& reply) {
if (auto okField = reply.asTempObj()["ok"]) {
// If ok is present, use its truthiness.
return okField.trueValue();
}
// Missing "ok" field is an implied success.
CommandHelpers::appendSimpleCommandStatus(reply, true);
return true;
}
StatusWith<BSONObj> ShardKeyPattern::extractShardKeyFromQuery(const BSONObj& basicQuery) const {
if (!isValid())
return StatusWith<BSONObj>(BSONObj());
// Extract equalities from query
CanonicalQuery* rawQuery;
Status queryStatus =
CanonicalQuery::canonicalize("", basicQuery, &rawQuery, WhereCallbackNoop());
if (!queryStatus.isOK())
return StatusWith<BSONObj>(queryStatus);
scoped_ptr<CanonicalQuery> query(rawQuery);
EqualityMatches equalities;
// TODO: Build the path set initially?
FieldRefSet keyPatternPathSet(_keyPatternPaths.vector());
// We only care about extracting the full key pattern paths - if they don't exist (or are
// conflicting), we don't contain the shard key.
Status eqStatus = pathsupport::extractFullEqualityMatches(*query->root(),
keyPatternPathSet,
&equalities);
// NOTE: Failure to extract equality matches just means we return no shard key - it's not
// an error we propagate
if (!eqStatus.isOK())
return StatusWith<BSONObj>(BSONObj());
// Extract key from equalities
// NOTE: The method below is equivalent to constructing a BSONObj and running
// extractShardKeyFromMatchable, but doesn't require creating the doc.
BSONObjBuilder keyBuilder;
// Iterate the parsed paths to avoid re-parsing
for (OwnedPointerVector<FieldRef>::const_iterator it = _keyPatternPaths.begin();
it != _keyPatternPaths.end(); ++it) {
const FieldRef& patternPath = **it;
BSONElement equalEl = findEqualityElement(equalities, patternPath);
if (!isShardKeyElement(equalEl, false))
return StatusWith<BSONObj>(BSONObj());
if (isHashedPattern()) {
keyBuilder.append(patternPath.dottedField(),
BSONElementHasher::hash64(equalEl,
BSONElementHasher::DEFAULT_HASH_SEED));
}
else {
// NOTE: The equal element may *not* have the same field name as the path -
// nested $and, $eq, for example
keyBuilder.appendAs(equalEl, patternPath.dottedField());
}
}
dassert(isShardKey(keyBuilder.asTempObj()));
return StatusWith<BSONObj>(keyBuilder.obj());
}
void Command::appendCommandStatus(BSONObjBuilder& result, bool ok, const std::string& errmsg) {
BSONObj tmp = result.asTempObj();
bool have_ok = tmp.hasField("ok");
bool have_errmsg = tmp.hasField("errmsg");
if (!have_ok)
result.append( "ok" , ok ? 1.0 : 0.0 );
if (!ok && !have_errmsg) {
result.append("errmsg", errmsg);
}
}
bool CommandHelpers::appendCommandStatus(BSONObjBuilder& result, const Status& status) {
appendCommandStatus(result, status.isOK(), status.reason());
BSONObj tmp = result.asTempObj();
if (!status.isOK() && !tmp.hasField("code")) {
result.append("code", status.code());
result.append("codeName", ErrorCodes::errorString(status.code()));
}
if (auto extraInfo = status.extraInfo()) {
extraInfo->serialize(&result);
}
return status.isOK();
}
bool Command::runAgainstRegistered(const char *ns, BSONObj& jsobj, BSONObjBuilder& anObjBuilder) {
const char *p = strchr(ns, '.');
if ( !p ) return false;
if ( strcmp(p, ".$cmd") != 0 ) return false;
bool ok = false;
bool valid = false;
BSONElement e;
e = jsobj.firstElement();
map<string,Command*>::iterator i;
if ( e.eoo() )
;
/* check for properly registered command objects. Note that all the commands below should be
migrated over to the command object format.
*/
else if ( (i = _commands->find(e.fieldName())) != _commands->end() ) {
valid = true;
string errmsg;
Command *c = i->second;
if ( c->adminOnly() && strncmp(ns, "admin", 5) != 0 ) {
ok = false;
errmsg = "access denied";
}
else if ( jsobj.getBoolField( "help" ) ){
stringstream help;
help << "help for: " << e.fieldName() << " ";
c->help( help );
anObjBuilder.append( "help" , help.str() );
}
else {
ok = c->run(ns, jsobj, errmsg, anObjBuilder, false);
}
BSONObj tmp = anObjBuilder.asTempObj();
bool have_ok = tmp.hasField("ok");
bool have_errmsg = tmp.hasField("errmsg");
if (!have_ok)
anObjBuilder.append( "ok" , ok ? 1.0 : 0.0 );
if ( !ok && !have_errmsg) {
anObjBuilder.append("errmsg", errmsg);
uassert_nothrow(errmsg.c_str());
}
return true;
}
return false;
}
bool run(OperationContext* txn, const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result,
bool fromRepl ) {
//
// Correct behavior here is very finicky.
//
// 1. The first step is to append the error that occurred on the previous operation.
// This adds an "err" field to the command, which is *not* the command failing.
//
// 2. Next we parse and validate write concern options. If these options are invalid
// the command fails no matter what, even if we actually had an error earlier. The
// reason for checking here is to match legacy behavior on these kind of failures -
// we'll still get an "err" field for the write error.
//
// 3. If we had an error on the previous operation, we then return immediately.
//
// 4. Finally, we actually enforce the write concern. All errors *except* timeout are
// reported with ok : 0.0, to match legacy behavior.
//
// There is a special case when "wOpTime" and "wElectionId" are explicitly provided by
// the client (mongos) - in this case we *only* enforce the write concern if it is
// valid.
//
// We always need to either report "err" (if ok : 1) or "errmsg" (if ok : 0), even if
// err is null.
//
LastError *le = lastError.disableForCommand();
// Always append lastOp and connectionId
Client& c = cc();
c.appendLastOp( result );
// for sharding; also useful in general for debugging
result.appendNumber( "connectionId" , c.getConnectionId() );
OpTime lastOpTime;
BSONField<OpTime> wOpTimeField("wOpTime");
FieldParser::FieldState extracted = FieldParser::extract(cmdObj, wOpTimeField,
&lastOpTime, &errmsg);
if (!extracted) {
result.append("badGLE", cmdObj);
appendCommandStatus(result, false, errmsg);
return false;
}
bool lastOpTimePresent = extracted != FieldParser::FIELD_NONE;
if (!lastOpTimePresent) {
// Use the client opTime if no wOpTime is specified
lastOpTime = cc().getLastOp();
}
OID electionId;
BSONField<OID> wElectionIdField("wElectionId");
extracted = FieldParser::extract(cmdObj, wElectionIdField,
&electionId, &errmsg);
if (!extracted) {
result.append("badGLE", cmdObj);
appendCommandStatus(result, false, errmsg);
return false;
}
bool electionIdPresent = extracted != FieldParser::FIELD_NONE;
bool errorOccurred = false;
// Errors aren't reported when wOpTime is used
if ( !lastOpTimePresent ) {
if ( le->nPrev != 1 ) {
errorOccurred = LastError::noError.appendSelf( result, false );
le->appendSelfStatus( result );
}
else {
errorOccurred = le->appendSelf( result, false );
}
}
BSONObj writeConcernDoc = cmdObj;
// Use the default options if we have no gle options aside from wOpTime/wElectionId
const int nFields = cmdObj.nFields();
bool useDefaultGLEOptions = (nFields == 1) ||
(nFields == 2 && lastOpTimePresent) ||
(nFields == 3 && lastOpTimePresent && electionIdPresent);
if ( useDefaultGLEOptions && getLastErrorDefault ) {
writeConcernDoc = *getLastErrorDefault;
}
//
// Validate write concern no matter what, this matches 2.4 behavior
//
WriteConcernOptions writeConcern;
Status status = writeConcern.parse( writeConcernDoc );
if ( status.isOK() ) {
// Ensure options are valid for this host
status = validateWriteConcern( writeConcern );
}
if ( !status.isOK() ) {
result.append( "badGLE", writeConcernDoc );
return appendCommandStatus( result, status );
}
// Don't wait for replication if there was an error reported - this matches 2.4 behavior
if ( errorOccurred ) {
dassert( !lastOpTimePresent );
return true;
}
// No error occurred, so we won't duplicate these fields with write concern errors
dassert( result.asTempObj()["err"].eoo() );
dassert( result.asTempObj()["code"].eoo() );
// If we got an electionId, make sure it matches
if (electionIdPresent) {
if (!theReplSet) {
// Ignore electionIds of 0 from mongos.
if (electionId != OID()) {
errmsg = "wElectionId passed but no replication active";
result.append("code", ErrorCodes::BadValue);
return false;
}
}
else {
if (electionId != theReplSet->getElectionId()) {
LOG(3) << "oid passed in is " << electionId
<< ", but our id is " << theReplSet->getElectionId();
errmsg = "election occurred after write";
result.append("code", ErrorCodes::WriteConcernFailed);
return false;
}
}
}
cc().curop()->setMessage( "waiting for write concern" );
WriteConcernResult wcResult;
status = waitForWriteConcern( txn, writeConcern, lastOpTime, &wcResult );
wcResult.appendTo( writeConcern, &result );
// For backward compatibility with 2.4, wtimeout returns ok : 1.0
if ( wcResult.wTimedOut ) {
dassert( !wcResult.err.empty() ); // so we always report err
dassert( !status.isOK() );
result.append( "errmsg", "timed out waiting for slaves" );
result.append( "code", status.code() );
return true;
}
return appendCommandStatus( result, status );
}
bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int options,
string& errmsg,
BSONObjBuilder& result) {
const string ns = parseNs(dbname, cmdObj);
const NamespaceString nss(ns);
const ExtensionsCallbackReal extensionsCallback(txn, &nss);
auto parsedDistinct = ParsedDistinct::parse(txn, nss, cmdObj, extensionsCallback, false);
if (!parsedDistinct.isOK()) {
return appendCommandStatus(result, parsedDistinct.getStatus());
}
if (!parsedDistinct.getValue().getQuery()->getQueryRequest().getCollation().isEmpty() &&
serverGlobalParams.featureCompatibility.version.load() ==
ServerGlobalParams::FeatureCompatibility::Version::k32) {
return appendCommandStatus(
result,
Status(ErrorCodes::InvalidOptions,
"The featureCompatibilityVersion must be 3.4 to use collation. See "
"http://dochub.mongodb.org/core/3.4-feature-compatibility."));
}
AutoGetCollectionOrViewForRead ctx(txn, ns);
Collection* collection = ctx.getCollection();
if (ctx.getView()) {
ctx.releaseLocksForView();
auto viewAggregation = parsedDistinct.getValue().asAggregationCommand();
if (!viewAggregation.isOK()) {
return appendCommandStatus(result, viewAggregation.getStatus());
}
BSONObjBuilder aggResult;
(void)Command::findCommand("aggregate")
->run(txn, dbname, viewAggregation.getValue(), options, errmsg, aggResult);
if (ResolvedView::isResolvedViewErrorResponse(aggResult.asTempObj())) {
result.appendElements(aggResult.obj());
return false;
}
ViewResponseFormatter formatter(aggResult.obj());
Status formatStatus = formatter.appendAsDistinctResponse(&result);
if (!formatStatus.isOK()) {
return appendCommandStatus(result, formatStatus);
}
return true;
}
auto executor = getExecutorDistinct(
txn, collection, ns, &parsedDistinct.getValue(), PlanExecutor::YIELD_AUTO);
if (!executor.isOK()) {
return appendCommandStatus(result, executor.getStatus());
}
{
stdx::lock_guard<Client>(*txn->getClient());
CurOp::get(txn)->setPlanSummary_inlock(
Explain::getPlanSummary(executor.getValue().get()));
}
string key = cmdObj[ParsedDistinct::kKeyField].valuestrsafe();
int bufSize = BSONObjMaxUserSize - 4096;
BufBuilder bb(bufSize);
char* start = bb.buf();
BSONArrayBuilder arr(bb);
BSONElementSet values(executor.getValue()->getCanonicalQuery()->getCollator());
BSONObj obj;
PlanExecutor::ExecState state;
while (PlanExecutor::ADVANCED == (state = executor.getValue()->getNext(&obj, NULL))) {
// Distinct expands arrays.
//
// If our query is covered, each value of the key should be in the index key and
// available to us without this. If a collection scan is providing the data, we may
// have to expand an array.
BSONElementSet elts;
dps::extractAllElementsAlongPath(obj, key, elts);
for (BSONElementSet::iterator it = elts.begin(); it != elts.end(); ++it) {
BSONElement elt = *it;
if (values.count(elt)) {
continue;
}
int currentBufPos = bb.len();
uassert(17217,
"distinct too big, 16mb cap",
(currentBufPos + elt.size() + 1024) < bufSize);
arr.append(elt);
BSONElement x(start + currentBufPos);
values.insert(x);
}
}
// Return an error if execution fails for any reason.
if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) {
log() << "Plan executor error during distinct command: "
<< redact(PlanExecutor::statestr(state))
<< ", stats: " << redact(Explain::getWinningPlanStats(executor.getValue().get()));
return appendCommandStatus(result,
Status(ErrorCodes::OperationFailed,
str::stream()
<< "Executor error during distinct command: "
<< WorkingSetCommon::toStatusString(obj)));
}
auto curOp = CurOp::get(txn);
// Get summary information about the plan.
PlanSummaryStats stats;
Explain::getSummaryStats(*executor.getValue(), &stats);
if (collection) {
collection->infoCache()->notifyOfQuery(txn, stats.indexesUsed);
}
curOp->debug().setPlanSummaryMetrics(stats);
if (curOp->shouldDBProfile()) {
BSONObjBuilder execStatsBob;
Explain::getWinningPlanStats(executor.getValue().get(), &execStatsBob);
curOp->debug().execStats = execStatsBob.obj();
}
verify(start == bb.buf());
result.appendArray("values", arr.done());
return true;
}
bool run(OperationContext* opCtx,
const string& dbname,
const BSONObj& cmdObj,
BSONObjBuilder& result) final {
CommandHelpers::handleMarkKillOnClientDisconnect(opCtx);
IDLParserErrorContext ctx("listDatabases");
auto cmd = ListDatabasesCommand::parse(ctx, cmdObj);
auto* as = AuthorizationSession::get(opCtx->getClient());
// {nameOnly: bool} - default false.
const bool nameOnly = cmd.getNameOnly();
// {authorizedDatabases: bool} - Dynamic default based on permissions.
const bool authorizedDatabases = ([as](const boost::optional<bool>& authDB) {
const bool mayListAllDatabases = as->isAuthorizedForActionsOnResource(
ResourcePattern::forClusterResource(), ActionType::listDatabases);
if (authDB) {
uassert(ErrorCodes::Unauthorized,
"Insufficient permissions to list all databases",
authDB.get() || mayListAllDatabases);
return authDB.get();
}
// By default, list all databases if we can, otherwise
// only those we're allowed to find on.
return !mayListAllDatabases;
})(cmd.getAuthorizedDatabases());
// {filter: matchExpression}.
std::unique_ptr<MatchExpression> filter;
if (auto filterObj = cmd.getFilter()) {
// The collator is null because database metadata objects are compared using simple
// binary comparison.
const CollatorInterface* collator = nullptr;
boost::intrusive_ptr<ExpressionContext> expCtx(new ExpressionContext(opCtx, collator));
auto matcher =
uassertStatusOK(MatchExpressionParser::parse(filterObj.get(), std::move(expCtx)));
filter = std::move(matcher);
}
vector<string> dbNames;
StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine();
{
Lock::GlobalLock lk(opCtx, MODE_IS);
CurOpFailpointHelpers::waitWhileFailPointEnabled(
&hangBeforeListDatabases, opCtx, "hangBeforeListDatabases", []() {});
dbNames = storageEngine->listDatabases();
}
vector<BSONObj> dbInfos;
const bool filterNameOnly = filter &&
filter->getCategory() == MatchExpression::MatchCategory::kLeaf &&
filter->path() == kNameField;
intmax_t totalSize = 0;
for (const auto& dbname : dbNames) {
if (authorizedDatabases && !as->isAuthorizedForAnyActionOnAnyResourceInDB(dbname)) {
// We don't have listDatabases on the cluser or find on this database.
continue;
}
BSONObjBuilder b;
b.append("name", dbname);
int64_t size = 0;
if (!nameOnly) {
// Filtering on name only should not require taking locks on filtered-out names.
if (filterNameOnly && !filter->matchesBSON(b.asTempObj()))
continue;
AutoGetDb autoDb(opCtx, dbname, MODE_IS);
Database* const db = autoDb.getDb();
if (!db)
continue;
writeConflictRetry(opCtx, "sizeOnDisk", dbname, [&] {
size = storageEngine->sizeOnDiskForDb(opCtx, dbname);
});
b.append("sizeOnDisk", static_cast<double>(size));
b.appendBool(
"empty",
CollectionCatalog::get(opCtx).getAllCollectionUUIDsFromDb(dbname).empty());
}
BSONObj curDbObj = b.obj();
if (!filter || filter->matchesBSON(curDbObj)) {
totalSize += size;
dbInfos.push_back(curDbObj);
}
}
result.append("databases", dbInfos);
if (!nameOnly) {
result.append("totalSize", double(totalSize));
}
return true;
}