/* called on a reconfig AND on initiate
throws
@param initial true when initiating
*/
void checkMembersUpForConfigChange(const ReplSetConfig& cfg, BSONObjBuilder& result, bool initial) {
int failures = 0, allVotes = 0, allowableFailures = 0;
int me = 0;
stringstream selfs;
for( vector<ReplSetConfig::MemberCfg>::const_iterator i = cfg.members.begin(); i != cfg.members.end(); i++ ) {
if( i->h.isSelf() ) {
me++;
if( me > 1 )
selfs << ',';
selfs << i->h.toString();
if( !i->potentiallyHot() ) {
uasserted(13420, "initiation and reconfiguration of a replica set must be sent to a node that can become primary");
}
}
allVotes += i->votes;
}
allowableFailures = allVotes - (allVotes/2 + 1);
uassert(13278, "bad config: isSelf is true for multiple hosts: " + selfs.str(), me <= 1); // dups?
if( me != 1 ) {
stringstream ss;
ss << "can't find self in the replset config";
if( !cmdLine.isDefaultPort() ) ss << " my port: " << cmdLine.port;
if( me != 0 ) ss << " found: " << me;
uasserted(13279, ss.str());
}
vector<string> down;
for( vector<ReplSetConfig::MemberCfg>::const_iterator i = cfg.members.begin(); i != cfg.members.end(); i++ ) {
// we know we're up
if (i->h.isSelf()) {
continue;
}
BSONObj res;
{
bool ok = false;
try {
int theirVersion = -1000;
ok = requestHeartbeat(cfg._id, "", i->h.toString(), res, -1, theirVersion, initial/*check if empty*/);
if( theirVersion >= cfg.version ) {
stringstream ss;
ss << "replSet member " << i->h.toString() << " has too new a config version (" << theirVersion << ") to reconfigure";
uasserted(13259, ss.str());
}
}
catch(DBException& e) {
log() << "replSet cmufcc requestHeartbeat " << i->h.toString() << " : " << e.toString() << rsLog;
}
catch(...) {
log() << "replSet cmufcc error exception in requestHeartbeat?" << rsLog;
}
if( res.getBoolField("mismatch") )
uasserted(13145, "set name does not match the set name host " + i->h.toString() + " expects");
if( *res.getStringField("set") ) {
if( cfg.version <= 1 ) {
// this was to be initiation, no one should be initiated already.
uasserted(13256, "member " + i->h.toString() + " is already initiated");
}
else {
// Assure no one has a newer config.
if( res["v"].Int() >= cfg.version ) {
uasserted(13341, "member " + i->h.toString() + " has a config version >= to the new cfg version; cannot change config");
}
}
}
if( !ok && !res["rs"].trueValue() ) {
down.push_back(i->h.toString());
if( !res.isEmpty() ) {
/* strange. got a response, but not "ok". log it. */
log() << "replSet warning " << i->h.toString() << " replied: " << res.toString() << rsLog;
}
bool allowFailure = false;
failures += i->votes;
if( !initial && failures <= allowableFailures ) {
const Member* m = theReplSet->findById( i->_id );
if( m ) {
verify( m->h().toString() == i->h.toString() );
}
// it's okay if the down member isn't part of the config,
// we might be adding a new member that isn't up yet
allowFailure = true;
}
if( !allowFailure ) {
string msg = string("need all members up to initiate, not ok : ") + i->h.toString();
if( !initial )
msg = string("need most members up to reconfigure, not ok : ") + i->h.toString();
uasserted(13144, msg);
}
}
}
if( initial ) {
bool hasData = res["hasData"].Bool();
uassert(13311, "member " + i->h.toString() + " has data already, cannot initiate set. All members except initiator must be empty.",
!hasData || i->h.isSelf());
}
}
if (down.size() > 0) {
result.append("down", down);
}
}
int run() {
bool usingMongos = isMongos();
int serverAuthzVersion = 0;
BSONObj dumpQuery;
if (mongoDumpGlobalParams.repair){
return repair();
}
{
if (mongoDumpGlobalParams.query.size()) {
dumpQuery = fromjson(mongoDumpGlobalParams.query);
}
}
if (mongoDumpGlobalParams.dumpUsersAndRoles) {
uassertStatusOK(auth::getRemoteStoredAuthorizationVersion(&conn(true),
&serverAuthzVersion));
uassert(17369,
mongoutils::str::stream() << "Backing up users and roles is only supported for "
"clusters with auth schema versions 1 or 3, found: " <<
serverAuthzVersion,
serverAuthzVersion == AuthorizationManager::schemaVersion24 ||
serverAuthzVersion == AuthorizationManager::schemaVersion26Final);
}
string opLogName = "";
unsigned long long opLogStart = 0;
if (mongoDumpGlobalParams.useOplog) {
BSONObj isMaster;
conn("true").simpleCommand("admin", &isMaster, "isMaster");
if (isMaster.hasField("hosts")) { // if connected to replica set member
opLogName = "local.oplog.rs";
}
else {
opLogName = "local.oplog.$main";
if ( ! isMaster["ismaster"].trueValue() ) {
toolError() << "oplog mode is only supported on master or replica set member"
<< std::endl;
return -1;
}
}
BSONObj op = conn(true).findOne(opLogName, Query().sort("$natural", -1), 0, QueryOption_SlaveOk);
if (op.isEmpty()) {
toolError() << "No operations in oplog. Please ensure you are connecting to a "
<< "master." << std::endl;
return -1;
}
verify(op["ts"].type() == Timestamp);
opLogStart = op["ts"]._numberLong();
}
// check if we're outputting to stdout
if (mongoDumpGlobalParams.outputDirectory == "-") {
if (toolGlobalParams.db != "" && toolGlobalParams.coll != "") {
writeCollectionStdout(toolGlobalParams.db + "." + toolGlobalParams.coll, dumpQuery,
usingMongos);
return 0;
}
else {
toolError() << "You must specify database and collection to print to stdout"
<< std::endl;
return -1;
}
}
boost::filesystem::path root(mongoDumpGlobalParams.outputDirectory);
if (toolGlobalParams.db == "") {
if (toolGlobalParams.coll != "") {
toolError() << "--db must be specified with --collection" << std::endl;
return -1;
}
toolInfoLog() << "all dbs" << std::endl;
BSONObj res = conn( true ).findOne( "admin.$cmd" , BSON( "listDatabases" << 1 ) );
if ( ! res["databases"].isABSONObj() ) {
toolError() << "output of listDatabases isn't what we expected, no 'databases' "
<< "field:\n" << res << std::endl;
return -2;
}
BSONObj dbs = res["databases"].embeddedObjectUserCheck();
set<string> keys;
dbs.getFieldNames( keys );
for ( set<string>::iterator i = keys.begin() ; i != keys.end() ; i++ ) {
string key = *i;
if ( ! dbs[key].isABSONObj() ) {
toolError() << "database field not an document key: " << key << " value: "
<< dbs[key] << std::endl;
return -3;
}
BSONObj dbobj = dbs[key].embeddedObjectUserCheck();
const char * dbName = dbobj.getField( "name" ).valuestr();
if ( (string)dbName == "local" )
continue;
boost::filesystem::path outdir = root / dbName;
toolInfoLog() << "DATABASE: " << dbName << "\t to \t" << outdir.string()
<< std::endl;
go ( dbName , "", dumpQuery, outdir, "", usingMongos );
}
}
else {
boost::filesystem::path outdir = root / toolGlobalParams.db;
toolInfoLog() << "DATABASE: " << toolGlobalParams.db << "\t to \t" << outdir.string()
<< std::endl;
go(toolGlobalParams.db, toolGlobalParams.coll, dumpQuery, outdir, "", usingMongos);
if (mongoDumpGlobalParams.dumpUsersAndRoles &&
serverAuthzVersion == AuthorizationManager::schemaVersion26Final &&
toolGlobalParams.db != "admin") {
toolInfoLog() << "Backing up user and role data for the " << toolGlobalParams.db <<
" database";
Query query = Query(BSON("db" << toolGlobalParams.db));
go("admin", "system.users", query, outdir, "$admin.system.users", usingMongos);
go("admin", "system.roles", query, outdir, "$admin.system.roles", usingMongos);
}
}
if (!opLogName.empty()) {
BSONObjBuilder b;
b.appendTimestamp("$gt", opLogStart);
dumpQuery = BSON("ts" << b.obj());
writeCollectionFile( opLogName , dumpQuery, root / "oplog.bson", usingMongos );
}
return 0;
}
Status ClusterAggregate::runAggregate(OperationContext* txn,
const Namespaces& namespaces,
BSONObj cmdObj,
int options,
BSONObjBuilder* result) {
auto dbname = namespaces.executionNss.db().toString();
auto status = grid.catalogCache()->getDatabase(txn, dbname);
if (!status.isOK()) {
appendEmptyResultSet(*result, status.getStatus(), namespaces.requestedNss.ns());
return Status::OK();
}
std::shared_ptr<DBConfig> conf = status.getValue();
if (!conf->isShardingEnabled()) {
return aggPassthrough(txn, namespaces, conf, cmdObj, result, options);
}
auto request = AggregationRequest::parseFromBSON(namespaces.executionNss, cmdObj);
if (!request.isOK()) {
return request.getStatus();
}
boost::intrusive_ptr<ExpressionContext> mergeCtx =
new ExpressionContext(txn, request.getValue());
mergeCtx->inRouter = true;
// explicitly *not* setting mergeCtx->tempDir
// Parse and optimize the pipeline specification.
auto pipeline = Pipeline::parse(request.getValue().getPipeline(), mergeCtx);
if (!pipeline.isOK()) {
return pipeline.getStatus();
}
for (auto&& ns : pipeline.getValue()->getInvolvedCollections()) {
uassert(28769, str::stream() << ns.ns() << " cannot be sharded", !conf->isSharded(ns.ns()));
// We won't try to execute anything on a mongos, but we still have to populate this map
// so that any $lookups etc will be able to have a resolved view definition. It's okay
// that this is incorrect, we will repopulate the real resolved namespace map on the
// mongod.
// TODO SERVER-25038 This should become unnecessary once we can get the involved
// namespaces before parsing.
mergeCtx->resolvedNamespaces[ns.coll()] = {ns, std::vector<BSONObj>{}};
}
if (!conf->isSharded(namespaces.executionNss.ns())) {
return aggPassthrough(txn, namespaces, conf, cmdObj, result, options);
}
ChunkManagerPtr chunkMgr = conf->getChunkManager(txn, namespaces.executionNss.ns());
// If there was no collation specified, but there is a default collation for the collation,
// use that.
if (request.getValue().getCollation().isEmpty() && chunkMgr->getDefaultCollator()) {
mergeCtx->setCollator(chunkMgr->getDefaultCollator()->clone());
}
// Now that we know the collation we'll be using, inject the ExpressionContext and optimize.
// TODO SERVER-25038: this must happen before we parse the pipeline, since we can make
// string comparisons during parse time.
pipeline.getValue()->injectExpressionContext(mergeCtx);
pipeline.getValue()->optimizePipeline();
// If the first $match stage is an exact match on the shard key (with a simple collation or
// no string matching), we only have to send it to one shard, so send the command to that
// shard.
BSONObj firstMatchQuery = pipeline.getValue()->getInitialQuery();
BSONObj shardKeyMatches;
shardKeyMatches = uassertStatusOK(
chunkMgr->getShardKeyPattern().extractShardKeyFromQuery(txn, firstMatchQuery));
bool singleShard = false;
if (!shardKeyMatches.isEmpty()) {
auto chunk = chunkMgr->findIntersectingChunk(
txn, shardKeyMatches, request.getValue().getCollation());
if (chunk.isOK()) {
singleShard = true;
}
}
// Don't need to split pipeline if the first $match is an exact match on shard key, unless
// there is a stage that needs to be run on the primary shard.
const bool needPrimaryShardMerger = pipeline.getValue()->needsPrimaryShardMerger();
const bool needSplit = !singleShard || needPrimaryShardMerger;
// Split the pipeline into pieces for mongod(s) and this mongos. If needSplit is true,
// 'pipeline' will become the merger side.
boost::intrusive_ptr<Pipeline> shardPipeline(needSplit ? pipeline.getValue()->splitForSharded()
: pipeline.getValue());
// Create the command for the shards. The 'fromRouter' field means produce output to be
// merged.
MutableDocument commandBuilder(request.getValue().serializeToCommandObj());
commandBuilder[AggregationRequest::kPipelineName] = Value(shardPipeline->serialize());
if (needSplit) {
commandBuilder[AggregationRequest::kFromRouterName] = Value(true);
commandBuilder[AggregationRequest::kCursorName] =
Value(DOC(AggregationRequest::kBatchSizeName << 0));
}
// These fields are not part of the AggregationRequest since they are not handled by the
// aggregation subsystem, so we serialize them separately.
const std::initializer_list<StringData> fieldsToPropagateToShards = {
"$queryOptions", "readConcern", QueryRequest::cmdOptionMaxTimeMS,
};
for (auto&& field : fieldsToPropagateToShards) {
commandBuilder[field] = Value(cmdObj[field]);
}
BSONObj shardedCommand = commandBuilder.freeze().toBson();
BSONObj shardQuery = shardPipeline->getInitialQuery();
// Run the command on the shards
// TODO need to make sure cursors are killed if a retry is needed
std::vector<Strategy::CommandResult> shardResults;
Strategy::commandOp(txn,
dbname,
shardedCommand,
options,
namespaces.executionNss.ns(),
shardQuery,
request.getValue().getCollation(),
&shardResults);
if (mergeCtx->isExplain) {
// This must be checked before we start modifying result.
uassertAllShardsSupportExplain(shardResults);
if (needSplit) {
*result << "needsPrimaryShardMerger" << needPrimaryShardMerger << "splitPipeline"
<< DOC("shardsPart" << shardPipeline->writeExplainOps() << "mergerPart"
<< pipeline.getValue()->writeExplainOps());
} else {
*result << "splitPipeline" << BSONNULL;
}
BSONObjBuilder shardExplains(result->subobjStart("shards"));
for (size_t i = 0; i < shardResults.size(); i++) {
shardExplains.append(shardResults[i].shardTargetId,
BSON("host" << shardResults[i].target.toString() << "stages"
<< shardResults[i].result["stages"]));
}
return Status::OK();
}
if (!needSplit) {
invariant(shardResults.size() == 1);
invariant(shardResults[0].target.getServers().size() == 1);
auto executorPool = grid.getExecutorPool();
const BSONObj reply =
uassertStatusOK(storePossibleCursor(shardResults[0].target.getServers()[0],
shardResults[0].result,
namespaces.requestedNss,
executorPool->getArbitraryExecutor(),
grid.getCursorManager()));
result->appendElements(reply);
return getStatusFromCommandResult(reply);
}
pipeline.getValue()->addInitialSource(
DocumentSourceMergeCursors::create(parseCursors(shardResults), mergeCtx));
MutableDocument mergeCmd(request.getValue().serializeToCommandObj());
mergeCmd["pipeline"] = Value(pipeline.getValue()->serialize());
mergeCmd["cursor"] = Value(cmdObj["cursor"]);
if (cmdObj.hasField("$queryOptions")) {
mergeCmd["$queryOptions"] = Value(cmdObj["$queryOptions"]);
}
if (cmdObj.hasField(QueryRequest::cmdOptionMaxTimeMS)) {
mergeCmd[QueryRequest::cmdOptionMaxTimeMS] =
Value(cmdObj[QueryRequest::cmdOptionMaxTimeMS]);
}
mergeCmd.setField("writeConcern", Value(cmdObj["writeConcern"]));
// Not propagating readConcern to merger since it doesn't do local reads.
// If the user didn't specify a collation already, make sure there's a collation attached to
// the merge command, since the merging shard may not have the collection metadata.
if (mergeCmd.peek()["collation"].missing()) {
mergeCmd.setField("collation",
mergeCtx->getCollator()
? Value(mergeCtx->getCollator()->getSpec().toBSON())
: Value(Document{CollationSpec::kSimpleSpec}));
}
std::string outputNsOrEmpty;
if (DocumentSourceOut* out =
dynamic_cast<DocumentSourceOut*>(pipeline.getValue()->getSources().back().get())) {
outputNsOrEmpty = out->getOutputNs().ns();
}
// Run merging command on random shard, unless a stage needs the primary shard. Need to use
// ShardConnection so that the merging mongod is sent the config servers on connection init.
auto& prng = txn->getClient()->getPrng();
const auto& mergingShardId = needPrimaryShardMerger
? conf->getPrimaryId()
: shardResults[prng.nextInt32(shardResults.size())].shardTargetId;
const auto mergingShard = uassertStatusOK(grid.shardRegistry()->getShard(txn, mergingShardId));
ShardConnection conn(mergingShard->getConnString(), outputNsOrEmpty);
BSONObj mergedResults =
aggRunCommand(conn.get(), namespaces, mergeCmd.freeze().toBson(), options);
conn.done();
if (auto wcErrorElem = mergedResults["writeConcernError"]) {
appendWriteConcernErrorToCmdResponse(mergingShardId, wcErrorElem, *result);
}
// Copy output from merging (primary) shard to the output object from our command.
// Also, propagates errmsg and code if ok == false.
result->appendElementsUnique(mergedResults);
return getStatusFromCommandResult(result->asTempObj());
}
/**
* Run a query -- includes checking for and running a Command.
* @return points to ns if exhaust mode. 0=normal mode
* @locks the db mutex for reading (and potentially for writing temporarily to create a new db).
* @yields the db mutex periodically after acquiring it.
* @asserts on scan and order memory exhaustion and other cases.
*/
const char *runQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) {
shared_ptr<ParsedQuery> pq_shared( new ParsedQuery(q) );
ParsedQuery& pq( *pq_shared );
BSONObj jsobj = q.query;
int queryOptions = q.queryOptions;
const char *ns = q.ns;
if( logLevel >= 2 )
log() << "runQuery called " << ns << " " << jsobj << endl;
curop.debug().ns = ns;
curop.debug().ntoreturn = pq.getNumToReturn();
curop.debug().query = jsobj;
curop.setQuery(jsobj);
// Run a command.
if ( pq.couldBeCommand() ) {
BufBuilder bb;
bb.skip(sizeof(QueryResult));
BSONObjBuilder cmdResBuf;
if ( runCommands(ns, jsobj, curop, bb, cmdResBuf, false, queryOptions) ) {
curop.debug().iscommand = true;
curop.debug().query = jsobj;
curop.markCommand();
auto_ptr< QueryResult > qr;
qr.reset( (QueryResult *) bb.buf() );
bb.decouple();
qr->setResultFlagsToOk();
qr->len = bb.len();
curop.debug().responseLength = bb.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
result.setData( qr.release(), true );
}
else {
uasserted(13530, "bad or malformed command request?");
}
return 0;
}
bool explain = pq.isExplain();
BSONObj order = pq.getOrder();
BSONObj query = pq.getFilter();
/* The ElemIter will not be happy if this isn't really an object. So throw exception
here when that is true.
(Which may indicate bad data from client.)
*/
if ( query.objsize() == 0 ) {
out() << "Bad query object?\n jsobj:";
out() << jsobj.toString() << "\n query:";
out() << query.toString() << endl;
uassert( 10110 , "bad query object", false);
}
Client::ReadContext ctx( ns , dbpath ); // read locks
const ConfigVersion shardingVersionAtStart = shardingState.getVersion( ns );
replVerifyReadsOk(&pq);
if ( pq.hasOption( QueryOption_CursorTailable ) ) {
NamespaceDetails *d = nsdetails( ns );
uassert( 13051, "tailable cursor requested on non capped collection", d && d->capped );
const BSONObj nat1 = BSON( "$natural" << 1 );
if ( order.isEmpty() ) {
order = nat1;
}
else {
uassert( 13052, "only {$natural:1} order allowed for tailable cursor", order == nat1 );
}
}
// Run a simple id query.
if ( ! (explain || pq.showDiskLoc()) && isSimpleIdQuery( query ) && !pq.hasOption( QueryOption_CursorTailable ) ) {
int n = 0;
bool nsFound = false;
bool indexFound = false;
BSONObj resObject;
Client& c = cc();
bool found = Helpers::findById( c, ns , query , resObject , &nsFound , &indexFound );
if ( nsFound == false || indexFound == true ) {
BufBuilder bb(sizeof(QueryResult)+resObject.objsize()+32);
bb.skip(sizeof(QueryResult));
curop.debug().idhack = true;
if ( found ) {
n = 1;
fillQueryResultFromObj( bb , pq.getFields() , resObject );
}
auto_ptr< QueryResult > qr;
qr.reset( (QueryResult *) bb.buf() );
bb.decouple();
qr->setResultFlagsToOk();
qr->len = bb.len();
curop.debug().responseLength = bb.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = n;
result.setData( qr.release(), true );
return NULL;
}
}
// Run a regular query.
BSONObj oldPlan;
if ( explain && ! pq.hasIndexSpecifier() ) {
MultiPlanScanner mps( ns, query, shared_ptr<Projection>(), order );
if ( mps.usingCachedPlan() ) {
oldPlan =
mps.oldExplain().firstElement().embeddedObject()
.firstElement().embeddedObject().getOwned();
}
}
// In some cases the query may be retried if there is an in memory sort size assertion.
for( int retry = 0; retry < 2; ++retry ) {
try {
return queryWithQueryOptimizer( m, queryOptions, ns, jsobj, curop, query, order,
pq_shared, oldPlan, shardingVersionAtStart, result );
} catch ( const QueryRetryException & ) {
verify( 16088, retry == 0 );
}
}
verify( 16082, false );
return 0;
}
bool run( const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result,
bool ) {
string ns = parseNs(dbname, cmdObj);
if ( ns.size() == 0 ) {
errmsg = "no namespace specified";
return false;
}
vector<BSONObj> bounds;
if ( !FieldParser::extract( cmdObj, boundsField, &bounds, &errmsg ) ) {
return false;
}
if ( bounds.size() == 0 ) {
errmsg = "no bounds were specified";
return false;
}
if ( bounds.size() != 2 ) {
errmsg = "only a min and max bound may be specified";
return false;
}
BSONObj minKey = bounds[0];
BSONObj maxKey = bounds[1];
if ( minKey.isEmpty() ) {
errmsg = "no min key specified";
return false;
}
if ( maxKey.isEmpty() ) {
errmsg = "no max key specified";
return false;
}
// This refreshes the chunk metadata if stale.
refreshChunkCache( NamespaceString( ns ) );
ShardPtr mergeShard = guessMergeShard( NamespaceString( ns ), minKey );
if ( !mergeShard ) {
errmsg = (string)"could not find shard for merge range starting at "
+ minKey.toString();
return false;
}
BSONObjBuilder remoteCmdObjB;
remoteCmdObjB.append( cmdObj[ ClusterMergeChunksCommand::nsField() ] );
remoteCmdObjB.append( cmdObj[ ClusterMergeChunksCommand::boundsField() ] );
remoteCmdObjB.append( ClusterMergeChunksCommand::configField(),
configServer.getPrimary().getAddress().toString() );
remoteCmdObjB.append( ClusterMergeChunksCommand::shardNameField(),
mergeShard->getName() );
BSONObj remoteResult;
// Throws, but handled at level above. Don't want to rewrap to preserve exception
// formatting.
ScopedDbConnection conn( mergeShard->getAddress() );
bool ok = conn->runCommand( "admin", remoteCmdObjB.obj(), remoteResult );
conn.done();
result.appendElements( remoteResult );
return ok;
}
void ReplicationCoordinatorImpl::_handleHeartbeatResponse(
const ReplicationExecutor::RemoteCommandCallbackData& cbData, int targetIndex) {
// remove handle from queued heartbeats
_untrackHeartbeatHandle(cbData.myHandle);
// Parse and validate the response. At the end of this step, if responseStatus is OK then
// hbResponse is valid.
Status responseStatus = cbData.response.getStatus();
if (responseStatus == ErrorCodes::CallbackCanceled) {
return;
}
const HostAndPort& target = cbData.request.target;
ReplSetHeartbeatResponse hbResponse;
BSONObj resp;
if (responseStatus.isOK()) {
resp = cbData.response.getValue().data;
responseStatus = hbResponse.initialize(resp, _topCoord->getTerm());
}
const Date_t now = _replExecutor.now();
const OpTime lastApplied = getMyLastOptime(); // Locks and unlocks _mutex.
Milliseconds networkTime(0);
StatusWith<ReplSetHeartbeatResponse> hbStatusResponse(hbResponse);
if (responseStatus.isOK()) {
networkTime = cbData.response.getValue().elapsedMillis;
_updateTerm_incallback(hbStatusResponse.getValue().getTerm(), nullptr);
}
else {
log() << "Error in heartbeat request to " << target << "; " << responseStatus;
if (!resp.isEmpty()) {
LOG(3) << "heartbeat response: " << resp;
}
hbStatusResponse = StatusWith<ReplSetHeartbeatResponse>(responseStatus);
}
HeartbeatResponseAction action =
_topCoord->processHeartbeatResponse(
now,
networkTime,
target,
hbStatusResponse,
lastApplied);
if (action.getAction() == HeartbeatResponseAction::NoAction &&
hbStatusResponse.isOK() &&
hbStatusResponse.getValue().hasOpTime() &&
targetIndex >= 0 &&
hbStatusResponse.getValue().hasState() &&
hbStatusResponse.getValue().getState() != MemberState::RS_PRIMARY) {
boost::unique_lock<boost::mutex> lk(_mutex);
if (hbStatusResponse.getValue().getConfigVersion() == _rsConfig.getConfigVersion()) {
_updateOpTimeFromHeartbeat_inlock(targetIndex,
hbStatusResponse.getValue().getOpTime());
// TODO: Enable with Data Replicator
//lk.unlock();
//_dr.slavesHaveProgressed();
}
}
_signalStepDownWaiters();
_scheduleHeartbeatToTarget(
target,
targetIndex,
std::max(now, action.getNextHeartbeatStartDate()));
_handleHeartbeatResponseAction(action, hbStatusResponse);
}
bool IndexType::scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const {
return ! order.isEmpty();
}
/* ****************************************************************************
*
* mongoUnsubscribeContext -
*/
HttpStatusCode mongoUnsubscribeContext(UnsubscribeContextRequest* requestP, UnsubscribeContextResponse* responseP, const std::string& tenant)
{
bool reqSemTaken;
std::string err;
reqSemTake(__FUNCTION__, "ngsi10 unsubscribe request", SemWriteOp, &reqSemTaken);
LM_T(LmtMongo, ("Unsubscribe Context"));
/* No matter if success or failure, the subscriptionId in the response is always the one
* in the request */
responseP->subscriptionId = requestP->subscriptionId;
if (responseP->subscriptionId.get() == "")
{
responseP->statusCode.fill(SccContextElementNotFound);
LM_W(("Bad Input (no subscriptionId)"));
return SccOk;
}
/* Look for document */
BSONObj sub;
OID id;
try
{
id = OID(requestP->subscriptionId.get());
}
catch (const AssertionException &e)
{
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo assertion exception)", reqSemTaken);
//
// This happens when OID format is wrong
// FIXME: this checking should be done at parsing stage, without progressing to
// mongoBackend. For the moment we can live this here, but we should remove in the future
// (old issue #95)
//
responseP->statusCode.fill(SccContextElementNotFound);
LM_W(("Bad Input (invalid OID format)"));
return SccOk;
}
if (!collectionFindOne(getSubscribeContextCollectionName(tenant), BSON("_id" << id), &sub, &err))
{
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo db exception)", reqSemTaken);
responseP->statusCode.fill(SccReceiverInternalError, err);
return SccOk;
}
if (sub.isEmpty())
{
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (no subscriptions found)", reqSemTaken);
responseP->statusCode.fill(SccContextElementNotFound, std::string("subscriptionId: /") + requestP->subscriptionId.get() + "/");
return SccOk;
}
/* Remove document in MongoDB */
// FIXME: I would prefer to do the find and remove in a single operation. Is there something similar
// to findAndModify for this?
if (!collectionRemove(getSubscribeContextCollectionName(tenant), BSON("_id" << OID(requestP->subscriptionId.get())), &err))
{
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo db exception)", reqSemTaken);
responseP->statusCode.fill(SccReceiverInternalError, err);
return SccOk;
}
/* Destroy any previous ONTIMEINTERVAL thread */
getNotifier()->destroyOntimeIntervalThreads(requestP->subscriptionId.get());
// FIXME P7: mongoSubCache stuff could be avoided if subscription is not patterned
//
// Removing subscription from mongo subscription cache
//
LM_T(LmtMongoSubCache, ("removing subscription '%s' (tenant '%s') from mongo subscription cache", requestP->subscriptionId.get().c_str(), tenant.c_str()));
cacheSemTake(__FUNCTION__, "Removing subscription from cache");
CachedSubscription* cSubP = mongoSubCacheItemLookup(tenant.c_str(), requestP->subscriptionId.get().c_str());
if (cSubP != NULL) // Will only enter here if wildcard subscription
{
mongoSubCacheItemRemove(cSubP);
}
cacheSemGive(__FUNCTION__, "Removing subscription from cache");
reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request", reqSemTaken);
responseP->statusCode.fill(SccOk);
return SccOk;
}
Status LegacyReplicationCoordinator::processReplSetInitiate(OperationContext* txn,
const BSONObj& givenConfig,
BSONObjBuilder* resultObj) {
log() << "replSet replSetInitiate admin command received from client" << rsLog;
if( !replSet ) {
return Status(ErrorCodes::NoReplicationEnabled, "server is not running with --replSet");
}
if( theReplSet ) {
resultObj->append("info",
"try querying " + rsConfigNs + " to see current configuration");
return Status(ErrorCodes::AlreadyInitialized, "already initialized");
}
try {
{
// just make sure we can get a write lock before doing anything else. we'll
// reacquire one later. of course it could be stuck then, but this check lowers the
// risk if weird things are up.
time_t t = time(0);
Lock::GlobalWrite lk(txn->lockState());
if( time(0)-t > 10 ) {
return Status(ErrorCodes::ExceededTimeLimit,
"took a long time to get write lock, so not initiating. "
"Initiate when server less busy?");
}
/* check that we don't already have an oplog. that could cause issues.
it is ok if the initiating member has *other* data than that.
*/
BSONObj o;
if( Helpers::getFirst(txn, rsoplog, o) ) {
return Status(ErrorCodes::AlreadyInitialized,
rsoplog + string(" is not empty on the initiating member. "
"cannot initiate."));
}
}
if( ReplSet::startupStatus == ReplSet::BADCONFIG ) {
resultObj->append("info", ReplSet::startupStatusMsg.get());
return Status(ErrorCodes::InvalidReplicaSetConfig,
"server already in BADCONFIG state (check logs); not initiating");
}
if( ReplSet::startupStatus != ReplSet::EMPTYCONFIG ) {
resultObj->append("startupStatus", ReplSet::startupStatus);
resultObj->append("info", replSettings.replSet);
return Status(ErrorCodes::InvalidReplicaSetConfig,
"all members and seeds must be reachable to initiate set");
}
BSONObj configObj;
if (!givenConfig.isEmpty()) {
configObj = givenConfig;
} else {
resultObj->append("info2", "no configuration explicitly specified -- making one");
log() << "replSet info initiate : no configuration specified. "
"Using a default configuration for the set" << rsLog;
string name;
vector<HostAndPort> seeds;
set<HostAndPort> seedSet;
parseReplSetSeedList(replSettings.replSet, name, seeds, seedSet); // may throw...
BSONObjBuilder b;
b.append("_id", name);
BSONObjBuilder members;
members.append("0", BSON( "_id" << 0 << "host" << HostAndPort::me().toString() ));
resultObj->append("me", HostAndPort::me().toString());
for( unsigned i = 0; i < seeds.size(); i++ ) {
members.append(BSONObjBuilder::numStr(i+1),
BSON( "_id" << i+1 << "host" << seeds[i].toString()));
}
b.appendArray("members", members.obj());
configObj = b.obj();
log() << "replSet created this configuration for initiation : " <<
configObj.toString() << rsLog;
}
scoped_ptr<ReplSetConfig> newConfig;
try {
newConfig.reset(ReplSetConfig::make(configObj));
} catch (const DBException& e) {
log() << "replSet replSetInitiate exception: " << e.what() << rsLog;
return Status(ErrorCodes::InvalidReplicaSetConfig,
mongoutils::str::stream() << "couldn't parse cfg object " << e.what());
}
if( newConfig->version > 1 ) {
return Status(ErrorCodes::InvalidReplicaSetConfig,
"can't initiate with a version number greater than 1");
}
log() << "replSet replSetInitiate config object parses ok, " <<
newConfig->members.size() << " members specified" << rsLog;
checkMembersUpForConfigChange(*newConfig, *resultObj, true);
log() << "replSet replSetInitiate all members seem up" << rsLog;
createOplog();
Lock::GlobalWrite lk(txn->lockState());
BSONObj comment = BSON( "msg" << "initiating set");
newConfig->saveConfigLocally(comment);
log() << "replSet replSetInitiate config now saved locally. "
"Should come online in about a minute." << rsLog;
resultObj->append("info",
"Config now saved locally. Should come online in about a minute.");
ReplSet::startupStatus = ReplSet::SOON;
ReplSet::startupStatusMsg.set("Received replSetInitiate - "
"should come online shortly.");
}
catch(const DBException& e ) {
return e.toStatus();
}
return Status::OK();
}
Status ModifierPush::init(const BSONElement& modExpr, const Options& opts, bool* positional) {
//
// field name analysis
//
// Break down the field name into its 'dotted' components (aka parts) and check that
// the field is fit for updates.
_fieldRef.parse(modExpr.fieldName());
Status status = fieldchecker::isUpdatable(_fieldRef);
if (!status.isOK()) {
return status;
}
// If a $-positional operator was used, get the index in which it occurred
// and ensure only one occurrence.
size_t foundCount;
bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount);
if (positional)
*positional = foundDollar;
if (foundDollar && foundCount > 1) {
return Status(ErrorCodes::BadValue,
str::stream() << "Too many positional (i.e. '$') elements found in path '"
<< _fieldRef.dottedField()
<< "'");
}
//
// value analysis
//
// Are the target push values safe to store?
BSONElement sliceElem;
BSONElement sortElem;
BSONElement positionElem;
switch (modExpr.type()) {
case Array:
if (_pushMode == PUSH_ALL) {
_eachMode = true;
Status status = parseEachMode(
PUSH_ALL, modExpr, &_eachElem, &sliceElem, &sortElem, &positionElem);
if (!status.isOK()) {
return status;
}
} else {
_val = modExpr;
}
break;
case Object:
if (_pushMode == PUSH_ALL) {
return Status(ErrorCodes::BadValue,
str::stream() << "$pushAll requires an array of values "
"but was given an embedded document.");
}
// If any known clause ($each, $slice, or $sort) is present, we'd assume
// we're using the $each variation of push and would parse accodingly.
_eachMode = inEachMode(modExpr);
if (_eachMode) {
Status status = parseEachMode(
PUSH_NORMAL, modExpr, &_eachElem, &sliceElem, &sortElem, &positionElem);
if (!status.isOK()) {
return status;
}
} else {
_val = modExpr;
}
break;
default:
if (_pushMode == PUSH_ALL) {
return Status(ErrorCodes::BadValue,
str::stream() << "$pushAll requires an array of values "
"but was given type: "
<< typeName(modExpr.type()));
}
_val = modExpr;
break;
}
// Is slice present and correct?
if (sliceElem.type() != EOO) {
if (_pushMode == PUSH_ALL) {
return Status(ErrorCodes::BadValue, "cannot use $slice in $pushAll");
}
if (!sliceElem.isNumber()) {
return Status(ErrorCodes::BadValue,
str::stream() << "The value for $slice must "
"be a numeric value but was given type: "
<< typeName(sliceElem.type()));
}
// TODO: Cleanup and unify numbers wrt getting int32/64 bson values (from doubles)
// If the value of slice is not fraction, even if it's a double, we allow it. The
// reason here is that the shell will use doubles by default unless told otherwise.
const double doubleVal = sliceElem.numberDouble();
if (doubleVal - static_cast<int64_t>(doubleVal) != 0) {
return Status(ErrorCodes::BadValue, "The $slice value in $push cannot be fractional");
}
_slice = sliceElem.numberLong();
_slicePresent = true;
}
// Is position present and correct?
if (positionElem.type() != EOO) {
if (_pushMode == PUSH_ALL) {
return Status(ErrorCodes::BadValue, "cannot use $position in $pushAll");
}
// Check that $position can be represented by a 32-bit integer.
switch (positionElem.type()) {
case NumberInt:
break;
case NumberLong:
if (positionElem.numberInt() != positionElem.numberLong()) {
return Status(
ErrorCodes::BadValue,
"The $position value in $push must be representable as a 32-bit integer.");
}
break;
case NumberDouble: {
const auto doubleVal = positionElem.numberDouble();
if (doubleVal != 0.0) {
if (!std::isnormal(doubleVal) || (doubleVal != positionElem.numberInt())) {
return Status(ErrorCodes::BadValue,
"The $position value in $push must be representable as a "
"32-bit integer.");
}
}
break;
}
default:
return Status(ErrorCodes::BadValue,
str::stream() << "The value for $position must "
"be a non-negative numeric value, not of type: "
<< typeName(positionElem.type()));
}
if (positionElem.numberInt() < 0) {
return {
Status(ErrorCodes::BadValue, "The $position value in $push must be non-negative.")};
}
_startPosition = size_t(positionElem.numberInt());
}
// Is sort present and correct?
if (sortElem.type() != EOO) {
if (_pushMode == PUSH_ALL) {
return Status(ErrorCodes::BadValue, "cannot use $sort in $pushAll");
}
if (sortElem.type() != Object && !sortElem.isNumber()) {
return Status(ErrorCodes::BadValue,
"The $sort is invalid: use 1/-1 to sort the whole element, "
"or {field:1/-1} to sort embedded fields");
}
if (sortElem.isABSONObj()) {
BSONObj sortObj = sortElem.embeddedObject();
if (sortObj.isEmpty()) {
return Status(ErrorCodes::BadValue,
"The $sort pattern is empty when it should be a set of fields.");
}
// Check if the sort pattern is sound.
BSONObjIterator sortIter(sortObj);
while (sortIter.more()) {
BSONElement sortPatternElem = sortIter.next();
// We require either <field>: 1 or -1 for asc and desc.
if (!isPatternElement(sortPatternElem)) {
return Status(ErrorCodes::BadValue,
"The $sort element value must be either 1 or -1");
}
// All fields parts must be valid.
FieldRef sortField(sortPatternElem.fieldName());
if (sortField.numParts() == 0) {
return Status(ErrorCodes::BadValue, "The $sort field cannot be empty");
}
for (size_t i = 0; i < sortField.numParts(); i++) {
if (sortField.getPart(i).size() == 0) {
return Status(ErrorCodes::BadValue,
str::stream() << "The $sort field is a dotted field "
"but has an empty part: "
<< sortField.dottedField());
}
}
}
_sort = PatternElementCmp(sortElem.embeddedObject(), opts.collator);
} else {
// Ensure the sortElem number is valid.
if (!isPatternElement(sortElem)) {
return Status(ErrorCodes::BadValue,
"The $sort element value must be either 1 or -1");
}
_sort = PatternElementCmp(BSON("" << sortElem.number()), opts.collator);
}
_sortPresent = true;
}
return Status::OK();
}
