Пример #1
0
    /* called on a reconfig AND on initiate
       throws
       @param initial true when initiating
    */
    void checkMembersUpForConfigChange(const ReplSetConfig& cfg, BSONObjBuilder& result, bool initial) {
        int failures = 0, allVotes = 0, allowableFailures = 0;
        int me = 0;
        stringstream selfs;
        for( vector<ReplSetConfig::MemberCfg>::const_iterator i = cfg.members.begin(); i != cfg.members.end(); i++ ) {
            if( i->h.isSelf() ) {
                me++;
                if( me > 1 )
                    selfs << ',';
                selfs << i->h.toString();
                if( !i->potentiallyHot() ) {
                    uasserted(13420, "initiation and reconfiguration of a replica set must be sent to a node that can become primary");
                }
            }
            allVotes += i->votes;
        }
        allowableFailures = allVotes - (allVotes/2 + 1);

        uassert(13278, "bad config: isSelf is true for multiple hosts: " + selfs.str(), me <= 1); // dups?
        if( me != 1 ) {
            stringstream ss;
            ss << "can't find self in the replset config";
            if( !cmdLine.isDefaultPort() ) ss << " my port: " << cmdLine.port;
            if( me != 0 ) ss << " found: " << me;
            uasserted(13279, ss.str());
        }

        vector<string> down;
        for( vector<ReplSetConfig::MemberCfg>::const_iterator i = cfg.members.begin(); i != cfg.members.end(); i++ ) {
            // we know we're up
            if (i->h.isSelf()) {
                continue;
            }

            BSONObj res;
            {
                bool ok = false;
                try {
                    int theirVersion = -1000;
                    ok = requestHeartbeat(cfg._id, "", i->h.toString(), res, -1, theirVersion, initial/*check if empty*/);
                    if( theirVersion >= cfg.version ) {
                        stringstream ss;
                        ss << "replSet member " << i->h.toString() << " has too new a config version (" << theirVersion << ") to reconfigure";
                        uasserted(13259, ss.str());
                    }
                }
                catch(DBException& e) {
                    log() << "replSet cmufcc requestHeartbeat " << i->h.toString() << " : " << e.toString() << rsLog;
                }
                catch(...) {
                    log() << "replSet cmufcc error exception in requestHeartbeat?" << rsLog;
                }
                if( res.getBoolField("mismatch") )
                    uasserted(13145, "set name does not match the set name host " + i->h.toString() + " expects");
                if( *res.getStringField("set") ) {
                    if( cfg.version <= 1 ) {
                        // this was to be initiation, no one should be initiated already.
                        uasserted(13256, "member " + i->h.toString() + " is already initiated");
                    }
                    else {
                        // Assure no one has a newer config.
                        if( res["v"].Int() >= cfg.version ) {
                            uasserted(13341, "member " + i->h.toString() + " has a config version >= to the new cfg version; cannot change config");
                        }
                    }
                }
                if( !ok && !res["rs"].trueValue() ) {
                    down.push_back(i->h.toString());

                    if( !res.isEmpty() ) {
                        /* strange.  got a response, but not "ok". log it. */
                        log() << "replSet warning " << i->h.toString() << " replied: " << res.toString() << rsLog;
                    }

                    bool allowFailure = false;
                    failures += i->votes;
                    if( !initial && failures <= allowableFailures ) {
                        const Member* m = theReplSet->findById( i->_id );
                        if( m ) {
                            verify( m->h().toString() == i->h.toString() );
                        }
                        // it's okay if the down member isn't part of the config,
                        // we might be adding a new member that isn't up yet
                        allowFailure = true;
                    }

                    if( !allowFailure ) {
                        string msg = string("need all members up to initiate, not ok : ") + i->h.toString();
                        if( !initial )
                            msg = string("need most members up to reconfigure, not ok : ") + i->h.toString();
                        uasserted(13144, msg);
                    }
                }
            }
            if( initial ) {
                bool hasData = res["hasData"].Bool();
                uassert(13311, "member " + i->h.toString() + " has data already, cannot initiate set.  All members except initiator must be empty.",
                        !hasData || i->h.isSelf());
            }
        }
        if (down.size() > 0) {
            result.append("down", down);
        }
    }
Пример #2
0
    int run() {
        bool usingMongos = isMongos();
        int serverAuthzVersion = 0;
        BSONObj dumpQuery;

        if (mongoDumpGlobalParams.repair){
            return repair();
        }

        {
            if (mongoDumpGlobalParams.query.size()) {
                dumpQuery = fromjson(mongoDumpGlobalParams.query);
            }
        }

        if (mongoDumpGlobalParams.dumpUsersAndRoles) {
            uassertStatusOK(auth::getRemoteStoredAuthorizationVersion(&conn(true),
                                                                      &serverAuthzVersion));
            uassert(17369,
                    mongoutils::str::stream() << "Backing up users and roles is only supported for "
                            "clusters with auth schema versions 1 or 3, found: " <<
                            serverAuthzVersion,
                    serverAuthzVersion == AuthorizationManager::schemaVersion24 ||
                    serverAuthzVersion == AuthorizationManager::schemaVersion26Final);
        }

        string opLogName = "";
        unsigned long long opLogStart = 0;
        if (mongoDumpGlobalParams.useOplog) {

            BSONObj isMaster;
            conn("true").simpleCommand("admin", &isMaster, "isMaster");

            if (isMaster.hasField("hosts")) { // if connected to replica set member
                opLogName = "local.oplog.rs";
            }
            else {
                opLogName = "local.oplog.$main";
                if ( ! isMaster["ismaster"].trueValue() ) {
                    toolError() << "oplog mode is only supported on master or replica set member"
                              << std::endl;
                    return -1;
                }
            }

            BSONObj op = conn(true).findOne(opLogName, Query().sort("$natural", -1), 0, QueryOption_SlaveOk);
            if (op.isEmpty()) {
                toolError() << "No operations in oplog. Please ensure you are connecting to a "
                            << "master." << std::endl;
                return -1;
            }

            verify(op["ts"].type() == Timestamp);
            opLogStart = op["ts"]._numberLong();
        }

        // check if we're outputting to stdout
        if (mongoDumpGlobalParams.outputDirectory == "-") {
            if (toolGlobalParams.db != "" && toolGlobalParams.coll != "") {
                writeCollectionStdout(toolGlobalParams.db + "." + toolGlobalParams.coll, dumpQuery,
                                      usingMongos);
                return 0;
            }
            else {
                toolError() << "You must specify database and collection to print to stdout"
                          << std::endl;
                return -1;
            }
        }

        boost::filesystem::path root(mongoDumpGlobalParams.outputDirectory);

        if (toolGlobalParams.db == "") {
            if (toolGlobalParams.coll != "") {
                toolError() << "--db must be specified with --collection" << std::endl;
                return -1;
            }

            toolInfoLog() << "all dbs" << std::endl;

            BSONObj res = conn( true ).findOne( "admin.$cmd" , BSON( "listDatabases" << 1 ) );
            if ( ! res["databases"].isABSONObj() ) {
                toolError() << "output of listDatabases isn't what we expected, no 'databases' "
                          << "field:\n" << res << std::endl;
                return -2;
            }
            BSONObj dbs = res["databases"].embeddedObjectUserCheck();
            set<string> keys;
            dbs.getFieldNames( keys );
            for ( set<string>::iterator i = keys.begin() ; i != keys.end() ; i++ ) {
                string key = *i;
                
                if ( ! dbs[key].isABSONObj() ) {
                    toolError() << "database field not an document key: " << key << " value: "
                              << dbs[key] << std::endl;
                    return -3;
                }

                BSONObj dbobj = dbs[key].embeddedObjectUserCheck();

                const char * dbName = dbobj.getField( "name" ).valuestr();
                if ( (string)dbName == "local" )
                    continue;

                boost::filesystem::path outdir = root / dbName;
                toolInfoLog() << "DATABASE: " << dbName << "\t to \t" << outdir.string()
                        << std::endl;
                go ( dbName , "", dumpQuery, outdir, "", usingMongos );
            }
        }
        else {
            boost::filesystem::path outdir = root / toolGlobalParams.db;
            toolInfoLog() << "DATABASE: " << toolGlobalParams.db << "\t to \t" << outdir.string()
                    << std::endl;
            go(toolGlobalParams.db, toolGlobalParams.coll, dumpQuery, outdir, "", usingMongos);
            if (mongoDumpGlobalParams.dumpUsersAndRoles &&
                    serverAuthzVersion == AuthorizationManager::schemaVersion26Final &&
                    toolGlobalParams.db != "admin") {
                toolInfoLog() << "Backing up user and role data for the " << toolGlobalParams.db <<
                        " database";
                Query query = Query(BSON("db" << toolGlobalParams.db));
                go("admin", "system.users", query, outdir, "$admin.system.users", usingMongos);
                go("admin", "system.roles", query, outdir, "$admin.system.roles", usingMongos);
            }
        }

        if (!opLogName.empty()) {
            BSONObjBuilder b;
            b.appendTimestamp("$gt", opLogStart);

            dumpQuery = BSON("ts" << b.obj());

            writeCollectionFile( opLogName , dumpQuery, root / "oplog.bson", usingMongos );
        }

        return 0;
    }
Пример #3
0
Status ClusterAggregate::runAggregate(OperationContext* txn,
                                      const Namespaces& namespaces,
                                      BSONObj cmdObj,
                                      int options,
                                      BSONObjBuilder* result) {
    auto dbname = namespaces.executionNss.db().toString();
    auto status = grid.catalogCache()->getDatabase(txn, dbname);
    if (!status.isOK()) {
        appendEmptyResultSet(*result, status.getStatus(), namespaces.requestedNss.ns());
        return Status::OK();
    }

    std::shared_ptr<DBConfig> conf = status.getValue();

    if (!conf->isShardingEnabled()) {
        return aggPassthrough(txn, namespaces, conf, cmdObj, result, options);
    }

    auto request = AggregationRequest::parseFromBSON(namespaces.executionNss, cmdObj);
    if (!request.isOK()) {
        return request.getStatus();
    }

    boost::intrusive_ptr<ExpressionContext> mergeCtx =
        new ExpressionContext(txn, request.getValue());
    mergeCtx->inRouter = true;
    // explicitly *not* setting mergeCtx->tempDir

    // Parse and optimize the pipeline specification.
    auto pipeline = Pipeline::parse(request.getValue().getPipeline(), mergeCtx);
    if (!pipeline.isOK()) {
        return pipeline.getStatus();
    }

    for (auto&& ns : pipeline.getValue()->getInvolvedCollections()) {
        uassert(28769, str::stream() << ns.ns() << " cannot be sharded", !conf->isSharded(ns.ns()));
        // We won't try to execute anything on a mongos, but we still have to populate this map
        // so that any $lookups etc will be able to have a resolved view definition. It's okay
        // that this is incorrect, we will repopulate the real resolved namespace map on the
        // mongod.
        // TODO SERVER-25038 This should become unnecessary once we can get the involved
        // namespaces before parsing.
        mergeCtx->resolvedNamespaces[ns.coll()] = {ns, std::vector<BSONObj>{}};
    }

    if (!conf->isSharded(namespaces.executionNss.ns())) {
        return aggPassthrough(txn, namespaces, conf, cmdObj, result, options);
    }

    ChunkManagerPtr chunkMgr = conf->getChunkManager(txn, namespaces.executionNss.ns());

    // If there was no collation specified, but there is a default collation for the collation,
    // use that.
    if (request.getValue().getCollation().isEmpty() && chunkMgr->getDefaultCollator()) {
        mergeCtx->setCollator(chunkMgr->getDefaultCollator()->clone());
    }

    // Now that we know the collation we'll be using, inject the ExpressionContext and optimize.
    // TODO SERVER-25038: this must happen before we parse the pipeline, since we can make
    // string comparisons during parse time.
    pipeline.getValue()->injectExpressionContext(mergeCtx);
    pipeline.getValue()->optimizePipeline();

    // If the first $match stage is an exact match on the shard key (with a simple collation or
    // no string matching), we only have to send it to one shard, so send the command to that
    // shard.
    BSONObj firstMatchQuery = pipeline.getValue()->getInitialQuery();
    BSONObj shardKeyMatches;
    shardKeyMatches = uassertStatusOK(
        chunkMgr->getShardKeyPattern().extractShardKeyFromQuery(txn, firstMatchQuery));
    bool singleShard = false;
    if (!shardKeyMatches.isEmpty()) {
        auto chunk = chunkMgr->findIntersectingChunk(
            txn, shardKeyMatches, request.getValue().getCollation());
        if (chunk.isOK()) {
            singleShard = true;
        }
    }

    // Don't need to split pipeline if the first $match is an exact match on shard key, unless
    // there is a stage that needs to be run on the primary shard.
    const bool needPrimaryShardMerger = pipeline.getValue()->needsPrimaryShardMerger();
    const bool needSplit = !singleShard || needPrimaryShardMerger;

    // Split the pipeline into pieces for mongod(s) and this mongos. If needSplit is true,
    // 'pipeline' will become the merger side.
    boost::intrusive_ptr<Pipeline> shardPipeline(needSplit ? pipeline.getValue()->splitForSharded()
                                                           : pipeline.getValue());

    // Create the command for the shards. The 'fromRouter' field means produce output to be
    // merged.
    MutableDocument commandBuilder(request.getValue().serializeToCommandObj());
    commandBuilder[AggregationRequest::kPipelineName] = Value(shardPipeline->serialize());
    if (needSplit) {
        commandBuilder[AggregationRequest::kFromRouterName] = Value(true);
        commandBuilder[AggregationRequest::kCursorName] =
            Value(DOC(AggregationRequest::kBatchSizeName << 0));
    }

    // These fields are not part of the AggregationRequest since they are not handled by the
    // aggregation subsystem, so we serialize them separately.
    const std::initializer_list<StringData> fieldsToPropagateToShards = {
        "$queryOptions", "readConcern", QueryRequest::cmdOptionMaxTimeMS,
    };
    for (auto&& field : fieldsToPropagateToShards) {
        commandBuilder[field] = Value(cmdObj[field]);
    }

    BSONObj shardedCommand = commandBuilder.freeze().toBson();
    BSONObj shardQuery = shardPipeline->getInitialQuery();

    // Run the command on the shards
    // TODO need to make sure cursors are killed if a retry is needed
    std::vector<Strategy::CommandResult> shardResults;
    Strategy::commandOp(txn,
                        dbname,
                        shardedCommand,
                        options,
                        namespaces.executionNss.ns(),
                        shardQuery,
                        request.getValue().getCollation(),
                        &shardResults);

    if (mergeCtx->isExplain) {
        // This must be checked before we start modifying result.
        uassertAllShardsSupportExplain(shardResults);

        if (needSplit) {
            *result << "needsPrimaryShardMerger" << needPrimaryShardMerger << "splitPipeline"
                    << DOC("shardsPart" << shardPipeline->writeExplainOps() << "mergerPart"
                                        << pipeline.getValue()->writeExplainOps());
        } else {
            *result << "splitPipeline" << BSONNULL;
        }

        BSONObjBuilder shardExplains(result->subobjStart("shards"));
        for (size_t i = 0; i < shardResults.size(); i++) {
            shardExplains.append(shardResults[i].shardTargetId,
                                 BSON("host" << shardResults[i].target.toString() << "stages"
                                             << shardResults[i].result["stages"]));
        }

        return Status::OK();
    }

    if (!needSplit) {
        invariant(shardResults.size() == 1);
        invariant(shardResults[0].target.getServers().size() == 1);
        auto executorPool = grid.getExecutorPool();
        const BSONObj reply =
            uassertStatusOK(storePossibleCursor(shardResults[0].target.getServers()[0],
                                                shardResults[0].result,
                                                namespaces.requestedNss,
                                                executorPool->getArbitraryExecutor(),
                                                grid.getCursorManager()));
        result->appendElements(reply);
        return getStatusFromCommandResult(reply);
    }

    pipeline.getValue()->addInitialSource(
        DocumentSourceMergeCursors::create(parseCursors(shardResults), mergeCtx));

    MutableDocument mergeCmd(request.getValue().serializeToCommandObj());
    mergeCmd["pipeline"] = Value(pipeline.getValue()->serialize());
    mergeCmd["cursor"] = Value(cmdObj["cursor"]);

    if (cmdObj.hasField("$queryOptions")) {
        mergeCmd["$queryOptions"] = Value(cmdObj["$queryOptions"]);
    }

    if (cmdObj.hasField(QueryRequest::cmdOptionMaxTimeMS)) {
        mergeCmd[QueryRequest::cmdOptionMaxTimeMS] =
            Value(cmdObj[QueryRequest::cmdOptionMaxTimeMS]);
    }

    mergeCmd.setField("writeConcern", Value(cmdObj["writeConcern"]));

    // Not propagating readConcern to merger since it doesn't do local reads.

    // If the user didn't specify a collation already, make sure there's a collation attached to
    // the merge command, since the merging shard may not have the collection metadata.
    if (mergeCmd.peek()["collation"].missing()) {
        mergeCmd.setField("collation",
                          mergeCtx->getCollator()
                              ? Value(mergeCtx->getCollator()->getSpec().toBSON())
                              : Value(Document{CollationSpec::kSimpleSpec}));
    }

    std::string outputNsOrEmpty;
    if (DocumentSourceOut* out =
            dynamic_cast<DocumentSourceOut*>(pipeline.getValue()->getSources().back().get())) {
        outputNsOrEmpty = out->getOutputNs().ns();
    }

    // Run merging command on random shard, unless a stage needs the primary shard. Need to use
    // ShardConnection so that the merging mongod is sent the config servers on connection init.
    auto& prng = txn->getClient()->getPrng();
    const auto& mergingShardId = needPrimaryShardMerger
        ? conf->getPrimaryId()
        : shardResults[prng.nextInt32(shardResults.size())].shardTargetId;
    const auto mergingShard = uassertStatusOK(grid.shardRegistry()->getShard(txn, mergingShardId));

    ShardConnection conn(mergingShard->getConnString(), outputNsOrEmpty);
    BSONObj mergedResults =
        aggRunCommand(conn.get(), namespaces, mergeCmd.freeze().toBson(), options);
    conn.done();

    if (auto wcErrorElem = mergedResults["writeConcernError"]) {
        appendWriteConcernErrorToCmdResponse(mergingShardId, wcErrorElem, *result);
    }

    // Copy output from merging (primary) shard to the output object from our command.
    // Also, propagates errmsg and code if ok == false.
    result->appendElementsUnique(mergedResults);

    return getStatusFromCommandResult(result->asTempObj());
}
Пример #4
0
    /**
     * Run a query -- includes checking for and running a Command.
     * @return points to ns if exhaust mode. 0=normal mode
     * @locks the db mutex for reading (and potentially for writing temporarily to create a new db).
     * @yields the db mutex periodically after acquiring it.
     * @asserts on scan and order memory exhaustion and other cases.
     */
    const char *runQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) {
        shared_ptr<ParsedQuery> pq_shared( new ParsedQuery(q) );
        ParsedQuery& pq( *pq_shared );
        BSONObj jsobj = q.query;
        int queryOptions = q.queryOptions;
        const char *ns = q.ns;

        if( logLevel >= 2 )
            log() << "runQuery called " << ns << " " << jsobj << endl;

        curop.debug().ns = ns;
        curop.debug().ntoreturn = pq.getNumToReturn();
        curop.debug().query = jsobj;
        curop.setQuery(jsobj);

        // Run a command.
        
        if ( pq.couldBeCommand() ) {
            BufBuilder bb;
            bb.skip(sizeof(QueryResult));
            BSONObjBuilder cmdResBuf;
            if ( runCommands(ns, jsobj, curop, bb, cmdResBuf, false, queryOptions) ) {
                curop.debug().iscommand = true;
                curop.debug().query = jsobj;
                curop.markCommand();

                auto_ptr< QueryResult > qr;
                qr.reset( (QueryResult *) bb.buf() );
                bb.decouple();
                qr->setResultFlagsToOk();
                qr->len = bb.len();
                curop.debug().responseLength = bb.len();
                qr->setOperation(opReply);
                qr->cursorId = 0;
                qr->startingFrom = 0;
                qr->nReturned = 1;
                result.setData( qr.release(), true );
            }
            else {
                uasserted(13530, "bad or malformed command request?");
            }
            return 0;
        }

        bool explain = pq.isExplain();
        BSONObj order = pq.getOrder();
        BSONObj query = pq.getFilter();

        /* The ElemIter will not be happy if this isn't really an object. So throw exception
           here when that is true.
           (Which may indicate bad data from client.)
        */
        if ( query.objsize() == 0 ) {
            out() << "Bad query object?\n  jsobj:";
            out() << jsobj.toString() << "\n  query:";
            out() << query.toString() << endl;
            uassert( 10110 , "bad query object", false);
        }

        Client::ReadContext ctx( ns , dbpath ); // read locks
        const ConfigVersion shardingVersionAtStart = shardingState.getVersion( ns );

        replVerifyReadsOk(&pq);

        if ( pq.hasOption( QueryOption_CursorTailable ) ) {
            NamespaceDetails *d = nsdetails( ns );
            uassert( 13051, "tailable cursor requested on non capped collection", d && d->capped );
            const BSONObj nat1 = BSON( "$natural" << 1 );
            if ( order.isEmpty() ) {
                order = nat1;
            }
            else {
                uassert( 13052, "only {$natural:1} order allowed for tailable cursor", order == nat1 );
            }
        }

        // Run a simple id query.
        
        if ( ! (explain || pq.showDiskLoc()) && isSimpleIdQuery( query ) && !pq.hasOption( QueryOption_CursorTailable ) ) {

            int n = 0;
            bool nsFound = false;
            bool indexFound = false;

            BSONObj resObject;
            Client& c = cc();
            bool found = Helpers::findById( c, ns , query , resObject , &nsFound , &indexFound );
            if ( nsFound == false || indexFound == true ) {
                BufBuilder bb(sizeof(QueryResult)+resObject.objsize()+32);
                bb.skip(sizeof(QueryResult));
                
                curop.debug().idhack = true;
                if ( found ) {
                    n = 1;
                    fillQueryResultFromObj( bb , pq.getFields() , resObject );
                }
                auto_ptr< QueryResult > qr;
                qr.reset( (QueryResult *) bb.buf() );
                bb.decouple();
                qr->setResultFlagsToOk();
                qr->len = bb.len();
                
                curop.debug().responseLength = bb.len();
                qr->setOperation(opReply);
                qr->cursorId = 0;
                qr->startingFrom = 0;
                qr->nReturned = n;
                result.setData( qr.release(), true );
                return NULL;
            }
        }
        
        // Run a regular query.
        
        BSONObj oldPlan;
        if ( explain && ! pq.hasIndexSpecifier() ) {
            MultiPlanScanner mps( ns, query, shared_ptr<Projection>(), order );
            if ( mps.usingCachedPlan() ) {
                oldPlan =
                mps.oldExplain().firstElement().embeddedObject()
                .firstElement().embeddedObject().getOwned();
            }
        }

        // In some cases the query may be retried if there is an in memory sort size assertion.
        for( int retry = 0; retry < 2; ++retry ) {
            try {
                return queryWithQueryOptimizer( m, queryOptions, ns, jsobj, curop, query, order,
                                               pq_shared, oldPlan, shardingVersionAtStart, result );
            } catch ( const QueryRetryException & ) {
                verify( 16088, retry == 0 );
            }
        }
        verify( 16082, false );
        return 0;
    }
        bool run( const string& dbname,
                  BSONObj& cmdObj,
                  int,
                  string& errmsg,
                  BSONObjBuilder& result,
                  bool ) {

            string ns = parseNs(dbname, cmdObj);

            if ( ns.size() == 0 ) {
                errmsg = "no namespace specified";
                return false;
            }

            vector<BSONObj> bounds;
            if ( !FieldParser::extract( cmdObj, boundsField, &bounds, &errmsg ) ) {
                return false;
            }

            if ( bounds.size() == 0 ) {
                errmsg = "no bounds were specified";
                return false;
            }

            if ( bounds.size() != 2 ) {
                errmsg = "only a min and max bound may be specified";
                return false;
            }

            BSONObj minKey = bounds[0];
            BSONObj maxKey = bounds[1];

            if ( minKey.isEmpty() ) {
                errmsg = "no min key specified";
                return false;
            }

            if ( maxKey.isEmpty() ) {
                errmsg = "no max key specified";
                return false;
            }

            // This refreshes the chunk metadata if stale.
            refreshChunkCache( NamespaceString( ns ) );

            ShardPtr mergeShard = guessMergeShard( NamespaceString( ns ), minKey );

            if ( !mergeShard ) {
                errmsg = (string)"could not find shard for merge range starting at "
                                 + minKey.toString();
                return false;
            }

            BSONObjBuilder remoteCmdObjB;
            remoteCmdObjB.append( cmdObj[ ClusterMergeChunksCommand::nsField() ] );
            remoteCmdObjB.append( cmdObj[ ClusterMergeChunksCommand::boundsField() ] );
            remoteCmdObjB.append( ClusterMergeChunksCommand::configField(),
                                  configServer.getPrimary().getAddress().toString() );
            remoteCmdObjB.append( ClusterMergeChunksCommand::shardNameField(),
                                  mergeShard->getName() );

            BSONObj remoteResult;
            // Throws, but handled at level above.  Don't want to rewrap to preserve exception
            // formatting.
            ScopedDbConnection conn( mergeShard->getAddress() );
            bool ok = conn->runCommand( "admin", remoteCmdObjB.obj(), remoteResult );
            conn.done();

            result.appendElements( remoteResult );
            return ok;
        }
    void ReplicationCoordinatorImpl::_handleHeartbeatResponse(
            const ReplicationExecutor::RemoteCommandCallbackData& cbData, int targetIndex) {

        // remove handle from queued heartbeats
        _untrackHeartbeatHandle(cbData.myHandle);

        // Parse and validate the response.  At the end of this step, if responseStatus is OK then
        // hbResponse is valid.
        Status responseStatus = cbData.response.getStatus();
        if (responseStatus == ErrorCodes::CallbackCanceled) {
            return;
        }

        const HostAndPort& target = cbData.request.target;
        ReplSetHeartbeatResponse hbResponse;
        BSONObj resp;
        if (responseStatus.isOK()) {
            resp = cbData.response.getValue().data;
            responseStatus = hbResponse.initialize(resp, _topCoord->getTerm());
        }
        const Date_t now = _replExecutor.now();
        const OpTime lastApplied = getMyLastOptime();  // Locks and unlocks _mutex.
        Milliseconds networkTime(0);
        StatusWith<ReplSetHeartbeatResponse> hbStatusResponse(hbResponse);

        if (responseStatus.isOK()) {
            networkTime = cbData.response.getValue().elapsedMillis;
            _updateTerm_incallback(hbStatusResponse.getValue().getTerm(), nullptr);
        }
        else {
            log() << "Error in heartbeat request to " << target << "; " << responseStatus;
            if (!resp.isEmpty()) {
                LOG(3) << "heartbeat response: " << resp;
            }

            hbStatusResponse = StatusWith<ReplSetHeartbeatResponse>(responseStatus);
        }

        HeartbeatResponseAction action =
            _topCoord->processHeartbeatResponse(
                    now,
                    networkTime,
                    target,
                    hbStatusResponse,
                    lastApplied);

        if (action.getAction() == HeartbeatResponseAction::NoAction &&
                hbStatusResponse.isOK() &&
                hbStatusResponse.getValue().hasOpTime() &&
                targetIndex >= 0 &&
                hbStatusResponse.getValue().hasState() &&
                hbStatusResponse.getValue().getState() != MemberState::RS_PRIMARY) {
            boost::unique_lock<boost::mutex> lk(_mutex);
            if (hbStatusResponse.getValue().getConfigVersion() == _rsConfig.getConfigVersion()) {
                _updateOpTimeFromHeartbeat_inlock(targetIndex,
                                                  hbStatusResponse.getValue().getOpTime());
                // TODO: Enable with Data Replicator
                //lk.unlock();
                //_dr.slavesHaveProgressed();
            }
        }

        _signalStepDownWaiters();

        _scheduleHeartbeatToTarget(
                target,
                targetIndex,
                std::max(now, action.getNextHeartbeatStartDate()));

        _handleHeartbeatResponseAction(action, hbStatusResponse);
    }
Пример #7
0
 bool IndexType::scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const {
     return ! order.isEmpty();
 }
/* ****************************************************************************
*
* mongoUnsubscribeContext - 
*/
HttpStatusCode mongoUnsubscribeContext(UnsubscribeContextRequest* requestP, UnsubscribeContextResponse* responseP, const std::string& tenant)
{
    bool         reqSemTaken;
    std::string  err;

    reqSemTake(__FUNCTION__, "ngsi10 unsubscribe request", SemWriteOp, &reqSemTaken);

    LM_T(LmtMongo, ("Unsubscribe Context"));

    /* No matter if success or failure, the subscriptionId in the response is always the one
     * in the request */
    responseP->subscriptionId = requestP->subscriptionId;

    if (responseP->subscriptionId.get() == "")
    {
        responseP->statusCode.fill(SccContextElementNotFound);
        LM_W(("Bad Input (no subscriptionId)"));
        return SccOk;
    }

    /* Look for document */
    BSONObj sub;
    OID     id;
    try
    {
      id = OID(requestP->subscriptionId.get());
    }
    catch (const AssertionException &e)
    {
      reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo assertion exception)", reqSemTaken);
      //
      // This happens when OID format is wrong
      // FIXME: this checking should be done at parsing stage, without progressing to
      // mongoBackend. For the moment we can live this here, but we should remove in the future
      // (old issue #95)
      //
      responseP->statusCode.fill(SccContextElementNotFound);
      LM_W(("Bad Input (invalid OID format)"));
      return SccOk;
    }

    if (!collectionFindOne(getSubscribeContextCollectionName(tenant), BSON("_id" << id), &sub, &err))
    {
      reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo db exception)", reqSemTaken);
      responseP->statusCode.fill(SccReceiverInternalError, err);
      return SccOk;
    }

    if (sub.isEmpty())
    {
       reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (no subscriptions found)", reqSemTaken);

       responseP->statusCode.fill(SccContextElementNotFound, std::string("subscriptionId: /") + requestP->subscriptionId.get() + "/");
       return SccOk;
    }

    /* Remove document in MongoDB */
    // FIXME: I would prefer to do the find and remove in a single operation. Is there something similar
    // to findAndModify for this?    
    if (!collectionRemove(getSubscribeContextCollectionName(tenant), BSON("_id" << OID(requestP->subscriptionId.get())), &err))
    {
      reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request (mongo db exception)", reqSemTaken);
      responseP->statusCode.fill(SccReceiverInternalError, err);
      return SccOk;
    }

    /* Destroy any previous ONTIMEINTERVAL thread */
    getNotifier()->destroyOntimeIntervalThreads(requestP->subscriptionId.get());


    // FIXME P7: mongoSubCache stuff could be avoided if subscription is not patterned

    //
    // Removing subscription from mongo subscription cache
    //
    LM_T(LmtMongoSubCache, ("removing subscription '%s' (tenant '%s') from mongo subscription cache", requestP->subscriptionId.get().c_str(), tenant.c_str()));

    cacheSemTake(__FUNCTION__, "Removing subscription from cache");

    CachedSubscription* cSubP = mongoSubCacheItemLookup(tenant.c_str(), requestP->subscriptionId.get().c_str());

    if (cSubP != NULL)  // Will only enter here if wildcard subscription
    {
      mongoSubCacheItemRemove(cSubP);
    }

    cacheSemGive(__FUNCTION__, "Removing subscription from cache");

    reqSemGive(__FUNCTION__, "ngsi10 unsubscribe request", reqSemTaken);

    responseP->statusCode.fill(SccOk);
    return SccOk;
}
Пример #9
0
    Status LegacyReplicationCoordinator::processReplSetInitiate(OperationContext* txn,
                                                                const BSONObj& givenConfig,
                                                                BSONObjBuilder* resultObj) {

        log() << "replSet replSetInitiate admin command received from client" << rsLog;

        if( !replSet ) {
            return Status(ErrorCodes::NoReplicationEnabled, "server is not running with --replSet");
        }

        if( theReplSet ) {
            resultObj->append("info",
                              "try querying " + rsConfigNs + " to see current configuration");
            return Status(ErrorCodes::AlreadyInitialized, "already initialized");
        }

        try {
            {
                // just make sure we can get a write lock before doing anything else.  we'll
                // reacquire one later.  of course it could be stuck then, but this check lowers the
                // risk if weird things are up.
                time_t t = time(0);
                Lock::GlobalWrite lk(txn->lockState());
                if( time(0)-t > 10 ) {
                    return Status(ErrorCodes::ExceededTimeLimit,
                                  "took a long time to get write lock, so not initiating.  "
                                          "Initiate when server less busy?");
                }

                /* check that we don't already have an oplog.  that could cause issues.
                   it is ok if the initiating member has *other* data than that.
                   */
                BSONObj o;
                if( Helpers::getFirst(txn, rsoplog, o) ) {
                    return Status(ErrorCodes::AlreadyInitialized,
                                  rsoplog + string(" is not empty on the initiating member.  "
                                          "cannot initiate."));
                }
            }

            if( ReplSet::startupStatus == ReplSet::BADCONFIG ) {
                resultObj->append("info", ReplSet::startupStatusMsg.get());
                return Status(ErrorCodes::InvalidReplicaSetConfig,
                              "server already in BADCONFIG state (check logs); not initiating");
            }
            if( ReplSet::startupStatus != ReplSet::EMPTYCONFIG ) {
                resultObj->append("startupStatus", ReplSet::startupStatus);
                resultObj->append("info", replSettings.replSet);
                return Status(ErrorCodes::InvalidReplicaSetConfig,
                              "all members and seeds must be reachable to initiate set");
            }

            BSONObj configObj;
            if (!givenConfig.isEmpty()) {
                configObj = givenConfig;
            } else {
                resultObj->append("info2", "no configuration explicitly specified -- making one");
                log() << "replSet info initiate : no configuration specified.  "
                        "Using a default configuration for the set" << rsLog;

                string name;
                vector<HostAndPort> seeds;
                set<HostAndPort> seedSet;
                parseReplSetSeedList(replSettings.replSet, name, seeds, seedSet); // may throw...

                BSONObjBuilder b;
                b.append("_id", name);
                BSONObjBuilder members;
                members.append("0", BSON( "_id" << 0 << "host" << HostAndPort::me().toString() ));
                resultObj->append("me", HostAndPort::me().toString());
                for( unsigned i = 0; i < seeds.size(); i++ ) {
                    members.append(BSONObjBuilder::numStr(i+1),
                                   BSON( "_id" << i+1 << "host" << seeds[i].toString()));
                }
                b.appendArray("members", members.obj());
                configObj = b.obj();
                log() << "replSet created this configuration for initiation : " <<
                        configObj.toString() << rsLog;
            }

            scoped_ptr<ReplSetConfig> newConfig;
            try {
                newConfig.reset(ReplSetConfig::make(configObj));
            } catch (const DBException& e) {
                log() << "replSet replSetInitiate exception: " << e.what() << rsLog;
                return Status(ErrorCodes::InvalidReplicaSetConfig,
                              mongoutils::str::stream() << "couldn't parse cfg object " << e.what());
            }

            if( newConfig->version > 1 ) {
                return Status(ErrorCodes::InvalidReplicaSetConfig,
                              "can't initiate with a version number greater than 1");
            }

            log() << "replSet replSetInitiate config object parses ok, " <<
                    newConfig->members.size() << " members specified" << rsLog;

            checkMembersUpForConfigChange(*newConfig, *resultObj, true);

            log() << "replSet replSetInitiate all members seem up" << rsLog;

            createOplog();

            Lock::GlobalWrite lk(txn->lockState());
            BSONObj comment = BSON( "msg" << "initiating set");
            newConfig->saveConfigLocally(comment);
            log() << "replSet replSetInitiate config now saved locally.  "
                "Should come online in about a minute." << rsLog;
            resultObj->append("info",
                              "Config now saved locally.  Should come online in about a minute.");
            ReplSet::startupStatus = ReplSet::SOON;
            ReplSet::startupStatusMsg.set("Received replSetInitiate - "
                                          "should come online shortly.");
        }
        catch(const DBException& e ) {
            return e.toStatus();
        }

        return Status::OK();
    }
Пример #10
0
Status ModifierPush::init(const BSONElement& modExpr, const Options& opts, bool* positional) {
    //
    // field name analysis
    //

    // Break down the field name into its 'dotted' components (aka parts) and check that
    // the field is fit for updates.
    _fieldRef.parse(modExpr.fieldName());
    Status status = fieldchecker::isUpdatable(_fieldRef);
    if (!status.isOK()) {
        return status;
    }

    // If a $-positional operator was used, get the index in which it occurred
    // and ensure only one occurrence.
    size_t foundCount;
    bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount);

    if (positional)
        *positional = foundDollar;

    if (foundDollar && foundCount > 1) {
        return Status(ErrorCodes::BadValue,
                      str::stream() << "Too many positional (i.e. '$') elements found in path '"
                                    << _fieldRef.dottedField()
                                    << "'");
    }

    //
    // value analysis
    //

    // Are the target push values safe to store?
    BSONElement sliceElem;
    BSONElement sortElem;
    BSONElement positionElem;
    switch (modExpr.type()) {
        case Array:
            if (_pushMode == PUSH_ALL) {
                _eachMode = true;
                Status status = parseEachMode(
                    PUSH_ALL, modExpr, &_eachElem, &sliceElem, &sortElem, &positionElem);
                if (!status.isOK()) {
                    return status;
                }
            } else {
                _val = modExpr;
            }
            break;

        case Object:
            if (_pushMode == PUSH_ALL) {
                return Status(ErrorCodes::BadValue,
                              str::stream() << "$pushAll requires an array of values "
                                               "but was given an embedded document.");
            }

            // If any known clause ($each, $slice, or $sort) is present, we'd assume
            // we're using the $each variation of push and would parse accodingly.
            _eachMode = inEachMode(modExpr);
            if (_eachMode) {
                Status status = parseEachMode(
                    PUSH_NORMAL, modExpr, &_eachElem, &sliceElem, &sortElem, &positionElem);
                if (!status.isOK()) {
                    return status;
                }
            } else {
                _val = modExpr;
            }
            break;

        default:
            if (_pushMode == PUSH_ALL) {
                return Status(ErrorCodes::BadValue,
                              str::stream() << "$pushAll requires an array of values "
                                               "but was given type: "
                                            << typeName(modExpr.type()));
            }

            _val = modExpr;
            break;
    }

    // Is slice present and correct?
    if (sliceElem.type() != EOO) {
        if (_pushMode == PUSH_ALL) {
            return Status(ErrorCodes::BadValue, "cannot use $slice in $pushAll");
        }

        if (!sliceElem.isNumber()) {
            return Status(ErrorCodes::BadValue,
                          str::stream() << "The value for $slice must "
                                           "be a numeric value but was given type: "
                                        << typeName(sliceElem.type()));
        }

        // TODO: Cleanup and unify numbers wrt getting int32/64 bson values (from doubles)

        // If the value of slice is not fraction, even if it's a double, we allow it. The
        // reason here is that the shell will use doubles by default unless told otherwise.
        const double doubleVal = sliceElem.numberDouble();
        if (doubleVal - static_cast<int64_t>(doubleVal) != 0) {
            return Status(ErrorCodes::BadValue, "The $slice value in $push cannot be fractional");
        }

        _slice = sliceElem.numberLong();
        _slicePresent = true;
    }

    // Is position present and correct?
    if (positionElem.type() != EOO) {
        if (_pushMode == PUSH_ALL) {
            return Status(ErrorCodes::BadValue, "cannot use $position in $pushAll");
        }

        // Check that $position can be represented by a 32-bit integer.
        switch (positionElem.type()) {
            case NumberInt:
                break;
            case NumberLong:
                if (positionElem.numberInt() != positionElem.numberLong()) {
                    return Status(
                        ErrorCodes::BadValue,
                        "The $position value in $push must be representable as a 32-bit integer.");
                }
                break;
            case NumberDouble: {
                const auto doubleVal = positionElem.numberDouble();
                if (doubleVal != 0.0) {
                    if (!std::isnormal(doubleVal) || (doubleVal != positionElem.numberInt())) {
                        return Status(ErrorCodes::BadValue,
                                      "The $position value in $push must be representable as a "
                                      "32-bit integer.");
                    }
                }
                break;
            }
            default:
                return Status(ErrorCodes::BadValue,
                              str::stream() << "The value for $position must "
                                               "be a non-negative numeric value, not of type: "
                                            << typeName(positionElem.type()));
        }

        if (positionElem.numberInt() < 0) {
            return {
                Status(ErrorCodes::BadValue, "The $position value in $push must be non-negative.")};
        }

        _startPosition = size_t(positionElem.numberInt());
    }

    // Is sort present and correct?
    if (sortElem.type() != EOO) {
        if (_pushMode == PUSH_ALL) {
            return Status(ErrorCodes::BadValue, "cannot use $sort in $pushAll");
        }

        if (sortElem.type() != Object && !sortElem.isNumber()) {
            return Status(ErrorCodes::BadValue,
                          "The $sort is invalid: use 1/-1 to sort the whole element, "
                          "or {field:1/-1} to sort embedded fields");
        }

        if (sortElem.isABSONObj()) {
            BSONObj sortObj = sortElem.embeddedObject();
            if (sortObj.isEmpty()) {
                return Status(ErrorCodes::BadValue,
                              "The $sort pattern is empty when it should be a set of fields.");
            }

            // Check if the sort pattern is sound.
            BSONObjIterator sortIter(sortObj);
            while (sortIter.more()) {
                BSONElement sortPatternElem = sortIter.next();

                // We require either <field>: 1 or -1 for asc and desc.
                if (!isPatternElement(sortPatternElem)) {
                    return Status(ErrorCodes::BadValue,
                                  "The $sort element value must be either 1 or -1");
                }

                // All fields parts must be valid.
                FieldRef sortField(sortPatternElem.fieldName());
                if (sortField.numParts() == 0) {
                    return Status(ErrorCodes::BadValue, "The $sort field cannot be empty");
                }

                for (size_t i = 0; i < sortField.numParts(); i++) {
                    if (sortField.getPart(i).size() == 0) {
                        return Status(ErrorCodes::BadValue,
                                      str::stream() << "The $sort field is a dotted field "
                                                       "but has an empty part: "
                                                    << sortField.dottedField());
                    }
                }
            }

            _sort = PatternElementCmp(sortElem.embeddedObject(), opts.collator);
        } else {
            // Ensure the sortElem number is valid.
            if (!isPatternElement(sortElem)) {
                return Status(ErrorCodes::BadValue,
                              "The $sort element value must be either 1 or -1");
            }

            _sort = PatternElementCmp(BSON("" << sortElem.number()), opts.collator);
        }

        _sortPresent = true;
    }

    return Status::OK();
}