void LinearRegression::calculate() { if (enough() && fabs(double(_count)*_sumXsquared - _sumX*_sumX) > DBL_EPSILON) { _b = (double(_count)*_sumXY - _sumY*_sumX)/(double(_count)*_sumXsquared - _sumX*_sumX); _a = (_sumY - _b*_sumX)/double(_count); double sx = _b*(_sumXY - _sumX*_sumY/double(_count)); double sy2 = _sumYsquared - _sumY*_sumY/double(_count); double sy = sy2 - sx; _coefD = sx/sy2; _coefC = sqrt(_coefD); _stdError = sqrt(sy/double(_count - 2)); } else { _a = 0.0; _b = 0.0; _coefD = 0.0; _coefC = 0.0; _stdError = 0.0; } }
void MainWindow::rankVPN() { for( int i = 0; i < vpnList.count(); ++i ) { connect(vpnList[i], SIGNAL(enough()), this, SLOT(downloadVPN())); vpnList[i]->rank(); } }
int main(){ int i; double up=2000000.0,lw=0,mid; scanf("%d%d",&N,&F); for(i=1;i<=N;i++) scanf("%d",org+i),org[i]*=1000; while(up-lw > .5){ mid = (up+lw)/2.0; if(enough(mid)) lw=mid; else up=mid; } printf("%d\n",(int)up); return 0; }
bool test(int type_num, int deep, int last) { int i, j; printf("test(%d %d %d)...\n", type_num, deep, last); ///////// for (i = last+1; i < g; i++) { types[deep] = i; if (deep == type_num - 1) { if (enough(type_num)) { fprintf (fout, "%d", type_num); for (j = 0; j < type_num; j++) fprintf(fout, " %d", types[j]+1); fprintf(fout, "\n"); return true; } } else { if(test(type_num, deep+1, i)) return true; } } return false; }
std::string newRunQuery(OperationContext* txn, Message& m, QueryMessage& q, CurOp& curop, Message &result, bool fromDBDirectClient) { // Validate the namespace. const char *ns = q.ns; uassert(16332, "can't have an empty ns", ns[0]); const NamespaceString nsString(ns); uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid()); // Set curop information. curop.debug().ns = ns; curop.debug().ntoreturn = q.ntoreturn; curop.debug().query = q.query; curop.setQuery(q.query); // If the query is really a command, run it. if (nsString.isCommand()) { int nToReturn = q.ntoreturn; uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn << ") for $cmd type ns - can only be 1 or -1", nToReturn == 1 || nToReturn == -1); curop.markCommand(); BufBuilder bb; bb.skip(sizeof(QueryResult::Value)); BSONObjBuilder cmdResBuf; if (!runCommands(txn, ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) { uasserted(13530, "bad or malformed command request?"); } curop.debug().iscommand = true; // TODO: Does this get overwritten/do we really need to set this twice? curop.debug().query = q.query; QueryResult::View qr = bb.buf(); bb.decouple(); qr.setResultFlagsToOk(); qr.msgdata().setLen(bb.len()); curop.debug().responseLength = bb.len(); qr.msgdata().setOperation(opReply); qr.setCursorId(0); qr.setStartingFrom(0); qr.setNReturned(1); result.setData(qr.view2ptr(), true); return ""; } const NamespaceString nss(q.ns); // Parse the qm into a CanonicalQuery. CanonicalQuery* cq; Status canonStatus = CanonicalQuery::canonicalize( q, &cq, WhereCallbackReal(txn, StringData(nss.db()))); if (!canonStatus.isOK()) { uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString()); } QLOG() << "Running query:\n" << cq->toString(); LOG(2) << "Running query: " << cq->toStringShort(); // Parse, canonicalize, plan, transcribe, and get a plan executor. PlanExecutor* rawExec = NULL; // We use this a lot below. const LiteParsedQuery& pq = cq->getParsed(); AutoGetCollectionForRead ctx(txn, nss); const int dbProfilingLevel = (ctx.getDb() != NULL) ? ctx.getDb()->getProfilingLevel() : serverGlobalParams.defaultProfile; Collection* collection = ctx.getCollection(); // We'll now try to get the query executor that will execute this query for us. There // are a few cases in which we know upfront which executor we should get and, therefore, // we shortcut the selection process here. // // (a) If the query is over a collection that doesn't exist, we use an EOFStage. // // (b) if the query is a replication's initial sync one, we use a specifically designed // stage that skips extents faster (see details in exec/oplogstart.h). // // Otherwise we go through the selection of which executor is most suited to the // query + run-time context at hand. Status status = Status::OK(); if (NULL != collection && pq.getOptions().oplogReplay) { // Takes ownership of 'cq'. status = getOplogStartHack(txn, collection, cq, &rawExec); } else { size_t options = QueryPlannerParams::DEFAULT; if (shardingState.needCollectionMetadata(pq.ns())) { options |= QueryPlannerParams::INCLUDE_SHARD_FILTER; } // Takes ownership of 'cq'. status = getExecutor(txn, collection, cq, PlanExecutor::YIELD_AUTO, &rawExec, options); } if (!status.isOK()) { // NOTE: Do not access cq as getExecutor has deleted it. uasserted(17007, "Unable to execute query: " + status.reason()); } verify(NULL != rawExec); auto_ptr<PlanExecutor> exec(rawExec); // If it's actually an explain, do the explain and return rather than falling through // to the normal query execution loop. if (pq.isExplain()) { BufBuilder bb; bb.skip(sizeof(QueryResult::Value)); BSONObjBuilder explainBob; Explain::explainStages(exec.get(), ExplainCommon::EXEC_ALL_PLANS, &explainBob); // Add the resulting object to the return buffer. BSONObj explainObj = explainBob.obj(); bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize()); curop.debug().iscommand = true; // TODO: Does this get overwritten/do we really need to set this twice? curop.debug().query = q.query; // Set query result fields. QueryResult::View qr = bb.buf(); bb.decouple(); qr.setResultFlagsToOk(); qr.msgdata().setLen(bb.len()); curop.debug().responseLength = bb.len(); qr.msgdata().setOperation(opReply); qr.setCursorId(0); qr.setStartingFrom(0); qr.setNReturned(1); result.setData(qr.view2ptr(), true); return ""; } // We freak out later if this changes before we're done with the query. const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns()); // Handle query option $maxTimeMS (not used with commands). curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000); txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. // uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set. bool slaveOK = pq.getOptions().slaveOk || pq.hasReadPref(); status = repl::getGlobalReplicationCoordinator()->checkCanServeReadsFor( txn, NamespaceString(cq->ns()), slaveOK); uassertStatusOK(status); // If this exists, the collection is sharded. // If it doesn't exist, we can assume we're not sharded. // If we're sharded, we might encounter data that is not consistent with our sharding state. // We must ignore this data. CollectionMetadataPtr collMetadata; if (!shardingState.needCollectionMetadata(pq.ns())) { collMetadata = CollectionMetadataPtr(); } else { collMetadata = shardingState.getCollectionMetadata(pq.ns()); } // Run the query. // bb is used to hold query results // this buffer should contain either requested documents per query or // explain information, but not both BufBuilder bb(32768); bb.skip(sizeof(QueryResult::Value)); // How many results have we obtained from the executor? int numResults = 0; // If we're replaying the oplog, we save the last time that we read. OpTime slaveReadTill; // Do we save the PlanExecutor in a ClientCursor for getMore calls later? bool saveClientCursor = false; BSONObj obj; PlanExecutor::ExecState state; // uint64_t numMisplacedDocs = 0; // Get summary info about which plan the executor is using. curop.debug().planSummary = Explain::getPlanSummary(exec.get()); while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) { // Add result to output buffer. bb.appendBuf((void*)obj.objdata(), obj.objsize()); // Count the result. ++numResults; // Possibly note slave's position in the oplog. if (pq.getOptions().oplogReplay) { BSONElement e = obj["ts"]; if (Date == e.type() || Timestamp == e.type()) { slaveReadTill = e._opTime(); } } // TODO: only one type of 2d search doesn't support this. We need a way to pull it out // of CanonicalQuery. :( const bool supportsGetMore = true; if (!supportsGetMore && (enough(pq, numResults) || bb.len() >= MaxBytesToReturnToClientAtOnce)) { break; } else if (enoughForFirstBatch(pq, numResults, bb.len())) { QLOG() << "Enough for first batch, wantMore=" << pq.wantMore() << " numToReturn=" << pq.getNumToReturn() << " numResults=" << numResults << endl; // If only one result requested assume it's a findOne() and don't save the cursor. if (pq.wantMore() && 1 != pq.getNumToReturn()) { QLOG() << " executor EOF=" << exec->isEOF() << endl; saveClientCursor = !exec->isEOF(); } break; } } // If we cache the executor later, we want to deregister it as it receives notifications // anyway by virtue of being cached. // // If we don't cache the executor later, we are deleting it, so it must be deregistered. // // So, no matter what, deregister the executor. exec->deregisterExec(); // Caller expects exceptions thrown in certain cases. if (PlanExecutor::EXEC_ERROR == state) { scoped_ptr<PlanStageStats> stats(exec->getStats()); error() << "Plan executor error, stats: " << Explain::statsToBSON(*stats); uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj)); } // Why save a dead executor? if (PlanExecutor::DEAD == state) { saveClientCursor = false; } else if (pq.getOptions().tailable) { // If we're tailing a capped collection, we don't bother saving the cursor if the // collection is empty. Otherwise, the semantics of the tailable cursor is that the // client will keep trying to read from it. So we'll keep it around. if (collection && collection->numRecords(txn) != 0 && pq.getNumToReturn() != 1) { saveClientCursor = true; } } // TODO(greg): This will go away soon. if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) { // if the version changed during the query we might be missing some data and its safe to // send this as mongos can resend at this point throw SendStaleConfigException(pq.ns(), "version changed during initial query", shardingVersionAtStart, shardingState.getVersion(pq.ns())); } const logger::LogComponent queryLogComponent = logger::LogComponent::kQuery; const logger::LogSeverity logLevelOne = logger::LogSeverity::Debug(1); PlanSummaryStats summaryStats; Explain::getSummaryStats(exec.get(), &summaryStats); curop.debug().ntoskip = pq.getSkip(); curop.debug().nreturned = numResults; curop.debug().scanAndOrder = summaryStats.hasSortStage; curop.debug().nscanned = summaryStats.totalKeysExamined; curop.debug().nscannedObjects = summaryStats.totalDocsExamined; curop.debug().idhack = summaryStats.isIdhack; // Set debug information for consumption by the profiler. if (dbProfilingLevel > 0 || curop.elapsedMillis() > serverGlobalParams.slowMS || logger::globalLogDomain()->shouldLog(queryLogComponent, logLevelOne)) { // Get BSON stats. scoped_ptr<PlanStageStats> execStats(exec->getStats()); BSONObjBuilder statsBob; Explain::statsToBSON(*execStats, &statsBob); curop.debug().execStats.set(statsBob.obj()); // Replace exec stats with plan summary if stats cannot fit into CachedBSONObj. if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) { BSONObjBuilder bob; bob.append("summary", curop.debug().planSummary.toString()); curop.debug().execStats.set(bob.done()); } } long long ccId = 0; if (saveClientCursor) { // We won't use the executor until it's getMore'd. exec->saveState(); // Allocate a new ClientCursor. We don't have to worry about leaking it as it's // inserted into a global map by its ctor. ClientCursor* cc = new ClientCursor(collection, exec.get(), cq->getParsed().getOptions().toInt(), cq->getParsed().getFilter()); ccId = cc->cursorid(); if (fromDBDirectClient) { cc->setUnownedRecoveryUnit(txn->recoveryUnit()); } else if (state == PlanExecutor::IS_EOF && pq.getOptions().tailable) { // Don't stash the RU for tailable cursors at EOF, let them get a new RU on their // next getMore. } else { // We stash away the RecoveryUnit in the ClientCursor. It's used for subsequent // getMore requests. The calling OpCtx gets a fresh RecoveryUnit. cc->setOwnedRecoveryUnit(txn->releaseRecoveryUnit()); StorageEngine* storageEngine = getGlobalEnvironment()->getGlobalStorageEngine(); txn->setRecoveryUnit(storageEngine->newRecoveryUnit(txn)); } QLOG() << "caching executor with cursorid " << ccId << " after returning " << numResults << " results" << endl; // ClientCursor takes ownership of executor. Release to make sure it's not deleted. exec.release(); // TODO document if (pq.getOptions().oplogReplay && !slaveReadTill.isNull()) { cc->slaveReadTill(slaveReadTill); } // TODO document if (pq.getOptions().exhaust) { curop.debug().exhaust = true; } // Set attributes for getMore. cc->setCollMetadata(collMetadata); cc->setPos(numResults); // If the query had a time limit, remaining time is "rolled over" to the cursor (for // use by future getmore ops). cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros()); } else { QLOG() << "Not caching executor but returning " << numResults << " results.\n"; } // Add the results from the query into the output buffer. result.appendData(bb.buf(), bb.len()); bb.decouple(); // Fill out the output buffer's header. QueryResult::View qr = result.header().view2ptr(); qr.setCursorId(ccId); curop.debug().cursorid = (0 == ccId ? -1 : ccId); qr.setResultFlagsToOk(); qr.msgdata().setOperation(opReply); qr.setStartingFrom(0); qr.setNReturned(numResults); // curop.debug().exhaust is set above. return curop.debug().exhaust ? pq.ns() : ""; }
/** * This is called by db/ops/query.cpp. This is the entry point for answering a query. */ std::string newRunQuery(CanonicalQuery* cq, CurOp& curop, Message &result) { QLOG() << "Running query on new system: " << cq->toString(); // This is a read lock. Client::ReadContext ctx(cq->ns(), storageGlobalParams.dbpath); // Parse, canonicalize, plan, transcribe, and get a runner. Runner* rawRunner = NULL; // We use this a lot below. const LiteParsedQuery& pq = cq->getParsed(); // Need to call cq->toString() now, since upon error getRunner doesn't guarantee // cq is in a consistent state. string cqStr = cq->toString(); // We'll now try to get the query runner that will execute this query for us. There // are a few cases in which we know upfront which runner we should get and, therefore, // we shortcut the selection process here. // // (a) If the query is over a collection that doesn't exist, we get a special runner // that's is so (a runner) which doesn't return results, the EOFRunner. // // (b) if the query is a replication's initial sync one, we get a SingleSolutinRunner // that uses a specifically designed stage that skips extents faster (see details in // exec/oplogstart.h) // // Otherwise we go through the selection of which runner is most suited to the // query + run-time context at hand. Status status = Status::OK(); if (ctx.ctx().db()->getCollection(cq->ns()) == NULL) { rawRunner = new EOFRunner(cq, cq->ns()); } else if (pq.hasOption(QueryOption_OplogReplay)) { status = getOplogStartHack(cq, &rawRunner); } else { // Takes ownership of cq. size_t options = QueryPlannerParams::DEFAULT; if (shardingState.needCollectionMetadata(pq.ns())) { options |= QueryPlannerParams::INCLUDE_SHARD_FILTER; } status = getRunner(cq, &rawRunner, options); } if (!status.isOK()) { uasserted(17007, "Couldn't get runner for query because: " + status.reason() + " query is " + cqStr); } verify(NULL != rawRunner); auto_ptr<Runner> runner(rawRunner); // We freak out later if this changes before we're done with the query. const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns()); // Handle query option $maxTimeMS (not used with commands). curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000); killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. // uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set. replVerifyReadsOk(&pq); // If this exists, the collection is sharded. // If it doesn't exist, we can assume we're not sharded. // If we're sharded, we might encounter data that is not consistent with our sharding state. // We must ignore this data. CollectionMetadataPtr collMetadata; if (!shardingState.needCollectionMetadata(pq.ns())) { collMetadata = CollectionMetadataPtr(); } else { collMetadata = shardingState.getCollectionMetadata(pq.ns()); } // Run the query. // bb is used to hold query results // this buffer should contain either requested documents per query or // explain information, but not both BufBuilder bb(32768); bb.skip(sizeof(QueryResult)); // How many results have we obtained from the runner? int numResults = 0; // If we're replaying the oplog, we save the last time that we read. OpTime slaveReadTill; // Do we save the Runner in a ClientCursor for getMore calls later? bool saveClientCursor = false; // We turn on auto-yielding for the runner here. The runner registers itself with the // active runners list in ClientCursor. ClientCursor::registerRunner(runner.get()); runner->setYieldPolicy(Runner::YIELD_AUTO); auto_ptr<DeregisterEvenIfUnderlyingCodeThrows> safety( new DeregisterEvenIfUnderlyingCodeThrows(runner.get())); BSONObj obj; Runner::RunnerState state; // uint64_t numMisplacedDocs = 0; // set this outside loop. we will need to use this both within loop and when deciding // to fill in explain information const bool isExplain = pq.isExplain(); while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { // Add result to output buffer. This is unnecessary if explain info is requested if (!isExplain) { bb.appendBuf((void*)obj.objdata(), obj.objsize()); } // Count the result. ++numResults; // Possibly note slave's position in the oplog. if (pq.hasOption(QueryOption_OplogReplay)) { BSONElement e = obj["ts"]; if (Date == e.type() || Timestamp == e.type()) { slaveReadTill = e._opTime(); } } // TODO: only one type of 2d search doesn't support this. We need a way to pull it out // of CanonicalQuery. :( const bool supportsGetMore = true; if (isExplain) { if (enoughForExplain(pq, numResults)) { break; } } else if (!supportsGetMore && (enough(pq, numResults) || bb.len() >= MaxBytesToReturnToClientAtOnce)) { break; } else if (enoughForFirstBatch(pq, numResults, bb.len())) { QLOG() << "Enough for first batch, wantMore=" << pq.wantMore() << " numToReturn=" << pq.getNumToReturn() << " numResults=" << numResults << endl; // If only one result requested assume it's a findOne() and don't save the cursor. if (pq.wantMore() && 1 != pq.getNumToReturn()) { QLOG() << " runner EOF=" << runner->isEOF() << endl; saveClientCursor = !runner->isEOF(); } break; } } // If we cache the runner later, we want to deregister it as it receives notifications // anyway by virtue of being cached. // // If we don't cache the runner later, we are deleting it, so it must be deregistered. // // So, no matter what, deregister the runner. safety.reset(); // Caller expects exceptions thrown in certain cases: // * in-memory sort using too much RAM. if (Runner::RUNNER_ERROR == state) { uasserted(17144, "Runner error, memory limit for sort probably exceeded"); } // Why save a dead runner? if (Runner::RUNNER_DEAD == state) { saveClientCursor = false; } else if (pq.hasOption(QueryOption_CursorTailable)) { // If we're tailing a capped collection, we don't bother saving the cursor if the // collection is empty. Otherwise, the semantics of the tailable cursor is that the // client will keep trying to read from it. So we'll keep it around. Collection* collection = ctx.ctx().db()->getCollection(cq->ns()); if (collection && collection->numRecords() != 0 && pq.getNumToReturn() != 1) { saveClientCursor = true; } } // TODO(greg): This will go away soon. if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) { // if the version changed during the query we might be missing some data and its safe to // send this as mongos can resend at this point throw SendStaleConfigException(pq.ns(), "version changed during initial query", shardingVersionAtStart, shardingState.getVersion(pq.ns())); } // Append explain information to query results by asking the runner to produce them. if (isExplain) { TypeExplain* bareExplain; Status res = runner->getExplainPlan(&bareExplain); if (!res.isOK()) { error() << "could not produce explain of query '" << pq.getFilter() << "', error: " << res.reason(); // If numResults and the data in bb don't correspond, we'll crash later when rooting // through the reply msg. BSONObj emptyObj; bb.appendBuf((void*)emptyObj.objdata(), emptyObj.objsize()); // The explain output is actually a result. numResults = 1; // TODO: we can fill out millis etc. here just fine even if the plan screwed up. } else { boost::scoped_ptr<TypeExplain> explain(bareExplain); // Fill in the missing run-time fields in explain, starting with propeties of // the process running the query. std::string server = mongoutils::str::stream() << getHostNameCached() << ":" << serverGlobalParams.port; explain->setServer(server); // We might have skipped some results due to chunk migration etc. so our count is // correct. explain->setN(numResults); // Clock the whole operation. explain->setMillis(curop.elapsedMillis()); BSONObj explainObj = explain->toBSON(); bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize()); // The explain output is actually a result. numResults = 1; } } long long ccId = 0; if (saveClientCursor) { // We won't use the runner until it's getMore'd. runner->saveState(); // Allocate a new ClientCursor. We don't have to worry about leaking it as it's // inserted into a global map by its ctor. ClientCursor* cc = new ClientCursor(runner.get(), cq->getParsed().getOptions(), cq->getParsed().getFilter()); ccId = cc->cursorid(); QLOG() << "caching runner with cursorid " << ccId << " after returning " << numResults << " results" << endl; // ClientCursor takes ownership of runner. Release to make sure it's not deleted. runner.release(); // TODO document if (pq.hasOption(QueryOption_OplogReplay) && !slaveReadTill.isNull()) { cc->slaveReadTill(slaveReadTill); } // TODO document if (pq.hasOption(QueryOption_Exhaust)) { curop.debug().exhaust = true; } // Set attributes for getMore. cc->setCollMetadata(collMetadata); cc->setPos(numResults); // If the query had a time limit, remaining time is "rolled over" to the cursor (for // use by future getmore ops). cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros()); } else { QLOG() << "not caching runner but returning " << numResults << " results\n"; } // Add the results from the query into the output buffer. result.appendData(bb.buf(), bb.len()); bb.decouple(); // Fill out the output buffer's header. QueryResult* qr = static_cast<QueryResult*>(result.header()); qr->cursorId = ccId; curop.debug().cursorid = (0 == ccId ? -1 : ccId); qr->setResultFlagsToOk(); qr->setOperation(opReply); qr->startingFrom = 0; qr->nReturned = numResults; curop.debug().ntoskip = pq.getSkip(); curop.debug().nreturned = numResults; // curop.debug().exhaust is set above. return curop.debug().exhaust ? pq.ns() : ""; }
/** * This is called by db/ops/query.cpp. This is the entry point for answering a query. */ string newRunQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) { log() << "Running query on new system: " << q.query.toString() << endl; // This is a read lock. Client::ReadContext ctx(q.ns, dbpath); // Parse, canonicalize, plan, transcribe, and get a runner. Runner* rawRunner; CanonicalQuery* cq; Status status = getRunner(q, &rawRunner, &cq); if (!status.isOK()) { uasserted(17007, "Couldn't process query " + q.query.toString() + " why: " + status.reason()); } verify(NULL != rawRunner); auto_ptr<Runner> runner(rawRunner); // We freak out later if this changes before we're done with the query. const ChunkVersion shardingVersionAtStart = shardingState.getVersion(q.ns); // We use this a lot below. const LiteParsedQuery& pq = cq->getParsed(); // TODO: Document why we do this. // TODO: do this when we can pass in our own parsed query //replVerifyReadsOk(&pq); // If this exists, the collection is sharded. // If it doesn't exist, we can assume we're not sharded. // If we're sharded, we might encounter data that is not consistent with our sharding state. // We must ignore this data. CollectionMetadataPtr collMetadata; if (!shardingState.needCollectionMetadata(pq.ns())) { collMetadata = CollectionMetadataPtr(); } else { collMetadata = shardingState.getCollectionMetadata(pq.ns()); } // Run the query. BufBuilder bb(32768); bb.skip(sizeof(QueryResult)); // How many results have we obtained from the runner? int numResults = 0; // If we're replaying the oplog, we save the last time that we read. OpTime slaveReadTill; // Do we save the Runner in a ClientCursor for getMore calls later? bool saveClientCursor = false; // We turn on auto-yielding for the runner here, so we must register it with the active // runners list in ClientCursor. ClientCursor::registerRunner(runner.get()); runner->setYieldPolicy(Runner::YIELD_AUTO); BSONObj obj; Runner::RunnerState state; while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { // If we're sharded make sure that we don't return any data that hasn't been migrated // off of our shared yet. if (collMetadata) { // This information can change if we yield and as such we must make sure to re-fetch // it if we yield. KeyPattern kp(collMetadata->getKeyPattern()); // This performs excessive BSONObj creation but that's OK for now. if (!collMetadata->keyBelongsToMe(kp.extractSingleKey(obj))) { continue; } } // Add result to output buffer. bb.appendBuf((void*)obj.objdata(), obj.objsize()); // Count the result. ++numResults; // Possibly note slave's position in the oplog. if (pq.hasOption(QueryOption_OplogReplay)) { BSONElement e = obj["ts"]; if (Date == e.type() || Timestamp == e.type()) { slaveReadTill = e._opTime(); } } // TODO: only one type of 2d search doesn't support this. We need a way to pull it out // of CanonicalQuery. :( const bool supportsGetMore = true; const bool isExplain = pq.isExplain(); if (isExplain && enoughForExplain(pq, numResults)) { break; } else if (!supportsGetMore && (enough(pq, numResults) || bb.len() >= MaxBytesToReturnToClientAtOnce)) { break; } else if (enoughForFirstBatch(pq, numResults, bb.len())) { // If only one result requested assume it's a findOne() and don't save the cursor. if (pq.wantMore() && 1 != pq.getNumToReturn()) { saveClientCursor = true; } break; } } // If we cache the runner later, we want to deregister it as it receives notifications // anyway by virtue of being cached. // // If we don't cache the runner later, we are deleting it, so it must be deregistered. // // So, no matter what, deregister the runner. ClientCursor::deregisterRunner(runner.get()); // Why save a dead runner? if (Runner::RUNNER_DEAD == state) { saveClientCursor = false; } // TODO: Stage creation can set tailable depending on what's in the parsed query. We have // the full parsed query available during planning...set it there. // // TODO: If we're tailable we want to save the client cursor. Make sure we do this later. //if (pq.hasOption(QueryOption_CursorTailable) && pq.getNumToReturn() != 1) { ... } // TODO(greg): This will go away soon. if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) { // if the version changed during the query we might be missing some data and its safe to // send this as mongos can resend at this point throw SendStaleConfigException(pq.ns(), "version changed during initial query", shardingVersionAtStart, shardingState.getVersion(pq.ns())); } long long ccId = 0; if (saveClientCursor) { // We won't use the runner until it's getMore'd. runner->saveState(); // Allocate a new ClientCursor. We don't have to worry about leaking it as it's // inserted into a global map by its ctor. ClientCursor* cc = new ClientCursor(runner.get(), cq->getParsed().getOptions(), cq->getParsed().getFilter()); ccId = cc->cursorid(); log() << "caching runner with cursorid " << ccId << endl; // ClientCursor takes ownership of runner. Release to make sure it's not deleted. runner.release(); // TODO document if (pq.hasOption(QueryOption_OplogReplay) && !slaveReadTill.isNull()) { cc->slaveReadTill(slaveReadTill); } // TODO document if (pq.hasOption(QueryOption_Exhaust)) { curop.debug().exhaust = true; } // Set attributes for getMore. cc->setCollMetadata(collMetadata); cc->setPos(numResults); // If the query had a time limit, remaining time is "rolled over" to the cursor (for // use by future getmore ops). cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros()); } // Add the results from the query into the output buffer. result.appendData(bb.buf(), bb.len()); bb.decouple(); // Fill out the output buffer's header. QueryResult* qr = static_cast<QueryResult*>(result.header()); qr->cursorId = ccId; curop.debug().cursorid = (0 == ccId ? -1 : ccId); qr->setResultFlagsToOk(); qr->setOperation(opReply); qr->startingFrom = 0; qr->nReturned = numResults; // TODO: nscanned is bogus. // curop.debug().nscanned = ( cursor ? cursor->nscanned() : 0LL ); curop.debug().ntoskip = pq.getSkip(); curop.debug().nreturned = numResults; // curop.debug().exhaust is set above. return curop.debug().exhaust ? pq.ns() : ""; }
std::string newRunQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) { // Validate the namespace. const char *ns = q.ns; uassert(16332, "can't have an empty ns", ns[0]); const NamespaceString nsString(ns); uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid()); // Set curop information. curop.debug().ns = ns; curop.debug().ntoreturn = q.ntoreturn; curop.debug().query = q.query; curop.setQuery(q.query); // If the query is really a command, run it. if (nsString.isCommand()) { int nToReturn = q.ntoreturn; uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn << ") for $cmd type ns - can only be 1 or -1", nToReturn == 1 || nToReturn == -1); curop.markCommand(); BufBuilder bb; bb.skip(sizeof(QueryResult)); BSONObjBuilder cmdResBuf; if (!runCommands(ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) { uasserted(13530, "bad or malformed command request?"); } curop.debug().iscommand = true; // TODO: Does this get overwritten/do we really need to set this twice? curop.debug().query = q.query; QueryResult* qr = reinterpret_cast<QueryResult*>(bb.buf()); bb.decouple(); qr->setResultFlagsToOk(); qr->len = bb.len(); curop.debug().responseLength = bb.len(); qr->setOperation(opReply); qr->cursorId = 0; qr->startingFrom = 0; qr->nReturned = 1; result.setData(qr, true); return ""; } // This is a read lock. We require this because if we're parsing a $where, the // where-specific parsing code assumes we have a lock and creates execution machinery that // requires it. Client::ReadContext ctx(q.ns); Collection* collection = ctx.ctx().db()->getCollection( ns ); // Parse the qm into a CanonicalQuery. CanonicalQuery* cq; Status canonStatus = CanonicalQuery::canonicalize(q, &cq); if (!canonStatus.isOK()) { uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString()); } verify(cq); QLOG() << "Running query:\n" << cq->toString(); LOG(2) << "Running query: " << cq->toStringShort(); // Parse, canonicalize, plan, transcribe, and get a runner. Runner* rawRunner = NULL; // We use this a lot below. const LiteParsedQuery& pq = cq->getParsed(); // We'll now try to get the query runner that will execute this query for us. There // are a few cases in which we know upfront which runner we should get and, therefore, // we shortcut the selection process here. // // (a) If the query is over a collection that doesn't exist, we get a special runner // that's is so (a runner) which doesn't return results, the EOFRunner. // // (b) if the query is a replication's initial sync one, we get a SingleSolutinRunner // that uses a specifically designed stage that skips extents faster (see details in // exec/oplogstart.h) // // Otherwise we go through the selection of which runner is most suited to the // query + run-time context at hand. Status status = Status::OK(); if (collection == NULL) { rawRunner = new EOFRunner(cq, cq->ns()); } else if (pq.hasOption(QueryOption_OplogReplay)) { status = getOplogStartHack(collection, cq, &rawRunner); } else { // Takes ownership of cq. size_t options = QueryPlannerParams::DEFAULT; if (shardingState.needCollectionMetadata(pq.ns())) { options |= QueryPlannerParams::INCLUDE_SHARD_FILTER; } status = getRunner(cq, &rawRunner, options); } if (!status.isOK()) { // NOTE: Do not access cq as getRunner has deleted it. uasserted(17007, "Unable to execute query: " + status.reason()); } verify(NULL != rawRunner); auto_ptr<Runner> runner(rawRunner); // We freak out later if this changes before we're done with the query. const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns()); // Handle query option $maxTimeMS (not used with commands). curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000); killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. // uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set. replVerifyReadsOk(&pq); // If this exists, the collection is sharded. // If it doesn't exist, we can assume we're not sharded. // If we're sharded, we might encounter data that is not consistent with our sharding state. // We must ignore this data. CollectionMetadataPtr collMetadata; if (!shardingState.needCollectionMetadata(pq.ns())) { collMetadata = CollectionMetadataPtr(); } else { collMetadata = shardingState.getCollectionMetadata(pq.ns()); } // Run the query. // bb is used to hold query results // this buffer should contain either requested documents per query or // explain information, but not both BufBuilder bb(32768); bb.skip(sizeof(QueryResult)); // How many results have we obtained from the runner? int numResults = 0; // If we're replaying the oplog, we save the last time that we read. OpTime slaveReadTill; // Do we save the Runner in a ClientCursor for getMore calls later? bool saveClientCursor = false; // We turn on auto-yielding for the runner here. The runner registers itself with the // active runners list in ClientCursor. auto_ptr<ScopedRunnerRegistration> safety(new ScopedRunnerRegistration(runner.get())); runner->setYieldPolicy(Runner::YIELD_AUTO); BSONObj obj; Runner::RunnerState state; // uint64_t numMisplacedDocs = 0; // set this outside loop. we will need to use this both within loop and when deciding // to fill in explain information const bool isExplain = pq.isExplain(); // Have we retrieved info about which plan the runner will // use to execute the query yet? bool gotPlanInfo = false; PlanInfo* rawInfo; boost::scoped_ptr<PlanInfo> planInfo; while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { // Add result to output buffer. This is unnecessary if explain info is requested if (!isExplain) { bb.appendBuf((void*)obj.objdata(), obj.objsize()); } // Count the result. ++numResults; // In the case of the multi plan runner, we may not be able to // successfully retrieve plan info until after the query starts // to run. This is because the multi plan runner doesn't know what // plan it will end up using until it runs candidates and selects // the best. // // TODO: Do we ever want to output what the MPR is comparing? if (!gotPlanInfo) { Status infoStatus = runner->getInfo(NULL, &rawInfo); if (infoStatus.isOK()) { gotPlanInfo = true; planInfo.reset(rawInfo); // planSummary is really a ThreadSafeString which copies the data from // the provided pointer. curop.debug().planSummary = planInfo->planSummary.c_str(); } } // Possibly note slave's position in the oplog. if (pq.hasOption(QueryOption_OplogReplay)) { BSONElement e = obj["ts"]; if (Date == e.type() || Timestamp == e.type()) { slaveReadTill = e._opTime(); } } // TODO: only one type of 2d search doesn't support this. We need a way to pull it out // of CanonicalQuery. :( const bool supportsGetMore = true; if (isExplain) { if (enoughForExplain(pq, numResults)) { break; } } else if (!supportsGetMore && (enough(pq, numResults) || bb.len() >= MaxBytesToReturnToClientAtOnce)) { break; } else if (enoughForFirstBatch(pq, numResults, bb.len())) { QLOG() << "Enough for first batch, wantMore=" << pq.wantMore() << " numToReturn=" << pq.getNumToReturn() << " numResults=" << numResults << endl; // If only one result requested assume it's a findOne() and don't save the cursor. if (pq.wantMore() && 1 != pq.getNumToReturn()) { QLOG() << " runner EOF=" << runner->isEOF() << endl; saveClientCursor = !runner->isEOF(); } break; } } // Try to get information about the plan which the runner // will use to execute the query, it we don't have it already. if (!gotPlanInfo) { Status infoStatus = runner->getInfo(NULL, &rawInfo); if (infoStatus.isOK()) { gotPlanInfo = true; planInfo.reset(rawInfo); // planSummary is really a ThreadSafeString which copies the data from // the provided pointer. curop.debug().planSummary = planInfo->planSummary.c_str(); } } // If we cache the runner later, we want to deregister it as it receives notifications // anyway by virtue of being cached. // // If we don't cache the runner later, we are deleting it, so it must be deregistered. // // So, no matter what, deregister the runner. safety.reset(); // Caller expects exceptions thrown in certain cases. if (Runner::RUNNER_ERROR == state) { TypeExplain* bareExplain; Status res = runner->getInfo(&bareExplain, NULL); if (res.isOK()) { boost::scoped_ptr<TypeExplain> errorExplain(bareExplain); error() << "Runner error, stats:\n" << errorExplain->stats.jsonString(Strict, true); } uasserted(17144, "Runner error: " + WorkingSetCommon::toStatusString(obj)); } // Why save a dead runner? if (Runner::RUNNER_DEAD == state) { saveClientCursor = false; } else if (pq.hasOption(QueryOption_CursorTailable)) { // If we're tailing a capped collection, we don't bother saving the cursor if the // collection is empty. Otherwise, the semantics of the tailable cursor is that the // client will keep trying to read from it. So we'll keep it around. Collection* collection = ctx.ctx().db()->getCollection(cq->ns()); if (collection && collection->numRecords() != 0 && pq.getNumToReturn() != 1) { saveClientCursor = true; } } // TODO(greg): This will go away soon. if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) { // if the version changed during the query we might be missing some data and its safe to // send this as mongos can resend at this point throw SendStaleConfigException(pq.ns(), "version changed during initial query", shardingVersionAtStart, shardingState.getVersion(pq.ns())); } // Used to fill in explain and to determine if the query is slow enough to be logged. int elapsedMillis = curop.elapsedMillis(); // Get explain information if: // 1) it is needed by an explain query; // 2) profiling is enabled; or // 3) profiling is disabled but we still need explain details to log a "slow" query. // Producing explain information is expensive and should be done only if we are certain // the information will be used. boost::scoped_ptr<TypeExplain> explain(NULL); if (isExplain || ctx.ctx().db()->getProfilingLevel() > 0 || elapsedMillis > serverGlobalParams.slowMS) { // Ask the runner to produce explain information. TypeExplain* bareExplain; Status res = runner->getInfo(&bareExplain, NULL); if (res.isOK()) { explain.reset(bareExplain); } else if (isExplain) { error() << "could not produce explain of query '" << pq.getFilter() << "', error: " << res.reason(); // If numResults and the data in bb don't correspond, we'll crash later when rooting // through the reply msg. BSONObj emptyObj; bb.appendBuf((void*)emptyObj.objdata(), emptyObj.objsize()); // The explain output is actually a result. numResults = 1; // TODO: we can fill out millis etc. here just fine even if the plan screwed up. } } // Fill in the missing run-time fields in explain, starting with propeties of // the process running the query. if (isExplain && NULL != explain.get()) { std::string server = mongoutils::str::stream() << getHostNameCached() << ":" << serverGlobalParams.port; explain->setServer(server); // We might have skipped some results due to chunk migration etc. so our count is // correct. explain->setN(numResults); // Clock the whole operation. explain->setMillis(elapsedMillis); BSONObj explainObj = explain->toBSON(); bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize()); // The explain output is actually a result. numResults = 1; } long long ccId = 0; if (saveClientCursor) { // We won't use the runner until it's getMore'd. runner->saveState(); // Allocate a new ClientCursor. We don't have to worry about leaking it as it's // inserted into a global map by its ctor. ClientCursor* cc = new ClientCursor(collection, runner.get(), cq->getParsed().getOptions(), cq->getParsed().getFilter()); ccId = cc->cursorid(); QLOG() << "caching runner with cursorid " << ccId << " after returning " << numResults << " results" << endl; // ClientCursor takes ownership of runner. Release to make sure it's not deleted. runner.release(); // TODO document if (pq.hasOption(QueryOption_OplogReplay) && !slaveReadTill.isNull()) { cc->slaveReadTill(slaveReadTill); } // TODO document if (pq.hasOption(QueryOption_Exhaust)) { curop.debug().exhaust = true; } // Set attributes for getMore. cc->setCollMetadata(collMetadata); cc->setPos(numResults); // If the query had a time limit, remaining time is "rolled over" to the cursor (for // use by future getmore ops). cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros()); } else { QLOG() << "Not caching runner but returning " << numResults << " results.\n"; } // Add the results from the query into the output buffer. result.appendData(bb.buf(), bb.len()); bb.decouple(); // Fill out the output buffer's header. QueryResult* qr = static_cast<QueryResult*>(result.header()); qr->cursorId = ccId; curop.debug().cursorid = (0 == ccId ? -1 : ccId); qr->setResultFlagsToOk(); qr->setOperation(opReply); qr->startingFrom = 0; qr->nReturned = numResults; // Set debug information for consumption by the profiler. curop.debug().ntoskip = pq.getSkip(); curop.debug().nreturned = numResults; if (NULL != explain.get()) { if (explain->isScanAndOrderSet()) { curop.debug().scanAndOrder = explain->getScanAndOrder(); } else { curop.debug().scanAndOrder = false; } if (explain->isNScannedSet()) { curop.debug().nscanned = explain->getNScanned(); } if (explain->isNScannedObjectsSet()) { curop.debug().nscannedObjects = explain->getNScannedObjects(); } if (explain->isIDHackSet()) { curop.debug().idhack = explain->getIDHack(); } if (!explain->stats.isEmpty()) { // execStats is a CachedBSONObj because it lives in the race-prone // curop. curop.debug().execStats.set(explain->stats); // Replace exec stats with plan summary if stats cannot fit into CachedBSONObj. if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) { BSONObjBuilder bob; bob.append("summary", curop.debug().planSummary.toString()); curop.debug().execStats.set(bob.done()); } } } // curop.debug().exhaust is set above. return curop.debug().exhaust ? pq.ns() : ""; }
int Assemble(struct supertype *st, char *mddev, int mdfd, mddev_ident_t ident, char *conffile, mddev_dev_t devlist, int readonly, int runstop, char *update, int verbose, int force) { /* * The task of Assemble is to find a collection of * devices that should (according to their superblocks) * form an array, and to give this collection to the MD driver. * In Linux-2.4 and later, this involves submitting a * SET_ARRAY_INFO ioctl with no arg - to prepare * the array - and then submit a number of * ADD_NEW_DISK ioctls to add disks into * the array. Finally RUN_ARRAY might * be submitted to start the array. * * Much of the work of Assemble is in finding and/or * checking the disks to make sure they look right. * * If mddev is not set, then scan must be set and we * read through the config file for dev+uuid mapping * We recurse, setting mddev, for each device that * - isn't running * - has a valid uuid (or any uuid if !uuidset) * * If mddev is set, we try to determine state of md. * check version - must be at least 0.90.0 * check kernel version. must be at least 2.4. * If not, we can possibly fall back on START_ARRAY * Try to GET_ARRAY_INFO. * If possible, give up * If not, try to STOP_ARRAY just to make sure * * If !uuidset and scan, look in conf-file for uuid * If not found, give up * If !devlist and scan and uuidset, get list of devs from conf-file * * For each device: * Check superblock - discard if bad * Check uuid (set if we don't have one) - discard if no match * Check superblock similarity if we have a superblock - discard if different * Record events, devicenum * This should give us a list of devices for the array * We should collect the most recent event number * * Count disks with recent enough event count * While force && !enough disks * Choose newest rejected disks, update event count * mark clean and rewrite superblock * If recent kernel: * SET_ARRAY_INFO * foreach device with recent events : ADD_NEW_DISK * if runstop == 1 || "enough" disks and runstop==0 -> RUN_ARRAY * If old kernel: * Check the device numbers in superblock are right * update superblock if any changes * START_ARRAY * */ int old_linux = 0; int vers; void *first_super = NULL, *super = NULL; struct { char *devname; unsigned int major, minor; unsigned int oldmajor, oldminor; long long events; int uptodate; int state; int raid_disk; int disk_nr; } *devices; int *best = NULL; /* indexed by raid_disk */ unsigned int bestcnt = 0; int devcnt = 0; unsigned int okcnt, sparecnt; unsigned int req_cnt; unsigned int i; int most_recent = 0; int chosen_drive; int change = 0; int inargv = 0; int start_partial_ok = force || devlist==NULL; unsigned int num_devs; mddev_dev_t tmpdev; struct mdinfo info; struct mddev_ident_s ident2; char *avail; int nextspare = 0; vers = md_get_version(mdfd); if (vers <= 0) { fprintf(stderr, Name ": %s appears not to be an md device.\n", mddev); return 1; } if (vers < 9000) { fprintf(stderr, Name ": Assemble requires driver version 0.90.0 or later.\n" " Upgrade your kernel or try --build\n"); return 1; } if (get_linux_version() < 2004000) old_linux = 1; if (ioctl(mdfd, GET_ARRAY_INFO, &info.array)>=0) { fprintf(stderr, Name ": device %s already active - cannot assemble it\n", mddev); return 1; } ioctl(mdfd, STOP_ARRAY, NULL); /* just incase it was started but has no content */ /* * If any subdevs are listed, then any that don't * match ident are discarded. Remainder must all match and * become the array. * If no subdevs, then we scan all devices in the config file, but * there must be something in the identity */ if (!devlist && ident->uuid_set == 0 && ident->super_minor < 0 && ident->devices == NULL) { fprintf(stderr, Name ": No identity information available for %s - cannot assemble.\n", mddev); return 1; } if (devlist == NULL) devlist = conf_get_devs(conffile); else inargv = 1; tmpdev = devlist; num_devs = 0; while (tmpdev) { num_devs++; tmpdev = tmpdev->next; } devices = malloc(num_devs * sizeof(*devices)); if (!st && ident->st) st = ident->st; if (verbose>0) fprintf(stderr, Name ": looking for devices for %s\n", mddev); while ( devlist) { char *devname; int dfd; struct stat stb; struct supertype *tst = st; devname = devlist->devname; devlist = devlist->next; if (ident->devices && !match_oneof(ident->devices, devname)) { if ((inargv && verbose>=0) || verbose > 0) fprintf(stderr, Name ": %s is not one of %s\n", devname, ident->devices); continue; } if (super) { free(super); super = NULL; } dfd = dev_open(devname, O_RDONLY|O_EXCL); if (dfd < 0) { if ((inargv && verbose >= 0) || verbose > 0) fprintf(stderr, Name ": cannot open device %s: %s\n", devname, strerror(errno)); } else if (fstat(dfd, &stb)< 0) { /* Impossible! */ fprintf(stderr, Name ": fstat failed for %s: %s\n", devname, strerror(errno)); } else if ((stb.st_mode & S_IFMT) != S_IFBLK) { fprintf(stderr, Name ": %s is not a block device.\n", devname); } else if (!tst && (tst = guess_super(dfd)) == NULL) { if ((inargv && verbose >= 0) || verbose > 0) fprintf(stderr, Name ": no recogniseable superblock\n"); } else if (tst->ss->load_super(tst,dfd, &super, NULL)) { if ((inargv && verbose >= 0) || verbose > 0) fprintf( stderr, Name ": no RAID superblock on %s\n", devname); } else { tst->ss->getinfo_super(&info, &ident2, super); } if (dfd >= 0) close(dfd); if (ident->uuid_set && (!update || strcmp(update, "uuid")!= 0) && (!super || same_uuid(info.uuid, ident->uuid, tst->ss->swapuuid)==0)) { if ((inargv && verbose >= 0) || verbose > 0) fprintf(stderr, Name ": %s has wrong uuid.\n", devname); continue; } if (ident->name[0] && (!super || strncmp(ident2.name, ident->name, 32)!=0)) { if ((inargv && verbose >= 0) || verbose > 0) fprintf(stderr, Name ": %s has wrong name.\n", devname); continue; } if (ident->super_minor != UnSet && (!super || ident->super_minor != info.array.md_minor)) { if ((inargv && verbose >= 0) || verbose > 0) fprintf(stderr, Name ": %s has wrong super-minor.\n", devname); continue; } if (ident->level != UnSet && (!super|| ident->level != info.array.level)) { if ((inargv && verbose >= 0) || verbose > 0) fprintf(stderr, Name ": %s has wrong raid level.\n", devname); continue; } if (ident->raid_disks != UnSet && (!super || ident->raid_disks!= info.array.raid_disks)) { if ((inargv && verbose >= 0) || verbose > 0) fprintf(stderr, Name ": %s requires wrong number of drives.\n", devname); continue; } /* If we are this far, then we are commited to this device. * If the super_block doesn't exist, or doesn't match others, * then we cannot continue */ if (!super) { fprintf(stderr, Name ": %s has no superblock - assembly aborted\n", devname); free(first_super); return 1; } st = tst; /* commit to this format, if haven't already */ if (st->ss->compare_super(&first_super, super)) { fprintf(stderr, Name ": superblock on %s doesn't match others - assembly aborted\n", devname); free(super); free(first_super); return 1; } /* looks like a good enough match to update the super block if needed */ if (update) { /* prepare useful information in info structures */ struct stat stb2; fstat(mdfd, &stb2); info.array.md_minor = minor(stb2.st_rdev); if (strcmp(update, "uuid")==0 && !ident->uuid_set) { int rfd; if ((rfd = open("/dev/urandom", O_RDONLY)) < 0 || read(rfd, ident->uuid, 16) != 16) { *(__u32*)(ident->uuid) = random(); *(__u32*)(ident->uuid+1) = random(); *(__u32*)(ident->uuid+2) = random(); *(__u32*)(ident->uuid+3) = random(); } if (rfd >= 0) close(rfd); ident->uuid_set = 1; } memcpy(info.uuid, ident->uuid, 16); st->ss->update_super(&info, super, update, devname, verbose); dfd = dev_open(devname, O_RDWR|O_EXCL); if (dfd < 0) fprintf(stderr, Name ": Cannot open %s for superblock update\n", devname); else if (st->ss->store_super(st, dfd, super)) fprintf(stderr, Name ": Could not re-write superblock on %s.\n", devname); if (dfd >= 0) close(dfd); } if (verbose > 0) fprintf(stderr, Name ": %s is identified as a member of %s, slot %d.\n", devname, mddev, info.disk.raid_disk); devices[devcnt].devname = devname; devices[devcnt].major = major(stb.st_rdev); devices[devcnt].minor = minor(stb.st_rdev); devices[devcnt].oldmajor = info.disk.major; devices[devcnt].oldminor = info.disk.minor; devices[devcnt].events = info.events; devices[devcnt].raid_disk = info.disk.raid_disk; devices[devcnt].disk_nr = info.disk.number; devices[devcnt].uptodate = 0; devices[devcnt].state = info.disk.state; if (most_recent < devcnt) { if (devices[devcnt].events > devices[most_recent].events) most_recent = devcnt; } if (info.array.level == -4) /* with multipath, the raid_disk from the superblock is meaningless */ i = devcnt; else i = devices[devcnt].raid_disk; if (i+1 == 0) { if (nextspare < info.array.raid_disks) nextspare = info.array.raid_disks; i = nextspare++; } if (i < 10000) { if (i >= bestcnt) { unsigned int newbestcnt = i+10; int *newbest = malloc(sizeof(int)*newbestcnt); unsigned int c; for (c=0; c < newbestcnt; c++) if (c < bestcnt) newbest[c] = best[c]; else newbest[c] = -1; if (best)free(best); best = newbest; bestcnt = newbestcnt; } if (best[i] == -1 || devices[best[i]].events < devices[devcnt].events) best[i] = devcnt; } devcnt++; } if (super) free(super); super = NULL; if (update && strcmp(update, "byteorder")==0) st->minor_version = 90; if (devcnt == 0) { fprintf(stderr, Name ": no devices found for %s\n", mddev); free(first_super); return 1; } st->ss->getinfo_super(&info, &ident2, first_super); /* now we have some devices that might be suitable. * I wonder how many */ avail = malloc(info.array.raid_disks); memset(avail, 0, info.array.raid_disks); okcnt = 0; sparecnt=0; for (i=0; i< bestcnt ;i++) { int j = best[i]; int event_margin = 1; /* always allow a difference of '1' * like the kernel does */ if (j < 0) continue; /* note: we ignore error flags in multipath arrays * as they don't make sense */ if (info.array.level != -4) if (!(devices[j].state & (1<<MD_DISK_SYNC))) { if (!(devices[j].state & (1<<MD_DISK_FAULTY))) sparecnt++; continue; } if (devices[j].events+event_margin >= devices[most_recent].events) { devices[j].uptodate = 1; if (i < info.array.raid_disks) { okcnt++; avail[i]=1; } else sparecnt++; } } while (force && !enough(info.array.level, info.array.raid_disks, info.array.layout, avail, okcnt)) { /* Choose the newest best drive which is * not up-to-date, update the superblock * and add it. */ int fd; chosen_drive = -1; for (i=0; i<info.array.raid_disks && i < bestcnt; i++) { int j = best[i]; if (j>=0 && !devices[j].uptodate && devices[j].events > 0 && (chosen_drive < 0 || devices[j].events > devices[chosen_drive].events)) chosen_drive = j; } if (chosen_drive < 0) break; if (verbose >= 0) fprintf(stderr, Name ": forcing event count in %s(%d) from %d upto %d\n", devices[chosen_drive].devname, devices[chosen_drive].raid_disk, (int)(devices[chosen_drive].events), (int)(devices[most_recent].events)); fd = dev_open(devices[chosen_drive].devname, O_RDWR|O_EXCL); if (fd < 0) { fprintf(stderr, Name ": Couldn't open %s for write - not updating\n", devices[chosen_drive].devname); devices[chosen_drive].events = 0; continue; } if (st->ss->load_super(st,fd, &super, NULL)) { close(fd); fprintf(stderr, Name ": RAID superblock disappeared from %s - not updating.\n", devices[chosen_drive].devname); devices[chosen_drive].events = 0; continue; } info.events = devices[most_recent].events; st->ss->update_super(&info, super, "force", devices[chosen_drive].devname, verbose); if (st->ss->store_super(st, fd, super)) { close(fd); fprintf(stderr, Name ": Could not re-write superblock on %s\n", devices[chosen_drive].devname); devices[chosen_drive].events = 0; free(super); continue; } close(fd); devices[chosen_drive].events = devices[most_recent].events; devices[chosen_drive].uptodate = 1; avail[chosen_drive] = 1; okcnt++; free(super); } /* Now we want to look at the superblock which the kernel will base things on * and compare the devices that we think are working with the devices that the * superblock thinks are working. * If there are differences and --force is given, then update this chosen * superblock. */ chosen_drive = -1; super = NULL; for (i=0; chosen_drive < 0 && i<bestcnt; i++) { int j = best[i]; int fd; if (j<0) continue; if (!devices[j].uptodate) continue; chosen_drive = j; if ((fd=dev_open(devices[j].devname, O_RDONLY|O_EXCL))< 0) { fprintf(stderr, Name ": Cannot open %s: %s\n", devices[j].devname, strerror(errno)); return 1; } if (st->ss->load_super(st,fd, &super, NULL)) { close(fd); fprintf(stderr, Name ": RAID superblock has disappeared from %s\n", devices[j].devname); return 1; } close(fd); } if (super == NULL) { fprintf(stderr, Name ": No suitable drives found for %s\n", mddev); return 1; } st->ss->getinfo_super(&info, &ident2, super); for (i=0; i<bestcnt; i++) { int j = best[i]; unsigned int desired_state; if (i < info.array.raid_disks) desired_state = (1<<MD_DISK_ACTIVE) | (1<<MD_DISK_SYNC); else desired_state = 0; if (j<0) continue; if (!devices[j].uptodate) continue; info.disk.number = devices[j].disk_nr; info.disk.raid_disk = i; info.disk.state = desired_state; if (devices[j].uptodate && st->ss->update_super(&info, super, "assemble", NULL, verbose)) { if (force) { if (verbose >= 0) fprintf(stderr, Name ": " "clearing FAULTY flag for device %d in %s for %s\n", j, mddev, devices[j].devname); change = 1; } else { if (verbose >= -1) fprintf(stderr, Name ": " "device %d in %s has wrong state in superblock, but %s seems ok\n", i, mddev, devices[j].devname); } } #if 0 if (!devices[j].uptodate && !(super.disks[i].state & (1 << MD_DISK_FAULTY))) { fprintf(stderr, Name ": devices %d of %s is not marked FAULTY in superblock, but cannot be found\n", i, mddev); } #endif } if (force && okcnt == info.array.raid_disks-1) { /* FIXME check event count */ change += st->ss->update_super(&info, super, "force", devices[chosen_drive].devname, verbose); } if (change) { int fd; fd = dev_open(devices[chosen_drive].devname, O_RDWR|O_EXCL); if (fd < 0) { fprintf(stderr, Name ": Could open %s for write - cannot Assemble array.\n", devices[chosen_drive].devname); return 1; } if (st->ss->store_super(st, fd, super)) { close(fd); fprintf(stderr, Name ": Could not re-write superblock on %s\n", devices[chosen_drive].devname); return 1; } close(fd); } /* count number of in-sync devices according to the superblock. * We must have this number to start the array without -s or -R */ req_cnt = info.array.working_disks; /* Almost ready to actually *do* something */ if (!old_linux) { int rv; if ((vers % 100) >= 1) { /* can use different versions */ mdu_array_info_t inf; memset(&inf, 0, sizeof(inf)); inf.major_version = st->ss->major; inf.minor_version = st->minor_version; rv = ioctl(mdfd, SET_ARRAY_INFO, &inf); } else rv = ioctl(mdfd, SET_ARRAY_INFO, NULL); if (rv) { fprintf(stderr, Name ": SET_ARRAY_INFO failed for %s: %s\n", mddev, strerror(errno)); return 1; } if (ident->bitmap_fd >= 0) { if (ioctl(mdfd, SET_BITMAP_FILE, ident->bitmap_fd) != 0) { fprintf(stderr, Name ": SET_BITMAP_FILE failed.\n"); return 1; } } /* First, add the raid disks, but add the chosen one last */ for (i=0; i<= bestcnt; i++) { int j; if (i < bestcnt) { j = best[i]; if (j == chosen_drive) continue; } else j = chosen_drive; if (j >= 0 /* && devices[j].uptodate */) { mdu_disk_info_t disk; memset(&disk, 0, sizeof(disk)); disk.major = devices[j].major; disk.minor = devices[j].minor; if (ioctl(mdfd, ADD_NEW_DISK, &disk)!=0) { fprintf(stderr, Name ": failed to add %s to %s: %s\n", devices[j].devname, mddev, strerror(errno)); if (i < info.array.raid_disks || i == bestcnt) okcnt--; else sparecnt--; } else if (verbose > 0) fprintf(stderr, Name ": added %s to %s as %d\n", devices[j].devname, mddev, devices[j].raid_disk); } else if (verbose > 0 && i < info.array.raid_disks) fprintf(stderr, Name ": no uptodate device for slot %d of %s\n", i, mddev); } if (runstop == 1 || (runstop == 0 && ( enough(info.array.level, info.array.raid_disks, info.array.layout, avail, okcnt) && (okcnt >= req_cnt || start_partial_ok) ))) { if (ioctl(mdfd, RUN_ARRAY, NULL)==0) { if (verbose >= 0) { fprintf(stderr, Name ": %s has been started with %d drive%s", mddev, okcnt, okcnt==1?"":"s"); if (okcnt < info.array.raid_disks) fprintf(stderr, " (out of %d)", info.array.raid_disks); if (sparecnt) fprintf(stderr, " and %d spare%s", sparecnt, sparecnt==1?"":"s"); fprintf(stderr, ".\n"); } return 0; } fprintf(stderr, Name ": failed to RUN_ARRAY %s: %s\n", mddev, strerror(errno)); return 1; } if (runstop == -1) { fprintf(stderr, Name ": %s assembled from %d drive%s, but not started.\n", mddev, okcnt, okcnt==1?"":"s"); return 0; } if (verbose >= 0) { fprintf(stderr, Name ": %s assembled from %d drive%s", mddev, okcnt, okcnt==1?"":"s"); if (sparecnt) fprintf(stderr, " and %d spare%s", sparecnt, sparecnt==1?"":"s"); if (!enough(info.array.level, info.array.raid_disks, info.array.layout, avail, okcnt)) fprintf(stderr, " - not enough to start the array.\n"); else { if (req_cnt == info.array.raid_disks) fprintf(stderr, " - need all %d to start it", req_cnt); else fprintf(stderr, " - need %d of %d to start", req_cnt, info.array.raid_disks); fprintf(stderr, " (use --run to insist).\n"); } } return 1; } else { /* The "chosen_drive" is a good choice, and if necessary, the superblock has * been updated to point to the current locations of devices. * so we can just start the array */ unsigned long dev; dev = makedev(devices[chosen_drive].major, devices[chosen_drive].minor); if (ioctl(mdfd, START_ARRAY, dev)) { fprintf(stderr, Name ": Cannot start array: %s\n", strerror(errno)); } } return 0; }