void LinearRegression::calculate()
{
if (enough() && fabs(double(_count)*_sumXsquared - _sumX*_sumX) > DBL_EPSILON)
{
_b = (double(_count)*_sumXY - _sumY*_sumX)/(double(_count)*_sumXsquared - _sumX*_sumX);
_a = (_sumY - _b*_sumX)/double(_count);
double sx = _b*(_sumXY - _sumX*_sumY/double(_count));
double sy2 = _sumYsquared - _sumY*_sumY/double(_count);
double sy = sy2 - sx;
_coefD = sx/sy2;
_coefC = sqrt(_coefD);
_stdError = sqrt(sy/double(_count - 2));
}
else
{
_a = 0.0;
_b = 0.0;
_coefD = 0.0;
_coefC = 0.0;
_stdError = 0.0;
}
}
void MainWindow::rankVPN()
{
for( int i = 0; i < vpnList.count(); ++i )
{
connect(vpnList[i], SIGNAL(enough()), this, SLOT(downloadVPN()));
vpnList[i]->rank();
}
}
int main(){
int i;
double up=2000000.0,lw=0,mid;
scanf("%d%d",&N,&F);
for(i=1;i<=N;i++)
scanf("%d",org+i),org[i]*=1000;
while(up-lw > .5){
mid = (up+lw)/2.0;
if(enough(mid)) lw=mid;
else up=mid;
}
printf("%d\n",(int)up);
return 0;
}
bool test(int type_num, int deep, int last)
{
int i, j;
printf("test(%d %d %d)...\n", type_num, deep, last); /////////
for (i = last+1; i < g; i++) {
types[deep] = i;
if (deep == type_num - 1) {
if (enough(type_num)) {
fprintf (fout, "%d", type_num);
for (j = 0; j < type_num; j++)
fprintf(fout, " %d", types[j]+1);
fprintf(fout, "\n");
return true;
}
} else {
if(test(type_num, deep+1, i))
return true;
}
}
return false;
}
std::string newRunQuery(OperationContext* txn,
Message& m,
QueryMessage& q,
CurOp& curop,
Message &result,
bool fromDBDirectClient) {
// Validate the namespace.
const char *ns = q.ns;
uassert(16332, "can't have an empty ns", ns[0]);
const NamespaceString nsString(ns);
uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid());
// Set curop information.
curop.debug().ns = ns;
curop.debug().ntoreturn = q.ntoreturn;
curop.debug().query = q.query;
curop.setQuery(q.query);
// If the query is really a command, run it.
if (nsString.isCommand()) {
int nToReturn = q.ntoreturn;
uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn
<< ") for $cmd type ns - can only be 1 or -1",
nToReturn == 1 || nToReturn == -1);
curop.markCommand();
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder cmdResBuf;
if (!runCommands(txn, ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) {
uasserted(13530, "bad or malformed command request?");
}
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
const NamespaceString nss(q.ns);
// Parse the qm into a CanonicalQuery.
CanonicalQuery* cq;
Status canonStatus = CanonicalQuery::canonicalize(
q, &cq, WhereCallbackReal(txn, StringData(nss.db())));
if (!canonStatus.isOK()) {
uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString());
}
QLOG() << "Running query:\n" << cq->toString();
LOG(2) << "Running query: " << cq->toStringShort();
// Parse, canonicalize, plan, transcribe, and get a plan executor.
PlanExecutor* rawExec = NULL;
// We use this a lot below.
const LiteParsedQuery& pq = cq->getParsed();
AutoGetCollectionForRead ctx(txn, nss);
const int dbProfilingLevel = (ctx.getDb() != NULL) ? ctx.getDb()->getProfilingLevel() :
serverGlobalParams.defaultProfile;
Collection* collection = ctx.getCollection();
// We'll now try to get the query executor that will execute this query for us. There
// are a few cases in which we know upfront which executor we should get and, therefore,
// we shortcut the selection process here.
//
// (a) If the query is over a collection that doesn't exist, we use an EOFStage.
//
// (b) if the query is a replication's initial sync one, we use a specifically designed
// stage that skips extents faster (see details in exec/oplogstart.h).
//
// Otherwise we go through the selection of which executor is most suited to the
// query + run-time context at hand.
Status status = Status::OK();
if (NULL != collection && pq.getOptions().oplogReplay) {
// Takes ownership of 'cq'.
status = getOplogStartHack(txn, collection, cq, &rawExec);
}
else {
size_t options = QueryPlannerParams::DEFAULT;
if (shardingState.needCollectionMetadata(pq.ns())) {
options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
// Takes ownership of 'cq'.
status = getExecutor(txn, collection, cq, PlanExecutor::YIELD_AUTO, &rawExec, options);
}
if (!status.isOK()) {
// NOTE: Do not access cq as getExecutor has deleted it.
uasserted(17007, "Unable to execute query: " + status.reason());
}
verify(NULL != rawExec);
auto_ptr<PlanExecutor> exec(rawExec);
// If it's actually an explain, do the explain and return rather than falling through
// to the normal query execution loop.
if (pq.isExplain()) {
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder explainBob;
Explain::explainStages(exec.get(), ExplainCommon::EXEC_ALL_PLANS, &explainBob);
// Add the resulting object to the return buffer.
BSONObj explainObj = explainBob.obj();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
// Set query result fields.
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns());
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
bool slaveOK = pq.getOptions().slaveOk || pq.hasReadPref();
status = repl::getGlobalReplicationCoordinator()->checkCanServeReadsFor(
txn,
NamespaceString(cq->ns()),
slaveOK);
uassertStatusOK(status);
// If this exists, the collection is sharded.
// If it doesn't exist, we can assume we're not sharded.
// If we're sharded, we might encounter data that is not consistent with our sharding state.
// We must ignore this data.
CollectionMetadataPtr collMetadata;
if (!shardingState.needCollectionMetadata(pq.ns())) {
collMetadata = CollectionMetadataPtr();
}
else {
collMetadata = shardingState.getCollectionMetadata(pq.ns());
}
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult::Value));
// How many results have we obtained from the executor?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
OpTime slaveReadTill;
// Do we save the PlanExecutor in a ClientCursor for getMore calls later?
bool saveClientCursor = false;
BSONObj obj;
PlanExecutor::ExecState state;
// uint64_t numMisplacedDocs = 0;
// Get summary info about which plan the executor is using.
curop.debug().planSummary = Explain::getPlanSummary(exec.get());
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
// Add result to output buffer.
bb.appendBuf((void*)obj.objdata(), obj.objsize());
// Count the result.
++numResults;
// Possibly note slave's position in the oplog.
if (pq.getOptions().oplogReplay) {
BSONElement e = obj["ts"];
if (Date == e.type() || Timestamp == e.type()) {
slaveReadTill = e._opTime();
}
}
// TODO: only one type of 2d search doesn't support this. We need a way to pull it out
// of CanonicalQuery. :(
const bool supportsGetMore = true;
if (!supportsGetMore && (enough(pq, numResults)
|| bb.len() >= MaxBytesToReturnToClientAtOnce)) {
break;
}
else if (enoughForFirstBatch(pq, numResults, bb.len())) {
QLOG() << "Enough for first batch, wantMore=" << pq.wantMore()
<< " numToReturn=" << pq.getNumToReturn()
<< " numResults=" << numResults
<< endl;
// If only one result requested assume it's a findOne() and don't save the cursor.
if (pq.wantMore() && 1 != pq.getNumToReturn()) {
QLOG() << " executor EOF=" << exec->isEOF() << endl;
saveClientCursor = !exec->isEOF();
}
break;
}
}
// If we cache the executor later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the executor later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the executor.
exec->deregisterExec();
// Caller expects exceptions thrown in certain cases.
if (PlanExecutor::EXEC_ERROR == state) {
scoped_ptr<PlanStageStats> stats(exec->getStats());
error() << "Plan executor error, stats: "
<< Explain::statsToBSON(*stats);
uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj));
}
// Why save a dead executor?
if (PlanExecutor::DEAD == state) {
saveClientCursor = false;
}
else if (pq.getOptions().tailable) {
// If we're tailing a capped collection, we don't bother saving the cursor if the
// collection is empty. Otherwise, the semantics of the tailable cursor is that the
// client will keep trying to read from it. So we'll keep it around.
if (collection && collection->numRecords(txn) != 0 && pq.getNumToReturn() != 1) {
saveClientCursor = true;
}
}
// TODO(greg): This will go away soon.
if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(pq.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(pq.ns()));
}
const logger::LogComponent queryLogComponent = logger::LogComponent::kQuery;
const logger::LogSeverity logLevelOne = logger::LogSeverity::Debug(1);
PlanSummaryStats summaryStats;
Explain::getSummaryStats(exec.get(), &summaryStats);
curop.debug().ntoskip = pq.getSkip();
curop.debug().nreturned = numResults;
curop.debug().scanAndOrder = summaryStats.hasSortStage;
curop.debug().nscanned = summaryStats.totalKeysExamined;
curop.debug().nscannedObjects = summaryStats.totalDocsExamined;
curop.debug().idhack = summaryStats.isIdhack;
// Set debug information for consumption by the profiler.
if (dbProfilingLevel > 0 ||
curop.elapsedMillis() > serverGlobalParams.slowMS ||
logger::globalLogDomain()->shouldLog(queryLogComponent, logLevelOne)) {
// Get BSON stats.
scoped_ptr<PlanStageStats> execStats(exec->getStats());
BSONObjBuilder statsBob;
Explain::statsToBSON(*execStats, &statsBob);
curop.debug().execStats.set(statsBob.obj());
// Replace exec stats with plan summary if stats cannot fit into CachedBSONObj.
if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) {
BSONObjBuilder bob;
bob.append("summary", curop.debug().planSummary.toString());
curop.debug().execStats.set(bob.done());
}
}
long long ccId = 0;
if (saveClientCursor) {
// We won't use the executor until it's getMore'd.
exec->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(collection, exec.get(),
cq->getParsed().getOptions().toInt(),
cq->getParsed().getFilter());
ccId = cc->cursorid();
if (fromDBDirectClient) {
cc->setUnownedRecoveryUnit(txn->recoveryUnit());
}
else if (state == PlanExecutor::IS_EOF && pq.getOptions().tailable) {
// Don't stash the RU for tailable cursors at EOF, let them get a new RU on their
// next getMore.
}
else {
// We stash away the RecoveryUnit in the ClientCursor. It's used for subsequent
// getMore requests. The calling OpCtx gets a fresh RecoveryUnit.
cc->setOwnedRecoveryUnit(txn->releaseRecoveryUnit());
StorageEngine* storageEngine = getGlobalEnvironment()->getGlobalStorageEngine();
txn->setRecoveryUnit(storageEngine->newRecoveryUnit(txn));
}
QLOG() << "caching executor with cursorid " << ccId
<< " after returning " << numResults << " results" << endl;
// ClientCursor takes ownership of executor. Release to make sure it's not deleted.
exec.release();
// TODO document
if (pq.getOptions().oplogReplay && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.getOptions().exhaust) {
curop.debug().exhaust = true;
}
// Set attributes for getMore.
cc->setCollMetadata(collMetadata);
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
}
else {
QLOG() << "Not caching executor but returning " << numResults << " results.\n";
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult::View qr = result.header().view2ptr();
qr.setCursorId(ccId);
curop.debug().cursorid = (0 == ccId ? -1 : ccId);
qr.setResultFlagsToOk();
qr.msgdata().setOperation(opReply);
qr.setStartingFrom(0);
qr.setNReturned(numResults);
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? pq.ns() : "";
}
/**
* This is called by db/ops/query.cpp. This is the entry point for answering a query.
*/
std::string newRunQuery(CanonicalQuery* cq, CurOp& curop, Message &result) {
QLOG() << "Running query on new system: " << cq->toString();
// This is a read lock.
Client::ReadContext ctx(cq->ns(), storageGlobalParams.dbpath);
// Parse, canonicalize, plan, transcribe, and get a runner.
Runner* rawRunner = NULL;
// We use this a lot below.
const LiteParsedQuery& pq = cq->getParsed();
// Need to call cq->toString() now, since upon error getRunner doesn't guarantee
// cq is in a consistent state.
string cqStr = cq->toString();
// We'll now try to get the query runner that will execute this query for us. There
// are a few cases in which we know upfront which runner we should get and, therefore,
// we shortcut the selection process here.
//
// (a) If the query is over a collection that doesn't exist, we get a special runner
// that's is so (a runner) which doesn't return results, the EOFRunner.
//
// (b) if the query is a replication's initial sync one, we get a SingleSolutinRunner
// that uses a specifically designed stage that skips extents faster (see details in
// exec/oplogstart.h)
//
// Otherwise we go through the selection of which runner is most suited to the
// query + run-time context at hand.
Status status = Status::OK();
if (ctx.ctx().db()->getCollection(cq->ns()) == NULL) {
rawRunner = new EOFRunner(cq, cq->ns());
}
else if (pq.hasOption(QueryOption_OplogReplay)) {
status = getOplogStartHack(cq, &rawRunner);
}
else {
// Takes ownership of cq.
size_t options = QueryPlannerParams::DEFAULT;
if (shardingState.needCollectionMetadata(pq.ns())) {
options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
status = getRunner(cq, &rawRunner, options);
}
if (!status.isOK()) {
uasserted(17007, "Couldn't get runner for query because: " + status.reason() + " query is " + cqStr);
}
verify(NULL != rawRunner);
auto_ptr<Runner> runner(rawRunner);
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns());
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
replVerifyReadsOk(&pq);
// If this exists, the collection is sharded.
// If it doesn't exist, we can assume we're not sharded.
// If we're sharded, we might encounter data that is not consistent with our sharding state.
// We must ignore this data.
CollectionMetadataPtr collMetadata;
if (!shardingState.needCollectionMetadata(pq.ns())) {
collMetadata = CollectionMetadataPtr();
}
else {
collMetadata = shardingState.getCollectionMetadata(pq.ns());
}
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult));
// How many results have we obtained from the runner?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
OpTime slaveReadTill;
// Do we save the Runner in a ClientCursor for getMore calls later?
bool saveClientCursor = false;
// We turn on auto-yielding for the runner here. The runner registers itself with the
// active runners list in ClientCursor.
ClientCursor::registerRunner(runner.get());
runner->setYieldPolicy(Runner::YIELD_AUTO);
auto_ptr<DeregisterEvenIfUnderlyingCodeThrows> safety(
new DeregisterEvenIfUnderlyingCodeThrows(runner.get()));
BSONObj obj;
Runner::RunnerState state;
// uint64_t numMisplacedDocs = 0;
// set this outside loop. we will need to use this both within loop and when deciding
// to fill in explain information
const bool isExplain = pq.isExplain();
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) {
// Add result to output buffer. This is unnecessary if explain info is requested
if (!isExplain) {
bb.appendBuf((void*)obj.objdata(), obj.objsize());
}
// Count the result.
++numResults;
// Possibly note slave's position in the oplog.
if (pq.hasOption(QueryOption_OplogReplay)) {
BSONElement e = obj["ts"];
if (Date == e.type() || Timestamp == e.type()) {
slaveReadTill = e._opTime();
}
}
// TODO: only one type of 2d search doesn't support this. We need a way to pull it out
// of CanonicalQuery. :(
const bool supportsGetMore = true;
if (isExplain) {
if (enoughForExplain(pq, numResults)) {
break;
}
}
else if (!supportsGetMore && (enough(pq, numResults)
|| bb.len() >= MaxBytesToReturnToClientAtOnce)) {
break;
}
else if (enoughForFirstBatch(pq, numResults, bb.len())) {
QLOG() << "Enough for first batch, wantMore=" << pq.wantMore()
<< " numToReturn=" << pq.getNumToReturn()
<< " numResults=" << numResults
<< endl;
// If only one result requested assume it's a findOne() and don't save the cursor.
if (pq.wantMore() && 1 != pq.getNumToReturn()) {
QLOG() << " runner EOF=" << runner->isEOF() << endl;
saveClientCursor = !runner->isEOF();
}
break;
}
}
// If we cache the runner later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the runner later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the runner.
safety.reset();
// Caller expects exceptions thrown in certain cases:
// * in-memory sort using too much RAM.
if (Runner::RUNNER_ERROR == state) {
uasserted(17144, "Runner error, memory limit for sort probably exceeded");
}
// Why save a dead runner?
if (Runner::RUNNER_DEAD == state) {
saveClientCursor = false;
}
else if (pq.hasOption(QueryOption_CursorTailable)) {
// If we're tailing a capped collection, we don't bother saving the cursor if the
// collection is empty. Otherwise, the semantics of the tailable cursor is that the
// client will keep trying to read from it. So we'll keep it around.
Collection* collection = ctx.ctx().db()->getCollection(cq->ns());
if (collection && collection->numRecords() != 0 && pq.getNumToReturn() != 1) {
saveClientCursor = true;
}
}
// TODO(greg): This will go away soon.
if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(pq.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(pq.ns()));
}
// Append explain information to query results by asking the runner to produce them.
if (isExplain) {
TypeExplain* bareExplain;
Status res = runner->getExplainPlan(&bareExplain);
if (!res.isOK()) {
error() << "could not produce explain of query '" << pq.getFilter()
<< "', error: " << res.reason();
// If numResults and the data in bb don't correspond, we'll crash later when rooting
// through the reply msg.
BSONObj emptyObj;
bb.appendBuf((void*)emptyObj.objdata(), emptyObj.objsize());
// The explain output is actually a result.
numResults = 1;
// TODO: we can fill out millis etc. here just fine even if the plan screwed up.
}
else {
boost::scoped_ptr<TypeExplain> explain(bareExplain);
// Fill in the missing run-time fields in explain, starting with propeties of
// the process running the query.
std::string server = mongoutils::str::stream()
<< getHostNameCached() << ":" << serverGlobalParams.port;
explain->setServer(server);
// We might have skipped some results due to chunk migration etc. so our count is
// correct.
explain->setN(numResults);
// Clock the whole operation.
explain->setMillis(curop.elapsedMillis());
BSONObj explainObj = explain->toBSON();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
// The explain output is actually a result.
numResults = 1;
}
}
long long ccId = 0;
if (saveClientCursor) {
// We won't use the runner until it's getMore'd.
runner->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(runner.get(), cq->getParsed().getOptions(),
cq->getParsed().getFilter());
ccId = cc->cursorid();
QLOG() << "caching runner with cursorid " << ccId
<< " after returning " << numResults << " results" << endl;
// ClientCursor takes ownership of runner. Release to make sure it's not deleted.
runner.release();
// TODO document
if (pq.hasOption(QueryOption_OplogReplay) && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.hasOption(QueryOption_Exhaust)) {
curop.debug().exhaust = true;
}
// Set attributes for getMore.
cc->setCollMetadata(collMetadata);
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
}
else {
QLOG() << "not caching runner but returning " << numResults << " results\n";
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult* qr = static_cast<QueryResult*>(result.header());
qr->cursorId = ccId;
curop.debug().cursorid = (0 == ccId ? -1 : ccId);
qr->setResultFlagsToOk();
qr->setOperation(opReply);
qr->startingFrom = 0;
qr->nReturned = numResults;
curop.debug().ntoskip = pq.getSkip();
curop.debug().nreturned = numResults;
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? pq.ns() : "";
}
/**
* This is called by db/ops/query.cpp. This is the entry point for answering a query.
*/
string newRunQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) {
log() << "Running query on new system: " << q.query.toString() << endl;
// This is a read lock.
Client::ReadContext ctx(q.ns, dbpath);
// Parse, canonicalize, plan, transcribe, and get a runner.
Runner* rawRunner;
CanonicalQuery* cq;
Status status = getRunner(q, &rawRunner, &cq);
if (!status.isOK()) {
uasserted(17007, "Couldn't process query " + q.query.toString()
+ " why: " + status.reason());
}
verify(NULL != rawRunner);
auto_ptr<Runner> runner(rawRunner);
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(q.ns);
// We use this a lot below.
const LiteParsedQuery& pq = cq->getParsed();
// TODO: Document why we do this.
// TODO: do this when we can pass in our own parsed query
//replVerifyReadsOk(&pq);
// If this exists, the collection is sharded.
// If it doesn't exist, we can assume we're not sharded.
// If we're sharded, we might encounter data that is not consistent with our sharding state.
// We must ignore this data.
CollectionMetadataPtr collMetadata;
if (!shardingState.needCollectionMetadata(pq.ns())) {
collMetadata = CollectionMetadataPtr();
}
else {
collMetadata = shardingState.getCollectionMetadata(pq.ns());
}
// Run the query.
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult));
// How many results have we obtained from the runner?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
OpTime slaveReadTill;
// Do we save the Runner in a ClientCursor for getMore calls later?
bool saveClientCursor = false;
// We turn on auto-yielding for the runner here, so we must register it with the active
// runners list in ClientCursor.
ClientCursor::registerRunner(runner.get());
runner->setYieldPolicy(Runner::YIELD_AUTO);
BSONObj obj;
Runner::RunnerState state;
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) {
// If we're sharded make sure that we don't return any data that hasn't been migrated
// off of our shared yet.
if (collMetadata) {
// This information can change if we yield and as such we must make sure to re-fetch
// it if we yield.
KeyPattern kp(collMetadata->getKeyPattern());
// This performs excessive BSONObj creation but that's OK for now.
if (!collMetadata->keyBelongsToMe(kp.extractSingleKey(obj))) { continue; }
}
// Add result to output buffer.
bb.appendBuf((void*)obj.objdata(), obj.objsize());
// Count the result.
++numResults;
// Possibly note slave's position in the oplog.
if (pq.hasOption(QueryOption_OplogReplay)) {
BSONElement e = obj["ts"];
if (Date == e.type() || Timestamp == e.type()) {
slaveReadTill = e._opTime();
}
}
// TODO: only one type of 2d search doesn't support this. We need a way to pull it out
// of CanonicalQuery. :(
const bool supportsGetMore = true;
const bool isExplain = pq.isExplain();
if (isExplain && enoughForExplain(pq, numResults)) {
break;
}
else if (!supportsGetMore && (enough(pq, numResults)
|| bb.len() >= MaxBytesToReturnToClientAtOnce)) {
break;
}
else if (enoughForFirstBatch(pq, numResults, bb.len())) {
// If only one result requested assume it's a findOne() and don't save the cursor.
if (pq.wantMore() && 1 != pq.getNumToReturn()) {
saveClientCursor = true;
}
break;
}
}
// If we cache the runner later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the runner later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the runner.
ClientCursor::deregisterRunner(runner.get());
// Why save a dead runner?
if (Runner::RUNNER_DEAD == state) { saveClientCursor = false; }
// TODO: Stage creation can set tailable depending on what's in the parsed query. We have
// the full parsed query available during planning...set it there.
//
// TODO: If we're tailable we want to save the client cursor. Make sure we do this later.
//if (pq.hasOption(QueryOption_CursorTailable) && pq.getNumToReturn() != 1) { ... }
// TODO(greg): This will go away soon.
if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(pq.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(pq.ns()));
}
long long ccId = 0;
if (saveClientCursor) {
// We won't use the runner until it's getMore'd.
runner->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(runner.get(), cq->getParsed().getOptions(),
cq->getParsed().getFilter());
ccId = cc->cursorid();
log() << "caching runner with cursorid " << ccId << endl;
// ClientCursor takes ownership of runner. Release to make sure it's not deleted.
runner.release();
// TODO document
if (pq.hasOption(QueryOption_OplogReplay) && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.hasOption(QueryOption_Exhaust)) {
curop.debug().exhaust = true;
}
// Set attributes for getMore.
cc->setCollMetadata(collMetadata);
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult* qr = static_cast<QueryResult*>(result.header());
qr->cursorId = ccId;
curop.debug().cursorid = (0 == ccId ? -1 : ccId);
qr->setResultFlagsToOk();
qr->setOperation(opReply);
qr->startingFrom = 0;
qr->nReturned = numResults;
// TODO: nscanned is bogus.
// curop.debug().nscanned = ( cursor ? cursor->nscanned() : 0LL );
curop.debug().ntoskip = pq.getSkip();
curop.debug().nreturned = numResults;
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? pq.ns() : "";
}
std::string newRunQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) {
// Validate the namespace.
const char *ns = q.ns;
uassert(16332, "can't have an empty ns", ns[0]);
const NamespaceString nsString(ns);
uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid());
// Set curop information.
curop.debug().ns = ns;
curop.debug().ntoreturn = q.ntoreturn;
curop.debug().query = q.query;
curop.setQuery(q.query);
// If the query is really a command, run it.
if (nsString.isCommand()) {
int nToReturn = q.ntoreturn;
uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn
<< ") for $cmd type ns - can only be 1 or -1",
nToReturn == 1 || nToReturn == -1);
curop.markCommand();
BufBuilder bb;
bb.skip(sizeof(QueryResult));
BSONObjBuilder cmdResBuf;
if (!runCommands(ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) {
uasserted(13530, "bad or malformed command request?");
}
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
QueryResult* qr = reinterpret_cast<QueryResult*>(bb.buf());
bb.decouple();
qr->setResultFlagsToOk();
qr->len = bb.len();
curop.debug().responseLength = bb.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
result.setData(qr, true);
return "";
}
// This is a read lock. We require this because if we're parsing a $where, the
// where-specific parsing code assumes we have a lock and creates execution machinery that
// requires it.
Client::ReadContext ctx(q.ns);
Collection* collection = ctx.ctx().db()->getCollection( ns );
// Parse the qm into a CanonicalQuery.
CanonicalQuery* cq;
Status canonStatus = CanonicalQuery::canonicalize(q, &cq);
if (!canonStatus.isOK()) {
uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString());
}
verify(cq);
QLOG() << "Running query:\n" << cq->toString();
LOG(2) << "Running query: " << cq->toStringShort();
// Parse, canonicalize, plan, transcribe, and get a runner.
Runner* rawRunner = NULL;
// We use this a lot below.
const LiteParsedQuery& pq = cq->getParsed();
// We'll now try to get the query runner that will execute this query for us. There
// are a few cases in which we know upfront which runner we should get and, therefore,
// we shortcut the selection process here.
//
// (a) If the query is over a collection that doesn't exist, we get a special runner
// that's is so (a runner) which doesn't return results, the EOFRunner.
//
// (b) if the query is a replication's initial sync one, we get a SingleSolutinRunner
// that uses a specifically designed stage that skips extents faster (see details in
// exec/oplogstart.h)
//
// Otherwise we go through the selection of which runner is most suited to the
// query + run-time context at hand.
Status status = Status::OK();
if (collection == NULL) {
rawRunner = new EOFRunner(cq, cq->ns());
}
else if (pq.hasOption(QueryOption_OplogReplay)) {
status = getOplogStartHack(collection, cq, &rawRunner);
}
else {
// Takes ownership of cq.
size_t options = QueryPlannerParams::DEFAULT;
if (shardingState.needCollectionMetadata(pq.ns())) {
options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
status = getRunner(cq, &rawRunner, options);
}
if (!status.isOK()) {
// NOTE: Do not access cq as getRunner has deleted it.
uasserted(17007, "Unable to execute query: " + status.reason());
}
verify(NULL != rawRunner);
auto_ptr<Runner> runner(rawRunner);
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns());
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
replVerifyReadsOk(&pq);
// If this exists, the collection is sharded.
// If it doesn't exist, we can assume we're not sharded.
// If we're sharded, we might encounter data that is not consistent with our sharding state.
// We must ignore this data.
CollectionMetadataPtr collMetadata;
if (!shardingState.needCollectionMetadata(pq.ns())) {
collMetadata = CollectionMetadataPtr();
}
else {
collMetadata = shardingState.getCollectionMetadata(pq.ns());
}
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult));
// How many results have we obtained from the runner?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
OpTime slaveReadTill;
// Do we save the Runner in a ClientCursor for getMore calls later?
bool saveClientCursor = false;
// We turn on auto-yielding for the runner here. The runner registers itself with the
// active runners list in ClientCursor.
auto_ptr<ScopedRunnerRegistration> safety(new ScopedRunnerRegistration(runner.get()));
runner->setYieldPolicy(Runner::YIELD_AUTO);
BSONObj obj;
Runner::RunnerState state;
// uint64_t numMisplacedDocs = 0;
// set this outside loop. we will need to use this both within loop and when deciding
// to fill in explain information
const bool isExplain = pq.isExplain();
// Have we retrieved info about which plan the runner will
// use to execute the query yet?
bool gotPlanInfo = false;
PlanInfo* rawInfo;
boost::scoped_ptr<PlanInfo> planInfo;
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) {
// Add result to output buffer. This is unnecessary if explain info is requested
if (!isExplain) {
bb.appendBuf((void*)obj.objdata(), obj.objsize());
}
// Count the result.
++numResults;
// In the case of the multi plan runner, we may not be able to
// successfully retrieve plan info until after the query starts
// to run. This is because the multi plan runner doesn't know what
// plan it will end up using until it runs candidates and selects
// the best.
//
// TODO: Do we ever want to output what the MPR is comparing?
if (!gotPlanInfo) {
Status infoStatus = runner->getInfo(NULL, &rawInfo);
if (infoStatus.isOK()) {
gotPlanInfo = true;
planInfo.reset(rawInfo);
// planSummary is really a ThreadSafeString which copies the data from
// the provided pointer.
curop.debug().planSummary = planInfo->planSummary.c_str();
}
}
// Possibly note slave's position in the oplog.
if (pq.hasOption(QueryOption_OplogReplay)) {
BSONElement e = obj["ts"];
if (Date == e.type() || Timestamp == e.type()) {
slaveReadTill = e._opTime();
}
}
// TODO: only one type of 2d search doesn't support this. We need a way to pull it out
// of CanonicalQuery. :(
const bool supportsGetMore = true;
if (isExplain) {
if (enoughForExplain(pq, numResults)) {
break;
}
}
else if (!supportsGetMore && (enough(pq, numResults)
|| bb.len() >= MaxBytesToReturnToClientAtOnce)) {
break;
}
else if (enoughForFirstBatch(pq, numResults, bb.len())) {
QLOG() << "Enough for first batch, wantMore=" << pq.wantMore()
<< " numToReturn=" << pq.getNumToReturn()
<< " numResults=" << numResults
<< endl;
// If only one result requested assume it's a findOne() and don't save the cursor.
if (pq.wantMore() && 1 != pq.getNumToReturn()) {
QLOG() << " runner EOF=" << runner->isEOF() << endl;
saveClientCursor = !runner->isEOF();
}
break;
}
}
// Try to get information about the plan which the runner
// will use to execute the query, it we don't have it already.
if (!gotPlanInfo) {
Status infoStatus = runner->getInfo(NULL, &rawInfo);
if (infoStatus.isOK()) {
gotPlanInfo = true;
planInfo.reset(rawInfo);
// planSummary is really a ThreadSafeString which copies the data from
// the provided pointer.
curop.debug().planSummary = planInfo->planSummary.c_str();
}
}
// If we cache the runner later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the runner later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the runner.
safety.reset();
// Caller expects exceptions thrown in certain cases.
if (Runner::RUNNER_ERROR == state) {
TypeExplain* bareExplain;
Status res = runner->getInfo(&bareExplain, NULL);
if (res.isOK()) {
boost::scoped_ptr<TypeExplain> errorExplain(bareExplain);
error() << "Runner error, stats:\n"
<< errorExplain->stats.jsonString(Strict, true);
}
uasserted(17144, "Runner error: " + WorkingSetCommon::toStatusString(obj));
}
// Why save a dead runner?
if (Runner::RUNNER_DEAD == state) {
saveClientCursor = false;
}
else if (pq.hasOption(QueryOption_CursorTailable)) {
// If we're tailing a capped collection, we don't bother saving the cursor if the
// collection is empty. Otherwise, the semantics of the tailable cursor is that the
// client will keep trying to read from it. So we'll keep it around.
Collection* collection = ctx.ctx().db()->getCollection(cq->ns());
if (collection && collection->numRecords() != 0 && pq.getNumToReturn() != 1) {
saveClientCursor = true;
}
}
// TODO(greg): This will go away soon.
if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(pq.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(pq.ns()));
}
// Used to fill in explain and to determine if the query is slow enough to be logged.
int elapsedMillis = curop.elapsedMillis();
// Get explain information if:
// 1) it is needed by an explain query;
// 2) profiling is enabled; or
// 3) profiling is disabled but we still need explain details to log a "slow" query.
// Producing explain information is expensive and should be done only if we are certain
// the information will be used.
boost::scoped_ptr<TypeExplain> explain(NULL);
if (isExplain ||
ctx.ctx().db()->getProfilingLevel() > 0 ||
elapsedMillis > serverGlobalParams.slowMS) {
// Ask the runner to produce explain information.
TypeExplain* bareExplain;
Status res = runner->getInfo(&bareExplain, NULL);
if (res.isOK()) {
explain.reset(bareExplain);
}
else if (isExplain) {
error() << "could not produce explain of query '" << pq.getFilter()
<< "', error: " << res.reason();
// If numResults and the data in bb don't correspond, we'll crash later when rooting
// through the reply msg.
BSONObj emptyObj;
bb.appendBuf((void*)emptyObj.objdata(), emptyObj.objsize());
// The explain output is actually a result.
numResults = 1;
// TODO: we can fill out millis etc. here just fine even if the plan screwed up.
}
}
// Fill in the missing run-time fields in explain, starting with propeties of
// the process running the query.
if (isExplain && NULL != explain.get()) {
std::string server = mongoutils::str::stream()
<< getHostNameCached() << ":" << serverGlobalParams.port;
explain->setServer(server);
// We might have skipped some results due to chunk migration etc. so our count is
// correct.
explain->setN(numResults);
// Clock the whole operation.
explain->setMillis(elapsedMillis);
BSONObj explainObj = explain->toBSON();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
// The explain output is actually a result.
numResults = 1;
}
long long ccId = 0;
if (saveClientCursor) {
// We won't use the runner until it's getMore'd.
runner->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(collection, runner.get(),
cq->getParsed().getOptions(),
cq->getParsed().getFilter());
ccId = cc->cursorid();
QLOG() << "caching runner with cursorid " << ccId
<< " after returning " << numResults << " results" << endl;
// ClientCursor takes ownership of runner. Release to make sure it's not deleted.
runner.release();
// TODO document
if (pq.hasOption(QueryOption_OplogReplay) && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.hasOption(QueryOption_Exhaust)) {
curop.debug().exhaust = true;
}
// Set attributes for getMore.
cc->setCollMetadata(collMetadata);
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
}
else {
QLOG() << "Not caching runner but returning " << numResults << " results.\n";
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult* qr = static_cast<QueryResult*>(result.header());
qr->cursorId = ccId;
curop.debug().cursorid = (0 == ccId ? -1 : ccId);
qr->setResultFlagsToOk();
qr->setOperation(opReply);
qr->startingFrom = 0;
qr->nReturned = numResults;
// Set debug information for consumption by the profiler.
curop.debug().ntoskip = pq.getSkip();
curop.debug().nreturned = numResults;
if (NULL != explain.get()) {
if (explain->isScanAndOrderSet()) {
curop.debug().scanAndOrder = explain->getScanAndOrder();
}
else {
curop.debug().scanAndOrder = false;
}
if (explain->isNScannedSet()) {
curop.debug().nscanned = explain->getNScanned();
}
if (explain->isNScannedObjectsSet()) {
curop.debug().nscannedObjects = explain->getNScannedObjects();
}
if (explain->isIDHackSet()) {
curop.debug().idhack = explain->getIDHack();
}
if (!explain->stats.isEmpty()) {
// execStats is a CachedBSONObj because it lives in the race-prone
// curop.
curop.debug().execStats.set(explain->stats);
// Replace exec stats with plan summary if stats cannot fit into CachedBSONObj.
if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) {
BSONObjBuilder bob;
bob.append("summary", curop.debug().planSummary.toString());
curop.debug().execStats.set(bob.done());
}
}
}
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? pq.ns() : "";
}
int Assemble(struct supertype *st, char *mddev, int mdfd,
mddev_ident_t ident, char *conffile,
mddev_dev_t devlist,
int readonly, int runstop,
char *update,
int verbose, int force)
{
/*
* The task of Assemble is to find a collection of
* devices that should (according to their superblocks)
* form an array, and to give this collection to the MD driver.
* In Linux-2.4 and later, this involves submitting a
* SET_ARRAY_INFO ioctl with no arg - to prepare
* the array - and then submit a number of
* ADD_NEW_DISK ioctls to add disks into
* the array. Finally RUN_ARRAY might
* be submitted to start the array.
*
* Much of the work of Assemble is in finding and/or
* checking the disks to make sure they look right.
*
* If mddev is not set, then scan must be set and we
* read through the config file for dev+uuid mapping
* We recurse, setting mddev, for each device that
* - isn't running
* - has a valid uuid (or any uuid if !uuidset)
*
* If mddev is set, we try to determine state of md.
* check version - must be at least 0.90.0
* check kernel version. must be at least 2.4.
* If not, we can possibly fall back on START_ARRAY
* Try to GET_ARRAY_INFO.
* If possible, give up
* If not, try to STOP_ARRAY just to make sure
*
* If !uuidset and scan, look in conf-file for uuid
* If not found, give up
* If !devlist and scan and uuidset, get list of devs from conf-file
*
* For each device:
* Check superblock - discard if bad
* Check uuid (set if we don't have one) - discard if no match
* Check superblock similarity if we have a superblock - discard if different
* Record events, devicenum
* This should give us a list of devices for the array
* We should collect the most recent event number
*
* Count disks with recent enough event count
* While force && !enough disks
* Choose newest rejected disks, update event count
* mark clean and rewrite superblock
* If recent kernel:
* SET_ARRAY_INFO
* foreach device with recent events : ADD_NEW_DISK
* if runstop == 1 || "enough" disks and runstop==0 -> RUN_ARRAY
* If old kernel:
* Check the device numbers in superblock are right
* update superblock if any changes
* START_ARRAY
*
*/
int old_linux = 0;
int vers;
void *first_super = NULL, *super = NULL;
struct {
char *devname;
unsigned int major, minor;
unsigned int oldmajor, oldminor;
long long events;
int uptodate;
int state;
int raid_disk;
int disk_nr;
} *devices;
int *best = NULL; /* indexed by raid_disk */
unsigned int bestcnt = 0;
int devcnt = 0;
unsigned int okcnt, sparecnt;
unsigned int req_cnt;
unsigned int i;
int most_recent = 0;
int chosen_drive;
int change = 0;
int inargv = 0;
int start_partial_ok = force || devlist==NULL;
unsigned int num_devs;
mddev_dev_t tmpdev;
struct mdinfo info;
struct mddev_ident_s ident2;
char *avail;
int nextspare = 0;
vers = md_get_version(mdfd);
if (vers <= 0) {
fprintf(stderr, Name ": %s appears not to be an md device.\n", mddev);
return 1;
}
if (vers < 9000) {
fprintf(stderr, Name ": Assemble requires driver version 0.90.0 or later.\n"
" Upgrade your kernel or try --build\n");
return 1;
}
if (get_linux_version() < 2004000)
old_linux = 1;
if (ioctl(mdfd, GET_ARRAY_INFO, &info.array)>=0) {
fprintf(stderr, Name ": device %s already active - cannot assemble it\n",
mddev);
return 1;
}
ioctl(mdfd, STOP_ARRAY, NULL); /* just incase it was started but has no content */
/*
* If any subdevs are listed, then any that don't
* match ident are discarded. Remainder must all match and
* become the array.
* If no subdevs, then we scan all devices in the config file, but
* there must be something in the identity
*/
if (!devlist &&
ident->uuid_set == 0 &&
ident->super_minor < 0 &&
ident->devices == NULL) {
fprintf(stderr, Name ": No identity information available for %s - cannot assemble.\n",
mddev);
return 1;
}
if (devlist == NULL)
devlist = conf_get_devs(conffile);
else inargv = 1;
tmpdev = devlist; num_devs = 0;
while (tmpdev) {
num_devs++;
tmpdev = tmpdev->next;
}
devices = malloc(num_devs * sizeof(*devices));
if (!st && ident->st) st = ident->st;
if (verbose>0)
fprintf(stderr, Name ": looking for devices for %s\n",
mddev);
while ( devlist) {
char *devname;
int dfd;
struct stat stb;
struct supertype *tst = st;
devname = devlist->devname;
devlist = devlist->next;
if (ident->devices &&
!match_oneof(ident->devices, devname)) {
if ((inargv && verbose>=0) || verbose > 0)
fprintf(stderr, Name ": %s is not one of %s\n", devname, ident->devices);
continue;
}
if (super) {
free(super);
super = NULL;
}
dfd = dev_open(devname, O_RDONLY|O_EXCL);
if (dfd < 0) {
if ((inargv && verbose >= 0) || verbose > 0)
fprintf(stderr, Name ": cannot open device %s: %s\n",
devname, strerror(errno));
} else if (fstat(dfd, &stb)< 0) {
/* Impossible! */
fprintf(stderr, Name ": fstat failed for %s: %s\n",
devname, strerror(errno));
} else if ((stb.st_mode & S_IFMT) != S_IFBLK) {
fprintf(stderr, Name ": %s is not a block device.\n",
devname);
} else if (!tst && (tst = guess_super(dfd)) == NULL) {
if ((inargv && verbose >= 0) || verbose > 0)
fprintf(stderr, Name ": no recogniseable superblock\n");
} else if (tst->ss->load_super(tst,dfd, &super, NULL)) {
if ((inargv && verbose >= 0) || verbose > 0)
fprintf( stderr, Name ": no RAID superblock on %s\n",
devname);
} else {
tst->ss->getinfo_super(&info, &ident2, super);
}
if (dfd >= 0) close(dfd);
if (ident->uuid_set && (!update || strcmp(update, "uuid")!= 0) &&
(!super || same_uuid(info.uuid, ident->uuid, tst->ss->swapuuid)==0)) {
if ((inargv && verbose >= 0) || verbose > 0)
fprintf(stderr, Name ": %s has wrong uuid.\n",
devname);
continue;
}
if (ident->name[0] &&
(!super || strncmp(ident2.name, ident->name, 32)!=0)) {
if ((inargv && verbose >= 0) || verbose > 0)
fprintf(stderr, Name ": %s has wrong name.\n",
devname);
continue;
}
if (ident->super_minor != UnSet &&
(!super || ident->super_minor != info.array.md_minor)) {
if ((inargv && verbose >= 0) || verbose > 0)
fprintf(stderr, Name ": %s has wrong super-minor.\n",
devname);
continue;
}
if (ident->level != UnSet &&
(!super|| ident->level != info.array.level)) {
if ((inargv && verbose >= 0) || verbose > 0)
fprintf(stderr, Name ": %s has wrong raid level.\n",
devname);
continue;
}
if (ident->raid_disks != UnSet &&
(!super || ident->raid_disks!= info.array.raid_disks)) {
if ((inargv && verbose >= 0) || verbose > 0)
fprintf(stderr, Name ": %s requires wrong number of drives.\n",
devname);
continue;
}
/* If we are this far, then we are commited to this device.
* If the super_block doesn't exist, or doesn't match others,
* then we cannot continue
*/
if (!super) {
fprintf(stderr, Name ": %s has no superblock - assembly aborted\n",
devname);
free(first_super);
return 1;
}
st = tst; /* commit to this format, if haven't already */
if (st->ss->compare_super(&first_super, super)) {
fprintf(stderr, Name ": superblock on %s doesn't match others - assembly aborted\n",
devname);
free(super);
free(first_super);
return 1;
}
/* looks like a good enough match to update the super block if needed */
if (update) {
/* prepare useful information in info structures */
struct stat stb2;
fstat(mdfd, &stb2);
info.array.md_minor = minor(stb2.st_rdev);
if (strcmp(update, "uuid")==0 &&
!ident->uuid_set) {
int rfd;
if ((rfd = open("/dev/urandom", O_RDONLY)) < 0 ||
read(rfd, ident->uuid, 16) != 16) {
*(__u32*)(ident->uuid) = random();
*(__u32*)(ident->uuid+1) = random();
*(__u32*)(ident->uuid+2) = random();
*(__u32*)(ident->uuid+3) = random();
}
if (rfd >= 0) close(rfd);
ident->uuid_set = 1;
}
memcpy(info.uuid, ident->uuid, 16);
st->ss->update_super(&info, super, update, devname, verbose);
dfd = dev_open(devname, O_RDWR|O_EXCL);
if (dfd < 0)
fprintf(stderr, Name ": Cannot open %s for superblock update\n",
devname);
else if (st->ss->store_super(st, dfd, super))
fprintf(stderr, Name ": Could not re-write superblock on %s.\n",
devname);
if (dfd >= 0)
close(dfd);
}
if (verbose > 0)
fprintf(stderr, Name ": %s is identified as a member of %s, slot %d.\n",
devname, mddev, info.disk.raid_disk);
devices[devcnt].devname = devname;
devices[devcnt].major = major(stb.st_rdev);
devices[devcnt].minor = minor(stb.st_rdev);
devices[devcnt].oldmajor = info.disk.major;
devices[devcnt].oldminor = info.disk.minor;
devices[devcnt].events = info.events;
devices[devcnt].raid_disk = info.disk.raid_disk;
devices[devcnt].disk_nr = info.disk.number;
devices[devcnt].uptodate = 0;
devices[devcnt].state = info.disk.state;
if (most_recent < devcnt) {
if (devices[devcnt].events
> devices[most_recent].events)
most_recent = devcnt;
}
if (info.array.level == -4)
/* with multipath, the raid_disk from the superblock is meaningless */
i = devcnt;
else
i = devices[devcnt].raid_disk;
if (i+1 == 0) {
if (nextspare < info.array.raid_disks)
nextspare = info.array.raid_disks;
i = nextspare++;
}
if (i < 10000) {
if (i >= bestcnt) {
unsigned int newbestcnt = i+10;
int *newbest = malloc(sizeof(int)*newbestcnt);
unsigned int c;
for (c=0; c < newbestcnt; c++)
if (c < bestcnt)
newbest[c] = best[c];
else
newbest[c] = -1;
if (best)free(best);
best = newbest;
bestcnt = newbestcnt;
}
if (best[i] == -1
|| devices[best[i]].events < devices[devcnt].events)
best[i] = devcnt;
}
devcnt++;
}
if (super)
free(super);
super = NULL;
if (update && strcmp(update, "byteorder")==0)
st->minor_version = 90;
if (devcnt == 0) {
fprintf(stderr, Name ": no devices found for %s\n",
mddev);
free(first_super);
return 1;
}
st->ss->getinfo_super(&info, &ident2, first_super);
/* now we have some devices that might be suitable.
* I wonder how many
*/
avail = malloc(info.array.raid_disks);
memset(avail, 0, info.array.raid_disks);
okcnt = 0;
sparecnt=0;
for (i=0; i< bestcnt ;i++) {
int j = best[i];
int event_margin = 1; /* always allow a difference of '1'
* like the kernel does
*/
if (j < 0) continue;
/* note: we ignore error flags in multipath arrays
* as they don't make sense
*/
if (info.array.level != -4)
if (!(devices[j].state & (1<<MD_DISK_SYNC))) {
if (!(devices[j].state & (1<<MD_DISK_FAULTY)))
sparecnt++;
continue;
}
if (devices[j].events+event_margin >=
devices[most_recent].events) {
devices[j].uptodate = 1;
if (i < info.array.raid_disks) {
okcnt++;
avail[i]=1;
} else
sparecnt++;
}
}
while (force && !enough(info.array.level, info.array.raid_disks,
info.array.layout,
avail, okcnt)) {
/* Choose the newest best drive which is
* not up-to-date, update the superblock
* and add it.
*/
int fd;
chosen_drive = -1;
for (i=0; i<info.array.raid_disks && i < bestcnt; i++) {
int j = best[i];
if (j>=0 &&
!devices[j].uptodate &&
devices[j].events > 0 &&
(chosen_drive < 0 ||
devices[j].events > devices[chosen_drive].events))
chosen_drive = j;
}
if (chosen_drive < 0)
break;
if (verbose >= 0)
fprintf(stderr, Name ": forcing event count in %s(%d) from %d upto %d\n",
devices[chosen_drive].devname, devices[chosen_drive].raid_disk,
(int)(devices[chosen_drive].events),
(int)(devices[most_recent].events));
fd = dev_open(devices[chosen_drive].devname, O_RDWR|O_EXCL);
if (fd < 0) {
fprintf(stderr, Name ": Couldn't open %s for write - not updating\n",
devices[chosen_drive].devname);
devices[chosen_drive].events = 0;
continue;
}
if (st->ss->load_super(st,fd, &super, NULL)) {
close(fd);
fprintf(stderr, Name ": RAID superblock disappeared from %s - not updating.\n",
devices[chosen_drive].devname);
devices[chosen_drive].events = 0;
continue;
}
info.events = devices[most_recent].events;
st->ss->update_super(&info, super, "force", devices[chosen_drive].devname, verbose);
if (st->ss->store_super(st, fd, super)) {
close(fd);
fprintf(stderr, Name ": Could not re-write superblock on %s\n",
devices[chosen_drive].devname);
devices[chosen_drive].events = 0;
free(super);
continue;
}
close(fd);
devices[chosen_drive].events = devices[most_recent].events;
devices[chosen_drive].uptodate = 1;
avail[chosen_drive] = 1;
okcnt++;
free(super);
}
/* Now we want to look at the superblock which the kernel will base things on
* and compare the devices that we think are working with the devices that the
* superblock thinks are working.
* If there are differences and --force is given, then update this chosen
* superblock.
*/
chosen_drive = -1;
super = NULL;
for (i=0; chosen_drive < 0 && i<bestcnt; i++) {
int j = best[i];
int fd;
if (j<0)
continue;
if (!devices[j].uptodate)
continue;
chosen_drive = j;
if ((fd=dev_open(devices[j].devname, O_RDONLY|O_EXCL))< 0) {
fprintf(stderr, Name ": Cannot open %s: %s\n",
devices[j].devname, strerror(errno));
return 1;
}
if (st->ss->load_super(st,fd, &super, NULL)) {
close(fd);
fprintf(stderr, Name ": RAID superblock has disappeared from %s\n",
devices[j].devname);
return 1;
}
close(fd);
}
if (super == NULL) {
fprintf(stderr, Name ": No suitable drives found for %s\n", mddev);
return 1;
}
st->ss->getinfo_super(&info, &ident2, super);
for (i=0; i<bestcnt; i++) {
int j = best[i];
unsigned int desired_state;
if (i < info.array.raid_disks)
desired_state = (1<<MD_DISK_ACTIVE) | (1<<MD_DISK_SYNC);
else
desired_state = 0;
if (j<0)
continue;
if (!devices[j].uptodate)
continue;
info.disk.number = devices[j].disk_nr;
info.disk.raid_disk = i;
info.disk.state = desired_state;
if (devices[j].uptodate &&
st->ss->update_super(&info, super, "assemble", NULL, verbose)) {
if (force) {
if (verbose >= 0)
fprintf(stderr, Name ": "
"clearing FAULTY flag for device %d in %s for %s\n",
j, mddev, devices[j].devname);
change = 1;
} else {
if (verbose >= -1)
fprintf(stderr, Name ": "
"device %d in %s has wrong state in superblock, but %s seems ok\n",
i, mddev, devices[j].devname);
}
}
#if 0
if (!devices[j].uptodate &&
!(super.disks[i].state & (1 << MD_DISK_FAULTY))) {
fprintf(stderr, Name ": devices %d of %s is not marked FAULTY in superblock, but cannot be found\n",
i, mddev);
}
#endif
}
if (force && okcnt == info.array.raid_disks-1) {
/* FIXME check event count */
change += st->ss->update_super(&info, super, "force",
devices[chosen_drive].devname, verbose);
}
if (change) {
int fd;
fd = dev_open(devices[chosen_drive].devname, O_RDWR|O_EXCL);
if (fd < 0) {
fprintf(stderr, Name ": Could open %s for write - cannot Assemble array.\n",
devices[chosen_drive].devname);
return 1;
}
if (st->ss->store_super(st, fd, super)) {
close(fd);
fprintf(stderr, Name ": Could not re-write superblock on %s\n",
devices[chosen_drive].devname);
return 1;
}
close(fd);
}
/* count number of in-sync devices according to the superblock.
* We must have this number to start the array without -s or -R
*/
req_cnt = info.array.working_disks;
/* Almost ready to actually *do* something */
if (!old_linux) {
int rv;
if ((vers % 100) >= 1) { /* can use different versions */
mdu_array_info_t inf;
memset(&inf, 0, sizeof(inf));
inf.major_version = st->ss->major;
inf.minor_version = st->minor_version;
rv = ioctl(mdfd, SET_ARRAY_INFO, &inf);
} else
rv = ioctl(mdfd, SET_ARRAY_INFO, NULL);
if (rv) {
fprintf(stderr, Name ": SET_ARRAY_INFO failed for %s: %s\n",
mddev, strerror(errno));
return 1;
}
if (ident->bitmap_fd >= 0) {
if (ioctl(mdfd, SET_BITMAP_FILE, ident->bitmap_fd) != 0) {
fprintf(stderr, Name ": SET_BITMAP_FILE failed.\n");
return 1;
}
}
/* First, add the raid disks, but add the chosen one last */
for (i=0; i<= bestcnt; i++) {
int j;
if (i < bestcnt) {
j = best[i];
if (j == chosen_drive)
continue;
} else
j = chosen_drive;
if (j >= 0 /* && devices[j].uptodate */) {
mdu_disk_info_t disk;
memset(&disk, 0, sizeof(disk));
disk.major = devices[j].major;
disk.minor = devices[j].minor;
if (ioctl(mdfd, ADD_NEW_DISK, &disk)!=0) {
fprintf(stderr, Name ": failed to add %s to %s: %s\n",
devices[j].devname,
mddev,
strerror(errno));
if (i < info.array.raid_disks || i == bestcnt)
okcnt--;
else
sparecnt--;
} else if (verbose > 0)
fprintf(stderr, Name ": added %s to %s as %d\n",
devices[j].devname, mddev, devices[j].raid_disk);
} else if (verbose > 0 && i < info.array.raid_disks)
fprintf(stderr, Name ": no uptodate device for slot %d of %s\n",
i, mddev);
}
if (runstop == 1 ||
(runstop == 0 &&
( enough(info.array.level, info.array.raid_disks, info.array.layout, avail, okcnt) &&
(okcnt >= req_cnt || start_partial_ok)
))) {
if (ioctl(mdfd, RUN_ARRAY, NULL)==0) {
if (verbose >= 0) {
fprintf(stderr, Name ": %s has been started with %d drive%s",
mddev, okcnt, okcnt==1?"":"s");
if (okcnt < info.array.raid_disks)
fprintf(stderr, " (out of %d)", info.array.raid_disks);
if (sparecnt)
fprintf(stderr, " and %d spare%s", sparecnt, sparecnt==1?"":"s");
fprintf(stderr, ".\n");
}
return 0;
}
fprintf(stderr, Name ": failed to RUN_ARRAY %s: %s\n",
mddev, strerror(errno));
return 1;
}
if (runstop == -1) {
fprintf(stderr, Name ": %s assembled from %d drive%s, but not started.\n",
mddev, okcnt, okcnt==1?"":"s");
return 0;
}
if (verbose >= 0) {
fprintf(stderr, Name ": %s assembled from %d drive%s", mddev, okcnt, okcnt==1?"":"s");
if (sparecnt)
fprintf(stderr, " and %d spare%s", sparecnt, sparecnt==1?"":"s");
if (!enough(info.array.level, info.array.raid_disks, info.array.layout, avail, okcnt))
fprintf(stderr, " - not enough to start the array.\n");
else {
if (req_cnt == info.array.raid_disks)
fprintf(stderr, " - need all %d to start it", req_cnt);
else
fprintf(stderr, " - need %d of %d to start", req_cnt, info.array.raid_disks);
fprintf(stderr, " (use --run to insist).\n");
}
}
return 1;
} else {
/* The "chosen_drive" is a good choice, and if necessary, the superblock has
* been updated to point to the current locations of devices.
* so we can just start the array
*/
unsigned long dev;
dev = makedev(devices[chosen_drive].major,
devices[chosen_drive].minor);
if (ioctl(mdfd, START_ARRAY, dev)) {
fprintf(stderr, Name ": Cannot start array: %s\n",
strerror(errno));
}
}
return 0;
}
