INT32 aggrProjectParser::buildNode( const BSONElement &elem, const CHAR *pCLName, qgmOptiTreeNode *&pNode, _qgmPtrTable *pTable, _qgmParamTable *pParamTable ) { INT32 rc = SDB_OK; BOOLEAN hasFunc = FALSE; BSONObj obj; qgmOptiSelect *pSelect = SDB_OSS_NEW qgmOptiSelect( pTable, pParamTable ); PD_CHECK( pSelect!=NULL, SDB_OOM, error, PDERROR, "malloc failed" ); PD_CHECK( elem.type() == Object, SDB_INVALIDARG, error, PDERROR, "failed to parse the parameter:%s(type=%d, expectType=%d)", elem.fieldName(), elem.type(), Object ); try { obj = elem.embeddedObject(); { PD_CHECK( !obj.isEmpty(), SDB_INVALIDARG, error, PDERROR, "Parameter-object can't be empty!" ); } BSONObjIterator iter( obj ); while ( iter.more() ) { BSONElement beField = iter.next(); const CHAR *pFieldName = beField.fieldName(); PD_CHECK( pFieldName[0] != AGGR_KEYWORD_PREFIX, SDB_INVALIDARG, error, PDERROR, "failed to parse \"project\", field name can't begin with\"$\"!" ); rc = parseSelectorField( beField, pCLName, pSelect->_selector, pTable, hasFunc ); PD_RC_CHECK( rc, PDERROR, "failed to parse the field:%s", pFieldName ); } if ( pSelect->_selector.empty() ) { qgmOpField selectAll; selectAll.type = SQL_GRAMMAR::WILDCARD; pSelect->_selector.push_back( selectAll ); } } catch ( std::exception &e ) { PD_CHECK( SDB_INVALIDARG, SDB_INVALIDARG, error, PDERROR, "failed to parse the Parameter-object, received unexpected error:%s", e.what() ); } pSelect->_limit = -1; pSelect->_skip = 0; pSelect->_type = QGM_OPTI_TYPE_SELECT; pSelect->_hasFunc = hasFunc; rc = pTable->getOwnField( AGGR_CL_DEFAULT_ALIAS, pSelect->_alias ); PD_RC_CHECK( rc, PDERROR, "failed to get the field(%s)", AGGR_CL_DEFAULT_ALIAS ); if ( pCLName != NULL ) { qgmField clValAttr; qgmField clValRelegation; rc = pTable->getOwnField( pCLName, clValAttr ); PD_RC_CHECK( rc, PDERROR, "failed to get the field(%s)", pCLName ); rc = pTable->getOwnField( AGGR_CL_DEFAULT_ALIAS, pSelect->_collection.alias ); PD_RC_CHECK( rc, PDERROR, "failed to get the field(%s)", AGGR_CL_DEFAULT_ALIAS ); pSelect->_collection.value = qgmDbAttr( clValRelegation, clValAttr ); pSelect->_collection.type = SQL_GRAMMAR::DBATTR; } pNode = pSelect; done: return rc; error: SAFE_OSS_DELETE( pSelect ); goto done; }
Config::Config( const string& _dbname , const BSONObj& cmdObj ) { dbname = _dbname; ns = dbname + "." + cmdObj.firstElement().valuestr(); verbose = cmdObj["verbose"].trueValue(); uassert( 13602 , "outType is no longer a valid option" , cmdObj["outType"].eoo() ); if ( cmdObj["out"].type() == String ) { finalShort = cmdObj["out"].String(); outType = REPLACE; } else if ( cmdObj["out"].type() == Object ) { BSONObj o = cmdObj["out"].embeddedObject(); BSONElement e = o.firstElement(); string t = e.fieldName(); if ( t == "normal" || t == "replace" ) { outType = REPLACE; finalShort = e.String(); } else if ( t == "merge" ) { outType = MERGE; finalShort = e.String(); } else if ( t == "reduce" ) { outType = REDUCE; finalShort = e.String(); } else if ( t == "inline" ) { outType = INMEMORY; } else { uasserted( 13522 , str::stream() << "unknown out specifier [" << t << "]" ); } if (o.hasElement("db")) { outDB = o["db"].String(); } } else { uasserted( 13606 , "'out' has to be a string or an object" ); } if ( outType != INMEMORY ) { // setup names tempLong = str::stream() << (outDB.empty() ? dbname : outDB) << ".tmp.mr." << cmdObj.firstElement().String() << "_" << finalShort << "_" << JOB_NUMBER++; incLong = tempLong + "_inc"; finalLong = str::stream() << (outDB.empty() ? dbname : outDB) << "." << finalShort; } { // scope and code if ( cmdObj["scope"].type() == Object ) scopeSetup = cmdObj["scope"].embeddedObjectUserCheck(); mapper.reset( new JSMapper( cmdObj["map"] ) ); reducer.reset( new JSReducer( cmdObj["reduce"] ) ); if ( cmdObj["finalize"].type() && cmdObj["finalize"].trueValue() ) finalizer.reset( new JSFinalizer( cmdObj["finalize"] ) ); if ( cmdObj["mapparams"].type() == Array ) { mapParams = cmdObj["mapparams"].embeddedObjectUserCheck(); } } { // query options BSONElement q = cmdObj["query"]; if ( q.type() == Object ) filter = q.embeddedObjectUserCheck(); else uassert( 13608 , "query has to be blank or an Object" , ! q.trueValue() ); BSONElement s = cmdObj["sort"]; if ( s.type() == Object ) sort = s.embeddedObjectUserCheck(); else uassert( 13609 , "sort has to be blank or an Object" , ! s.trueValue() ); if ( cmdObj["limit"].isNumber() ) limit = cmdObj["limit"].numberLong(); else limit = 0; } }
int run() { string ns; const bool csv = hasParam( "csv" ); const bool jsonArray = hasParam( "jsonArray" ); ostream *outPtr = &cout; string outfile = getParam( "out" ); auto_ptr<ofstream> fileStream; if ( hasParam( "out" ) ) { size_t idx = outfile.rfind( "/" ); if ( idx != string::npos ) { string dir = outfile.substr( 0 , idx + 1 ); boost::filesystem::create_directories( dir ); } ofstream * s = new ofstream( outfile.c_str() , ios_base::out ); fileStream.reset( s ); outPtr = s; if ( ! s->good() ) { cerr << "couldn't open [" << outfile << "]" << endl; return -1; } } ostream &out = *outPtr; BSONObj * fieldsToReturn = 0; BSONObj realFieldsToReturn; try { ns = getNS(); } catch (...) { printHelp(cerr); return 1; } auth(); if ( hasParam( "fields" ) || csv ) { needFields(); // we can't use just _fieldsObj since we support everything getFieldDotted does set<string> seen; BSONObjBuilder b; BSONObjIterator i( _fieldsObj ); while ( i.more() ){ BSONElement e = i.next(); string f = str::before( e.fieldName() , '.' ); if ( seen.insert( f ).second ) b.append( f , 1 ); } realFieldsToReturn = b.obj(); fieldsToReturn = &realFieldsToReturn; } if ( csv && _fields.size() == 0 ) { cerr << "csv mode requires a field list" << endl; return -1; } Query q( getParam( "query" , "" ) ); if ( q.getFilter().isEmpty() && !hasParam("dbpath") && !hasParam("forceTableScan") ) q.snapshot(); bool slaveOk = _params["slaveOk"].as<bool>(); auto_ptr<DBClientCursor> cursor = conn().query( ns.c_str() , q , 0 , 0 , fieldsToReturn , ( slaveOk ? QueryOption_SlaveOk : 0 ) | QueryOption_NoCursorTimeout ); if ( csv ) { for ( vector<string>::iterator i=_fields.begin(); i != _fields.end(); i++ ) { if ( i != _fields.begin() ) out << ","; out << *i; } out << endl; } if (jsonArray) out << '['; long long num = 0; while ( cursor->more() ) { num++; BSONObj obj = cursor->next(); if ( csv ) { for ( vector<string>::iterator i=_fields.begin(); i != _fields.end(); i++ ) { if ( i != _fields.begin() ) out << ","; const BSONElement & e = obj.getFieldDotted(i->c_str()); if ( ! e.eoo() ) { out << csvString(e); } } out << endl; } else { if (jsonArray && num != 1) out << ','; out << obj.jsonString(); if (!jsonArray) out << endl; } } if (jsonArray) out << ']' << endl; cerr << "exported " << num << " records" << endl; return 0; }
StatusWith<BSONObj> fixDocumentForInsert( const BSONObj& doc ) { if ( doc.objsize() > BSONObjMaxUserSize ) return StatusWith<BSONObj>( ErrorCodes::BadValue, str::stream() << "object to insert too large" << ". size in bytes: " << doc.objsize() << ", max size: " << BSONObjMaxUserSize ); bool firstElementIsId = doc.firstElement().fieldNameStringData() == "_id"; bool hasTimestampToFix = false; { BSONObjIterator i( doc ); while ( i.more() ) { BSONElement e = i.next(); if ( e.type() == Timestamp && e.timestampValue() == 0 ) { // we replace Timestamp(0,0) at the top level with a correct value // in the fast pass, we just mark that we want to swap hasTimestampToFix = true; } const char* fieldName = e.fieldName(); if ( fieldName[0] == '$' ) { return StatusWith<BSONObj>( ErrorCodes::BadValue, str::stream() << "Document can't have $ prefixed field names: " << e.fieldName() ); } // check no regexp for _id (SERVER-9502) // also, disallow undefined and arrays if ( str::equals( fieldName, "_id") ) { if ( e.type() == RegEx ) { return StatusWith<BSONObj>( ErrorCodes::BadValue, "can't use a regex for _id" ); } if ( e.type() == Undefined ) { return StatusWith<BSONObj>( ErrorCodes::BadValue, "can't use a undefined for _id" ); } if ( e.type() == Array ) { return StatusWith<BSONObj>( ErrorCodes::BadValue, "can't use an array for _id" ); } if ( e.type() == Object ) { BSONObj o = e.Obj(); Status s = o.storageValidEmbedded(); if ( !s.isOK() ) return StatusWith<BSONObj>( s ); } } } } if ( firstElementIsId && !hasTimestampToFix ) return StatusWith<BSONObj>( BSONObj() ); bool hadId = firstElementIsId; BSONObjIterator i( doc ); BSONObjBuilder b( doc.objsize() + 16 ); if ( firstElementIsId ) { b.append( doc.firstElement() ); i.next(); } else { BSONElement e = doc["_id"]; if ( e.type() ) { b.append( e ); hadId = true; } else { b.appendOID( "_id", NULL, true ); } } while ( i.more() ) { BSONElement e = i.next(); if ( hadId && e.fieldNameStringData() == "_id" ) { // no-op } else if ( e.type() == Timestamp && e.timestampValue() == 0 ) { mutex::scoped_lock lk(OpTime::m); b.append( e.fieldName(), OpTime::now(lk) ); } else { b.append( e ); } } return StatusWith<BSONObj>( b.obj() ); }
virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { string source = cmdObj.getStringField( name.c_str() ); string target = cmdObj.getStringField( "to" ); if ( !NamespaceString::validCollectionComponent(target.c_str()) ) { errmsg = "invalid collection name: " + target; return false; } if ( source.empty() || target.empty() ) { errmsg = "invalid command syntax"; return false; } if (!fromRepl) { // If it got through on the master, need to allow it here too Status sourceStatus = userAllowedWriteNS(source); if (!sourceStatus.isOK()) { errmsg = "error with source namespace: " + sourceStatus.reason(); return false; } Status targetStatus = userAllowedWriteNS(target); if (!targetStatus.isOK()) { errmsg = "error with target namespace: " + targetStatus.reason(); return false; } } string sourceDB = nsToDatabase(source); string targetDB = nsToDatabase(target); bool capped = false; long long size = 0; std::vector<BSONObj> indexesInProg; Lock::GlobalWrite globalWriteLock; DurTransaction txn; { Client::Context srcCtx( source ); Collection* sourceColl = srcCtx.db()->getCollection( source ); if ( !sourceColl ) { errmsg = "source namespace does not exist"; return false; } // Ensure that collection name does not exceed maximum length. // Ensure that index names do not push the length over the max. // Iterator includes unfinished indexes. IndexCatalog::IndexIterator sourceIndIt = sourceColl->getIndexCatalog()->getIndexIterator( true ); int longestIndexNameLength = 0; while ( sourceIndIt.more() ) { int thisLength = sourceIndIt.next()->indexName().length(); if ( thisLength > longestIndexNameLength ) longestIndexNameLength = thisLength; } unsigned int longestAllowed = min(int(Namespace::MaxNsColletionLen), int(Namespace::MaxNsLen) - 2/*strlen(".$")*/ - longestIndexNameLength); if (target.size() > longestAllowed) { StringBuilder sb; sb << "collection name length of " << target.size() << " exceeds maximum length of " << longestAllowed << ", allowing for index names"; errmsg = sb.str(); return false; } { indexesInProg = stopIndexBuilds( srcCtx.db(), cmdObj ); capped = sourceColl->isCapped(); if ( capped ) { size = sourceColl->storageSize(); } } } { Client::Context ctx( target ); // Check if the target namespace exists and if dropTarget is true. // If target exists and dropTarget is not true, return false. if ( ctx.db()->getCollection( target ) ) { if ( !cmdObj["dropTarget"].trueValue() ) { errmsg = "target namespace exists"; return false; } Status s = ctx.db()->dropCollection( &txn, target ); if ( !s.isOK() ) { errmsg = s.toString(); restoreIndexBuildsOnSource( indexesInProg, source ); return false; } } // If we are renaming in the same database, just // rename the namespace and we're done. if ( sourceDB == targetDB ) { Status s = ctx.db()->renameCollection( &txn, source, target, cmdObj["stayTemp"].trueValue() ); if ( !s.isOK() ) { errmsg = s.toString(); restoreIndexBuildsOnSource( indexesInProg, source ); return false; } return true; } // Otherwise, we are enaming across databases, so we must copy all // the data and then remove the source collection. // Create the target collection. Collection* targetColl = NULL; if ( capped ) { CollectionOptions options; options.capped = true; options.cappedSize = size; options.setNoIdIndex(); targetColl = ctx.db()->createCollection( &txn, target, options ); } else { CollectionOptions options; options.setNoIdIndex(); // No logOp necessary because the entire renameCollection command is one logOp. targetColl = ctx.db()->createCollection( &txn, target, options ); } if ( !targetColl ) { errmsg = "Failed to create target collection."; restoreIndexBuildsOnSource( indexesInProg, source ); return false; } } // Copy over all the data from source collection to target collection. bool insertSuccessful = true; boost::scoped_ptr<RecordIterator> sourceIt; Collection* sourceColl = NULL; { Client::Context srcCtx( source ); sourceColl = srcCtx.db()->getCollection( source ); sourceIt.reset( sourceColl->getIterator( DiskLoc(), false, CollectionScanParams::FORWARD ) ); } Collection* targetColl = NULL; while ( !sourceIt->isEOF() ) { BSONObj o; { Client::Context srcCtx( source ); o = sourceColl->docFor(sourceIt->getNext()); } // Insert and check return status of insert. { Client::Context ctx( target ); if ( !targetColl ) targetColl = ctx.db()->getCollection( target ); // No logOp necessary because the entire renameCollection command is one logOp. Status s = targetColl->insertDocument( &txn, o, true ).getStatus(); if ( !s.isOK() ) { insertSuccessful = false; errmsg = s.toString(); break; } } } // If inserts were unsuccessful, drop the target collection and return false. if ( !insertSuccessful ) { Client::Context ctx( target ); Status s = ctx.db()->dropCollection( &txn, target ); if ( !s.isOK() ) errmsg = s.toString(); restoreIndexBuildsOnSource( indexesInProg, source ); return false; } // Copy over the indexes to temp storage and then to the target.. vector<BSONObj> copiedIndexes; bool indexSuccessful = true; { Client::Context srcCtx( source ); IndexCatalog::IndexIterator sourceIndIt = sourceColl->getIndexCatalog()->getIndexIterator( true ); while ( sourceIndIt.more() ) { BSONObj currIndex = sourceIndIt.next()->infoObj(); // Process the source index. BSONObjBuilder b; BSONObjIterator i( currIndex ); while( i.moreWithEOO() ) { BSONElement e = i.next(); if ( e.eoo() ) break; else if ( strcmp( e.fieldName(), "ns" ) == 0 ) b.append( "ns", target ); else b.append( e ); } BSONObj newIndex = b.obj(); copiedIndexes.push_back( newIndex ); } } { Client::Context ctx( target ); if ( !targetColl ) targetColl = ctx.db()->getCollection( target ); for ( vector<BSONObj>::iterator it = copiedIndexes.begin(); it != copiedIndexes.end(); ++it ) { Status s = targetColl->getIndexCatalog()->createIndex( *it, true ); if ( !s.isOK() ) { indexSuccessful = false; errmsg = s.toString(); break; } } // If indexes were unsuccessful, drop the target collection and return false. if ( !indexSuccessful ) { Status s = ctx.db()->dropCollection( &txn, target ); if ( !s.isOK() ) errmsg = s.toString(); restoreIndexBuildsOnSource( indexesInProg, source ); return false; } } // Drop the source collection. { Client::Context srcCtx( source ); Status s = srcCtx.db()->dropCollection( &txn, source ); if ( !s.isOK() ) { errmsg = s.toString(); restoreIndexBuildsOnSource( indexesInProg, source ); return false; } } return true; }
Status LiteParsedQuery::init(const string& ns, int ntoskip, int ntoreturn, int queryOptions, const BSONObj& queryObj, const BSONObj& proj, bool fromQueryMessage) { _ns = ns; _ntoskip = ntoskip; _ntoreturn = ntoreturn; _options = queryOptions; _proj = proj.getOwned(); if (_ntoskip < 0) { return Status(ErrorCodes::BadValue, "bad skip value in query"); } if (_ntoreturn == std::numeric_limits<int>::min()) { // _ntoreturn is negative but can't be negated. return Status(ErrorCodes::BadValue, "bad limit value in query"); } if (_ntoreturn < 0) { // _ntoreturn greater than zero is simply a hint on how many objects to send back per // "cursor batch". A negative number indicates a hard limit. _wantMore = false; _ntoreturn = -_ntoreturn; } if (fromQueryMessage) { BSONElement queryField = queryObj["query"]; if (!queryField.isABSONObj()) { queryField = queryObj["$query"]; } if (queryField.isABSONObj()) { _filter = queryField.embeddedObject().getOwned(); Status status = initFullQuery(queryObj); if (!status.isOK()) { return status; } } else { // TODO: Does this ever happen? _filter = queryObj.getOwned(); } } else { // This is the debugging code path. _filter = queryObj.getOwned(); } _hasReadPref = queryObj.hasField("$readPreference"); if (!_sort.isEmpty()) { if (!isValidSortOrder(_sort)) { return Status(ErrorCodes::BadValue, "bad sort specification"); } _sort = normalizeSortOrder(_sort); } // Min and Max objects must have the same fields. if (!_min.isEmpty() && !_max.isEmpty()) { if (!_min.isFieldNamePrefixOf(_max) || (_min.nFields() != _max.nFields())) { return Status(ErrorCodes::BadValue, "min and max must have the same field names"); } } // Can't combine a normal sort and a $meta projection on the same field. BSONObjIterator projIt(_proj); while (projIt.more()) { BSONElement projElt = projIt.next(); if (isTextScoreMeta(projElt)) { BSONElement sortElt = _sort[projElt.fieldName()]; if (!sortElt.eoo() && !isTextScoreMeta(sortElt)) { return Status(ErrorCodes::BadValue, "can't have a non-$meta sort on a $meta projection"); } } } // All fields with a $meta sort must have a corresponding $meta projection. BSONObjIterator sortIt(_sort); while (sortIt.more()) { BSONElement sortElt = sortIt.next(); if (isTextScoreMeta(sortElt)) { BSONElement projElt = _proj[sortElt.fieldName()]; if (projElt.eoo() || !isTextScoreMeta(projElt)) { return Status(ErrorCodes::BadValue, "must have $meta projection for all $meta sort keys"); } } } return Status::OK(); }
bool handleSpecialNamespaces( Request& r , QueryMessage& q ) { const char * ns = r.getns(); ns = strstr( r.getns() , ".$cmd.sys." ); if ( ! ns ) return false; ns += 10; r.checkAuth( Auth::WRITE ); BSONObjBuilder b; vector<Shard> shards; if ( strcmp( ns , "inprog" ) == 0 ) { Shard::getAllShards( shards ); BSONArrayBuilder arr( b.subarrayStart( "inprog" ) ); for ( unsigned i=0; i<shards.size(); i++ ) { Shard shard = shards[i]; ScopedDbConnection conn( shard ); BSONObj temp = conn->findOne( r.getns() , BSONObj() ); if ( temp["inprog"].isABSONObj() ) { BSONObjIterator i( temp["inprog"].Obj() ); while ( i.more() ) { BSONObjBuilder x; BSONObjIterator j( i.next().Obj() ); while( j.more() ) { BSONElement e = j.next(); if ( str::equals( e.fieldName() , "opid" ) ) { stringstream ss; ss << shard.getName() << ':' << e.numberInt(); x.append( "opid" , ss.str() ); } else if ( str::equals( e.fieldName() , "client" ) ) { x.appendAs( e , "client_s" ); } else { x.append( e ); } } arr.append( x.obj() ); } } conn.done(); } arr.done(); } else if ( strcmp( ns , "killop" ) == 0 ) { BSONElement e = q.query["op"]; if ( strstr( r.getns() , "admin." ) == 0 ) { b.append( "err" , "unauthorized" ); } else if ( e.type() != String ) { b.append( "err" , "bad op" ); b.append( e ); } else { b.append( e ); string s = e.String(); string::size_type i = s.find( ':' ); if ( i == string::npos ) { b.append( "err" , "bad opid" ); } else { string shard = s.substr( 0 , i ); int opid = atoi( s.substr( i + 1 ).c_str() ); b.append( "shard" , shard ); b.append( "shardid" , opid ); log() << "want to kill op: " << e << endl; Shard s(shard); ScopedDbConnection conn( s ); conn->findOne( r.getns() , BSON( "op" << opid ) ); conn.done(); } } } else if ( strcmp( ns , "unlock" ) == 0 ) { b.append( "err" , "can't do unlock through mongos" ); } else { log( LL_WARNING ) << "unknown sys command [" << ns << "]" << endl; return false; } BSONObj x = b.done(); replyToQuery(0, r.p(), r.m(), x); return true; }
static void _initAndListen(int listenPort) { Client::initThread("initandlisten"); getGlobalServiceContext()->setOpObserver(stdx::make_unique<OpObserver>()); const repl::ReplSettings& replSettings = repl::getGlobalReplicationCoordinator()->getSettings(); { ProcessId pid = ProcessId::getCurrent(); LogstreamBuilder l = log(LogComponent::kControl); l << "MongoDB starting : pid=" << pid << " port=" << serverGlobalParams.port << " dbpath=" << storageGlobalParams.dbpath; if (replSettings.master) l << " master=" << replSettings.master; if (replSettings.slave) l << " slave=" << (int)replSettings.slave; const bool is32bit = sizeof(int*) == 4; l << (is32bit ? " 32" : " 64") << "-bit host=" << getHostNameCached() << endl; } DEV log(LogComponent::kControl) << "DEBUG build (which is slower)" << endl; #if defined(_WIN32) printTargetMinOS(); #endif logProcessDetails(); // Due to SERVER-15389, we must setupSockets first thing at startup in order to avoid // obtaining too high a file descriptor for our calls to select(). MessageServer::Options options; options.port = listenPort; options.ipList = serverGlobalParams.bind_ip; MessageServer* server = createServer(options, new MyMessageHandler()); server->setAsTimeTracker(); // This is what actually creates the sockets, but does not yet listen on them because we // do not want connections to just hang if recovery takes a very long time. if (!server->setupSockets()) { error() << "Failed to set up sockets during startup."; return; } std::shared_ptr<DbWebServer> dbWebServer; if (serverGlobalParams.isHttpInterfaceEnabled) { dbWebServer.reset(new DbWebServer( serverGlobalParams.bind_ip, serverGlobalParams.port + 1000, new RestAdminAccess())); if (!dbWebServer->setupSockets()) { error() << "Failed to set up sockets for HTTP interface during startup."; return; } } getGlobalServiceContext()->initializeGlobalStorageEngine(); #ifdef MONGO_CONFIG_WIREDTIGER_ENABLED if (WiredTigerCustomizationHooks::get(getGlobalServiceContext())->restartRequired()) { exitCleanly(EXIT_CLEAN); } #endif // Warn if we detect configurations for multiple registered storage engines in // the same configuration file/environment. if (serverGlobalParams.parsedOpts.hasField("storage")) { BSONElement storageElement = serverGlobalParams.parsedOpts.getField("storage"); invariant(storageElement.isABSONObj()); BSONObj storageParamsObj = storageElement.Obj(); BSONObjIterator i = storageParamsObj.begin(); while (i.more()) { BSONElement e = i.next(); // Ignore if field name under "storage" matches current storage engine. if (storageGlobalParams.engine == e.fieldName()) { continue; } // Warn if field name matches non-active registered storage engine. if (getGlobalServiceContext()->isRegisteredStorageEngine(e.fieldName())) { warning() << "Detected configuration for non-active storage engine " << e.fieldName() << " when current storage engine is " << storageGlobalParams.engine; } } } if (!getGlobalServiceContext()->getGlobalStorageEngine()->getSnapshotManager()) { if (moe::startupOptionsParsed.count("replication.enableMajorityReadConcern")) { // Note: we are intentionally only erroring if the user explicitly requested that we // enable majority read concern. We do not error if the they are implicitly enabled for // CSRS because a required step in the upgrade procedure can involve an mmapv1 node in // the CSRS in the REMOVED state. This is handled by the TopologyCoordinator. invariant(replSettings.majorityReadConcernEnabled); severe() << "Majority read concern requires a storage engine that supports" << "snapshots, such as wiredTiger. " << storageGlobalParams.engine << " does not support snapshots."; exitCleanly(EXIT_BADOPTIONS); } } logMongodStartupWarnings(storageGlobalParams, serverGlobalParams); { stringstream ss; ss << endl; ss << "*********************************************************************" << endl; ss << " ERROR: dbpath (" << storageGlobalParams.dbpath << ") does not exist." << endl; ss << " Create this directory or give existing directory in --dbpath." << endl; ss << " See http://dochub.mongodb.org/core/startingandstoppingmongo" << endl; ss << "*********************************************************************" << endl; uassert(10296, ss.str().c_str(), boost::filesystem::exists(storageGlobalParams.dbpath)); } { stringstream ss; ss << "repairpath (" << storageGlobalParams.repairpath << ") does not exist"; uassert(12590, ss.str().c_str(), boost::filesystem::exists(storageGlobalParams.repairpath)); } // TODO: This should go into a MONGO_INITIALIZER once we have figured out the correct // dependencies. if (snmpInit) { snmpInit(); } boost::filesystem::remove_all(storageGlobalParams.dbpath + "/_tmp/"); if (mmapv1GlobalOptions.journalOptions & MMAPV1Options::JournalRecoverOnly) return; if (mongodGlobalParams.scriptingEnabled) { ScriptEngine::setup(); } auto startupOpCtx = getGlobalServiceContext()->makeOperationContext(&cc()); repairDatabasesAndCheckVersion(startupOpCtx.get()); if (storageGlobalParams.upgrade) { log() << "finished checking dbs" << endl; exitCleanly(EXIT_CLEAN); } uassertStatusOK(getGlobalAuthorizationManager()->initialize(startupOpCtx.get())); /* this is for security on certain platforms (nonce generation) */ srand((unsigned)(curTimeMicros64() ^ startupSrandTimer.micros())); // The snapshot thread provides historical collection level and lock statistics for use // by the web interface. Only needed when HTTP is enabled. if (serverGlobalParams.isHttpInterfaceEnabled) { statsSnapshotThread.go(); invariant(dbWebServer); stdx::thread web(stdx::bind(&webServerListenThread, dbWebServer)); web.detach(); } { #ifndef _WIN32 mongo::signalForkSuccess(); #endif Status status = authindex::verifySystemIndexes(startupOpCtx.get()); if (!status.isOK()) { log() << status.reason(); exitCleanly(EXIT_NEED_UPGRADE); } // SERVER-14090: Verify that auth schema version is schemaVersion26Final. int foundSchemaVersion; status = getGlobalAuthorizationManager()->getAuthorizationVersion(startupOpCtx.get(), &foundSchemaVersion); if (!status.isOK()) { log() << "Auth schema version is incompatible: " << "User and role management commands require auth data to have " << "at least schema version " << AuthorizationManager::schemaVersion26Final << " but startup could not verify schema version: " << status.toString() << endl; exitCleanly(EXIT_NEED_UPGRADE); } if (foundSchemaVersion < AuthorizationManager::schemaVersion26Final) { log() << "Auth schema version is incompatible: " << "User and role management commands require auth data to have " << "at least schema version " << AuthorizationManager::schemaVersion26Final << " but found " << foundSchemaVersion << ". In order to upgrade " << "the auth schema, first downgrade MongoDB binaries to version " << "2.6 and then run the authSchemaUpgrade command." << endl; exitCleanly(EXIT_NEED_UPGRADE); } getDeleter()->startWorkers(); restartInProgressIndexesFromLastShutdown(startupOpCtx.get()); repl::getGlobalReplicationCoordinator()->startReplication(startupOpCtx.get()); const unsigned long long missingRepl = checkIfReplMissingFromCommandLine(startupOpCtx.get()); if (missingRepl) { log() << startupWarningsLog; log() << "** WARNING: mongod started without --replSet yet " << missingRepl << " documents are present in local.system.replset" << startupWarningsLog; log() << "** Restart with --replSet unless you are doing maintenance and " << " no other clients are connected." << startupWarningsLog; log() << "** The TTL collection monitor will not start because of this." << startupWarningsLog; log() << "** "; log() << " For more info see http://dochub.mongodb.org/core/ttlcollections"; log() << startupWarningsLog; } else { startTTLBackgroundJob(); } } startClientCursorMonitor(); PeriodicTask::startRunningPeriodicTasks(); HostnameCanonicalizationWorker::start(getGlobalServiceContext()); startFTDC(); if (!repl::getGlobalReplicationCoordinator()->isReplEnabled()) { uassertStatusOK(ShardingStateRecovery::recover(startupOpCtx.get())); } logStartup(startupOpCtx.get()); // MessageServer::run will return when exit code closes its socket and we don't need the // operation context anymore startupOpCtx.reset(); server->run(); }
Status GeoNearExpression::parseNewQuery(const BSONObj &obj) { bool hasGeometry = false; BSONObjIterator objIt(obj); if (!objIt.more()) { return Status(ErrorCodes::BadValue, "empty geo near query object"); } BSONElement e = objIt.next(); // Just one arg. to $geoNear. if (objIt.more()) { return Status(ErrorCodes::BadValue, mongoutils::str::stream() << "geo near accepts just one argument when querying for a GeoJSON " << "point. Extra field found: " << objIt.next()); } // Parse "new" near: // t.find({"geo" : {"$near" : {"$geometry": pointA, $minDistance: 1, $maxDistance: 3}}}) // t.find({"geo" : {"$geoNear" : {"$geometry": pointA, $minDistance: 1, $maxDistance: 3}}}) if (!e.isABSONObj()) { return Status(ErrorCodes::BadValue, "geo near query argument is not an object"); } BSONObj::MatchType matchType = static_cast<BSONObj::MatchType>(e.getGtLtOp()); if (BSONObj::opNEAR != matchType) { return Status(ErrorCodes::BadValue, mongoutils::str::stream() << "invalid geo near query operator: " << e.fieldName()); } // Iterate over the argument. BSONObjIterator it(e.embeddedObject()); while (it.more()) { BSONElement e = it.next(); if (equals(e.fieldName(), "$geometry")) { if (e.isABSONObj()) { BSONObj embeddedObj = e.embeddedObject(); Status status = GeoParser::parseQueryPoint(e, centroid.get()); if (!status.isOK()) { return Status(ErrorCodes::BadValue, str::stream() << "invalid point in geo near query $geometry argument: " << embeddedObj << " " << status.reason()); } uassert(16681, "$near requires geojson point, given " + embeddedObj.toString(), (SPHERE == centroid->crs)); hasGeometry = true; } } else if (equals(e.fieldName(), "$minDistance")) { uassert(16897, "$minDistance must be a number", e.isNumber()); minDistance = e.Number(); uassert(16898, "$minDistance must be non-negative", minDistance >= 0.0); } else if (equals(e.fieldName(), "$maxDistance")) { uassert(16899, "$maxDistance must be a number", e.isNumber()); maxDistance = e.Number(); uassert(16900, "$maxDistance must be non-negative", maxDistance >= 0.0); } } if (!hasGeometry) { return Status(ErrorCodes::BadValue, "$geometry is required for geo near query"); } return Status::OK(); }
StatusWith<BSONObj> FTSSpec::fixSpec(const BSONObj& spec) { if (spec["textIndexVersion"].numberInt() == TEXT_INDEX_VERSION_1) { return _fixSpecV1(spec); } map<string, int> m; BSONObj keyPattern; { BSONObjBuilder b; // Populate m and keyPattern. { bool addedFtsStuff = false; BSONObjIterator i(spec["key"].Obj()); while (i.more()) { BSONElement e = i.next(); if (e.fieldNameStringData() == "_fts") { if (INDEX_NAME != e.valuestrsafe()) { return {ErrorCodes::CannotCreateIndex, "expecting _fts:\"text\""}; } addedFtsStuff = true; b.append(e); } else if (e.fieldNameStringData() == "_ftsx") { if (e.numberInt() != 1) { return {ErrorCodes::CannotCreateIndex, "expecting _ftsx:1"}; } b.append(e); } else if (e.type() == String && INDEX_NAME == e.valuestr()) { if (!addedFtsStuff) { _addFTSStuff(&b); addedFtsStuff = true; } m[e.fieldName()] = 1; } else { if (e.numberInt() != 1 && e.numberInt() != -1) { return {ErrorCodes::CannotCreateIndex, "expected value 1 or -1 for non-text key in compound index"}; } b.append(e); } } verify(addedFtsStuff); } keyPattern = b.obj(); // Verify that index key is in the correct format: extraBefore fields, then text // fields, then extraAfter fields. { BSONObjIterator i(spec["key"].Obj()); verify(i.more()); BSONElement e = i.next(); // extraBefore fields while (String != e.type()) { Status notReservedStatus = verifyFieldNameNotReserved(e.fieldNameStringData()); if (!notReservedStatus.isOK()) { return notReservedStatus; } if (!i.more()) { return {ErrorCodes::CannotCreateIndex, "expected additional fields in text index key pattern"}; } e = i.next(); } // text fields bool alreadyFixed = (e.fieldNameStringData() == "_fts"); if (alreadyFixed) { if (!i.more()) { return {ErrorCodes::CannotCreateIndex, "expected _ftsx after _fts"}; } e = i.next(); if (e.fieldNameStringData() != "_ftsx") { return {ErrorCodes::CannotCreateIndex, "expected _ftsx after _fts"}; } e = i.next(); } else { do { Status notReservedStatus = verifyFieldNameNotReserved(e.fieldNameStringData()); if (!notReservedStatus.isOK()) { return notReservedStatus; } e = i.next(); } while (!e.eoo() && e.type() == String); } // extraAfterFields while (!e.eoo()) { if (e.type() == BSONType::String) { return {ErrorCodes::CannotCreateIndex, "'text' fields in index must all be adjacent"}; } Status notReservedStatus = verifyFieldNameNotReserved(e.fieldNameStringData()); if (!notReservedStatus.isOK()) { return notReservedStatus; } e = i.next(); } } } if (spec["weights"].type() == Object) { BSONObjIterator i(spec["weights"].Obj()); while (i.more()) { BSONElement e = i.next(); if (!e.isNumber()) { return {ErrorCodes::CannotCreateIndex, "weight for text index needs numeric type"}; } m[e.fieldName()] = e.numberInt(); } } else if (spec["weights"].str() == WILDCARD) { m[WILDCARD] = 1; } else if (!spec["weights"].eoo()) { return {ErrorCodes::CannotCreateIndex, "text index option 'weights' must be an object"}; } if (m.empty()) { return {ErrorCodes::CannotCreateIndex, "text index option 'weights' must specify fields or the wildcard"}; } BSONObj weights; { BSONObjBuilder b; for (map<string, int>::iterator i = m.begin(); i != m.end(); ++i) { if (i->second <= 0 || i->second >= MAX_WORD_WEIGHT) { return {ErrorCodes::CannotCreateIndex, str::stream() << "text index weight must be in the exclusive interval (0," << MAX_WORD_WEIGHT << ") but found: " << i->second}; } // Verify weight refers to a valid field. if (i->first != "$**") { FieldRef keyField(i->first); if (keyField.numParts() == 0) { return {ErrorCodes::CannotCreateIndex, "weight cannot be on an empty field"}; } for (size_t partNum = 0; partNum < keyField.numParts(); partNum++) { StringData part = keyField.getPart(partNum); if (part.empty()) { return {ErrorCodes::CannotCreateIndex, "weight cannot have empty path component"}; } if (part.startsWith("$")) { return {ErrorCodes::CannotCreateIndex, "weight cannot have path component with $ prefix"}; } } } b.append(i->first, i->second); } weights = b.obj(); } BSONElement default_language_elt = spec["default_language"]; string default_language(default_language_elt.str()); if (default_language_elt.eoo()) { default_language = moduleDefaultLanguage; } else if (default_language_elt.type() != BSONType::String) { return {ErrorCodes::CannotCreateIndex, "default_language needs a string type"}; } if (!FTSLanguage::make(default_language, TEXT_INDEX_VERSION_3).getStatus().isOK()) { return {ErrorCodes::CannotCreateIndex, "default_language is not valid"}; } BSONElement language_override_elt = spec["language_override"]; string language_override(language_override_elt.str()); if (language_override_elt.eoo()) { language_override = "language"; } else if (language_override_elt.type() != BSONType::String) { return {ErrorCodes::CannotCreateIndex, "language_override must be a string"}; } else if (!validateOverride(language_override)) { return {ErrorCodes::CannotCreateIndex, "language_override is not valid"}; } int version = -1; int textIndexVersion = TEXT_INDEX_VERSION_3; // default text index version BSONObjBuilder b; BSONObjIterator i(spec); while (i.more()) { BSONElement e = i.next(); StringData fieldName = e.fieldNameStringData(); if (fieldName == "key") { b.append("key", keyPattern); } else if (fieldName == "weights") { b.append("weights", weights); weights = BSONObj(); } else if (fieldName == "default_language") { b.append("default_language", default_language); default_language = ""; } else if (fieldName == "language_override") { b.append("language_override", language_override); language_override = ""; } else if (fieldName == "v") { version = e.numberInt(); } else if (fieldName == "textIndexVersion") { if (!e.isNumber()) { return {ErrorCodes::CannotCreateIndex, "text index option 'textIndexVersion' must be a number"}; } textIndexVersion = e.numberInt(); if (textIndexVersion != TEXT_INDEX_VERSION_2 && textIndexVersion != TEXT_INDEX_VERSION_3) { return {ErrorCodes::CannotCreateIndex, str::stream() << "bad textIndexVersion: " << textIndexVersion}; } } else { b.append(e); } } if (!weights.isEmpty()) { b.append("weights", weights); } if (!default_language.empty()) { b.append("default_language", default_language); } if (!language_override.empty()) { b.append("language_override", language_override); } if (version >= 0) { b.append("v", version); } b.append("textIndexVersion", textIndexVersion); return b.obj(); }
Status S2IndexCursor::seek(const BSONObj &position) { vector<GeoQuery> regions; bool isNearQuery = false; NearQuery nearQuery; // Go through the fields that we index, and for each geo one, make // a GeoQuery object for the S2*Cursor class to do intersection // testing/cover generating with. BSONObjIterator keyIt(_descriptor->keyPattern()); while (keyIt.more()) { BSONElement keyElt = keyIt.next(); if (keyElt.type() != String || IndexNames::GEO_2DSPHERE != keyElt.valuestr()) { continue; } BSONElement e = position.getFieldDotted(keyElt.fieldName()); if (e.eoo()) { continue; } if (!e.isABSONObj()) { continue; } BSONObj obj = e.Obj(); if (nearQuery.parseFrom(obj, _params.radius)) { if (isNearQuery) { return Status(ErrorCodes::BadValue, "Only one $near clause allowed: " + position.toString(), 16685); } isNearQuery = true; nearQuery.field = keyElt.fieldName(); continue; } GeoQuery geoQueryField(keyElt.fieldName()); if (!geoQueryField.parseFrom(obj)) { return Status(ErrorCodes::BadValue, "can't parse query (2dsphere): " + obj.toString(), 16535); } if (!geoQueryField.hasS2Region()) { return Status(ErrorCodes::BadValue, "Geometry unsupported: " + obj.toString(), 16684); } regions.push_back(geoQueryField); } // Remove all the indexed geo regions from the query. The s2*cursor will // instead create a covering for that key to speed up the search. // // One thing to note is that we create coverings for indexed geo keys during // a near search to speed it up further. BSONObjBuilder geoFieldsToNuke; if (isNearQuery) { geoFieldsToNuke.append(nearQuery.field, ""); } for (size_t i = 0; i < regions.size(); ++i) { geoFieldsToNuke.append(regions[i].getField(), ""); } // false means we want to filter OUT geoFieldsToNuke, not filter to include only that. BSONObj filteredQuery = position.filterFieldsUndotted(geoFieldsToNuke.obj(), false); if (isNearQuery) { S2NearIndexCursor* nearCursor = new S2NearIndexCursor(_descriptor, _params); _underlyingCursor.reset(nearCursor); nearCursor->seek(filteredQuery, nearQuery, regions); } else { S2SimpleCursor* simpleCursor = new S2SimpleCursor(_descriptor, _params); _underlyingCursor.reset(simpleCursor); simpleCursor->seek(filteredQuery, regions); } return Status::OK(); }
Status ModifierCurrentDate::init(const BSONElement& modExpr, const Options& opts) { _updatePath.parse(modExpr.fieldName()); Status status = fieldchecker::isUpdatable(_updatePath); if (!status.isOK()) { return status; } // If a $-positional operator was used, get the index in which it occurred // and ensure only one occurrence. size_t foundCount; fieldchecker::isPositional(_updatePath, &_pathReplacementPosition, &foundCount); if (_pathReplacementPosition && foundCount > 1) { return Status(ErrorCodes::BadValue, "too many positional($) elements found."); } // Validate and store the type to produce switch (modExpr.type()) { case Bool: _typeIsDate = true; break; case Object: { const BSONObj argObj = modExpr.embeddedObject(); const BSONElement typeElem = argObj.getField(kType); bool badInput = typeElem.eoo() || !(typeElem.type() == String); if (!badInput) { std::string typeVal = typeElem.String(); badInput = !(typeElem.String() == kDate || typeElem.String() == kTimestamp); if (!badInput) _typeIsDate = (typeVal == kDate); if (!badInput) { // Check to make sure only the $type field was given as an arg BSONObjIterator i( argObj ); const bool onlyHasTypeField = ((i.next().fieldNameStringData() == kType) && i.next().eoo()); if (!onlyHasTypeField) { return Status(ErrorCodes::BadValue, str::stream() << "The only valid field of the option is '$type': " "{$currentDate: {field : {$type: 'date/timestamp'}}}; " << "arg: " << argObj); } } } if (badInput) { return Status(ErrorCodes::BadValue, "The '$type' string field is required " "to be 'date' or 'timestamp': " "{$currentDate: {field : {$type: 'date'}}}"); } break; } default: return Status(ErrorCodes::BadValue, str::stream() << typeName(modExpr.type()) << " is not valid type. Please use boolean ('true') " "or a $type ({$type: 'timestamp/date'} expression. " ); } return Status::OK(); }
virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) { string source = cmdObj.getStringField( name.c_str() ); string target = cmdObj.getStringField( "to" ); if ( source.empty() || target.empty() ) { errmsg = "invalid command syntax"; return false; } setClient( source.c_str() ); NamespaceDetails *nsd = nsdetails( source.c_str() ); uassert( "source namespace does not exist", nsd ); bool capped = nsd->capped; long long size = 0; if ( capped ) for( DiskLoc i = nsd->firstExtent; !i.isNull(); i = i.ext()->xnext ) size += i.ext()->length; setClient( target.c_str() ); BSONObjBuilder spec; if ( capped ) { spec.appendBool( "capped", true ); spec.append( "size", double( size ) ); } if ( !userCreateNS( target.c_str(), spec.done(), errmsg, true ) ) return false; auto_ptr< DBClientCursor > c; DBDirectClient bridge; { c = bridge.query( source, BSONObj() ); } while( 1 ) { { if ( !c->more() ) break; } BSONObj o = c->next(); theDataFileMgr.insertAndLog( target.c_str(), o ); } char cl[256]; nsToClient( source.c_str(), cl ); string sourceIndexes = string( cl ) + ".system.indexes"; nsToClient( target.c_str(), cl ); string targetIndexes = string( cl ) + ".system.indexes"; { c = bridge.query( sourceIndexes, QUERY( "ns" << source ) ); } while( 1 ) { { if ( !c->more() ) break; } BSONObj o = c->next(); BSONObjBuilder b; BSONObjIterator i( o ); while( i.moreWithEOO() ) { BSONElement e = i.next(); if ( e.eoo() ) break; if ( strcmp( e.fieldName(), "ns" ) == 0 ) { b.append( "ns", target ); } else { b.append( e ); } } BSONObj n = b.done(); theDataFileMgr.insertAndLog( targetIndexes.c_str(), n ); } { if ( !bridge.dropCollection( source ) ) { errmsg = "failed to drop old name collection"; return false; } } return true; }
void IndexScanNode::computeProperties() { _sorts.clear(); BSONObj sortPattern; { BSONObjBuilder sortBob; BSONObj normalizedIndexKeyPattern(LiteParsedQuery::normalizeSortOrder(indexKeyPattern)); BSONObjIterator it(normalizedIndexKeyPattern); while (it.more()) { BSONElement elt = it.next(); // Zero is returned if elt is not a number. This happens when elt is hashed or // 2dsphere, our two projection indices. We want to drop those from the sort // pattern. int val = elt.numberInt() * direction; if (0 != val) { sortBob.append(elt.fieldName(), val); } } sortPattern = sortBob.obj(); } _sorts.insert(sortPattern); const int nFields = sortPattern.nFields(); if (nFields > 1) { // We're sorted not only by sortPattern but also by all prefixes of it. for (int i = 0; i < nFields; ++i) { // Make obj out of fields [0,i] BSONObjIterator it(sortPattern); BSONObjBuilder prefixBob; for (int j = 0; j <= i; ++j) { prefixBob.append(it.next()); } _sorts.insert(prefixBob.obj()); } } // If we are using the index {a:1, b:1} to answer the predicate {a: 10}, it's sorted // both by the index key pattern and by the pattern {b: 1}. // See if there are any fields with equalities for bounds. We can drop these // from any sort orders created. set<string> equalityFields; if (!bounds.isSimpleRange) { // Figure out how many fields are point intervals. for (size_t i = 0; i < bounds.fields.size(); ++i) { const OrderedIntervalList& oil = bounds.fields[i]; if (oil.intervals.size() != 1) { continue; } const Interval& ival = oil.intervals[0]; if (!ival.isPoint()) { continue; } equalityFields.insert(oil.name); } } if (equalityFields.empty()) { return; } // TODO: Each field in equalityFields could be dropped from the sort order since it is // a point interval. The full set of sort orders is as follows: // For each sort in _sorts: // For each drop in powerset(equalityFields): // Remove fields in 'drop' from 'sort' and add resulting sort to output. // Since this involves a powerset, we only remove point intervals that the prior sort // planning code removed, namely the contiguous prefix of the key pattern. BSONObjIterator it(sortPattern); BSONObjBuilder prefixBob; while (it.more()) { BSONElement elt = it.next(); // XXX string slowness. fix when bounds are stringdata not string. if (equalityFields.end() == equalityFields.find(string(elt.fieldName()))) { prefixBob.append(elt); // This field isn't a point interval, can't drop. break; } } while (it.more()) { prefixBob.append(it.next()); } // If we have an index {a:1} and an equality on 'a' don't append an empty sort order. BSONObj filterPointsObj = prefixBob.obj(); if (!filterPointsObj.isEmpty()) { _sorts.insert(filterPointsObj); } }
void BtreeKeyGeneratorV0::getKeysImpl(std::vector<const char*> fieldNames, std::vector<BSONElement> fixed, const BSONObj& obj, BSONObjSet* keys, MultikeyPaths* multikeyPaths) const { if (_isIdIndex) { // we special case for speed BSONElement e = obj["_id"]; if (e.eoo()) { keys->insert(_nullKey); } else { int size = e.size() + 5 /* bson over head*/ - 3 /* remove _id string */; BSONObjBuilder b(size); b.appendAs(e, ""); keys->insert(b.obj()); invariant(keys->begin()->objsize() == size); } return; } BSONElement arrElt; unsigned arrIdx = ~0; unsigned numNotFound = 0; for (unsigned i = 0; i < fieldNames.size(); ++i) { if (*fieldNames[i] == '\0') continue; BSONElement e = dps::extractElementAtPathOrArrayAlongPath(obj, fieldNames[i]); if (e.eoo()) { e = nullElt; // no matching field numNotFound++; } if (e.type() != Array) fieldNames[i] = ""; // no matching field or non-array match if (*fieldNames[i] == '\0') // no need for further object expansion (though array expansion still possible) fixed[i] = e; if (e.type() == Array && arrElt.eoo()) { // we only expand arrays on a single path -- track the path here arrIdx = i; arrElt = e; } // enforce single array path here if (e.type() == Array && e.rawdata() != arrElt.rawdata()) { assertParallelArrays(e.fieldName(), arrElt.fieldName()); } } bool allFound = true; // have we found elements for all field names in the key spec? for (std::vector<const char*>::const_iterator i = fieldNames.begin(); i != fieldNames.end(); ++i) { if (**i != '\0') { allFound = false; break; } } if (_isSparse && numNotFound == _fieldNames.size()) { // we didn't find any fields // so we're not going to index this document return; } bool insertArrayNull = false; if (allFound) { if (arrElt.eoo()) { // no terminal array element to expand BSONObjBuilder b(_sizeTracker); for (std::vector<BSONElement>::iterator i = fixed.begin(); i != fixed.end(); ++i) b.appendAs(*i, ""); keys->insert(b.obj()); } else { // terminal array element to expand, so generate all keys BSONObjIterator i(arrElt.embeddedObject()); if (i.more()) { while (i.more()) { BSONObjBuilder b(_sizeTracker); for (unsigned j = 0; j < fixed.size(); ++j) { if (j == arrIdx) b.appendAs(i.next(), ""); else b.appendAs(fixed[j], ""); } keys->insert(b.obj()); } } else if (fixed.size() > 1) { insertArrayNull = true; } } } else { // nonterminal array element to expand, so recurse verify(!arrElt.eoo()); BSONObjIterator i(arrElt.embeddedObject()); if (i.more()) { while (i.more()) { BSONElement e = i.next(); if (e.type() == Object) { getKeysImpl(fieldNames, fixed, e.embeddedObject(), keys, multikeyPaths); } } } else { insertArrayNull = true; } } if (insertArrayNull) { // x : [] - need to insert undefined BSONObjBuilder b(_sizeTracker); for (unsigned j = 0; j < fixed.size(); ++j) { if (j == arrIdx) { b.appendUndefined(""); } else { BSONElement e = fixed[j]; if (e.eoo()) b.appendNull(""); else b.appendAs(e, ""); } } keys->insert(b.obj()); } }
long long BSONTool::processFile( const path& root ){ string fileString = root.string(); long long fileLength = file_size( root ); if ( fileLength == 0 ) { out() << "file " << fileString << " empty, skipping" << endl; return 0; } ifstream file( fileString.c_str() , ios_base::in | ios_base::binary); if ( ! file.is_open() ){ log() << "error opening file: " << fileString << endl; return 0; } log(1) << "\t file size: " << fileLength << endl; long long read = 0; long long num = 0; long long processed = 0; const int BUF_SIZE = 1024 * 1024 * 5; boost::scoped_array<char> buf_holder(new char[BUF_SIZE]); char * buf = buf_holder.get(); ProgressMeter m( fileLength ); while ( read < fileLength ) { file.read( buf , 4 ); int size = ((int*)buf)[0]; if ( size >= BUF_SIZE ){ cerr << "got an object of size: " << size << " terminating..." << endl; } uassert( 10264 , "invalid object size" , size < BUF_SIZE ); file.read( buf + 4 , size - 4 ); BSONObj o( buf ); if ( _objcheck && ! o.valid() ){ cerr << "INVALID OBJECT - going try and pring out " << endl; cerr << "size: " << size << endl; BSONObjIterator i(o); while ( i.more() ){ BSONElement e = i.next(); try { e.validate(); } catch ( ... ){ cerr << "\t\t NEXT ONE IS INVALID" << endl; } cerr << "\t name : " << e.fieldName() << " " << e.type() << endl; cerr << "\t " << e << endl; } } if ( _matcher.get() == 0 || _matcher->matches( o ) ){ gotObject( o ); processed++; } read += o.objsize(); num++; m.hit( o.objsize() ); } uassert( 10265 , "counts don't match" , m.done() == fileLength ); out() << "\t " << m.hits() << " objects found" << endl; if ( _matcher.get() ) out() << "\t " << processed << " objects processed" << endl; return processed; }
void BtreeKeyGeneratorV1::getKeysImplWithArray( std::vector<const char*> fieldNames, std::vector<BSONElement> fixed, const BSONObj& obj, BSONObjSet* keys, unsigned numNotFound, const std::vector<PositionalPathInfo>& positionalInfo, MultikeyPaths* multikeyPaths) const { BSONElement arrElt; // A set containing the position of any indexed fields in the key pattern that traverse through // the 'arrElt' array value. std::set<size_t> arrIdxs; // A vector with size equal to the number of elements in the index key pattern. Each element in // the vector, if initialized, refers to the component within the indexed field that traverses // through the 'arrElt' array value. We say that this component within the indexed field // corresponds to a path that causes the index to be multikey if the 'arrElt' array value // contains multiple elements. // // For example, consider the index {'a.b': 1, 'a.c'} and the document // {a: [{b: 1, c: 'x'}, {b: 2, c: 'y'}]}. The path "a" causes the index to be multikey, so we'd // have a std::vector<boost::optional<size_t>>{{0U}, {0U}}. // // Furthermore, due to how positional key patterns are specified, it's possible for an indexed // field to cause the index to be multikey at a different component than another indexed field // that also traverses through the 'arrElt' array value. It's then also possible for an indexed // field not to cause the index to be multikey, even if it traverses through the 'arrElt' array // value, because only a particular element would be indexed. // // For example, consider the index {'a.b': 1, 'a.b.0'} and the document {a: {b: [1, 2]}}. The // path "a.b" causes the index to be multikey, but the key pattern "a.b.0" only indexes the // first element of the array, so we'd have a // std::vector<boost::optional<size_t>>{{1U}, boost::none}. std::vector<boost::optional<size_t>> arrComponents(fieldNames.size()); bool mayExpandArrayUnembedded = true; for (size_t i = 0; i < fieldNames.size(); ++i) { if (*fieldNames[i] == '\0') { continue; } bool arrayNestedArray; // Extract element matching fieldName[ i ] from object xor array. BSONElement e = extractNextElement(obj, positionalInfo[i], &fieldNames[i], &arrayNestedArray); if (e.eoo()) { // if field not present, set to null fixed[i] = nullElt; // done expanding this field name fieldNames[i] = ""; numNotFound++; } else if (e.type() == Array) { arrIdxs.insert(i); if (arrElt.eoo()) { // we only expand arrays on a single path -- track the path here arrElt = e; } else if (e.rawdata() != arrElt.rawdata()) { // enforce single array path here assertParallelArrays(e.fieldName(), arrElt.fieldName()); } if (arrayNestedArray) { mayExpandArrayUnembedded = false; } } else { // not an array - no need for further expansion fixed[i] = e; } } if (arrElt.eoo()) { // No array, so generate a single key. if (_isSparse && numNotFound == fieldNames.size()) { return; } BSONObjBuilder b(_sizeTracker); for (std::vector<BSONElement>::iterator i = fixed.begin(); i != fixed.end(); ++i) { CollationIndexKey::collationAwareIndexKeyAppend(*i, _collator, &b); } keys->insert(b.obj()); } else if (arrElt.embeddedObject().firstElement().eoo()) { // We've encountered an empty array. if (multikeyPaths && mayExpandArrayUnembedded) { // Any indexed path which traverses through the empty array must be recorded as an array // component. for (auto i : arrIdxs) { // We need to determine which component of the indexed field causes the index to be // multikey as a result of the empty array. Indexed empty arrays are considered // multikey and may occur mid-path. For instance, the indexed path "a.b.c" has // multikey components {0, 1} given the document {a: [{b: []}, {b: 1}]}. size_t fullPathLength = _pathLengths[i]; size_t suffixPathLength = FieldRef{fieldNames[i]}.numParts(); invariant(suffixPathLength < fullPathLength); arrComponents[i] = fullPathLength - suffixPathLength - 1; } } // For an empty array, set matching fields to undefined. _getKeysArrEltFixed(&fieldNames, &fixed, undefinedElt, keys, numNotFound, arrElt, arrIdxs, true, _emptyPositionalInfo, multikeyPaths); } else { BSONObj arrObj = arrElt.embeddedObject(); // For positional key patterns, e.g. {'a.1.b': 1}, we lookup the indexed array element // and then traverse the remainder of the field path up front. This prevents us from // having to look up the indexed element again on each recursive call (i.e. once per // array element). std::vector<PositionalPathInfo> subPositionalInfo(fixed.size()); for (size_t i = 0; i < fieldNames.size(); ++i) { const bool fieldIsArray = arrIdxs.find(i) != arrIdxs.end(); if (*fieldNames[i] == '\0') { // We've reached the end of the path. if (multikeyPaths && fieldIsArray && mayExpandArrayUnembedded) { // The 'arrElt' array value isn't expanded into multiple elements when the last // component of the indexed field is positional and 'arrElt' contains nested // array values. In all other cases, the 'arrElt' array value may be expanded // into multiple element and can therefore cause the index to be multikey. arrComponents[i] = _pathLengths[i] - 1; } continue; } // The earlier call to dps::extractElementAtPathOrArrayAlongPath(..., fieldNames[i]) // modified fieldNames[i] to refer to the suffix of the path immediately following the // 'arrElt' array value. If we haven't reached the end of this indexed field yet, then // we must have traversed through 'arrElt'. invariant(fieldIsArray); StringData part = fieldNames[i]; part = part.substr(0, part.find('.')); subPositionalInfo[i].positionallyIndexedElt = arrObj[part]; if (subPositionalInfo[i].positionallyIndexedElt.eoo()) { // We aren't indexing a particular element of the 'arrElt' array value, so it may be // expanded into multiple elements. It can therefore cause the index to be multikey. if (multikeyPaths) { // We need to determine which component of the indexed field causes the index to // be multikey as a result of the 'arrElt' array value. Since // // NumComponents("<pathPrefix>") + NumComponents("<pathSuffix>") // = NumComponents("<pathPrefix>.<pathSuffix>"), // // we can compute the number of components in a prefix of the indexed field by // subtracting the number of components in the suffix 'fieldNames[i]' from the // number of components in the indexed field '_fieldNames[i]'. // // For example, consider the indexed field "a.b.c" and the suffix "c". The path // "a.b.c" has 3 components and the suffix "c" has 1 component. Subtracting the // latter from the former yields the number of components in the prefix "a.b", // i.e. 2. size_t fullPathLength = _pathLengths[i]; size_t suffixPathLength = FieldRef{fieldNames[i]}.numParts(); invariant(suffixPathLength < fullPathLength); arrComponents[i] = fullPathLength - suffixPathLength - 1; } continue; } // We're indexing an array element by its position. Traverse the remainder of the // field path now. // // Indexing an array element by its position selects a particular element of the // 'arrElt' array value when generating keys. It therefore cannot cause the index to be // multikey. subPositionalInfo[i].arrayObj = arrObj; subPositionalInfo[i].remainingPath = fieldNames[i]; subPositionalInfo[i].dottedElt = dps::extractElementAtPathOrArrayAlongPath( arrObj, subPositionalInfo[i].remainingPath); } // Generate a key for each element of the indexed array. for (const auto arrObjElem : arrObj) { _getKeysArrEltFixed(&fieldNames, &fixed, arrObjElem, keys, numNotFound, arrElt, arrIdxs, mayExpandArrayUnembedded, subPositionalInfo, multikeyPaths); } } // Record multikey path components. if (multikeyPaths) { for (size_t i = 0; i < arrComponents.size(); ++i) { if (auto arrComponent = arrComponents[i]) { (*multikeyPaths)[i].insert(*arrComponent); } } } }
// static void QueryPlannerIXSelect::rateIndices(MatchExpression* node, string prefix, const vector<IndexEntry>& indices) { // Do not traverse tree beyond logical NOR node MatchExpression::MatchType exprtype = node->matchType(); if (exprtype == MatchExpression::NOR) { return; } // Every indexable node is tagged even when no compatible index is // available. if (Indexability::isBoundsGenerating(node)) { string fullPath; if (MatchExpression::NOT == node->matchType()) { fullPath = prefix + node->getChild(0)->path().toString(); } else { fullPath = prefix + node->path().toString(); } verify(NULL == node->getTag()); RelevantTag* rt = new RelevantTag(); node->setTag(rt); rt->path = fullPath; // TODO: This is slow, with all the string compares. for (size_t i = 0; i < indices.size(); ++i) { BSONObjIterator it(indices[i].keyPattern); BSONElement elt = it.next(); if (elt.fieldName() == fullPath && compatible(elt, indices[i], node)) { rt->first.push_back(i); } while (it.more()) { elt = it.next(); if (elt.fieldName() == fullPath && compatible(elt, indices[i], node)) { rt->notFirst.push_back(i); } } } // If this is a NOT, we have to clone the tag and attach // it to the NOT's child. if (MatchExpression::NOT == node->matchType()) { RelevantTag* childRt = static_cast<RelevantTag*>(rt->clone()); childRt->path = rt->path; node->getChild(0)->setTag(childRt); } } else if (Indexability::arrayUsesIndexOnChildren(node)) { // See comment in getFields about all/elemMatch and paths. if (!node->path().empty()) { prefix += node->path().toString() + "."; } for (size_t i = 0; i < node->numChildren(); ++i) { rateIndices(node->getChild(i), prefix, indices); } } else if (node->isLogical()) { for (size_t i = 0; i < node->numChildren(); ++i) { rateIndices(node->getChild(i), prefix, indices); } } }
Status LiteParsedQuery::initFullQuery(const BSONObj& top) { BSONObjIterator i(top); while (i.more()) { BSONElement e = i.next(); const char* name = e.fieldName(); if (0 == strcmp("$orderby", name) || 0 == strcmp("orderby", name)) { if (Object == e.type()) { _sort = e.embeddedObject(); } else if (Array == e.type()) { _sort = e.embeddedObject(); // TODO: Is this ever used? I don't think so. // Quote: // This is for languages whose "objects" are not well ordered (JSON is well // ordered). // [ { a : ... } , { b : ... } ] -> { a : ..., b : ... } // note: this is slow, but that is ok as order will have very few pieces BSONObjBuilder b; char p[2] = "0"; while (1) { BSONObj j = _sort.getObjectField(p); if (j.isEmpty()) { break; } BSONElement e = j.firstElement(); if (e.eoo()) { return Status(ErrorCodes::BadValue, "bad order array"); } if (!e.isNumber()) { return Status(ErrorCodes::BadValue, "bad order array [2]"); } b.append(e); (*p)++; if (!(*p <= '9')) { return Status(ErrorCodes::BadValue, "too many ordering elements"); } } _sort = b.obj(); } else { return Status(ErrorCodes::BadValue, "sort must be object or array"); } } else if ('$' == *name) { name++; if (str::equals("explain", name)) { // Won't throw. _explain = e.trueValue(); } else if (str::equals("snapshot", name)) { // Won't throw. _snapshot = e.trueValue(); } else if (str::equals("min", name)) { if (!e.isABSONObj()) { return Status(ErrorCodes::BadValue, "$min must be a BSONObj"); } _min = e.embeddedObject(); } else if (str::equals("max", name)) { if (!e.isABSONObj()) { return Status(ErrorCodes::BadValue, "$max must be a BSONObj"); } _max = e.embeddedObject(); } else if (str::equals("hint", name)) { if (e.isABSONObj()) { _hint = e.embeddedObject(); } else { // Hint can be specified as an object or as a string. Wrap takes care of // it. _hint = e.wrap(); } } else if (str::equals("returnKey", name)) { // Won't throw. if (e.trueValue()) { _returnKey = true; BSONObjBuilder projBob; projBob.appendElements(_proj); // We use $$ because it's never going to show up in a user's projection. // The exact text doesn't matter. BSONObj indexKey = BSON("$$" << BSON("$meta" << LiteParsedQuery::metaIndexKey)); projBob.append(indexKey.firstElement()); _proj = projBob.obj(); } } else if (str::equals("maxScan", name)) { // Won't throw. _maxScan = e.numberInt(); } else if (str::equals("showDiskLoc", name)) { // Won't throw. if (e.trueValue()) { BSONObjBuilder projBob; projBob.appendElements(_proj); BSONObj metaDiskLoc = BSON("$diskLoc" << BSON("$meta" << LiteParsedQuery::metaDiskLoc)); projBob.append(metaDiskLoc.firstElement()); _proj = projBob.obj(); } } else if (str::equals("maxTimeMS", name)) { StatusWith<int> maxTimeMS = parseMaxTimeMS(e); if (!maxTimeMS.isOK()) { return maxTimeMS.getStatus(); } _maxTimeMS = maxTimeMS.getValue(); } } } if (_snapshot) { if (!_sort.isEmpty()) { return Status(ErrorCodes::BadValue, "E12001 can't use sort with $snapshot"); } if (!_hint.isEmpty()) { return Status(ErrorCodes::BadValue, "E12002 can't use hint with $snapshot"); } } return Status::OK(); }
bool _compact(const char *ns, NamespaceDetails *d, string& errmsg, bool validate, BSONObjBuilder& result) { //int les = d->lastExtentSize; // this is a big job, so might as well make things tidy before we start just to be nice. getDur().commitNow(); list<DiskLoc> extents; for( DiskLoc L = d->firstExtent; !L.isNull(); L = L.ext()->xnext ) extents.push_back(L); log() << "compact " << extents.size() << " extents" << endl; ProgressMeterHolder pm( cc().curop()->setMessage( "compact extent" , extents.size() ) ); // same data, but might perform a little different after compact? NamespaceDetailsTransient::get_w(ns).clearQueryCache(); int nidx = d->nIndexes; scoped_array<IndexSpec> indexSpecs( new IndexSpec[nidx] ); scoped_array<SortPhaseOne> phase1( new SortPhaseOne[nidx] ); { NamespaceDetails::IndexIterator ii = d->ii(); int x = 0; while( ii.more() ) { BSONObjBuilder b; BSONObj::iterator i(ii.next().info.obj()); while( i.more() ) { BSONElement e = i.next(); if( !str::equals(e.fieldName(), "v") && !str::equals(e.fieldName(), "background") ) { b.append(e); } } BSONObj o = b.obj().getOwned(); phase1[x].sorter.reset( new BSONObjExternalSorter( o.getObjectField("key") ) ); phase1[x].sorter->hintNumObjects( d->stats.nrecords ); indexSpecs[x++].reset(o); } } log() << "compact orphan deleted lists" << endl; for( int i = 0; i < Buckets; i++ ) { d->deletedList[i].writing().Null(); } // before dropping indexes, at least make sure we can allocate one extent! uassert(14025, "compact error no space available to allocate", !allocateSpaceForANewRecord(ns, d, Record::HeaderSize+1).isNull()); // note that the drop indexes call also invalidates all clientcursors for the namespace, which is important and wanted here log() << "compact dropping indexes" << endl; BSONObjBuilder b; if( !dropIndexes(d, ns, "*", errmsg, b, true) ) { errmsg = "compact drop indexes failed"; log() << errmsg << endl; return false; } getDur().commitNow(); long long skipped = 0; int n = 0; for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) { skipped += compactExtent(ns, d, *i, n++, indexSpecs, phase1, nidx, validate); pm.hit(); } if( skipped ) { result.append("invalidObjects", skipped); } assert( d->firstExtent.ext()->xprev.isNull() ); // indexes will do their own progress meter? pm.finished(); // build indexes NamespaceString s(ns); string si = s.db + ".system.indexes"; for( int i = 0; i < nidx; i++ ) { killCurrentOp.checkForInterrupt(false); BSONObj info = indexSpecs[i].info; log() << "compact create index " << info["key"].Obj().toString() << endl; try { precalced = &phase1[i]; theDataFileMgr.insert(si.c_str(), info.objdata(), info.objsize()); } catch(...) { precalced = 0; throw; } precalced = 0; } return true; }
bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result) { if (!cmdObj["start"].eoo()) { errmsg = "using deprecated 'start' argument to geoNear"; return false; } const NamespaceString nss(parseNsCollectionRequired(dbname, cmdObj)); AutoGetCollectionForRead ctx(txn, nss); Collection* collection = ctx.getCollection(); if (!collection) { errmsg = "can't find ns"; return false; } IndexCatalog* indexCatalog = collection->getIndexCatalog(); // cout << "raw cmd " << cmdObj.toString() << endl; // We seek to populate this. string nearFieldName; bool using2DIndex = false; if (!getFieldName(txn, collection, indexCatalog, &nearFieldName, &errmsg, &using2DIndex)) { return false; } PointWithCRS point; uassert(17304, "'near' field must be point", GeoParser::parseQueryPoint(cmdObj["near"], &point).isOK()); bool isSpherical = cmdObj["spherical"].trueValue(); if (!using2DIndex) { uassert(17301, "2dsphere index must have spherical: true", isSpherical); } // Build the $near expression for the query. BSONObjBuilder nearBob; if (isSpherical) { nearBob.append("$nearSphere", cmdObj["near"].Obj()); } else { nearBob.append("$near", cmdObj["near"].Obj()); } if (!cmdObj["maxDistance"].eoo()) { uassert(17299, "maxDistance must be a number", cmdObj["maxDistance"].isNumber()); nearBob.append("$maxDistance", cmdObj["maxDistance"].number()); } if (!cmdObj["minDistance"].eoo()) { uassert(17298, "minDistance doesn't work on 2d index", !using2DIndex); uassert(17300, "minDistance must be a number", cmdObj["minDistance"].isNumber()); nearBob.append("$minDistance", cmdObj["minDistance"].number()); } if (!cmdObj["uniqueDocs"].eoo()) { warning() << nss << ": ignoring deprecated uniqueDocs option in geoNear command"; } // And, build the full query expression. BSONObjBuilder queryBob; queryBob.append(nearFieldName, nearBob.obj()); if (!cmdObj["query"].eoo() && cmdObj["query"].isABSONObj()) { queryBob.appendElements(cmdObj["query"].Obj()); } BSONObj rewritten = queryBob.obj(); // Extract the collation, if it exists. BSONObj collation; { BSONElement collationElt; Status collationEltStatus = bsonExtractTypedField(cmdObj, "collation", BSONType::Object, &collationElt); if (!collationEltStatus.isOK() && (collationEltStatus != ErrorCodes::NoSuchKey)) { return appendCommandStatus(result, collationEltStatus); } if (collationEltStatus.isOK()) { collation = collationElt.Obj(); } } if (!collation.isEmpty() && serverGlobalParams.featureCompatibility.version.load() == ServerGlobalParams::FeatureCompatibility::Version::k32) { return appendCommandStatus( result, Status(ErrorCodes::InvalidOptions, "The featureCompatibilityVersion must be 3.4 to use collation. See " "http://dochub.mongodb.org/core/3.4-feature-compatibility.")); } long long numWanted = 100; const char* limitName = !cmdObj["num"].eoo() ? "num" : "limit"; BSONElement eNumWanted = cmdObj[limitName]; if (!eNumWanted.eoo()) { uassert(17303, "limit must be number", eNumWanted.isNumber()); numWanted = eNumWanted.safeNumberLong(); uassert(17302, "limit must be >=0", numWanted >= 0); } bool includeLocs = false; if (!cmdObj["includeLocs"].eoo()) { includeLocs = cmdObj["includeLocs"].trueValue(); } double distanceMultiplier = 1.0; BSONElement eDistanceMultiplier = cmdObj["distanceMultiplier"]; if (!eDistanceMultiplier.eoo()) { uassert(17296, "distanceMultiplier must be a number", eDistanceMultiplier.isNumber()); distanceMultiplier = eDistanceMultiplier.number(); uassert(17297, "distanceMultiplier must be non-negative", distanceMultiplier >= 0); } BSONObj projObj = BSON("$pt" << BSON("$meta" << QueryRequest::metaGeoNearPoint) << "$dis" << BSON("$meta" << QueryRequest::metaGeoNearDistance)); auto qr = stdx::make_unique<QueryRequest>(nss); qr->setFilter(rewritten); qr->setProj(projObj); qr->setLimit(numWanted); qr->setCollation(collation); const ExtensionsCallbackReal extensionsCallback(txn, &nss); auto statusWithCQ = CanonicalQuery::canonicalize(txn, std::move(qr), extensionsCallback); if (!statusWithCQ.isOK()) { errmsg = "Can't parse filter / create query"; return false; } unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue()); // Prevent chunks from being cleaned up during yields - this allows us to only check the // version on initial entry into geoNear. RangePreserver preserver(collection); auto statusWithPlanExecutor = getExecutor(txn, collection, std::move(cq), PlanExecutor::YIELD_AUTO, 0); if (!statusWithPlanExecutor.isOK()) { errmsg = "can't get query executor"; return false; } unique_ptr<PlanExecutor> exec = std::move(statusWithPlanExecutor.getValue()); auto curOp = CurOp::get(txn); { stdx::lock_guard<Client> lk(*txn->getClient()); curOp->setPlanSummary_inlock(Explain::getPlanSummary(exec.get())); } double totalDistance = 0; BSONObjBuilder resultBuilder(result.subarrayStart("results")); double farthestDist = 0; BSONObj currObj; long long results = 0; PlanExecutor::ExecState state; while (PlanExecutor::ADVANCED == (state = exec->getNext(&currObj, NULL))) { // Come up with the correct distance. double dist = currObj["$dis"].number() * distanceMultiplier; totalDistance += dist; if (dist > farthestDist) { farthestDist = dist; } // Strip out '$dis' and '$pt' from the result obj. The rest gets added as 'obj' // in the command result. BSONObjIterator resIt(currObj); BSONObjBuilder resBob; while (resIt.more()) { BSONElement elt = resIt.next(); if (!mongoutils::str::equals("$pt", elt.fieldName()) && !mongoutils::str::equals("$dis", elt.fieldName())) { resBob.append(elt); } } BSONObj resObj = resBob.obj(); // Don't make a too-big result object. if (resultBuilder.len() + resObj.objsize() > BSONObjMaxUserSize) { warning() << "Too many geoNear results for query " << redact(rewritten) << ", truncating output."; break; } // Add the next result to the result builder. BSONObjBuilder oneResultBuilder( resultBuilder.subobjStart(BSONObjBuilder::numStr(results))); oneResultBuilder.append("dis", dist); if (includeLocs) { oneResultBuilder.appendAs(currObj["$pt"], "loc"); } oneResultBuilder.append("obj", resObj); oneResultBuilder.done(); ++results; // Break if we have the number of requested result documents. if (results >= numWanted) { break; } } resultBuilder.done(); // Return an error if execution fails for any reason. if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { log() << "Plan executor error during geoNear command: " << PlanExecutor::statestr(state) << ", stats: " << redact(Explain::getWinningPlanStats(exec.get())); return appendCommandStatus(result, Status(ErrorCodes::OperationFailed, str::stream() << "Executor error during geoNear command: " << WorkingSetCommon::toStatusString(currObj))); } PlanSummaryStats summary; Explain::getSummaryStats(*exec, &summary); // Fill out the stats subobj. BSONObjBuilder stats(result.subobjStart("stats")); stats.appendNumber("nscanned", summary.totalKeysExamined); stats.appendNumber("objectsLoaded", summary.totalDocsExamined); if (results > 0) { stats.append("avgDistance", totalDistance / results); } stats.append("maxDistance", farthestDist); stats.appendIntOrLL("time", curOp->elapsedMicros() / 1000); stats.done(); collection->infoCache()->notifyOfQuery(txn, summary.indexesUsed); curOp->debug().setPlanSummaryMetrics(summary); if (curOp->shouldDBProfile()) { BSONObjBuilder execStatsBob; Explain::getWinningPlanStats(exec.get(), &execStatsBob); curOp->debug().execStats = execStatsBob.obj(); } return true; }
virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { if ( cmdObj.firstElement().type() != Array ) { errmsg = "ops has to be an array"; return false; } BSONObj ops = cmdObj.firstElement().Obj(); { // check input BSONObjIterator i( ops ); while ( i.more() ) { BSONElement e = i.next(); if ( e.type() == Object ) continue; errmsg = "op not an object: "; errmsg += e.fieldName(); return false; } } if ( cmdObj["preCondition"].type() == Array ) { BSONObjIterator i( cmdObj["preCondition"].Obj() ); while ( i.more() ) { BSONObj f = i.next().Obj(); BSONObj realres = db.findOne( f["ns"].String() , f["q"].Obj() ); Matcher m( f["res"].Obj() ); if ( ! m.matches( realres ) ) { result.append( "got" , realres ); result.append( "whatFailed" , f ); errmsg = "pre-condition failed"; return false; } } } // apply int num = 0; int errors = 0; BSONObjIterator i( ops ); BSONArrayBuilder ab; const bool alwaysUpsert = cmdObj.hasField("alwaysUpsert") ? cmdObj["alwaysUpsert"].trueValue() : true; while ( i.more() ) { BSONElement e = i.next(); const BSONObj& temp = e.Obj(); Client::Context ctx(temp["ns"].String()); bool failed = applyOperation_inlock(temp, false, alwaysUpsert); ab.append(!failed); if ( failed ) errors++; num++; } result.append( "applied" , num ); result.append( "results" , ab.arr() ); if ( ! fromRepl ) { // We want this applied atomically on slaves // so we re-wrap without the pre-condition for speed string tempNS = str::stream() << dbname << ".$cmd"; // TODO: possibly use mutable BSON to remove preCondition field // once it is available BSONObjIterator iter(cmdObj); BSONObjBuilder cmdBuilder; while (iter.more()) { BSONElement elem(iter.next()); if (strcmp(elem.fieldName(), "preCondition") != 0) { cmdBuilder.append(elem); } } logOp("c", tempNS.c_str(), cmdBuilder.done()); } return errors == 0; }
void IndexScanNode::computeProperties() { _sorts.clear(); BSONObj sortPattern = QueryPlannerAnalysis::getSortPattern(indexKeyPattern); if (direction == -1) { sortPattern = QueryPlannerCommon::reverseSortObj(sortPattern); } _sorts.insert(sortPattern); const int nFields = sortPattern.nFields(); if (nFields > 1) { // We're sorted not only by sortPattern but also by all prefixes of it. for (int i = 0; i < nFields; ++i) { // Make obj out of fields [0,i] BSONObjIterator it(sortPattern); BSONObjBuilder prefixBob; for (int j = 0; j <= i; ++j) { prefixBob.append(it.next()); } _sorts.insert(prefixBob.obj()); } } // If we are using the index {a:1, b:1} to answer the predicate {a: 10}, it's sorted // both by the index key pattern and by the pattern {b: 1}. // See if there are any fields with equalities for bounds. We can drop these // from any sort orders created. set<string> equalityFields; if (!bounds.isSimpleRange) { // Figure out how many fields are point intervals. for (size_t i = 0; i < bounds.fields.size(); ++i) { const OrderedIntervalList& oil = bounds.fields[i]; if (oil.intervals.size() != 1) { continue; } const Interval& ival = oil.intervals[0]; if (!ival.isPoint()) { continue; } equalityFields.insert(oil.name); } } if (equalityFields.empty()) { return; } // TODO: Each field in equalityFields could be dropped from the sort order since it is // a point interval. The full set of sort orders is as follows: // For each sort in _sorts: // For each drop in powerset(equalityFields): // Remove fields in 'drop' from 'sort' and add resulting sort to output. // // Since this involves a powerset, we don't generate the full set of possibilities. // Instead, we generate sort orders by removing possible contiguous prefixes of equality // predicates. For example, if the key pattern is {a: 1, b: 1, c: 1, d: 1, e: 1} // and and there are equality predicates on 'a', 'b', and 'c', then here we add the sort // orders {b: 1, c: 1, d: 1, e: 1} and {c: 1, d: 1, e: 1}. (We also end up adding // {d: 1, e: 1} and {d: 1}, but this is done later on.) BSONObjIterator it(sortPattern); BSONObjBuilder suffixBob; while (it.more()) { BSONElement elt = it.next(); // TODO: string slowness. fix when bounds are stringdata not string. if (equalityFields.end() == equalityFields.find(string(elt.fieldName()))) { suffixBob.append(elt); // This field isn't a point interval, can't drop. break; } // We add the sort obtained by dropping 'elt' and all preceding elements from the index // key pattern. BSONObjIterator droppedPrefixIt = it; BSONObjBuilder droppedPrefixBob; while (droppedPrefixIt.more()) { droppedPrefixBob.append(droppedPrefixIt.next()); } _sorts.insert(droppedPrefixBob.obj()); } while (it.more()) { suffixBob.append(it.next()); } // We've found the suffix following the contiguous prefix of equality fields. // Ex. For index {a: 1, b: 1, c: 1, d: 1} and query {a: 3, b: 5}, this suffix // of the key pattern is {c: 1, d: 1}. // // Now we have to add all prefixes of this suffix as possible sort orders. // Ex. Continuing the example from above, we have to include sort orders // {c: 1} and {c: 1, d: 1}. BSONObj filterPointsObj = suffixBob.obj(); for (int i = 0; i < filterPointsObj.nFields(); ++i) { // Make obj out of fields [0,i] BSONObjIterator it(filterPointsObj); BSONObjBuilder prefixBob; for (int j = 0; j <= i; ++j) { prefixBob.append(it.next()); } _sorts.insert(prefixBob.obj()); } }
void Strategy::clientCommandOp(OperationContext* txn, Request& request) { QueryMessage q(request.d()); LOG(3) << "command: " << q.ns << " " << q.query << " ntoreturn: " << q.ntoreturn << " options: " << q.queryOptions; if (q.queryOptions & QueryOption_Exhaust) { uasserted(18527, string("the 'exhaust' query option is invalid for mongos commands: ") + q.ns + " " + q.query.toString()); } NamespaceString nss(request.getns()); // Regular queries are handled in strategy_shard.cpp verify(nss.isCommand() || nss.isSpecialCommand()); if (handleSpecialNamespaces(txn, request, q)) return; int loops = 5; bool cmChangeAttempted = false; while (true) { try { BSONObj cmdObj = q.query; { BSONElement e = cmdObj.firstElement(); if (e.type() == Object && (e.fieldName()[0] == '$' ? str::equals("query", e.fieldName() + 1) : str::equals("query", e.fieldName()))) { // Extract the embedded query object. if (cmdObj.hasField(Query::ReadPrefField.name())) { // The command has a read preference setting. We don't want // to lose this information so we copy this to a new field // called $queryOptions.$readPreference BSONObjBuilder finalCmdObjBuilder; finalCmdObjBuilder.appendElements(e.embeddedObject()); BSONObjBuilder queryOptionsBuilder( finalCmdObjBuilder.subobjStart("$queryOptions")); queryOptionsBuilder.append(cmdObj[Query::ReadPrefField.name()]); queryOptionsBuilder.done(); cmdObj = finalCmdObjBuilder.obj(); } else { cmdObj = e.embeddedObject(); } } } OpQueryReplyBuilder reply; { BSONObjBuilder builder(reply.bufBuilderForResults()); Command::runAgainstRegistered(txn, q.ns, cmdObj, builder, q.queryOptions); } reply.sendCommandReply(request.p(), request.m()); return; } catch (const StaleConfigException& e) { if (loops <= 0) throw e; loops--; log() << "retrying command: " << q.query; // For legacy reasons, ns may not actually be set in the exception :-( string staleNS = e.getns(); if (staleNS.size() == 0) staleNS = q.ns; ShardConnection::checkMyConnectionVersions(txn, staleNS); if (loops < 4) versionManager.forceRemoteCheckShardVersionCB(txn, staleNS); } catch (const DBException& e) { if (e.getCode() == ErrorCodes::IncompatibleCatalogManager) { fassert(28791, !cmChangeAttempted); cmChangeAttempted = true; grid.forwardingCatalogManager()->waitForCatalogManagerChange(txn); } else { OpQueryReplyBuilder reply; { BSONObjBuilder builder(reply.bufBuilderForResults()); Command::appendCommandStatus(builder, e.toStatus()); } reply.sendCommandReply(request.p(), request.m()); return; } } } }
// static Status QueryPlanner::plan(const CanonicalQuery& query, const QueryPlannerParams& params, std::vector<QuerySolution*>* out) { LOG(5) << "Beginning planning..." << endl << "=============================" << endl << "Options = " << optionString(params.options) << endl << "Canonical query:" << endl << query.toString() << "=============================" << endl; for (size_t i = 0; i < params.indices.size(); ++i) { LOG(5) << "Index " << i << " is " << params.indices[i].toString() << endl; } bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. TODO: This is a policy departure. Previously I think you // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we // can't provide one. Is this what we want? if (query.getParsed().isTailable()) { if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) { QuerySolution* soln = buildCollscanSoln(query, true, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } // The hint or sort can be $natural: 1. If this happens, output a collscan. If both // a $natural hint and a $natural sort are specified, then the direction of the collscan // is determined by the sign of the sort (not the sign of the hint). if (!query.getParsed().getHint().isEmpty() || !query.getParsed().getSort().isEmpty()) { BSONObj hintObj = query.getParsed().getHint(); BSONObj sortObj = query.getParsed().getSort(); BSONElement naturalHint = hintObj.getFieldDotted("$natural"); BSONElement naturalSort = sortObj.getFieldDotted("$natural"); // A hint overrides a $natural sort. This means that we don't force a table // scan if there is a $natural sort with a non-$natural hint. if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) { LOG(5) << "Forcing a table scan due to hinted $natural\n"; // min/max are incompatible with $natural. if (canTableScan && query.getParsed().getMin().isEmpty() && query.getParsed().getMax().isEmpty()) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } } // Figure out what fields we care about. unordered_set<string> fields; QueryPlannerIXSelect::getFields(query.root(), "", &fields); for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) { LOG(5) << "Predicate over field '" << *it << "'" << endl; } // Filter our indices so we only look at indices that are over our predicates. vector<IndexEntry> relevantIndices; // Hints require us to only consider the hinted index. // If index filters in the query settings were used to override // the allowed indices for planning, we should not use the hinted index // requested in the query. BSONObj hintIndex; if (!params.indexFiltersApplied) { hintIndex = query.getParsed().getHint(); } // Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real // plan. If that fails, just scan the _id index. if (query.getParsed().isSnapshot()) { // Find the ID index in indexKeyPatterns. It's our hint. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { hintIndex = params.indices[i].keyPattern; break; } } } size_t hintIndexNumber = numeric_limits<size_t>::max(); if (hintIndex.isEmpty()) { QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices); } else { // Sigh. If the hint is specified it might be using the index name. BSONElement firstHintElt = hintIndex.firstElement(); if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) { string hintName = firstHintElt.String(); for (size_t i = 0; i < params.indices.size(); ++i) { if (params.indices[i].name == hintName) { LOG(5) << "Hint by name specified, restricting indices to " << params.indices[i].keyPattern.toString() << endl; relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); hintIndexNumber = i; hintIndex = params.indices[i].keyPattern; break; } } } else { for (size_t i = 0; i < params.indices.size(); ++i) { if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) { relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); LOG(5) << "Hint specified, restricting indices to " << hintIndex.toString() << endl; hintIndexNumber = i; break; } } } if (hintIndexNumber == numeric_limits<size_t>::max()) { return Status(ErrorCodes::BadValue, "bad hint"); } } // Deal with the .min() and .max() query options. If either exist we can only use an index // that matches the object inside. if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) { BSONObj minObj = query.getParsed().getMin(); BSONObj maxObj = query.getParsed().getMax(); // The unfinished siblings of these objects may not be proper index keys because they // may be empty objects or have field names. When an index is picked to use for the // min/max query, these "finished" objects will always be valid index keys for the // index's key pattern. BSONObj finishedMinObj; BSONObj finishedMaxObj; // This is the index into params.indices[...] that we use. size_t idxNo = numeric_limits<size_t>::max(); // If there's an index hinted we need to be able to use it. if (!hintIndex.isEmpty()) { if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) { LOG(5) << "Minobj doesn't work with hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with min query"); } if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) { LOG(5) << "Maxobj doesn't work with hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with max query"); } const BSONObj& kp = params.indices[hintIndexNumber].keyPattern; finishedMinObj = finishMinObj(kp, minObj, maxObj); finishedMaxObj = finishMaxObj(kp, minObj, maxObj); // The min must be less than the max for the hinted index ordering. if (0 <= finishedMinObj.woCompare(finishedMaxObj, kp, false)) { LOG(5) << "Minobj/Maxobj don't work with hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with min/max query"); } idxNo = hintIndexNumber; } else { // No hinted index, look for one that is compatible (has same field names and // ordering thereof). for (size_t i = 0; i < params.indices.size(); ++i) { const BSONObj& kp = params.indices[i].keyPattern; BSONObj toUse = minObj.isEmpty() ? maxObj : minObj; if (indexCompatibleMaxMin(toUse, kp)) { // In order to be fully compatible, the min has to be less than the max // according to the index key pattern ordering. The first step in verifying // this is "finish" the min and max by replacing empty objects and stripping // field names. finishedMinObj = finishMinObj(kp, minObj, maxObj); finishedMaxObj = finishMaxObj(kp, minObj, maxObj); // Now we have the final min and max. This index is only relevant for // the min/max query if min < max. if (0 >= finishedMinObj.woCompare(finishedMaxObj, kp, false)) { // Found a relevant index. idxNo = i; break; } // This index is not relevant; move on to the next. } } } if (idxNo == numeric_limits<size_t>::max()) { LOG(5) << "Can't find relevant index to use for max/min query"; // Can't find an index to use, bail out. return Status(ErrorCodes::BadValue, "unable to find relevant index for max/min query"); } LOG(5) << "Max/min query using index " << params.indices[idxNo].toString() << endl; // Make our scan and output. QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan( params.indices[idxNo], query, params, finishedMinObj, finishedMaxObj); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } return Status::OK(); } for (size_t i = 0; i < relevantIndices.size(); ++i) { LOG(2) << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl; } // Figure out how useful each index is to each predicate. QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices); QueryPlannerIXSelect::stripInvalidAssignments(query.root(), relevantIndices); // Unless we have GEO_NEAR, TEXT, or a projection, we may be able to apply an optimization // in which we strip unnecessary index assignments. // // Disallowed with projection because assignment to a non-unique index can allow the plan // to be covered. // // TEXT and GEO_NEAR are special because they require the use of a text/geo index in order // to be evaluated correctly. Stripping these "mandatory assignments" is therefore invalid. if (query.getParsed().getProj().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { QueryPlannerIXSelect::stripUnneededAssignments(query.root(), relevantIndices); } // query.root() is now annotated with RelevantTag(s). LOG(5) << "Rated tree:" << endl << query.root()->toString(); // If there is a GEO_NEAR it must have an index it can use directly. MatchExpression* gnNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { // No index for GEO_NEAR? No query. RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { LOG(5) << "Unable to find index for $geoNear query." << endl; // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query"); } LOG(5) << "Rated tree after geonear processing:" << query.root()->toString(); } // Likewise, if there is a TEXT it must have an index it can use directly. MatchExpression* textNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) { RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag()); // Exactly one text index required for TEXT. We need to check this explicitly because // the text stage can't be built if no text index exists or there is an ambiguity as to // which one to use. size_t textIndexCount = 0; for (size_t i = 0; i < params.indices.size(); i++) { if (INDEX_TEXT == params.indices[i].type) { textIndexCount++; } } if (textIndexCount != 1) { // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "need exactly one text index for $text query"); } // Error if the text node is tagged with zero indices. if (0 == tag->first.size() && 0 == tag->notFirst.size()) { // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "failed to use text index to satisfy $text query (if text index is " "compound, are equality predicates given for all prefix fields?)"); } // At this point, we know that there is only one text index and that the TEXT node is // assigned to it. invariant(1 == tag->first.size() + tag->notFirst.size()); LOG(5) << "Rated tree after text processing:" << query.root()->toString(); } // If we have any relevant indices, we try to create indexed plans. if (0 < relevantIndices.size()) { // The enumerator spits out trees tagged with IndexTag(s). PlanEnumeratorParams enumParams; enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION; enumParams.root = query.root(); enumParams.indices = &relevantIndices; PlanEnumerator isp(enumParams); isp.init(); MatchExpression* rawTree; while (isp.getNext(&rawTree) && (out->size() < params.maxIndexedSolutions)) { LOG(5) << "About to build solntree from tagged tree:" << endl << rawTree->toString(); // The tagged tree produced by the plan enumerator is not guaranteed // to be canonically sorted. In order to be compatible with the cached // data, sort the tagged tree according to CanonicalQuery ordering. std::unique_ptr<MatchExpression> clone(rawTree->shallowClone()); CanonicalQuery::sortTree(clone.get()); PlanCacheIndexTree* cacheData; Status indexTreeStatus = cacheDataFromTaggedTree(clone.get(), relevantIndices, &cacheData); if (!indexTreeStatus.isOK()) { LOG(5) << "Query is not cachable: " << indexTreeStatus.reason() << endl; } unique_ptr<PlanCacheIndexTree> autoData(cacheData); // This can fail if enumeration makes a mistake. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess( query, rawTree, false, relevantIndices, params); if (NULL == solnRoot) { continue; } QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { LOG(5) << "Planner: adding solution:" << endl << soln->toString(); if (indexTreeStatus.isOK()) { SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(autoData.release()); soln->cacheData.reset(scd); } out->push_back(soln); } } } // Don't leave tags on query tree. query.root()->resetTag(); LOG(5) << "Planner: outputted " << out->size() << " indexed solutions.\n"; // Produce legible error message for failed OR planning with a TEXT child. // TODO: support collection scan for non-TEXT children of OR. if (out->size() == 0 && textNode != NULL && MatchExpression::OR == query.root()->matchType()) { MatchExpression* root = query.root(); for (size_t i = 0; i < root->numChildren(); ++i) { if (textNode == root->getChild(i)) { return Status(ErrorCodes::BadValue, "Failed to produce a solution for TEXT under OR - " "other non-TEXT clauses under OR have to be indexed as well."); } } } // An index was hinted. If there are any solutions, they use the hinted index. If not, we // scan the entire index to provide results and output that as our plan. This is the // desired behavior when an index is hinted that is not relevant to the query. if (!hintIndex.isEmpty()) { if (0 == out->size()) { QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params); verify(NULL != soln); LOG(5) << "Planner: outputting soln that uses hinted index as scan." << endl; out->push_back(soln); } return Status::OK(); } // If a sort order is requested, there may be an index that provides it, even if that // index is not over any predicates in the query. // if (!query.getParsed().getSort().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { // See if we have a sort provided from an index already. // This is implied by the presence of a non-blocking solution. bool usingIndexToSort = false; for (size_t i = 0; i < out->size(); ++i) { QuerySolution* soln = (*out)[i]; if (!soln->hasBlockingStage) { usingIndexToSort = true; break; } } if (!usingIndexToSort) { for (size_t i = 0; i < params.indices.size(); ++i) { const IndexEntry& index = params.indices[i]; // Only regular (non-plugin) indexes can be used to provide a sort, and only // non-sparse indexes can be used to provide a sort. // // TODO: Sparse indexes can't normally provide a sort, because non-indexed // documents could potentially be missing from the result set. However, if the // query predicate can be used to guarantee that all documents to be returned // are indexed, then the index should be able to provide the sort. // // For example: // - Sparse index {a: 1, b: 1} should be able to provide a sort for // find({b: 1}).sort({a: 1}). SERVER-13908. // - Index {a: 1, b: "2dsphere"} (which is "geo-sparse", if // 2dsphereIndexVersion=2) should be able to provide a sort for // find({b: GEO}).sort({a:1}). SERVER-10801. if (index.type != INDEX_BTREE) { continue; } if (index.sparse) { continue; } // Partial indexes can only be used to provide a sort only if the query predicate is // compatible. if (index.filterExpr && !expression::isSubsetOf(query.root(), index.filterExpr)) { continue; } const BSONObj kp = QueryPlannerAnalysis::getSortPattern(index.keyPattern); if (providesSort(query, kp)) { LOG(5) << "Planner: outputting soln that uses index to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params); if (NULL != soln) { PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); indexTree->setIndexEntry(params.indices[i]); SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(indexTree); scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN; scd->wholeIXSolnDir = 1; soln->cacheData.reset(scd); out->push_back(soln); break; } } if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) { LOG(5) << "Planner: outputting soln that uses (reverse) index " << "to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1); if (NULL != soln) { PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); indexTree->setIndexEntry(params.indices[i]); SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(indexTree); scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN; scd->wholeIXSolnDir = -1; soln->cacheData.reset(scd); out->push_back(soln); break; } } } } } // geoNear and text queries *require* an index. // Also, if a hint is specified it indicates that we MUST use it. bool possibleToCollscan = !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) && hintIndex.isEmpty(); // The caller can explicitly ask for a collscan. bool collscanRequested = (params.options & QueryPlannerParams::INCLUDE_COLLSCAN); // No indexed plans? We must provide a collscan if possible or else we can't run the query. bool collscanNeeded = (0 == out->size() && canTableScan); if (possibleToCollscan && (collscanRequested || collscanNeeded)) { QuerySolution* collscan = buildCollscanSoln(query, false, params); if (NULL != collscan) { SolutionCacheData* scd = new SolutionCacheData(); scd->solnType = SolutionCacheData::COLLSCAN_SOLN; collscan->cacheData.reset(scd); out->push_back(collscan); LOG(5) << "Planner: outputting a collscan:" << endl << collscan->toString(); } } return Status::OK(); }
bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { int s = 0; bool found = setParmsMongodSpecific(dbname, cmdObj, errmsg, result, fromRepl); if( cmdObj.hasElement("journalCommitInterval") ) { if( !cmdLine.dur ) { errmsg = "journaling is off"; return false; } int x = (int) cmdObj["journalCommitInterval"].Number(); verify( x > 1 && x < 500 ); cmdLine.journalCommitInterval = x; log() << "setParameter journalCommitInterval=" << x << endl; s++; } if( cmdObj.hasElement("notablescan") ) { verify( !cmdLine.isMongos() ); if( s == 0 ) result.append("was", cmdLine.noTableScan); cmdLine.noTableScan = cmdObj["notablescan"].Bool(); s++; } if( cmdObj.hasElement("quiet") ) { if( s == 0 ) result.append("was", cmdLine.quiet ); cmdLine.quiet = cmdObj["quiet"].Bool(); s++; } if( cmdObj.hasElement("syncdelay") ) { verify( !cmdLine.isMongos() ); if( s == 0 ) result.append("was", cmdLine.syncdelay ); cmdLine.syncdelay = cmdObj["syncdelay"].Number(); s++; } if( cmdObj.hasElement( "logLevel" ) ) { if( s == 0 ) result.append("was", logLevel ); logLevel = cmdObj["logLevel"].numberInt(); s++; } if( cmdObj.hasElement( "replApplyBatchSize" ) ) { if( s == 0 ) result.append("was", replApplyBatchSize ); BSONElement e = cmdObj["replApplyBatchSize"]; ParameterValidator * v = ParameterValidator::get( e.fieldName() ); verify( v ); if ( ! v->isValid( e , errmsg ) ) return false; replApplyBatchSize = e.numberInt(); s++; } if( cmdObj.hasElement( "traceExceptions" ) ) { if( s == 0 ) result.append( "was", DBException::traceExceptions ); DBException::traceExceptions = cmdObj["traceExceptions"].Bool(); s++; } if( cmdObj.hasElement( "replMonitorMaxFailedChecks" ) ) { if( s == 0 ) result.append( "was", ReplicaSetMonitor::getMaxFailedChecks() ); ReplicaSetMonitor::setMaxFailedChecks( cmdObj["replMonitorMaxFailedChecks"].numberInt() ); s++; } if( s == 0 && !found ) { errmsg = "no option found to set, use help:true to see options "; return false; } return true; }
Status ModifierPush::init(const BSONElement& modExpr) { // // field name analysis // // Break down the field name into its 'dotted' components (aka parts) and check that // the field is fit for updates. _fieldRef.parse(modExpr.fieldName()); Status status = fieldchecker::isUpdatable(_fieldRef); if (! status.isOK()) { return status; } // If a $-positional operator was used, get the index in which it occurred // and ensure only one occurrence. size_t foundCount; bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount); if (foundDollar && foundCount > 1) { return Status(ErrorCodes::BadValue, "too many positional($) elements found."); } // // value analysis // // Are the target push values safe to store? BSONElement sliceElem; BSONElement sortElem; switch (modExpr.type()) { case Array: if (! modExpr.Obj().okForStorage()) { return Status(ErrorCodes::BadValue, "cannot use '$' or '.' as values"); } if (_pushMode == PUSH_ALL) { _eachMode = true; Status status = parseEachMode(PUSH_ALL, modExpr, &_eachElem, &sliceElem, &sortElem); if (!status.isOK()) { return status; } } else { _val = modExpr; } break; case Object: if (_pushMode == PUSH_ALL) { return Status(ErrorCodes::BadValue, "$pushAll requires an array of values"); } // If any known clause ($each, $slice, or $sort) is present, we'd assume // we're using the $each variation of push and would parse accodingly. _eachMode = inEachMode(modExpr); if (_eachMode) { Status status = parseEachMode(PUSH_NORMAL, modExpr, &_eachElem, &sliceElem, &sortElem); if (!status.isOK()) { return status; } } else { if (! modExpr.Obj().okForStorage()) { return Status(ErrorCodes::BadValue, "cannot use '$' as values"); } _val = modExpr; } break; default: if (_pushMode == PUSH_ALL) { return Status(ErrorCodes::BadValue, "$pushAll requires an array of values"); } _val = modExpr; break; } // Is slice present and correct? if (sliceElem.type() != EOO) { if (_pushMode == PUSH_ALL) { return Status(ErrorCodes::BadValue, "cannot use $slice in $pushAll"); } if (!sliceElem.isNumber()) { return Status(ErrorCodes::BadValue, "$slice must be a numeric value"); } // If the value of slice is not fraction, even if it's a double, we allow it. The // reason here is that the shell will use doubles by default unless told otherwise. double fractional = sliceElem.numberDouble(); if (fractional - static_cast<int64_t>(fractional) != 0) { return Status(ErrorCodes::BadValue, "$slice in $push cannot be fractional"); } _slice = sliceElem.numberLong(); if (_slice > 0) { return Status(ErrorCodes::BadValue, "$slice in $push must be zero or negative"); } _slicePresent = true; } // Is sort present and correct? if (sortElem.type() != EOO) { if (_pushMode == PUSH_ALL) { return Status(ErrorCodes::BadValue, "cannot use $sort in $pushAll"); } if (!_slicePresent) { return Status(ErrorCodes::BadValue, "$sort requires $slice to be present"); } else if (sortElem.type() != Object) { return Status(ErrorCodes::BadValue, "invalid $sort clause"); } BSONObj sortObj = sortElem.embeddedObject(); if (sortObj.isEmpty()) { return Status(ErrorCodes::BadValue, "sort parttern is empty"); } // Check if the sort pattern is sound. BSONObjIterator sortIter(sortObj); while (sortIter.more()) { BSONElement sortPatternElem = sortIter.next(); // We require either <field>: 1 or -1 for asc and desc. if (!isPatternElement(sortPatternElem)) { return Status(ErrorCodes::BadValue, "$sort elements' must be either 1 or -1"); } // All fields parts must be valid. FieldRef sortField; sortField.parse(sortPatternElem.fieldName()); if (sortField.numParts() == 0) { return Status(ErrorCodes::BadValue, "$sort field cannot be empty"); } for (size_t i = 0; i < sortField.numParts(); i++) { if (sortField.getPart(i).size() == 0) { return Status(ErrorCodes::BadValue, "empty field in dotted sort pattern"); } } } _sort = PatternElementCmp(sortElem.embeddedObject()); _sortPresent = true; } return Status::OK(); }
/** * benchRun( { ops : [] , host : XXX , db : XXXX , parallel : 5 , seconds : 5 } */ BSONObj benchRun( const BSONObj& argsFake, void* data ) { assert( argsFake.firstElement().isABSONObj() ); BSONObj args = argsFake.firstElement().Obj(); // setup BenchRunConfig config; if ( args["host"].type() == String ) config.host = args["host"].String(); if ( args["db"].type() == String ) config.db = args["db"].String(); if ( args["parallel"].isNumber() ) config.parallel = args["parallel"].numberInt(); if ( args["seconds"].isNumber() ) config.seconds = args["seconds"].number(); config.ops = args["ops"].Obj(); // execute ScopedDbConnection conn( config.host ); // start threads vector<boost::thread*> all; for ( unsigned i=0; i<config.parallel; i++ ) all.push_back( new boost::thread( boost::bind( benchThread , &config ) ) ); // give them time to init while ( config.threadsReady < config.parallel ) sleepmillis( 1 ); BSONObj before; conn->simpleCommand( "admin" , &before , "serverStatus" ); sleepmillis( (int)(1000.0 * config.seconds) ); BSONObj after; conn->simpleCommand( "admin" , &after , "serverStatus" ); conn.done(); config.active = false; for ( unsigned i=0; i<all.size(); i++ ) all[i]->join(); if ( config.error ) return BSON( "err" << 1 ); // compute actual ops/sec before = before["opcounters"].Obj(); after = after["opcounters"].Obj(); bool totals = args["totals"].trueValue(); BSONObjBuilder buf; if ( ! totals ) buf.append( "note" , "values per second" ); { BSONObjIterator i( after ); while ( i.more() ) { BSONElement e = i.next(); double x = e.number(); x = x - before[e.fieldName()].number(); if ( ! totals ) x = x / config.seconds; buf.append( e.fieldName() , x ); } } BSONObj zoo = buf.obj(); return BSON( "" << zoo ); }
bool handleRESTQuery( const std::string& ns, const std::string& action, BSONObj & params, int & responseCode, stringstream & out ) { Timer t; int html = _getOption( params["html"] , 0 ); int skip = _getOption( params["skip"] , 0 ); int num = _getOption( params["limit"] , _getOption( params["count" ] , 1000 ) ); // count is old, limit is new int one = 0; if ( params["one"].type() == String && tolower( params["one"].valuestr()[0] ) == 't' ) { num = 1; one = 1; } BSONObjBuilder queryBuilder; BSONObjIterator i(params); while ( i.more() ) { BSONElement e = i.next(); string name = e.fieldName(); if ( name.find( "filter_" ) != 0 ) continue; string field = name.substr(7); const char * val = e.valuestr(); char * temp; // TODO: this is how i guess if something is a number. pretty lame right now double number = strtod( val , &temp ); if ( temp != val ) queryBuilder.append( field , number ); else queryBuilder.append( field , val ); } BSONObj query = queryBuilder.obj(); auto_ptr<DBClientCursor> cursor = db.query( ns.c_str() , query, num , skip ); uassert( 13085 , "query failed for dbwebserver" , cursor.get() ); if ( one ) { if ( cursor->more() ) { BSONObj obj = cursor->next(); out << obj.jsonString(Strict,html?1:0) << '\n'; } else { responseCode = 404; } return html != 0; } if( html ) { string title = string("query ") + ns; out << start(title) << p(title) << "<pre>"; } else { out << "{\n"; out << " \"offset\" : " << skip << ",\n"; out << " \"rows\": [\n"; } int howMany = 0; while ( cursor->more() ) { if ( howMany++ && html == 0 ) out << " ,\n"; BSONObj obj = cursor->next(); if( html ) { if( out.tellp() > 4 * 1024 * 1024 ) { out << "Stopping output: more than 4MB returned and in html mode\n"; break; } out << obj.jsonString(Strict, html?1:0) << "\n\n"; } else { if( out.tellp() > 50 * 1024 * 1024 ) // 50MB limit - we are using ram break; out << " " << obj.jsonString(); } } if( html ) { out << "</pre>\n"; if( howMany == 0 ) out << p("Collection is empty"); out << _end(); } else { out << "\n ],\n\n"; out << " \"total_rows\" : " << howMany << " ,\n"; out << " \"query\" : " << query.jsonString() << " ,\n"; out << " \"millis\" : " << t.millis() << '\n'; out << "}\n"; } return html != 0; }
INT32 aggrProjectParser::addObj( const CHAR *pAlias, const bson::BSONObj &Obj, const CHAR *pCLName, qgmOPFieldVec &selectorVec, _qgmPtrTable *pTable ) { INT32 rc = SDB_OK; CHAR *pFuncBuf = NULL; #define FUNC_NAME_BUILDOBJ "buildObj" try { UINT32 nameLen = ossStrlen( FUNC_NAME_BUILDOBJ ); UINT32 paramLen = 0; UINT32 fieldNum = 0; UINT32 curPos = 0; BSONObjIterator iter( Obj ); while ( iter.more() ) { BSONElement beField = iter.next(); PD_CHECK( beField.isNumber(), SDB_INVALIDARG, error, PDERROR, "field type must be number!" ); if ( beField.number() == 0 ) { continue; } ++fieldNum; paramLen += ossStrlen(AGGR_CL_DEFAULT_ALIAS) + 1 + ossStrlen( beField.fieldName() ); } PD_CHECK( fieldNum > 0, SDB_INVALIDARG, error, PDERROR, "Can't add empty obj!" ); paramLen += ( fieldNum - 1 ); pFuncBuf = ( CHAR * )SDB_OSS_MALLOC( nameLen + 3 + paramLen ); PD_CHECK( pFuncBuf != NULL, SDB_OOM, error, PDERROR, "malloc failed(size=%d)", ( nameLen + 3 + paramLen )); ossStrcpy( pFuncBuf, FUNC_NAME_BUILDOBJ ); pFuncBuf[ nameLen ] = '('; curPos = nameLen + 1; iter = BSONObjIterator( Obj ); while ( iter.more() ) { BSONElement beField = iter.next(); if ( beField.number() == 0 ) { continue; } ossStrcpy( ( pFuncBuf + curPos ), AGGR_CL_DEFAULT_ALIAS ); curPos += ossStrlen( AGGR_CL_DEFAULT_ALIAS ); pFuncBuf[ curPos ] = '.'; curPos += 1; ossStrcpy( ( pFuncBuf + curPos ), beField.fieldName() ); curPos += ossStrlen( beField.fieldName() ); if ( fieldNum > 1 ) { pFuncBuf[ curPos ] = ','; curPos += 1; } --fieldNum; } pFuncBuf[ curPos ] = ')'; curPos += 1; pFuncBuf[ curPos ] = 0; qgmField slValAttr; qgmField slValRelegation; rc = pTable->getOwnField( pFuncBuf, slValAttr ); PD_RC_CHECK( rc, PDERROR, "failed to get the field(%s)", pFuncBuf ); qgmDbAttr slVal( slValRelegation, slValAttr ); qgmField slAlias; rc = pTable->getOwnField( pAlias, slAlias ); PD_RC_CHECK( rc, PDERROR, "failed to get the field(%s)", pAlias ); qgmOpField selector; selector.alias = slAlias; selector.value = slVal; selector.type = SQL_GRAMMAR::FUNC; selectorVec.push_back( selector ); } catch ( std::exception &e ) { PD_CHECK( SDB_INVALIDARG, SDB_INVALIDARG, error, PDERROR, "failed to add function-field, received unexpected error:%s", e.what() ); } done: SDB_OSS_FREE( pFuncBuf ); return rc; error: goto done; }