JSBool mongo_find(JSContext *cx, JSObject *obj, uintN argc, jsval *argv, jsval *rval){ uassert( "mongo_find neesd 5 args" , argc == 5 ); shared_ptr< DBClientBase > * connHolder = (shared_ptr< DBClientBase >*)JS_GetPrivate( cx , obj ); uassert( "no connection!" , connHolder && connHolder->get() ); DBClientBase *conn = connHolder->get(); Convertor c( cx ); string ns = c.toString( argv[0] ); BSONObj q = c.toObject( argv[1] ); BSONObj f = c.toObject( argv[2] ); int nToReturn = (int) c.toNumber( argv[3] ); int nToSkip = (int) c.toNumber( argv[4] ); bool slaveOk = c.getBoolean( obj , "slaveOk" ); try { auto_ptr<DBClientCursor> cursor = conn->query( ns , q , nToReturn , nToSkip , f.nFields() ? &f : 0 , slaveOk ? Option_SlaveOk : 0 ); JSObject * mycursor = JS_NewObject( cx , &internal_cursor_class , 0 , 0 ); assert( JS_SetPrivate( cx , mycursor , new CursorHolder( cursor, *connHolder ) ) ); *rval = OBJECT_TO_JSVAL( mycursor ); return JS_TRUE; } catch ( ... ){ JS_ReportError( cx , "error doing query" ); return JS_FALSE; } }
BSONObj _look_up_info_hash(scoped_ptr<ScopedDbConnection> const& scoped_conn, sha1_hash const& info_hash) { BSONObj torrent_record; char ih_hex[41]; to_hex((char const*)&info_hash[0], sha1_hash::size, ih_hex); DBClientBase* conn = scoped_conn->get(); if (conn->isFailed()) { log_util::error() << "torrentdb::_look_up_info_hash: mongodb connection failed" << endl; } else { #ifdef _DEBUG log_util::debug() << "torrentdb::_look_up_info_hash: running mongodb query (" << ih_hex << ")" << endl; #endif std::auto_ptr<DBClientCursor> cursor = conn->query(_param_map["torrentdb_ns"], QUERY("info_hash" << ih_hex)); bool found_results = false; //if (conn->getLastError().empty()) { while (cursor->more()) { // TODO: verify no more than one record returned? torrent_record = cursor->next(); found_results = true; } //} #ifdef _DEBUG if (!found_results) { log_util::debug() << "torrentdb::_look_up_info_hash: torrent not found" << endl; } #endif } return torrent_record; }
BSONObj _look_up_seedbank(scoped_ptr<ScopedDbConnection> const& scoped_conn, int seedbank_id) { BSONObj sb_record; DBClientBase* conn = scoped_conn->get(); if (conn->isFailed()) { log_util::error() << "torrentdb::_look_up_seedbank: mongodb connection failed" << endl; } else { #ifdef _DEBUG log_util::debug() << "torrentdb::_look_up_seedbank: running mongodb query (" << seedbank_id << ")" << endl; #endif std::auto_ptr<DBClientCursor> cursor = conn->query(_param_map["seedbankdb_ns"], QUERY("seedbank_id" << seedbank_id)); bool found_results = false; //if (conn->getLastError().empty()) { while (cursor->more()) { // TODO: verify no more than one record returned? sb_record = cursor->next(); found_results = true; } //} if (!found_results) { log_util::error() << "torrentdb::_look_up_seedbank: mongodb result not found" << endl; } } return sb_record; }
void Scope::loadStored(bool ignoreNotConnected) { if (_localDBName.size() == 0) { if (ignoreNotConnected) return; uassert(10208, "need to have locallyConnected already", _localDBName.size()); } if (_loadedVersion == _lastVersion) return; _loadedVersion = _lastVersion; string coll = _localDBName + ".system.js"; DBClientBase* directDBClient = createDirectClient(); auto_ptr<DBClientCursor> c = directDBClient->query(coll, Query(), 0, 0, NULL, QueryOption_SlaveOk, 0); massert(16669, "unable to get db client cursor from query", c.get()); set<string> thisTime; while (c->more()) { BSONObj o = c->nextSafe(); BSONElement n = o["_id"]; BSONElement v = o["value"]; uassert(10209, str::stream() << "name has to be a string: " << n, n.type() == String); uassert(10210, "value has to be set", v.type() != EOO); try { setElement(n.valuestr(), v); thisTime.insert(n.valuestr()); _storedNames.insert(n.valuestr()); } catch (const DBException& setElemEx) { log() << "unable to load stored JavaScript function " << n.valuestr() << "(): " << setElemEx.what() << endl; } } // remove things from scope that were removed from the system.js collection for (set<string>::iterator i = _storedNames.begin(); i != _storedNames.end(); ) { if (thisTime.count(*i) == 0) { string toDelete = str::stream() << "delete " << *i; _storedNames.erase(i++); execSetup(toDelete, "clean up scope"); } else { ++i; } } }
int ConfigServer::dbConfigVersion( DBClientBase& conn ){ auto_ptr<DBClientCursor> c = conn.query( "config.version" , BSONObj() ); int version = 0; if ( c->more() ){ BSONObj o = c->next(); version = o["version"].numberInt(); uassert( 10189 , "should only have 1 thing in config.version" , ! c->more() ); } else { if ( conn.count( ShardNS::shard ) || conn.count( ShardNS::database ) ){ version = 1; } } return version; }
/* "tail" the specified namespace, outputting elements as they are added. _id values must be inserted in increasing order for this to work. (Some other field could also be used.) Note: one could use a capped collection and $natural order to do something similar, using sort({$natural:1}), and then not need to worry about _id's being in order. */ void tail(DBClientBase& conn, const char *ns) { conn.ensureIndex(ns, fromjson("{_id:1}")); BSONElement lastId; Query query = Query().sort("_id"); while( 1 ) { auto_ptr<DBClientCursor> c = conn.query(ns, query, 0, 0, 0, Option_CursorTailable); while( 1 ) { if( !c->more() ) { if( c->isDead() ) { // we need to requery break; } sleepsecs(1); } BSONObj o = c->next(); lastId = o["_id"]; cout << o.toString() << endl; } query = QUERY( "_id" << GT << lastId ).sort("_id"); } }
void tail(DBClientBase& conn, const char *ns) { BSONElement lastId = minKey.firstElement(); Query query = Query(); auto_ptr<DBClientCursor> c = conn.query(ns, query, 0, 0, 0, QueryOption_CursorTailable); while( 1 ) { if( !c->more() ) { if( c->isDead() ) { break; // we need to requery } // all data (so far) exhausted, wait for more sleepsecs(1); continue; } BSONObj o = c->next(); lastId = o["_id"]; cout << o.toString() << endl; } }
/* "tail" the namespace, outputting elements as they are added. For this to work something field -- _id in this case -- should be increasing when items are added. */ void tail(DBClientBase& conn, const char *ns) { BSONElement lastId = minKey.firstElement(); // minKey is smaller than any other possible value Query query = Query().sort("$natural"); // { $natural : 1 } means in forward capped collection insertion order while( 1 ) { auto_ptr<DBClientCursor> c = conn.query(ns, query, 0, 0, 0, QueryOption_CursorTailable); while( 1 ) { if( !c->more() ) { if( c->isDead() ) { // we need to requery break; } sleepsecs(1); // all data (so far) exhausted, wait for more continue; // we will try more() again } BSONObj o = c->next(); lastId = o["_id"]; cout << o.toString() << endl; } // prepare to requery from where we left off query = QUERY( "_id" << GT << lastId ).sort("$natural"); } }
void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) { verify( candidateChunks ); // // 1. Check whether there is any sharded collection to be balanced by querying // the ShardsNS::collections collection // auto_ptr<DBClientCursor> cursor = conn.query(CollectionType::ConfigNS, BSONObj()); if ( NULL == cursor.get() ) { warning() << "could not query " << CollectionType::ConfigNS << " while trying to balance" << endl; return; } vector< string > collections; while ( cursor->more() ) { BSONObj col = cursor->nextSafe(); // sharded collections will have a shard "key". if ( ! col[CollectionType::keyPattern()].eoo() && ! col[CollectionType::noBalance()].trueValue() ){ collections.push_back( col[CollectionType::ns()].String() ); } else if( col[CollectionType::noBalance()].trueValue() ){ LOG(1) << "not balancing collection " << col[CollectionType::ns()].String() << ", explicitly disabled" << endl; } } cursor.reset(); if ( collections.empty() ) { LOG(1) << "no collections to balance" << endl; return; } // // 2. Get a list of all the shards that are participating in this balance round // along with any maximum allowed quotas and current utilization. We get the // latter by issuing db.serverStatus() (mem.mapped) to all shards. // // TODO: skip unresponsive shards and mark information as stale. // ShardInfoMap shardInfo; Status loadStatus = DistributionStatus::populateShardInfoMap(&shardInfo); if (!loadStatus.isOK()) { warning() << "failed to load shard metadata" << causedBy(loadStatus) << endl; return; } if (shardInfo.size() < 2) { LOG(1) << "can't balance without more active shards" << endl; return; } OCCASIONALLY warnOnMultiVersion( shardInfo ); // // 3. For each collection, check if the balancing policy recommends moving anything around. // for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) { const string& ns = *it; OwnedPointerMap<string, OwnedPointerVector<ChunkType> > shardToChunksMap; cursor = conn.query(ChunkType::ConfigNS, QUERY(ChunkType::ns(ns)).sort(ChunkType::min())); set<BSONObj> allChunkMinimums; while ( cursor->more() ) { BSONObj chunkDoc = cursor->nextSafe().getOwned(); auto_ptr<ChunkType> chunk(new ChunkType()); string errmsg; if (!chunk->parseBSON(chunkDoc, &errmsg)) { error() << "bad chunk format for " << chunkDoc << ": " << errmsg << endl; return; } allChunkMinimums.insert(chunk->getMin().getOwned()); OwnedPointerVector<ChunkType>*& chunkList = shardToChunksMap.mutableMap()[chunk->getShard()]; if (chunkList == NULL) { chunkList = new OwnedPointerVector<ChunkType>(); } chunkList->mutableVector().push_back(chunk.release()); } cursor.reset(); if (shardToChunksMap.map().empty()) { LOG(1) << "skipping empty collection (" << ns << ")"; continue; } for (ShardInfoMap::const_iterator i = shardInfo.begin(); i != shardInfo.end(); ++i) { // this just makes sure there is an entry in shardToChunksMap for every shard OwnedPointerVector<ChunkType>*& chunkList = shardToChunksMap.mutableMap()[i->first]; if (chunkList == NULL) { chunkList = new OwnedPointerVector<ChunkType>(); } } DistributionStatus status(shardInfo, shardToChunksMap.map()); // load tags Status result = clusterCreateIndex(TagsType::ConfigNS, BSON(TagsType::ns() << 1 << TagsType::min() << 1), true, // unique WriteConcernOptions::AllConfigs, NULL); if ( !result.isOK() ) { warning() << "could not create index tags_1_min_1: " << result.reason() << endl; continue; } cursor = conn.query(TagsType::ConfigNS, QUERY(TagsType::ns(ns)).sort(TagsType::min())); vector<TagRange> ranges; while ( cursor->more() ) { BSONObj tag = cursor->nextSafe(); TagRange tr(tag[TagsType::min()].Obj().getOwned(), tag[TagsType::max()].Obj().getOwned(), tag[TagsType::tag()].String()); ranges.push_back(tr); uassert(16356, str::stream() << "tag ranges not valid for: " << ns, status.addTagRange(tr) ); } cursor.reset(); DBConfigPtr cfg = grid.getDBConfig( ns ); if ( !cfg ) { warning() << "could not load db config to balance " << ns << " collection" << endl; continue; } // This line reloads the chunk manager once if this process doesn't know the collection // is sharded yet. ChunkManagerPtr cm = cfg->getChunkManagerIfExists( ns, true ); if ( !cm ) { warning() << "could not load chunks to balance " << ns << " collection" << endl; continue; } // loop through tags to make sure no chunk spans tags; splits on tag min. for all chunks bool didAnySplits = false; for ( unsigned i = 0; i < ranges.size(); i++ ) { BSONObj min = ranges[i].min; min = cm->getShardKey().extendRangeBound( min, false ); if ( allChunkMinimums.count( min ) > 0 ) continue; didAnySplits = true; log() << "ns: " << ns << " need to split on " << min << " because there is a range there" << endl; ChunkPtr c = cm->findIntersectingChunk( min ); vector<BSONObj> splitPoints; splitPoints.push_back( min ); BSONObj res; if ( !c->multiSplit( splitPoints, res ) ) { error() << "split failed: " << res << endl; } else { LOG(1) << "split worked: " << res << endl; } break; } if ( didAnySplits ) { // state change, just wait till next round continue; } CandidateChunk* p = _policy->balance( ns, status, _balancedLastTime ); if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) ); } }
/* **************************************************************************** * * attributeType - * */ static std::string attributeType ( const std::string& tenant, const std::vector<std::string>& servicePathV, const std::string entityType, const std::string attrName ) { std::string idType = std::string("_id.") + ENT_ENTITY_TYPE; std::string idServicePath = std::string("_id.") + ENT_SERVICE_PATH; std::string attributeName = std::string(ENT_ATTRS) + "." + attrName; BSONObj query = BSON(idType << entityType << idServicePath << fillQueryServicePath(servicePathV) << attributeName << BSON("$exists" << true)); std::auto_ptr<DBClientCursor> cursor; DBClientBase* connection = NULL; LM_T(LmtMongo, ("query() in '%s' collection: '%s'", getEntitiesCollectionName(tenant).c_str(), query.toString().c_str())); try { connection = getMongoConnection(); cursor = connection->query(getEntitiesCollectionName(tenant).c_str(), query); /* * We have observed that in some cases of DB errors (e.g. the database daemon is down) instead of * raising an exception, the query() method sets the cursor to NULL. In this case, we raise the * exception ourselves */ if (cursor.get() == NULL) { throw DBException("Null cursor from mongo (details on this is found in the source code)", 0); } releaseMongoConnection(connection); LM_I(("Database Operation Successful (%s)", query.toString().c_str())); } catch (const DBException &e) { releaseMongoConnection(connection); LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), e.what())); return ""; } catch (...) { releaseMongoConnection(connection); LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), "generic exception")); return ""; } while (cursor->more()) { BSONObj r = cursor->next(); LM_T(LmtMongo, ("retrieved document: '%s'", r.toString().c_str())); /* It could happen that different entities within the same entity type may have attributes with the same name * but different types. In that case, one type (at random) is returned. A list could be returned but the * NGSIv2 operations only allow to set one type */ return r.getField(ENT_ATTRS).embeddedObject().getField(attrName).embeddedObject().getStringField(ENT_ATTRS_TYPE); } return ""; }
void ShardChunkManager::_init( const string& configServer , const string& ns , const string& shardName, ShardChunkManagerPtr oldManager ) { // have to get a connection to the config db // special case if I'm the configdb since I'm locked and if I connect to myself // its a deadlock scoped_ptr<ScopedDbConnection> scoped; scoped_ptr<DBDirectClient> direct; DBClientBase * conn; if ( configServer.empty() ) { direct.reset( new DBDirectClient() ); conn = direct.get(); } else { scoped.reset( ScopedDbConnection::getInternalScopedDbConnection( configServer, 30.0 ) ); conn = scoped->get(); } // get this collection's sharding key BSONObj collectionDoc = conn->findOne( "config.collections", BSON( "_id" << ns ) ); if( collectionDoc.isEmpty() ){ warning() << ns << " does not exist as a sharded collection" << endl; return; } if( collectionDoc["dropped"].Bool() ){ warning() << ns << " was dropped. Re-shard collection first." << endl; return; } _fillCollectionKey( collectionDoc ); map<string,ShardChunkVersion> versionMap; versionMap[ shardName ] = _version; _collVersion = ShardChunkVersion( 0, OID() ); // Check to see if we have an old ShardChunkManager to use if( oldManager && oldManager->_collVersion.isSet() ){ versionMap[ shardName ] = oldManager->_version; _collVersion = oldManager->_collVersion; // TODO: This could be made more efficient if copying not required, but not as // frequently reloaded as in mongos. _chunksMap = oldManager->_chunksMap; LOG(2) << "loading new chunks for collection " << ns << " using old chunk manager w/ version " << _collVersion << " and " << _chunksMap.size() << " chunks" << endl; } // Attach our config diff tracker to our range map and versions SCMConfigDiffTracker differ( shardName ); differ.attach( ns, _chunksMap, _collVersion, versionMap ); // Need to do the query ourselves, since we may use direct conns to the db Query query = differ.configDiffQuery(); auto_ptr<DBClientCursor> cursor = conn->query( "config.chunks" , query ); uassert( 16181, str::stream() << "could not initialize cursor to config server chunks collection for ns " << ns, cursor.get() ); // Diff tracker should *always* find at least one chunk if collection exists int diffsApplied = differ.calculateConfigDiff( *cursor ); if( diffsApplied > 0 ){ LOG(2) << "loaded " << diffsApplied << " chunks into new chunk manager for " << ns << " with version " << _collVersion << endl; // Save the new version of this shard _version = versionMap[ shardName ]; _fillRanges(); } else if( diffsApplied == 0 ){ // No chunks were found for the ns warning() << "no chunks found when reloading " << ns << ", previous version was " << _collVersion << endl; _version = ShardChunkVersion( 0, OID() ); _collVersion = ShardChunkVersion( 0, OID() ); _chunksMap.clear(); } else{ // TODO: make this impossible by making sure we don't migrate / split on this shard during the // reload // No chunks were found for the ns warning() << "invalid chunks found when reloading " << ns << ", previous version was " << _collVersion << ", this should be rare" << endl; // Handle the same way as a connectivity error, for now // TODO: handle inline uassert( 16229, str::stream() << "could not initialize cursor to config server chunks collection for ns " << ns, cursor.get() ); } if ( scoped.get() ) scoped->done(); if ( _chunksMap.empty() ) log() << "no chunk for collection " << ns << " on shard " << shardName << endl; }
/* **************************************************************************** * * associationsQuery - */ static bool associationsQuery ( EntityIdVector* enV, AttributeList* attrL, const std::string& scope, MetadataVector* mdV, std::string* err, const std::string& tenant, int offset, int limit, bool details, const std::vector<std::string>& servicePathV ) { DBClientBase* connection = getMongoConnection(); /* Note that SCOPE_VALUE_ASSOC_SOURCE means that the argument is a target (so we use ASSOC_TARGET_ENT and * ASSOC_ATTRS_TARGET in the query), while SCOPE_VALUE_ASSOC_TARGET means that the argument is a source (so we * use ASSOC_SOURCE_ENT and ASSOC_ATTRS_source in the query) */ BSONObjBuilder queryB; /* Build query (entity part) */ BSONArrayBuilder enArray; for (unsigned int ix = 0; ix < enV->size(); ++ix) { enArray.append(BSON(ASSOC_ENT_ID << enV->get(ix)->id << ASSOC_ENT_TYPE << enV->get(ix)->type)); } BSONObj queryEn; if (scope == SCOPE_VALUE_ASSOC_SOURCE) { queryB.append(ASSOC_TARGET_ENT, BSON("$in" << enArray.arr())); } else { // SCOPE_VALUE_ASSOC_TARGET queryB.append(ASSOC_SOURCE_ENT, BSON("$in" << enArray.arr())); } /* Build query (attribute part) */ BSONArrayBuilder attrArray; for (unsigned int ix = 0; ix < attrL->size() ; ++ix) { attrArray.append(attrL->get(ix)); } std::string attrField; if (scope == SCOPE_VALUE_ASSOC_SOURCE) { attrField = ASSOC_ATTRS "." ASSOC_ATTRS_TARGET; } else { // SCOPE_VALUE_ASSOC_TARGET attrField = ASSOC_ATTRS "." ASSOC_ATTRS_SOURCE; } // If there are no attributes specified we want them all if (attrArray.arrSize() != 0) { queryB.append(attrField, BSON("$in" << attrArray.arr())); } /* Do the query in MongoDB */ auto_ptr<DBClientCursor> cursor; Query query(queryB.obj()); Query sortCriteria = query.sort(BSON("_id" << 1)); try { LM_T(LmtMongo, ("query() in '%s' collection: '%s'", getAssociationsCollectionName(tenant).c_str(), query.toString().c_str())); mongoSemTake(__FUNCTION__, "query in AssociationsCollection"); cursor = connection->query(getAssociationsCollectionName(tenant).c_str(), query, limit, offset); /* * We have observed that in some cases of DB errors (e.g. the database daemon is down) instead of * raising an exception, the query() method sets the cursor to NULL. In this case, we raise the * exception ourselves */ if (cursor.get() == NULL) { throw DBException("Null cursor from mongo (details on this is found in the source code)", 0); } mongoSemGive(__FUNCTION__, "query in AssociationsCollection"); LM_I(("Database Operation Successful (%s)", query.toString().c_str())); } catch (const DBException &e) { mongoSemGive(__FUNCTION__, "query in AssociationsCollection (DBException)"); *err = std::string("collection: ") + getAssociationsCollectionName(tenant).c_str() + " - query(): " + query.toString() + " - exception: " + e.what(); LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), err->c_str())); return false; } catch (...) { mongoSemGive(__FUNCTION__, "query in AssociationsCollection (Generic Exception)"); *err = std::string("collection: ") + getAssociationsCollectionName(tenant).c_str() + " - query(): " + query.toString() + " - exception: " + "generic"; LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), err->c_str())); return false; } /* Process query result */ while (cursor->more()) { BSONObj r = cursor->next(); LM_T(LmtMongo, ("retrieved document: '%s'", r.toString().c_str())); std::string name = STR_FIELD(r, "_id"); std::string srcEnId = STR_FIELD(r.getField(ASSOC_SOURCE_ENT).embeddedObject(), ASSOC_ENT_ID); std::string srcEnType = STR_FIELD(r.getField(ASSOC_SOURCE_ENT).embeddedObject(), ASSOC_ENT_TYPE); std::string tgtEnId = STR_FIELD(r.getField(ASSOC_TARGET_ENT).embeddedObject(), ASSOC_ENT_ID); std::string tgtEnType = STR_FIELD(r.getField(ASSOC_TARGET_ENT).embeddedObject(), ASSOC_ENT_TYPE); Metadata* md = new Metadata(name, "Association"); md->association.entityAssociation.source.fill(srcEnId, srcEnType, "false"); md->association.entityAssociation.target.fill(tgtEnId, tgtEnType, "false"); std::vector<BSONElement> attrs = r.getField(ASSOC_ATTRS).Array(); for (unsigned int ix = 0; ix < attrs.size(); ++ix) { std::string srcAttr = STR_FIELD(attrs[ix].embeddedObject(), ASSOC_ATTRS_SOURCE); std::string tgtAttr = STR_FIELD(attrs[ix].embeddedObject(), ASSOC_ATTRS_TARGET); AttributeAssociation* attrAssoc = new AttributeAssociation(); attrAssoc->source = srcAttr; attrAssoc->target = tgtAttr; md->association.attributeAssociationList.push_back(attrAssoc); } mdV->push_back(md); } return true; }
ChunkMatcherPtr ShardingState::getChunkMatcher( const string& ns ){ if ( ! _enabled ) return ChunkMatcherPtr(); if ( ! ShardedConnectionInfo::get( false ) ) return ChunkMatcherPtr(); ConfigVersion version; { // check cache scoped_lock lk( _mutex ); version = _versions[ns]; if ( ! version ) return ChunkMatcherPtr(); ChunkMatcherPtr p = _chunks[ns]; if ( p && p->_version >= version ){ // our cached version is good, so just return return p; } } BSONObj q; { BSONObjBuilder b; b.append( "ns" , ns.c_str() ); b.append( "shard" , BSON( "$in" << BSON_ARRAY( _shardHost << _shardName ) ) ); q = b.obj(); } // have to get a connection to the config db // special case if i'm the congigdb since i'm locked and if i connect to myself // its a deadlock auto_ptr<ScopedDbConnection> scoped; auto_ptr<DBDirectClient> direct; DBClientBase * conn; if ( _configServer == _shardHost ){ direct.reset( new DBDirectClient() ); conn = direct.get(); } else { scoped.reset( new ScopedDbConnection( _configServer ) ); conn = scoped->get(); } // actually query all the chunks // sorting so we can efficiently bucket them auto_ptr<DBClientCursor> cursor = conn->query( "config.chunks" , Query(q).sort( "min" ) ); assert( cursor.get() ); if ( ! cursor->more() ){ // TODO: should we update the local version or cache this result? if ( scoped.get() ) scoped->done(); return ChunkMatcherPtr(); } ChunkMatcherPtr p( new ChunkMatcher( version ) ); BSONObj min,max; while ( cursor->more() ){ BSONObj d = cursor->next(); if ( min.isEmpty() ){ min = d["min"].Obj().getOwned(); max = d["max"].Obj().getOwned(); continue; } if ( max == d["min"].Obj() ){ max = d["max"].Obj().getOwned(); continue; } p->gotRange( min.getOwned() , max.getOwned() ); min = d["min"].Obj().getOwned(); max = d["max"].Obj().getOwned(); } assert( ! min.isEmpty() ); p->gotRange( min.getOwned() , max.getOwned() ); if ( scoped.get() ) scoped->done(); { scoped_lock lk( _mutex ); _chunks[ns] = p; } return p; }
ChunkMatcherPtr ShardingState::getChunkMatcher( const string& ns ){ if ( ! _enabled ) return ChunkMatcherPtr(); if ( ! ShardedConnectionInfo::get( false ) ) return ChunkMatcherPtr(); ConfigVersion version; { scoped_lock lk( _mutex ); version = _versions[ns]; if ( ! version ) return ChunkMatcherPtr(); ChunkMatcherPtr p = _chunks[ns]; if ( p && p->_version >= version ) return p; } BSONObj q; { BSONObjBuilder b; b.append( "ns" , ns.c_str() ); b.append( "shard" , BSON( "$in" << BSON_ARRAY( _shardHost << _shardName ) ) ); q = b.obj(); } auto_ptr<ScopedDbConnection> scoped; auto_ptr<DBDirectClient> direct; DBClientBase * conn; if ( _configServer == _shardHost ){ direct.reset( new DBDirectClient() ); conn = direct.get(); } else { scoped.reset( new ScopedDbConnection( _configServer ) ); conn = scoped->get(); } auto_ptr<DBClientCursor> cursor = conn->query( "config.chunks" , Query(q).sort( "min" ) ); assert( cursor.get() ); if ( ! cursor->more() ){ if ( scoped.get() ) scoped->done(); return ChunkMatcherPtr(); } ChunkMatcherPtr p( new ChunkMatcher( version ) ); BSONObj min,max; while ( cursor->more() ){ BSONObj d = cursor->next(); if ( min.isEmpty() ){ min = d["min"].Obj().getOwned(); max = d["max"].Obj().getOwned(); continue; } if ( max == d["min"].Obj() ){ max = d["max"].Obj().getOwned(); continue; } p->gotRange( min.getOwned() , max.getOwned() ); min = d["min"].Obj().getOwned(); max = d["max"].Obj().getOwned(); } assert( ! min.isEmpty() ); p->gotRange( min.getOwned() , max.getOwned() ); if ( scoped.get() ) scoped->done(); { scoped_lock lk( _mutex ); _chunks[ns] = p; } return p; }
void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ){ assert( candidateChunks ); // // 1. Check whether there is any sharded collection to be balanced by querying // the ShardsNS::database collection // // { "_id" : "test", "partitioned" : true, "primary" : "shard0", // "sharded" : { // "test.images" : { "key" : { "_id" : 1 }, "unique" : false }, // ... // } // } // auto_ptr<DBClientCursor> cursor = conn.query( ShardNS::database , BSON( "partitioned" << true ) ); vector< string > collections; while ( cursor->more() ){ BSONObj db = cursor->next(); // A database may be partitioned but not yet have a sharded collection. // 'cursor' will point to docs that do not contain the "sharded" key. Since // there'd be nothing to balance, we want to skip those here. BSONElement shardedColls = db["sharded"]; if ( shardedColls.eoo() ){ log(2) << "balancer: skipping database with no sharded collection (" << db["_id"].str() << ")" << endl; continue; } BSONObjIterator i( shardedColls.Obj() ); while ( i.more() ){ BSONElement e = i.next(); collections.push_back( e.fieldName() ); } } cursor.reset(); if ( collections.empty() ) { log(1) << "balancer: no collections to balance" << endl; return; } // // 2. Get a list of all the shards that are participating in this balance round // along with any maximum allowed quotas and current utilization. We get the // latter by issuing db.serverStatus() (mem.mapped) to all shards. // // TODO: skip unresponsive shards and mark information as stale. // vector<Shard> allShards; Shard::getAllShards( allShards ); if ( allShards.size() < 2) { log(1) << "balancer: can't balance without more active shards" << endl; return; } map< string, BSONObj > shardLimitsMap; for ( vector<Shard>::const_iterator it = allShards.begin(); it != allShards.end(); ++it ){ const Shard& s = *it; ShardStatus status = s.getStatus(); BSONObj limitsObj = BSON( ShardFields::maxSize( s.getMaxSize() ) << ShardFields::currSize( status.mapped() ) << ShardFields::draining( s.isDraining()) ); shardLimitsMap[ s.getName() ] = limitsObj; } // // 3. For each collection, check if the balancing policy recommends moving anything around. // for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) { const string& ns = *it; map< string,vector<BSONObj> > shardToChunksMap; cursor = conn.query( ShardNS::chunk , QUERY( "ns" << ns ).sort( "min" ) ); while ( cursor->more() ){ BSONObj chunk = cursor->next(); vector<BSONObj>& chunks = shardToChunksMap[chunk["shard"].String()]; chunks.push_back( chunk.getOwned() ); } cursor.reset(); if (shardToChunksMap.empty()) { log(1) << "balancer: skipping empty collection (" << ns << ")"; continue; } for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ){ // this just makes sure there is an entry in shardToChunksMap for every shard Shard s = *i; shardToChunksMap[s.getName()].size(); } CandidateChunk* p = _policy->balance( ns , shardLimitsMap , shardToChunksMap , _balancedLastTime ); if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) ); } }
void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) { verify( candidateChunks ); // // 1. Check whether there is any sharded collection to be balanced by querying // the ShardsNS::collections collection // auto_ptr<DBClientCursor> cursor = conn.query( ShardNS::collection , BSONObj() ); vector< string > collections; while ( cursor->more() ) { BSONObj col = cursor->nextSafe(); // sharded collections will have a shard "key". if ( ! col["key"].eoo() && ! col["noBalance"].trueValue() ){ collections.push_back( col["_id"].String() ); } else if( col["noBalance"].trueValue() ){ LOG(1) << "not balancing collection " << col["_id"].String() << ", explicitly disabled" << endl; } } cursor.reset(); if ( collections.empty() ) { LOG(1) << "no collections to balance" << endl; return; } // // 2. Get a list of all the shards that are participating in this balance round // along with any maximum allowed quotas and current utilization. We get the // latter by issuing db.serverStatus() (mem.mapped) to all shards. // // TODO: skip unresponsive shards and mark information as stale. // vector<Shard> allShards; Shard::getAllShards( allShards ); if ( allShards.size() < 2) { LOG(1) << "can't balance without more active shards" << endl; return; } ShardInfoMap shardInfo; for ( vector<Shard>::const_iterator it = allShards.begin(); it != allShards.end(); ++it ) { const Shard& s = *it; ShardStatus status = s.getStatus(); shardInfo[ s.getName() ] = ShardInfo( s.getMaxSize(), status.mapped(), s.isDraining(), status.hasOpsQueued(), s.tags() ); } // // 3. For each collection, check if the balancing policy recommends moving anything around. // for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) { const string& ns = *it; map< string,vector<BSONObj> > shardToChunksMap; cursor = conn.query( ShardNS::chunk , QUERY( "ns" << ns ).sort( "min" ) ); while ( cursor->more() ) { BSONObj chunk = cursor->nextSafe(); if ( chunk["jumbo"].trueValue() ) continue; vector<BSONObj>& chunks = shardToChunksMap[chunk["shard"].String()]; chunks.push_back( chunk.getOwned() ); } cursor.reset(); if (shardToChunksMap.empty()) { LOG(1) << "skipping empty collection (" << ns << ")"; continue; } for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ) { // this just makes sure there is an entry in shardToChunksMap for every shard Shard s = *i; shardToChunksMap[s.getName()].size(); } DistributionStatus status( shardInfo, shardToChunksMap ); // load tags conn.ensureIndex( ShardNS::tags, BSON( "ns" << 1 << "min" << 1 ), true ); cursor = conn.query( ShardNS::tags , QUERY( "ns" << ns ).sort( "min" ) ); while ( cursor->more() ) { BSONObj tag = cursor->nextSafe(); uassert( 16356 , str::stream() << "tag ranges not valid for: " << ns , status.addTagRange( TagRange( tag["min"].Obj().getOwned(), tag["max"].Obj().getOwned(), tag["tag"].String() ) ) ); } cursor.reset(); CandidateChunk* p = _policy->balance( ns, status, _balancedLastTime ); if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) ); } }
void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) { verify( candidateChunks ); // // 1. Check whether there is any sharded collection to be balanced by querying // the ShardsNS::collections collection // auto_ptr<DBClientCursor> cursor = conn.query(CollectionType::ConfigNS, BSONObj()); vector< string > collections; while ( cursor->more() ) { BSONObj col = cursor->nextSafe(); // sharded collections will have a shard "key". if ( ! col[CollectionType::keyPattern()].eoo() && ! col[CollectionType::noBalance()].trueValue() ){ collections.push_back( col[CollectionType::ns()].String() ); } else if( col[CollectionType::noBalance()].trueValue() ){ LOG(1) << "not balancing collection " << col[CollectionType::ns()].String() << ", explicitly disabled" << endl; } } cursor.reset(); if ( collections.empty() ) { LOG(1) << "no collections to balance" << endl; return; } // // 2. Get a list of all the shards that are participating in this balance round // along with any maximum allowed quotas and current utilization. We get the // latter by issuing db.serverStatus() (mem.mapped) to all shards. // // TODO: skip unresponsive shards and mark information as stale. // vector<Shard> allShards; Shard::getAllShards( allShards ); if ( allShards.size() < 2) { LOG(1) << "can't balance without more active shards" << endl; return; } ShardInfoMap shardInfo; for ( vector<Shard>::const_iterator it = allShards.begin(); it != allShards.end(); ++it ) { const Shard& s = *it; ShardStatus status = s.getStatus(); shardInfo[ s.getName() ] = ShardInfo( s.getMaxSize(), status.mapped(), s.isDraining(), status.hasOpsQueued(), s.tags() ); } // // 3. For each collection, check if the balancing policy recommends moving anything around. // for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) { const string& ns = *it; map< string,vector<BSONObj> > shardToChunksMap; cursor = conn.query(ChunkType::ConfigNS, QUERY(ChunkType::ns(ns)).sort(ChunkType::min())); set<BSONObj> allChunkMinimums; while ( cursor->more() ) { BSONObj chunk = cursor->nextSafe().getOwned(); vector<BSONObj>& chunks = shardToChunksMap[chunk[ChunkType::shard()].String()]; allChunkMinimums.insert( chunk[ChunkType::min()].Obj() ); chunks.push_back( chunk ); } cursor.reset(); if (shardToChunksMap.empty()) { LOG(1) << "skipping empty collection (" << ns << ")"; continue; } for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ) { // this just makes sure there is an entry in shardToChunksMap for every shard Shard s = *i; shardToChunksMap[s.getName()].size(); } DistributionStatus status( shardInfo, shardToChunksMap ); // load tags conn.ensureIndex(TagsType::ConfigNS, BSON(TagsType::ns() << 1 << TagsType::min() << 1), true); cursor = conn.query(TagsType::ConfigNS, QUERY(TagsType::ns(ns)).sort(TagsType::min())); vector<TagRange> ranges; while ( cursor->more() ) { BSONObj tag = cursor->nextSafe(); TagRange tr(tag[TagsType::min()].Obj().getOwned(), tag[TagsType::max()].Obj().getOwned(), tag[TagsType::tag()].String()); ranges.push_back(tr); uassert(16356, str::stream() << "tag ranges not valid for: " << ns, status.addTagRange(tr) ); } cursor.reset(); DBConfigPtr cfg = grid.getDBConfig( ns ); verify( cfg ); ChunkManagerPtr cm = cfg->getChunkManager( ns ); verify( cm ); // loop through tags to make sure no chunk spans tags; splits on tag min. for all chunks bool didAnySplits = false; for ( unsigned i = 0; i < ranges.size(); i++ ) { BSONObj min = ranges[i].min; min = cm->getShardKey().extendRangeBound( min, false ); if ( allChunkMinimums.count( min ) > 0 ) continue; didAnySplits = true; log() << "ns: " << ns << " need to split on " << min << " because there is a range there" << endl; ChunkPtr c = cm->findIntersectingChunk( min ); vector<BSONObj> splitPoints; splitPoints.push_back( min ); BSONObj res; if ( !c->multiSplit( splitPoints, res ) ) { error() << "split failed: " << res << endl; } else { LOG(1) << "split worked: " << res << endl; } break; } if ( didAnySplits ) { // state change, just wait till next round continue; } CandidateChunk* p = _policy->balance( ns, status, _balancedLastTime ); if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) ); } }
void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) { assert( candidateChunks ); // // 1. Check whether there is any sharded collection to be balanced by querying // the ShardsNS::collections collection // auto_ptr<DBClientCursor> cursor = conn.query( ShardNS::collection , BSONObj() ); vector< string > collections; while ( cursor->more() ) { BSONObj col = cursor->nextSafe(); // sharded collections will have a shard "key". if ( ! col["key"].eoo() ) collections.push_back( col["_id"].String() ); } cursor.reset(); if ( collections.empty() ) { LOG(1) << "no collections to balance" << endl; return; } // // 2. Get a list of all the shards that are participating in this balance round // along with any maximum allowed quotas and current utilization. We get the // latter by issuing db.serverStatus() (mem.mapped) to all shards. // // TODO: skip unresponsive shards and mark information as stale. // vector<Shard> allShards; Shard::getAllShards( allShards ); if ( allShards.size() < 2) { LOG(1) << "can't balance without more active shards" << endl; return; } map< string, BSONObj > shardLimitsMap; for ( vector<Shard>::const_iterator it = allShards.begin(); it != allShards.end(); ++it ) { const Shard& s = *it; ShardStatus status = s.getStatus(); BSONObj limitsObj = BSON( ShardFields::maxSize( s.getMaxSize() ) << LimitsFields::currSize( status.mapped() ) << ShardFields::draining( s.isDraining() ) << LimitsFields::hasOpsQueued( status.hasOpsQueued() ) ); shardLimitsMap[ s.getName() ] = limitsObj; } // // 3. For each collection, check if the balancing policy recommends moving anything around. // for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) { const string& ns = *it; map< string,vector<BSONObj> > shardToChunksMap; cursor = conn.query( ShardNS::chunk , QUERY( "ns" << ns ).sort( "min" ) ); while ( cursor->more() ) { BSONObj chunk = cursor->nextSafe(); if ( chunk["jumbo"].trueValue() ) continue; vector<BSONObj>& chunks = shardToChunksMap[chunk["shard"].String()]; chunks.push_back( chunk.getOwned() ); } cursor.reset(); if (shardToChunksMap.empty()) { LOG(1) << "skipping empty collection (" << ns << ")"; continue; } for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ) { // this just makes sure there is an entry in shardToChunksMap for every shard Shard s = *i; shardToChunksMap[s.getName()].size(); } CandidateChunk* p = _policy->balance( ns , shardLimitsMap , shardToChunksMap , _balancedLastTime ); if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) ); } }
/* **************************************************************************** * * addTriggeredSubscriptions * */ static bool addTriggeredSubscriptions(ContextRegistration cr, map<string, TriggeredSubscription*>& subs, std::string& err, std::string tenant) { DBClientBase* connection = NULL; BSONArrayBuilder entitiesNoPatternA; std::vector<std::string> idJsV; std::vector<std::string> typeJsV; for (unsigned int ix = 0; ix < cr.entityIdVector.size(); ++ix ) { //FIXME: take into account subscriptions with no type EntityId* enP = cr.entityIdVector.get(ix); /* The registration of isPattern=true entities is not supported, so we don't include them here */ if (enP->isPattern == "false") { entitiesNoPatternA.append(BSON(CASUB_ENTITY_ID << enP->id << CASUB_ENTITY_TYPE << enP->type << CASUB_ENTITY_ISPATTERN << "false")); idJsV.push_back(enP->id); typeJsV.push_back(enP->type); } } BSONArrayBuilder attrA; for (unsigned int ix = 0; ix < cr.contextRegistrationAttributeVector.size(); ++ix) { ContextRegistrationAttribute* craP = cr.contextRegistrationAttributeVector.get(ix); attrA.append(craP->name); } BSONObjBuilder queryNoPattern; queryNoPattern.append(CASUB_ENTITIES, BSON("$in" << entitiesNoPatternA.arr())); if (attrA.arrSize() > 0) { /* If we don't do this checking, the {$in: [] } in the attribute name part will * make the query fail*/ //queryB.append(CASUB_ATTRS, BSON("$in" << attrA.arr())); queryNoPattern.append("$or", BSON_ARRAY( BSON(CASUB_ATTRS << BSON("$in" << attrA.arr())) << BSON(CASUB_ATTRS << BSON("$size" << 0)) )); } else { queryNoPattern.append(CASUB_ATTRS, BSON("$size" << 0)); } queryNoPattern.append(CASUB_EXPIRATION, BSON("$gt" << (long long) getCurrentTime())); /* This is JavaScript code that runs in MongoDB engine. As far as I know, this is the only * way to do a "reverse regex" query in MongoDB (see * http://stackoverflow.com/questions/15966991/mongodb-reverse-regex/15989520). * Note that although we are using a isPattern=true in the MongoDB query besides $where, we * also need to check that in the if statement in the JavaScript function given that a given * sub document could include both isPattern=true and isPattern=false documents */ std::string idJsString = "[ "; for (unsigned int ix = 0; ix < idJsV.size(); ++ix ) { if (ix != idJsV.size()-1) { idJsString += "\""+idJsV[ix]+ "\" ,"; } else { idJsString += "\"" +idJsV[ix]+ "\""; } } idJsString += " ]"; std::string typeJsString = "[ "; for (unsigned int ix = 0; ix < typeJsV.size(); ++ix ) { if (ix != typeJsV.size()-1) { typeJsString += "\"" +typeJsV[ix] + "\" ,"; } else { typeJsString += "\"" + typeJsV[ix] + "\""; } } typeJsString += " ]"; std::string function = std::string("function()") + "{" + "enId = "+idJsString+ ";" + "enType = "+typeJsString+ ";" + "for (var i=0; i < this."+CASUB_ENTITIES+".length; i++) {" + "if (this."+CASUB_ENTITIES+"[i]."+CASUB_ENTITY_ISPATTERN+" == \"true\") {" + "for (var j=0; j < enId.length; j++) {" + "if (enId[j].match(this."+CASUB_ENTITIES+"[i]."+CASUB_ENTITY_ID+") && this."+CASUB_ENTITIES+"[i]."+CASUB_ENTITY_TYPE+" == enType[j]) {" + "return true; " + "}" + "}" + "}" + "}" + "return false; " + "}"; LM_T(LmtMongo, ("JS function: %s", function.c_str())); std::string entPatternQ = CSUB_ENTITIES "." CSUB_ENTITY_ISPATTERN; BSONObjBuilder queryPattern; queryPattern.append(entPatternQ, "true"); queryPattern.append(CASUB_EXPIRATION, BSON("$gt" << (long long) getCurrentTime())); queryPattern.appendCode("$where", function); BSONObj query = BSON("$or" << BSON_ARRAY(queryNoPattern.obj() << queryPattern.obj())); /* Do the query */ auto_ptr<DBClientCursor> cursor; LM_T(LmtMongo, ("query() in '%s' collection: '%s'", getSubscribeContextAvailabilityCollectionName(tenant).c_str(), query.toString().c_str())); try { connection = getMongoConnection(); cursor = connection->query(getSubscribeContextAvailabilityCollectionName(tenant).c_str(), query); /* * We have observed that in some cases of DB errors (e.g. the database daemon is down) instead of * raising an exception, the query() method sets the cursor to NULL. In this case, we raise the * exception ourselves */ if (cursor.get() == NULL) { throw DBException("Null cursor from mongo (details on this is found in the source code)", 0); } releaseMongoConnection(connection); LM_I(("Database Operation Successful (%s)", query.toString().c_str())); } catch (const DBException &e) { releaseMongoConnection(connection); err = std::string("collection: ") + getSubscribeContextAvailabilityCollectionName(tenant).c_str() + " - query(): " + query.toString() + " - exception: " + e.what(); LM_E(("Database Error (%s)", err.c_str())); return false; } catch (...) { releaseMongoConnection(connection); err = std::string("collection: ") + getSubscribeContextAvailabilityCollectionName(tenant).c_str() + " - query(): " + query.toString() + " - exception: " + "generic"; LM_E(("Database Error (%s)", err.c_str())); return false; } /* For each one of the subscriptions found, add it to the map (if not already there) */ while (cursor->more()) { BSONObj sub = cursor->next(); BSONElement idField = sub.getField("_id"); // // BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub' // // Now, if 'sub.getField("_id")' is not found, if we continue, calling OID() on it, then we get // an exception and the broker crashes. // if (idField.eoo() == true) { LM_E(("Database Error (error retrieving _id field in doc: %s)", sub.toString().c_str())); continue; } std::string subIdStr = idField.OID().toString(); if (subs.count(subIdStr) == 0) { LM_T(LmtMongo, ("adding subscription: '%s'", sub.toString().c_str())); TriggeredSubscription* trigs = new TriggeredSubscription(sub.hasField(CASUB_FORMAT) ? stringToFormat(STR_FIELD(sub, CASUB_FORMAT)) : XML, STR_FIELD(sub, CASUB_REFERENCE), subToAttributeList(sub)); subs.insert(std::pair<string, TriggeredSubscription*>(subIdStr, trigs)); } } return true; }