bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){ WriteBackListener::init( *conn_in ); DBClientBase* conn = getVersionable( conn_in ); verify( conn ); // errors thrown above BSONObjBuilder cmdBuilder; cmdBuilder.append( "setShardVersion" , "" ); cmdBuilder.appendBool( "init", true ); cmdBuilder.append( "configdb" , configServer.modelServer() ); cmdBuilder.appendOID( "serverID" , &serverID ); cmdBuilder.appendBool( "authoritative" , true ); BSONObj cmd = cmdBuilder.obj(); LOG(1) << "initializing shard connection to " << conn->toString() << endl; LOG(2) << "initial sharding settings : " << cmd << endl; bool ok = conn->runCommand( "admin" , cmd , result ); // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1 // Result is false, but will still initialize serverID and configdb if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ || result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ )) { ok = true; } LOG(3) << "initial sharding result : " << result << endl; return ok; }
bool setShardVersion(DBClientBase& conn, const string& ns, const string& configServerPrimary, ChunkVersion version, ChunkManager* manager, bool authoritative, BSONObj& result) { BSONObjBuilder cmdBuilder; cmdBuilder.append("setShardVersion", ns); cmdBuilder.append("configdb", configServerPrimary); Shard s = Shard::make(conn.getServerAddress()); cmdBuilder.append("shard", s.getName()); cmdBuilder.append("shardHost", s.getConnString()); if (ns.size() > 0) { version.addToBSON(cmdBuilder); } else { cmdBuilder.append("init", true); } if (authoritative) { cmdBuilder.appendBool("authoritative", 1); } BSONObj cmd = cmdBuilder.obj(); LOG(1) << " setShardVersion " << s.getName() << " " << conn.getServerAddress() << " " << ns << " " << cmd << (manager ? string(str::stream() << " " << manager->getSequenceNumber()) : ""); return conn.runCommand("admin", cmd, result, 0); }
bool initShardVersion( DBClientBase& conn_in, BSONObj& result ){ WriteBackListener::init( conn_in ); DBClientBase* conn = getVersionable( &conn_in ); assert( conn ); // errors thrown above BSONObjBuilder cmdBuilder; cmdBuilder.append( "setShardVersion" , "" ); cmdBuilder.appendBool( "init", true ); cmdBuilder.append( "configdb" , configServer.modelServer() ); cmdBuilder.appendOID( "serverID" , &serverID ); cmdBuilder.appendBool( "authoritative" , true ); BSONObj cmd = cmdBuilder.obj(); LOG(1) << "initializing shard connection to " << conn->toString() << endl; LOG(2) << "initial sharding settings : " << cmd << endl; bool ok = conn->runCommand( "admin" , cmd , result ); LOG(3) << "initial sharding result : " << result << endl; return ok; }
bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){ WriteBackListener::init( *conn_in ); bool ok; DBClientBase* conn = NULL; try { // May throw if replica set primary is down conn = getVersionable( conn_in ); dassert( conn ); // errors thrown above BSONObjBuilder cmdBuilder; cmdBuilder.append( "setShardVersion" , "" ); cmdBuilder.appendBool( "init", true ); cmdBuilder.append( "configdb" , configServer.modelServer() ); cmdBuilder.appendOID( "serverID" , &serverID ); cmdBuilder.appendBool( "authoritative" , true ); BSONObj cmd = cmdBuilder.obj(); LOG(1) << "initializing shard connection to " << conn->toString() << endl; LOG(2) << "initial sharding settings : " << cmd << endl; ok = conn->runCommand("admin", cmd, result, 0); } catch( const DBException& ex ) { bool ignoreFailure = ShardConnection::ignoreInitialVersionFailure && conn_in->type() == ConnectionString::SET; if ( !ignoreFailure ) throw; // Using initShardVersion is not strictly required when talking to replica sets - it is // preferred to do so because it registers mongos early with the mongod. This info is // also sent by checkShardVersion before a connection is used for a write or read. OCCASIONALLY { warning() << "failed to initialize new replica set connection version, " << "will initialize on first use" << endl; } return true; } // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1 // Result is false, but will still initialize serverID and configdb if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ || result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ )) { ok = true; } LOG(3) << "initial sharding result : " << result << endl; return ok; }
void DocumentSourceOutReplaceColl::initializeWriteNs() { DBClientBase* conn = pExpCtx->mongoProcessInterface->directClient(); const auto& outputNs = getOutputNs(); _tempNs = NamespaceString(str::stream() << outputNs.db() << ".tmp.agg_out." << aggOutCounter.addAndFetch(1)); // Save the original collection options and index specs so we can check they didn't change // during computation. _originalOutOptions = pExpCtx->mongoProcessInterface->getCollectionOptions(outputNs); _originalIndexes = conn->getIndexSpecs(outputNs.ns()); // Check if it's capped to make sure we have a chance of succeeding before we do all the work. // If the collection becomes capped during processing, the collection options will have changed, // and the $out will fail. uassert(17152, str::stream() << "namespace '" << outputNs.ns() << "' is capped so it can't be used for $out", _originalOutOptions["capped"].eoo()); // We will write all results into a temporary collection, then rename the temporary // collection to be the target collection once we are done. _tempNs = NamespaceString(str::stream() << outputNs.db() << ".tmp.agg_out." << aggOutCounter.addAndFetch(1)); // Create temp collection, copying options from the existing output collection if any. { BSONObjBuilder cmd; cmd << "create" << _tempNs.coll(); cmd << "temp" << true; cmd.appendElementsUnique(_originalOutOptions); BSONObj info; uassert(16994, str::stream() << "failed to create temporary $out collection '" << _tempNs.ns() << "': " << info.toString(), conn->runCommand(outputNs.db().toString(), cmd.done(), info)); } if (_originalIndexes.empty()) { return; } // Copy the indexes of the output collection to the temp collection. std::vector<BSONObj> tempNsIndexes; for (const auto& indexSpec : _originalIndexes) { // Replace the spec's 'ns' field value, which is the original collection, with the temp // collection. tempNsIndexes.push_back(indexSpec.addField(BSON("ns" << _tempNs.ns()).firstElement())); } try { conn->createIndexes(_tempNs.ns(), tempNsIndexes); } catch (DBException& ex) { ex.addContext("Copying indexes for $out failed"); throw; } };
boost::optional<Document> DocumentSourceOut::getNext() { pExpCtx->checkForInterrupt(); // make sure we only write out once if (_done) return boost::none; _done = true; verify(_mongod); DBClientBase* conn = _mongod->directClient(); prepTempCollection(); verify(_tempNs.size() != 0); vector<BSONObj> bufferedObjects; int bufferedBytes = 0; while (boost::optional<Document> next = pSource->getNext()) { BSONObj toInsert = next->toBson(); bufferedBytes += toInsert.objsize(); if (!bufferedObjects.empty() && bufferedBytes > BSONObjMaxUserSize) { spill(conn, bufferedObjects); bufferedObjects.clear(); bufferedBytes = toInsert.objsize(); } bufferedObjects.push_back(toInsert); } if (!bufferedObjects.empty()) spill(conn, bufferedObjects); // Checking again to make sure we didn't become sharded while running. uassert(17018, str::stream() << "namespace '" << _outputNs.ns() << "' became sharded so it can't be used for $out'", !_mongod->isSharded(_outputNs)); BSONObj rename = BSON("renameCollection" << _tempNs.ns() << "to" << _outputNs.ns() << "dropTarget" << true ); BSONObj info; bool ok = conn->runCommand("admin", rename, info); uassert(16997, str::stream() << "renameCollection for $out failed: " << info, ok); // We don't need to drop the temp collection in our destructor if the rename succeeded. _tempNs = NamespaceString(""); // This "DocumentSource" doesn't produce output documents. This can change in the future // if we support using $out in "tee" mode. return boost::none; }
bool setShardVersion( DBClientBase & conn , const string& ns , ShardChunkVersion version , bool authoritative , BSONObj& result ){ BSONObjBuilder cmdBuilder; cmdBuilder.append( "setShardVersion" , ns.c_str() ); cmdBuilder.append( "configdb" , configServer.modelServer() ); cmdBuilder.appendTimestamp( "version" , version ); cmdBuilder.appendOID( "serverID" , &serverID ); if ( authoritative ) cmdBuilder.appendBool( "authoritative" , 1 ); Shard s = Shard::make( conn.getServerAddress() ); cmdBuilder.append( "shard" , s.getName() ); cmdBuilder.append( "shardHost" , s.getConnString() ); BSONObj cmd = cmdBuilder.obj(); log(1) << " setShardVersion " << s.getName() << " " << conn.getServerAddress() << " " << ns << " " << cmd << " " << &conn << endl; return conn.runCommand( "admin" , cmd , result ); }
/* **************************************************************************** * * mongoEntityTypes - */ HttpStatusCode mongoEntityTypes ( EntityTypesResponse* responseP, const std::string& tenant, const std::vector<std::string>& servicePathV, std::map<std::string, std::string>& uriParams ) { unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str()); unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str()); std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS]; bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false; LM_T(LmtMongo, ("Query Entity Types")); LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false")); reqSemTake(__FUNCTION__, "query types request"); DBClientBase* connection = getMongoConnection(); /* Compose query based on this aggregation command: * * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the * aggregation query below is fully correct * * db.runCommand({aggregate: "entities", * pipeline: [ {$match: { "_id.servicePath": /.../ } }, * {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} }, * {$project: { "attrs" * {$cond: [ {$eq: [ "$attrs", [ ] ] }, [null], "$attrs"] } * } * }, * {$unwind: "$attrs"}, * {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} }, * {$sort: {_id: 1} } * ] * }) * * The $cond part is hard... more information at http://stackoverflow.com/questions/27510143/empty-array-prevents-document-to-appear-in-query * As a consequence, some "null" values may appear in the resulting attrs vector, which are prunned by the result processing logic. * * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the * following command can be used: * * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]}) * */ BSONObj result; // Building the projection part of the query that includes types that have no attributes // See bug: https://github.com/telefonicaid/fiware-orion/issues/686 BSONArrayBuilder emptyArrayBuilder; BSONArrayBuilder nulledArrayBuilder; nulledArrayBuilder.appendNull(); // We are using the $cond: [ .. ] and not the $cond: { .. } one, as the former is the only one valid in MongoDB 2.4 BSONObj projection = BSON( "$project" << BSON( "attrs" << BSON( "$cond" << BSON_ARRAY( BSON("$eq" << BSON_ARRAY(S_ATTRS << emptyArrayBuilder.arr()) ) << nulledArrayBuilder.arr() << S_ATTRS ) ) ) ); BSONObj cmd = BSON("aggregate" << COL_ENTITIES << "pipeline" << BSON_ARRAY( BSON("$match" << BSON(C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) << BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) << projection << BSON("$unwind" << S_ATTRS) << BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) << BSON("$sort" << BSON("_id" << 1)) ) ); LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str())); mongoSemTake(__FUNCTION__, "aggregation command"); try { connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result); mongoSemGive(__FUNCTION__, "aggregation command"); LM_I(("Database Operation Successful (%s)", cmd.toString().c_str())); } catch (const DBException& e) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + e.what(); LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } catch (...) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + "generic"; LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } /* Processing result to build response */ LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str())); std::vector<BSONElement> resultsArray = result.getField("result").Array(); /* Another strategy to implement pagination is to use the $skip and $limit operators in the * aggregation framework. However, doing so, we don't know the total number of results, which can * be needed in the case of details=on (using that approach, we need to do two queries: one to get * the count and other to get the actual results with $skip and $limit, in the same "transaction" to * avoid incoherence between both if some entity type is created or deleted in the process). * * However, considering that the number of types will be small compared with the number of entities, * the current approach seems to be ok */ for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix) { BSONObj resultItem = resultsArray[ix].embeddedObject(); TypeEntity* type = new TypeEntity(resultItem.getStringField("_id")); std::vector<BSONElement> attrsArray = resultItem.getField("attrs").Array(); if (!attrsArray[0].isNull()) { for (unsigned int jx = 0; jx < attrsArray.size(); ++jx) { /* This is the place in which null elements in the resulting attrs vector are prunned */ if (attrsArray[jx].isNull()) { continue; } BSONObj jAttr = attrsArray[jx].embeddedObject(); ContextAttribute* ca = new ContextAttribute(jAttr.getStringField(ENT_ATTRS_NAME), jAttr.getStringField(ENT_ATTRS_TYPE)); type->contextAttributeVector.push_back(ca); } } responseP->typeEntityVector.push_back(type); } char detailsMsg[256]; if (responseP->typeEntityVector.size() > 0) { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size()); responseP->statusCode.fill(SccOk, detailsMsg); } else { responseP->statusCode.fill(SccOk); } } else { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset); responseP->statusCode.fill(SccContextElementNotFound, detailsMsg); } else { responseP->statusCode.fill(SccContextElementNotFound); } } reqSemGive(__FUNCTION__, "query types request"); return SccOk; }
/* **************************************************************************** * * mongoAttributesForEntityType - */ HttpStatusCode mongoAttributesForEntityType ( std::string entityType, EntityTypeAttributesResponse* responseP, const std::string& tenant, const std::vector<std::string>& servicePathV, std::map<std::string, std::string>& uriParams ) { unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str()); unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str()); std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS]; bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false; // Setting the name of the entity type for the response responseP->entityType.type = entityType; LM_T(LmtMongo, ("Query Types Attribute for <%s>", entityType.c_str())); LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false")); reqSemTake(__FUNCTION__, "query types attributes request"); DBClientBase* connection = getMongoConnection(); /* Compose query based on this aggregation command: * * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the * aggregation query below is fully correct * * db.runCommand({aggregate: "entities", * pipeline: [ {$match: { "_id.type": "TYPE" , "_id.servicePath": /.../ } }, * {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} }, * {$unwind: "$attrs"}, * {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} }, * {$unwind: "$attrs"}, * {$group: {_id: "$attrs" }}, * {$sort: {_id.name: 1, _id.type: 1} } * ] * }) * */ BSONObj result; BSONObj cmd = BSON("aggregate" << COL_ENTITIES << "pipeline" << BSON_ARRAY( BSON("$match" << BSON(C_ID_ENTITY << entityType << C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) << BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) << BSON("$unwind" << S_ATTRS) << BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) << BSON("$unwind" << S_ATTRS) << BSON("$group" << BSON("_id" << S_ATTRS)) << BSON("$sort" << BSON(C_ID_NAME << 1 << C_ID_TYPE << 1)) ) ); LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str())); mongoSemTake(__FUNCTION__, "aggregation command"); try { connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result); mongoSemGive(__FUNCTION__, "aggregation command"); LM_I(("Database Operation Successful (%s)", cmd.toString().c_str())); } catch (const DBException& e) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + e.what(); LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } catch (...) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + "generic"; LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } /* Processing result to build response*/ LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str())); std::vector<BSONElement> resultsArray = result.getField("result").Array(); /* See comment above in the other method regarding this strategy to implement pagination */ for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix) { BSONElement idField = resultsArray[ix].embeddedObject().getField("_id"); // // BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub' // // Now, if 'resultsArray[ix].embeddedObject().getField("_id")' is not found, if we continue, // calling embeddedObject() on it, then we get an exception and the broker crashes. // if (idField.eoo() == true) { LM_E(("Database Error (error retrieving _id field in doc: %s)", resultsArray[ix].embeddedObject().toString().c_str())); continue; } BSONObj resultItem = idField.embeddedObject(); ContextAttribute* ca = new ContextAttribute(resultItem.getStringField(ENT_ATTRS_NAME), resultItem.getStringField(ENT_ATTRS_TYPE)); responseP->entityType.contextAttributeVector.push_back(ca); } char detailsMsg[256]; if (responseP->entityType.contextAttributeVector.size() > 0) { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size()); responseP->statusCode.fill(SccOk, detailsMsg); } else { responseP->statusCode.fill(SccOk); } } else { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Number of attributes: %d. Offset is %d", (int) resultsArray.size(), offset); responseP->statusCode.fill(SccContextElementNotFound, detailsMsg); } else { responseP->statusCode.fill(SccContextElementNotFound); } } reqSemGive(__FUNCTION__, "query types request"); return SccOk; }
void DocumentSourceOut::initialize() { DBClientBase* conn = pExpCtx->mongoProcessInterface->directClient(); // Save the original collection options and index specs so we can check they didn't change // during computation. _originalOutOptions = pExpCtx->mongoProcessInterface->getCollectionOptions(_outputNs); _originalIndexes = conn->getIndexSpecs(_outputNs.ns()); // Check if it's sharded or capped to make sure we have a chance of succeeding before we do all // the work. If the collection becomes capped during processing, the collection options will // have changed, and the $out will fail. If it becomes sharded during processing, the final // rename will fail. uassert(17017, str::stream() << "namespace '" << _outputNs.ns() << "' is sharded so it can't be used for $out'", !pExpCtx->mongoProcessInterface->isSharded(pExpCtx->opCtx, _outputNs)); uassert(17152, str::stream() << "namespace '" << _outputNs.ns() << "' is capped so it can't be used for $out", _originalOutOptions["capped"].eoo()); // We will write all results into a temporary collection, then rename the temporary collection // to be the target collection once we are done. _tempNs = NamespaceString(str::stream() << _outputNs.db() << ".tmp.agg_out." << aggOutCounter.addAndFetch(1)); // Create output collection, copying options from existing collection if any. { BSONObjBuilder cmd; cmd << "create" << _tempNs.coll(); cmd << "temp" << true; cmd.appendElementsUnique(_originalOutOptions); BSONObj info; bool ok = conn->runCommand(_outputNs.db().toString(), cmd.done(), info); uassert(16994, str::stream() << "failed to create temporary $out collection '" << _tempNs.ns() << "': " << info.toString(), ok); } // copy indexes to _tempNs for (std::list<BSONObj>::const_iterator it = _originalIndexes.begin(); it != _originalIndexes.end(); ++it) { MutableDocument index((Document(*it))); index.remove("_id"); // indexes shouldn't have _ids but some existing ones do index["ns"] = Value(_tempNs.ns()); BSONObj indexBson = index.freeze().toBson(); conn->insert(_tempNs.getSystemIndexesCollection(), indexBson); BSONObj err = conn->getLastErrorDetailed(); uassert(16995, str::stream() << "copying index for $out failed." << " index: " << indexBson << " error: " << err, DBClientBase::getLastErrorString(err).empty()); } _initialized = true; }
/* **************************************************************************** * * mongoEntityTypes - */ HttpStatusCode mongoEntityTypes ( EntityTypesResponse* responseP, const std::string& tenant, const std::vector<std::string>& servicePathV, std::map<std::string, std::string>& uriParams ) { unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str()); unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str()); std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS]; bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false; LM_T(LmtMongo, ("Query Entity Types")); LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false")); reqSemTake(__FUNCTION__, "query types request"); DBClientBase* connection = getMongoConnection(); /* Compose query based on this aggregation command: * * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the * aggregation query below is fully correct * * db.runCommand({aggregate: "entities", * pipeline: [ {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} }, * {$unwind: "$attrs"}, * {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} }, * {$sort: {_id: 1} } * ] * }) * * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the * following command can be used: * * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]}) * */ BSONObj result; BSONObj cmd = BSON("aggregate" << COL_ENTITIES << "pipeline" << BSON_ARRAY( BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) << BSON("$unwind" << S_ATTRS) << BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) << BSON("$sort" << BSON("_id" << 1)) ) ); LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str())); mongoSemTake(__FUNCTION__, "aggregation command"); try { connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result); mongoSemGive(__FUNCTION__, "aggregation command"); LM_I(("Database Operation Successful (%s)", cmd.toString().c_str())); } catch (const DBException& e) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + e.what(); LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } catch (...) { mongoSemGive(__FUNCTION__, "aggregation command"); std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() + " - command: " + cmd.toString() + " - exception: " + "generic"; LM_E(("Database Error (%s)", err.c_str())); responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request"); return SccOk; } /* Processing result to build response*/ LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str())); std::vector<BSONElement> resultsArray = result.getField("result").Array(); /* Another strategy to implement pagination is to use the $skip and $limit operators in the * aggregation framework. However, doing so, we don't know the total number of results, which can * be needed in the case of details=on (using that approach, we need to do two queries: one to get * the count and other to get the actual results with $skip and $limit, in the same "transaction" to * avoid incoherence between both if some entity type is created or deleted in the process). * * However, considering that the number of types will be small compared with the number of entities, * the current approach seems to be ok */ for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix) { BSONObj resultItem = resultsArray[ix].embeddedObject(); TypeEntity* type = new TypeEntity(resultItem.getStringField("_id")); std::vector<BSONElement> attrsArray = resultItem.getField("attrs").Array(); for (unsigned int jx = 0; jx < attrsArray.size(); ++jx) { BSONObj jAttr = attrsArray[jx].embeddedObject(); ContextAttribute* ca = new ContextAttribute(jAttr.getStringField(ENT_ATTRS_NAME), jAttr.getStringField(ENT_ATTRS_TYPE)); type->contextAttributeVector.push_back(ca); } responseP->typeEntityVector.push_back(type); } char detailsMsg[256]; if (responseP->typeEntityVector.size() > 0) { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size()); responseP->statusCode.fill(SccOk, detailsMsg); } else { responseP->statusCode.fill(SccOk); } } else { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset); responseP->statusCode.fill(SccContextElementNotFound, detailsMsg); } else { responseP->statusCode.fill(SccContextElementNotFound); } } reqSemGive(__FUNCTION__, "query types request"); return SccOk; }
/* **************************************************************************** * * mongoConnect - * * Default value for writeConcern == 1 (0: unacknowledged, 1: acknowledged) */ static DBClientBase* mongoConnect ( const char* host, const char* db, const char* rplSet, const char* username, const char* passwd, bool multitenant, int writeConcern, double timeout ) { std::string err; DBClientBase* connection = NULL; LM_T(LmtMongo, ("Connection info: dbName='%s', rplSet='%s', timeout=%f", db, rplSet, timeout)); bool connected = false; int retries = RECONNECT_RETRIES; if (strlen(rplSet) == 0) { /* Setting the first argument to true is to use autoreconnect */ connection = new DBClientConnection(true); /* Not sure of how to generalize the following code, given that DBClientBase class doesn't have a common connect() method (surprisingly) */ for (int tryNo = 0; tryNo < retries; ++tryNo) { if ( ((DBClientConnection*)connection)->connect(host, err)) { connected = true; break; } if (tryNo == 0) { LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)", retries, RECONNECT_DELAY)); } else { LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo)); } usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds } } else { LM_T(LmtMongo, ("Using replica set %s", rplSet)); // autoReconnect is always on for DBClientReplicaSet connections. std::vector<std::string> hostTokens; int components = stringSplit(host, ',', hostTokens); std::vector<HostAndPort> rplSetHosts; for (int ix = 0; ix < components; ix++) { LM_T(LmtMongo, ("rplSet host <%s>", hostTokens[ix].c_str())); rplSetHosts.push_back(HostAndPort(hostTokens[ix])); } connection = new DBClientReplicaSet(rplSet, rplSetHosts, timeout); /* Not sure of to generalize the following code, given that DBClientBase class hasn't a common connect() method (surprisingly) */ for (int tryNo = 0; tryNo < retries; ++tryNo) { if ( ((DBClientReplicaSet*)connection)->connect()) { connected = true; break; } if (tryNo == 0) { LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)", retries, RECONNECT_DELAY)); } else { LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo)); } usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds } } if (connected == false) { LM_E(("Database Error (connection failed, after %d retries: '%s')", retries, err.c_str())); return NULL; } LM_I(("Successful connection to database")); // // WriteConcern // mongo::WriteConcern writeConcernCheck; // In legacy driver writeConcern is no longer an int, but a class. We need a small // conversion step here mongo::WriteConcern wc = writeConcern == 1 ? mongo::WriteConcern::acknowledged : mongo::WriteConcern::unacknowledged; connection->setWriteConcern((mongo::WriteConcern) wc); writeConcernCheck = (mongo::WriteConcern) connection->getWriteConcern(); if (writeConcernCheck.nodes() != wc.nodes()) { LM_E(("Database Error (Write Concern not set as desired)")); return NULL; } LM_T(LmtMongo, ("Active DB Write Concern mode: %d", writeConcern)); /* Authentication is different depending if multiservice is used or not. In the case of not * using multiservice, we authenticate in the single-service database. In the case of using * multiservice, it isn't a default database that we know at contextBroker start time (when * this connection function is invoked) so we authenticate on the admin database, which provides * access to any database */ if (multitenant) { if (strlen(username) != 0 && strlen(passwd) != 0) { if (!connection->auth("admin", std::string(username), std::string(passwd), err)) { LM_E(("Database Startup Error (authentication: db='admin', username='******', password='******': %s)", username, err.c_str())); return NULL; } } } else { if (strlen(db) != 0 && strlen(username) != 0 && strlen(passwd) != 0) { if (!connection->auth(std::string(db), std::string(username), std::string(passwd), err)) { LM_E(("Database Startup Error (authentication: db='%s', username='******', password='******': %s)", db, username, err.c_str())); return NULL; } } } /* Get mongo version with the 'buildinfo' command */ BSONObj result; std::string extra; connection->runCommand("admin", BSON("buildinfo" << 1), result); std::string versionString = std::string(result.getStringField("version")); if (!versionParse(versionString, mongoVersionMayor, mongoVersionMinor, extra)) { LM_E(("Database Startup Error (invalid version format: %s)", versionString.c_str())); return NULL; } LM_T(LmtMongo, ("mongo version server: %s (mayor: %d, minor: %d, extra: %s)", versionString.c_str(), mongoVersionMayor, mongoVersionMinor, extra.c_str())); return connection; }
bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){ WriteBackListener::init( *conn_in ); bool ok; DBClientBase* conn = NULL; try { // May throw if replica set primary is down conn = getVersionable( conn_in ); dassert( conn ); // errors thrown above BSONObjBuilder cmdBuilder; cmdBuilder.append( "setShardVersion" , "" ); cmdBuilder.appendBool( "init", true ); cmdBuilder.append( "configdb" , configServer.modelServer() ); cmdBuilder.appendOID( "serverID" , &serverID ); cmdBuilder.appendBool( "authoritative" , true ); BSONObj cmd = cmdBuilder.obj(); LOG(1) << "initializing shard connection to " << conn->toString() << endl; LOG(2) << "initial sharding settings : " << cmd << endl; ok = conn->runCommand("admin", cmd, result, 0); } catch( const DBException& ) { if ( conn_in->type() != ConnectionString::SET ) { throw; } // NOTE: Only old-style cluster operations will talk via DBClientReplicaSets - using // checkShardVersion is required (which includes initShardVersion information) if these // connections are used. OCCASIONALLY { warning() << "failed to initialize new replica set connection version, " << "will initialize on first use" << endl; } return true; } // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1 // Result is false, but will still initialize serverID and configdb if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ || result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ )) { ok = true; } // Record the connection wire version if sent in the response, initShardVersion is a // handshake for mongos->mongod connections. if ( !result["minWireVersion"].eoo() ) { int minWireVersion = result["minWireVersion"].numberInt(); int maxWireVersion = result["maxWireVersion"].numberInt(); conn->setWireVersions( minWireVersion, maxWireVersion ); } LOG(3) << "initial sharding result : " << result << endl; return ok; }