/* **************************************************************************** * * mongoAttributesForEntityType - */ HttpStatusCode mongoAttributesForEntityType ( std::string entityType, EntityTypeResponse* responseP, const std::string& tenant, const std::vector<std::string>& servicePathV, std::map<std::string, std::string>& uriParams ) { unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str()); unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str()); std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS]; bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false; bool reqSemTaken = false; // Setting the name of the entity type for the response responseP->entityType.type = entityType; LM_T(LmtMongo, ("Query Types Attribute for <%s>", entityType.c_str())); LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false")); reqSemTake(__FUNCTION__, "query types attributes request", SemReadOp, &reqSemTaken); /* Compose query based on this aggregation command: * * db.runCommand({aggregate: "entities", * pipeline: [ {$match: { "_id.type": "TYPE" , "_id.servicePath": /.../ } }, * {$project: {_id: 1, "attrNames": 1} }, * {$unwind: "$attrNames"}, * {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrNames"}} }, * {$unwind: "$attrs"}, * {$group: {_id: "$attrs" }}, * {$sort: {_id: 1}} * ] * }) * */ BSONObj result; BSONObj cmd = BSON("aggregate" << COL_ENTITIES << "pipeline" << BSON_ARRAY( BSON("$match" << BSON(C_ID_ENTITY << entityType << C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) << BSON("$project" << BSON("_id" << 1 << ENT_ATTRNAMES << 1)) << BSON("$unwind" << S_ATTRNAMES) << BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRNAMES))) << BSON("$unwind" << "$attrs") << BSON("$group" << BSON("_id" << "$attrs")) << BSON("$sort" << BSON("_id" << 1)) ) ); std::string err; if (!runCollectionCommand(composeDatabaseName(tenant), cmd, &result, &err)) { responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; } /* Processing result to build response */ LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str())); std::vector<BSONElement> resultsArray = getField(result, "result").Array(); responseP->entityType.count = countEntities(tenant, servicePathV, entityType); if (resultsArray.size() == 0) { responseP->statusCode.fill(SccContextElementNotFound); reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; } /* See comment above in the other method regarding this strategy to implement pagination */ for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix) { BSONElement idField = resultsArray[ix].embeddedObject().getField("_id"); // // BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub' // // Now, if 'resultsArray[ix].embeddedObject().getField("_id")' is not found, if we continue, // calling embeddedObject() on it, then we get an exception and the broker crashes. // if (idField.eoo() == true) { LM_E(("Database Error (error retrieving _id field in doc: %s)", resultsArray[ix].embeddedObject().toString().c_str())); continue; } /* Note that we need and extra query() to the database (inside attributeType() function) to get each attribute type. * This could be unefficient, specially if the number of attributes is large */ std::string attrType = attributeType(tenant, servicePathV, entityType , idField.str()); ContextAttribute* ca = new ContextAttribute(idField.str(), attrType, ""); responseP->entityType.contextAttributeVector.push_back(ca); } char detailsMsg[256]; if (responseP->entityType.contextAttributeVector.size() > 0) { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size()); responseP->statusCode.fill(SccOk, detailsMsg); } else { responseP->statusCode.fill(SccOk); } } else { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Number of attributes: %d. Offset is %d", (int) resultsArray.size(), offset); responseP->statusCode.fill(SccContextElementNotFound, detailsMsg); } else { responseP->statusCode.fill(SccContextElementNotFound); } } reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; }
/* **************************************************************************** * * mongoConnect - * * Default value for writeConcern == 1 (0: unacknowledged, 1: acknowledged) */ static DBClientBase* mongoConnect ( const char* host, const char* db, const char* rplSet, const char* username, const char* passwd, bool multitenant, int writeConcern, double timeout ) { std::string err; DBClientBase* connection = NULL; LM_T(LmtMongo, ("Connection info: dbName='%s', rplSet='%s', timeout=%f", db, rplSet, timeout)); bool connected = false; int retries = RECONNECT_RETRIES; if (strlen(rplSet) == 0) { // Setting the first argument to true is to use autoreconnect connection = new DBClientConnection(true); // // Not sure of how to generalize the following code, // given that DBClientBase class doesn't have a common connect() method (surprisingly) // for (int tryNo = 0; tryNo < retries; ++tryNo) { if ( ((DBClientConnection*)connection)->connect(host, err)) { connected = true; break; } if (tryNo == 0) { LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)", retries, RECONNECT_DELAY)); } else { LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo)); } usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds } } else { LM_T(LmtMongo, ("Using replica set %s", rplSet)); // autoReconnect is always on for DBClientReplicaSet connections. std::vector<std::string> hostTokens; int components = stringSplit(host, ',', hostTokens); std::vector<HostAndPort> rplSetHosts; for (int ix = 0; ix < components; ix++) { LM_T(LmtMongo, ("rplSet host <%s>", hostTokens[ix].c_str())); rplSetHosts.push_back(HostAndPort(hostTokens[ix])); } connection = new DBClientReplicaSet(rplSet, rplSetHosts, timeout); // // Not sure of to generalize the following code, // given that DBClientBase class hasn't a common connect() method (surprisingly) // for (int tryNo = 0; tryNo < retries; ++tryNo) { if ( ((DBClientReplicaSet*)connection)->connect()) { connected = true; break; } if (tryNo == 0) { LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)", retries, RECONNECT_DELAY)); } else { LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo)); } usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds } } if (connected == false) { LM_E(("Database Error (connection failed, after %d retries: '%s')", retries, err.c_str())); return NULL; } LM_I(("Successful connection to database")); // // WriteConcern // mongo::WriteConcern writeConcernCheck; // // In the legacy driver, writeConcern is no longer an int, but a class. // We need a small conversion step here // mongo::WriteConcern wc = writeConcern == 1 ? mongo::WriteConcern::acknowledged : mongo::WriteConcern::unacknowledged; setWriteConcern(connection, wc, &err); getWriteConcern(connection, &writeConcernCheck, &err); if (writeConcernCheck.nodes() != wc.nodes()) { LM_E(("Database Error (Write Concern not set as desired)")); return NULL; } LM_T(LmtMongo, ("Active DB Write Concern mode: %d", writeConcern)); /* Authentication is different depending if multiservice is used or not. In the case of not * using multiservice, we authenticate in the single-service database. In the case of using * multiservice, it isn't a default database that we know at contextBroker start time (when * this connection function is invoked) so we authenticate on the admin database, which provides * access to any database */ if (multitenant) { if (strlen(username) != 0 && strlen(passwd) != 0) { if (!connectionAuth(connection, "admin", std::string(username), std::string(passwd), &err)) { return NULL; } } } else { if (strlen(db) != 0 && strlen(username) != 0 && strlen(passwd) != 0) { if (!connectionAuth(connection, std::string(db), std::string(username), std::string(passwd), &err)) { return NULL; } } } /* Get mongo version with the 'buildinfo' command */ BSONObj result; std::string extra; runCollectionCommand(connection, "admin", BSON("buildinfo" << 1), &result, &err); std::string versionString = std::string(getStringField(result, "version")); if (!versionParse(versionString, mongoVersionMayor, mongoVersionMinor, extra)) { LM_E(("Database Startup Error (invalid version format: %s)", versionString.c_str())); return NULL; } LM_T(LmtMongo, ("mongo version server: %s (mayor: %d, minor: %d, extra: %s)", versionString.c_str(), mongoVersionMayor, mongoVersionMinor, extra.c_str())); return connection; }
/* **************************************************************************** * * mongoEntityTypes - */ HttpStatusCode mongoEntityTypes ( EntityTypeVectorResponse* responseP, const std::string& tenant, const std::vector<std::string>& servicePathV, std::map<std::string, std::string>& uriParams ) { unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str()); unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str()); std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS]; bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false; bool reqSemTaken = false; LM_T(LmtMongo, ("Query Entity Types")); LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false")); reqSemTake(__FUNCTION__, "query types request", SemReadOp, &reqSemTaken); /* Compose query based on this aggregation command: * * db.runCommand({aggregate: "entities", * pipeline: [ {$match: { "_id.servicePath": /.../ } }, * {$project: {_id: 1, "attrNames": 1} }, * {$project: { "attrNames" * {$cond: [ {$eq: [ "$attrNames", [ ] ] }, [null], "$attrNames"] } * } * }, * {$unwind: "$attrNames"}, * {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrNames"}} }, * {$sort: {_id: 1} } * ] * }) * * The $cond part is hard... more information at http://stackoverflow.com/questions/27510143/empty-array-prevents-document-to-appear-in-query * As a consequence, some "null" values may appear in the resulting attrs vector, which are pruned by the result processing logic. * * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the * following command can be used: * * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]}) * * * */ BSONObj result; // // Building the projection part of the query that includes types that have no attributes // See bug: https://github.com/telefonicaid/fiware-orion/issues/686 // BSONArrayBuilder emptyArrayBuilder; BSONArrayBuilder nulledArrayBuilder; nulledArrayBuilder.appendNull(); // We are using the $cond: [ .. ] and not the $cond: { .. } one, as the former is the only one valid in MongoDB 2.4 BSONObj projection = BSON( "$project" << BSON( ENT_ATTRNAMES << BSON( "$cond" << BSON_ARRAY( BSON("$eq" << BSON_ARRAY(S_ATTRNAMES << emptyArrayBuilder.arr()) ) << nulledArrayBuilder.arr() << S_ATTRNAMES ) ) ) ); BSONObj cmd = BSON("aggregate" << COL_ENTITIES << "pipeline" << BSON_ARRAY( BSON("$match" << BSON(C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) << BSON("$project" << BSON("_id" << 1 << ENT_ATTRNAMES << 1)) << projection << BSON("$unwind" << S_ATTRNAMES) << BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRNAMES))) << BSON("$sort" << BSON("_id" << 1)) ) ); std::string err; if (!runCollectionCommand(composeDatabaseName(tenant), cmd, &result, &err)) { responseP->statusCode.fill(SccReceiverInternalError, err); reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; } // Processing result to build response LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str())); std::vector<BSONElement> resultsArray = getField(result, "result").Array(); if (resultsArray.size() == 0) { responseP->statusCode.fill(SccContextElementNotFound); reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; } /* Another strategy to implement pagination is to use the $skip and $limit operators in the * aggregation framework. However, doing so, we don't know the total number of results, which can * be needed in the case of details=on (using that approach, we need to do two queries: one to get * the count and other to get the actual results with $skip and $limit, in the same "transaction" to * avoid incoherence between both if some entity type is created or deleted in the process). * * However, considering that the number of types will be small compared with the number of entities, * the current approach seems to be ok */ for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix) { BSONObj resultItem = resultsArray[ix].embeddedObject(); EntityType* entityType = new EntityType(getStringField(resultItem, "_id")); std::vector<BSONElement> attrsArray = getField(resultItem, "attrs").Array(); entityType->count = countEntities(tenant, servicePathV, entityType->type); if (!attrsArray[0].isNull()) { for (unsigned int jx = 0; jx < attrsArray.size(); ++jx) { /* This is where NULL elements in the resulting attrs vector are pruned */ if (attrsArray[jx].isNull()) { continue; } /* Note that we need and extra query() to the database (inside attributeType() function) to get each attribute type. * This could be unefficient, specially if the number of attributes is large */ std::string attrType = attributeType(tenant, servicePathV, entityType->type , attrsArray[jx].str()); ContextAttribute* ca = new ContextAttribute(attrsArray[jx].str(), attrType, ""); entityType->contextAttributeVector.push_back(ca); } } responseP->entityTypeVector.push_back(entityType); } char detailsMsg[256]; if (responseP->entityTypeVector.size() > 0) { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size()); responseP->statusCode.fill(SccOk, detailsMsg); } else { responseP->statusCode.fill(SccOk); } } else { if (details) { snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset); responseP->statusCode.fill(SccContextElementNotFound, detailsMsg); } else { responseP->statusCode.fill(SccContextElementNotFound); } } reqSemGive(__FUNCTION__, "query types request", reqSemTaken); return SccOk; }