/* ****************************************************************************
*
* mongoAttributesForEntityType -
*/
HttpStatusCode mongoAttributesForEntityType
(
std::string entityType,
EntityTypeResponse* responseP,
const std::string& tenant,
const std::vector<std::string>& servicePathV,
std::map<std::string, std::string>& uriParams
)
{
unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str());
unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str());
std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS];
bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false;
bool reqSemTaken = false;
// Setting the name of the entity type for the response
responseP->entityType.type = entityType;
LM_T(LmtMongo, ("Query Types Attribute for <%s>", entityType.c_str()));
LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false"));
reqSemTake(__FUNCTION__, "query types attributes request", SemReadOp, &reqSemTaken);
/* Compose query based on this aggregation command:
*
* db.runCommand({aggregate: "entities",
* pipeline: [ {$match: { "_id.type": "TYPE" , "_id.servicePath": /.../ } },
* {$project: {_id: 1, "attrNames": 1} },
* {$unwind: "$attrNames"},
* {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrNames"}} },
* {$unwind: "$attrs"},
* {$group: {_id: "$attrs" }},
* {$sort: {_id: 1}}
* ]
* })
*
*/
BSONObj result;
BSONObj cmd = BSON("aggregate" << COL_ENTITIES <<
"pipeline" << BSON_ARRAY(
BSON("$match" << BSON(C_ID_ENTITY << entityType << C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) <<
BSON("$project" << BSON("_id" << 1 << ENT_ATTRNAMES << 1)) <<
BSON("$unwind" << S_ATTRNAMES) <<
BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRNAMES))) <<
BSON("$unwind" << "$attrs") <<
BSON("$group" << BSON("_id" << "$attrs")) <<
BSON("$sort" << BSON("_id" << 1))
)
);
std::string err;
if (!runCollectionCommand(composeDatabaseName(tenant), cmd, &result, &err))
{
responseP->statusCode.fill(SccReceiverInternalError, err);
reqSemGive(__FUNCTION__, "query types request", reqSemTaken);
return SccOk;
}
/* Processing result to build response */
LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str()));
std::vector<BSONElement> resultsArray = getField(result, "result").Array();
responseP->entityType.count = countEntities(tenant, servicePathV, entityType);
if (resultsArray.size() == 0)
{
responseP->statusCode.fill(SccContextElementNotFound);
reqSemGive(__FUNCTION__, "query types request", reqSemTaken);
return SccOk;
}
/* See comment above in the other method regarding this strategy to implement pagination */
for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix)
{
BSONElement idField = resultsArray[ix].embeddedObject().getField("_id");
//
// BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub'
//
// Now, if 'resultsArray[ix].embeddedObject().getField("_id")' is not found, if we continue,
// calling embeddedObject() on it, then we get an exception and the broker crashes.
//
if (idField.eoo() == true)
{
LM_E(("Database Error (error retrieving _id field in doc: %s)", resultsArray[ix].embeddedObject().toString().c_str()));
continue;
}
/* Note that we need and extra query() to the database (inside attributeType() function) to get each attribute type.
* This could be unefficient, specially if the number of attributes is large */
std::string attrType = attributeType(tenant, servicePathV, entityType , idField.str());
ContextAttribute* ca = new ContextAttribute(idField.str(), attrType, "");
responseP->entityType.contextAttributeVector.push_back(ca);
}
char detailsMsg[256];
if (responseP->entityType.contextAttributeVector.size() > 0)
{
if (details)
{
snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size());
responseP->statusCode.fill(SccOk, detailsMsg);
}
else
{
responseP->statusCode.fill(SccOk);
}
}
else
{
if (details)
{
snprintf(detailsMsg, sizeof(detailsMsg), "Number of attributes: %d. Offset is %d", (int) resultsArray.size(), offset);
responseP->statusCode.fill(SccContextElementNotFound, detailsMsg);
}
else
{
responseP->statusCode.fill(SccContextElementNotFound);
}
}
reqSemGive(__FUNCTION__, "query types request", reqSemTaken);
return SccOk;
}
/* ****************************************************************************
*
* mongoConnect -
*
* Default value for writeConcern == 1 (0: unacknowledged, 1: acknowledged)
*/
static DBClientBase* mongoConnect
(
const char* host,
const char* db,
const char* rplSet,
const char* username,
const char* passwd,
bool multitenant,
int writeConcern,
double timeout
)
{
std::string err;
DBClientBase* connection = NULL;
LM_T(LmtMongo, ("Connection info: dbName='%s', rplSet='%s', timeout=%f", db, rplSet, timeout));
bool connected = false;
int retries = RECONNECT_RETRIES;
if (strlen(rplSet) == 0)
{
// Setting the first argument to true is to use autoreconnect
connection = new DBClientConnection(true);
//
// Not sure of how to generalize the following code,
// given that DBClientBase class doesn't have a common connect() method (surprisingly)
//
for (int tryNo = 0; tryNo < retries; ++tryNo)
{
if ( ((DBClientConnection*)connection)->connect(host, err))
{
connected = true;
break;
}
if (tryNo == 0)
{
LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)",
retries,
RECONNECT_DELAY));
}
else
{
LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo));
}
usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds
}
}
else
{
LM_T(LmtMongo, ("Using replica set %s", rplSet));
// autoReconnect is always on for DBClientReplicaSet connections.
std::vector<std::string> hostTokens;
int components = stringSplit(host, ',', hostTokens);
std::vector<HostAndPort> rplSetHosts;
for (int ix = 0; ix < components; ix++)
{
LM_T(LmtMongo, ("rplSet host <%s>", hostTokens[ix].c_str()));
rplSetHosts.push_back(HostAndPort(hostTokens[ix]));
}
connection = new DBClientReplicaSet(rplSet, rplSetHosts, timeout);
//
// Not sure of to generalize the following code,
// given that DBClientBase class hasn't a common connect() method (surprisingly)
//
for (int tryNo = 0; tryNo < retries; ++tryNo)
{
if ( ((DBClientReplicaSet*)connection)->connect())
{
connected = true;
break;
}
if (tryNo == 0)
{
LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)",
retries,
RECONNECT_DELAY));
}
else
{
LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo));
}
usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds
}
}
if (connected == false)
{
LM_E(("Database Error (connection failed, after %d retries: '%s')", retries, err.c_str()));
return NULL;
}
LM_I(("Successful connection to database"));
//
// WriteConcern
//
mongo::WriteConcern writeConcernCheck;
//
// In the legacy driver, writeConcern is no longer an int, but a class.
// We need a small conversion step here
//
mongo::WriteConcern wc = writeConcern == 1 ? mongo::WriteConcern::acknowledged : mongo::WriteConcern::unacknowledged;
setWriteConcern(connection, wc, &err);
getWriteConcern(connection, &writeConcernCheck, &err);
if (writeConcernCheck.nodes() != wc.nodes())
{
LM_E(("Database Error (Write Concern not set as desired)"));
return NULL;
}
LM_T(LmtMongo, ("Active DB Write Concern mode: %d", writeConcern));
/* Authentication is different depending if multiservice is used or not. In the case of not
* using multiservice, we authenticate in the single-service database. In the case of using
* multiservice, it isn't a default database that we know at contextBroker start time (when
* this connection function is invoked) so we authenticate on the admin database, which provides
* access to any database */
if (multitenant)
{
if (strlen(username) != 0 && strlen(passwd) != 0)
{
if (!connectionAuth(connection, "admin", std::string(username), std::string(passwd), &err))
{
return NULL;
}
}
}
else
{
if (strlen(db) != 0 && strlen(username) != 0 && strlen(passwd) != 0)
{
if (!connectionAuth(connection, std::string(db), std::string(username), std::string(passwd), &err))
{
return NULL;
}
}
}
/* Get mongo version with the 'buildinfo' command */
BSONObj result;
std::string extra;
runCollectionCommand(connection, "admin", BSON("buildinfo" << 1), &result, &err);
std::string versionString = std::string(getStringField(result, "version"));
if (!versionParse(versionString, mongoVersionMayor, mongoVersionMinor, extra))
{
LM_E(("Database Startup Error (invalid version format: %s)", versionString.c_str()));
return NULL;
}
LM_T(LmtMongo, ("mongo version server: %s (mayor: %d, minor: %d, extra: %s)",
versionString.c_str(),
mongoVersionMayor,
mongoVersionMinor,
extra.c_str()));
return connection;
}
/* ****************************************************************************
*
* mongoEntityTypes -
*/
HttpStatusCode mongoEntityTypes
(
EntityTypeVectorResponse* responseP,
const std::string& tenant,
const std::vector<std::string>& servicePathV,
std::map<std::string, std::string>& uriParams
)
{
unsigned int offset = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str());
unsigned int limit = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str());
std::string detailsString = uriParams[URI_PARAM_PAGINATION_DETAILS];
bool details = (strcasecmp("on", detailsString.c_str()) == 0)? true : false;
bool reqSemTaken = false;
LM_T(LmtMongo, ("Query Entity Types"));
LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false"));
reqSemTake(__FUNCTION__, "query types request", SemReadOp, &reqSemTaken);
/* Compose query based on this aggregation command:
*
* db.runCommand({aggregate: "entities",
* pipeline: [ {$match: { "_id.servicePath": /.../ } },
* {$project: {_id: 1, "attrNames": 1} },
* {$project: { "attrNames"
* {$cond: [ {$eq: [ "$attrNames", [ ] ] }, [null], "$attrNames"] }
* }
* },
* {$unwind: "$attrNames"},
* {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrNames"}} },
* {$sort: {_id: 1} }
* ]
* })
*
* The $cond part is hard... more information at http://stackoverflow.com/questions/27510143/empty-array-prevents-document-to-appear-in-query
* As a consequence, some "null" values may appear in the resulting attrs vector, which are pruned by the result processing logic.
*
* FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the
* following command can be used:
*
* db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]})
*
*
*
*/
BSONObj result;
//
// Building the projection part of the query that includes types that have no attributes
// See bug: https://github.com/telefonicaid/fiware-orion/issues/686
//
BSONArrayBuilder emptyArrayBuilder;
BSONArrayBuilder nulledArrayBuilder;
nulledArrayBuilder.appendNull();
// We are using the $cond: [ .. ] and not the $cond: { .. } one, as the former is the only one valid in MongoDB 2.4
BSONObj projection = BSON(
"$project" << BSON(
ENT_ATTRNAMES << BSON(
"$cond" << BSON_ARRAY(
BSON("$eq" << BSON_ARRAY(S_ATTRNAMES << emptyArrayBuilder.arr()) ) <<
nulledArrayBuilder.arr() <<
S_ATTRNAMES
)
)
)
);
BSONObj cmd = BSON("aggregate" << COL_ENTITIES <<
"pipeline" << BSON_ARRAY(
BSON("$match" << BSON(C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) <<
BSON("$project" << BSON("_id" << 1 << ENT_ATTRNAMES << 1)) <<
projection <<
BSON("$unwind" << S_ATTRNAMES) <<
BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRNAMES))) <<
BSON("$sort" << BSON("_id" << 1))
)
);
std::string err;
if (!runCollectionCommand(composeDatabaseName(tenant), cmd, &result, &err))
{
responseP->statusCode.fill(SccReceiverInternalError, err);
reqSemGive(__FUNCTION__, "query types request", reqSemTaken);
return SccOk;
}
// Processing result to build response
LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str()));
std::vector<BSONElement> resultsArray = getField(result, "result").Array();
if (resultsArray.size() == 0)
{
responseP->statusCode.fill(SccContextElementNotFound);
reqSemGive(__FUNCTION__, "query types request", reqSemTaken);
return SccOk;
}
/* Another strategy to implement pagination is to use the $skip and $limit operators in the
* aggregation framework. However, doing so, we don't know the total number of results, which can
* be needed in the case of details=on (using that approach, we need to do two queries: one to get
* the count and other to get the actual results with $skip and $limit, in the same "transaction" to
* avoid incoherence between both if some entity type is created or deleted in the process).
*
* However, considering that the number of types will be small compared with the number of entities,
* the current approach seems to be ok
*/
for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix)
{
BSONObj resultItem = resultsArray[ix].embeddedObject();
EntityType* entityType = new EntityType(getStringField(resultItem, "_id"));
std::vector<BSONElement> attrsArray = getField(resultItem, "attrs").Array();
entityType->count = countEntities(tenant, servicePathV, entityType->type);
if (!attrsArray[0].isNull())
{
for (unsigned int jx = 0; jx < attrsArray.size(); ++jx)
{
/* This is where NULL elements in the resulting attrs vector are pruned */
if (attrsArray[jx].isNull())
{
continue;
}
/* Note that we need and extra query() to the database (inside attributeType() function) to get each attribute type.
* This could be unefficient, specially if the number of attributes is large */
std::string attrType = attributeType(tenant, servicePathV, entityType->type , attrsArray[jx].str());
ContextAttribute* ca = new ContextAttribute(attrsArray[jx].str(), attrType, "");
entityType->contextAttributeVector.push_back(ca);
}
}
responseP->entityTypeVector.push_back(entityType);
}
char detailsMsg[256];
if (responseP->entityTypeVector.size() > 0)
{
if (details)
{
snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size());
responseP->statusCode.fill(SccOk, detailsMsg);
}
else
{
responseP->statusCode.fill(SccOk);
}
}
else
{
if (details)
{
snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset);
responseP->statusCode.fill(SccContextElementNotFound, detailsMsg);
}
else
{
responseP->statusCode.fill(SccContextElementNotFound);
}
}
reqSemGive(__FUNCTION__, "query types request", reqSemTaken);
return SccOk;
}
