Beispiel #1
0
    bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){

        WriteBackListener::init( *conn_in );

        DBClientBase* conn = getVersionable( conn_in );
        verify( conn ); // errors thrown above

        BSONObjBuilder cmdBuilder;

        cmdBuilder.append( "setShardVersion" , "" );
        cmdBuilder.appendBool( "init", true );
        cmdBuilder.append( "configdb" , configServer.modelServer() );
        cmdBuilder.appendOID( "serverID" , &serverID );
        cmdBuilder.appendBool( "authoritative" , true );

        BSONObj cmd = cmdBuilder.obj();

        LOG(1) << "initializing shard connection to " << conn->toString() << endl;
        LOG(2) << "initial sharding settings : " << cmd << endl;

        bool ok = conn->runCommand( "admin" , cmd , result );

        // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
        // Result is false, but will still initialize serverID and configdb
        if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ ||
                                                  result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ ))
        {
            ok = true;
        }

        LOG(3) << "initial sharding result : " << result << endl;

        return ok;

    }
    bool setShardVersion(DBClientBase& conn,
                         const string& ns,
                         const string& configServerPrimary,
                         ChunkVersion version,
                         ChunkManager* manager,
                         bool authoritative,
                         BSONObj& result) {

        BSONObjBuilder cmdBuilder;
        cmdBuilder.append("setShardVersion", ns);
        cmdBuilder.append("configdb", configServerPrimary);

        Shard s = Shard::make(conn.getServerAddress());
        cmdBuilder.append("shard", s.getName());
        cmdBuilder.append("shardHost", s.getConnString());

        if (ns.size() > 0) {
            version.addToBSON(cmdBuilder);
        }
        else {
            cmdBuilder.append("init", true);
        }

        if (authoritative) {
            cmdBuilder.appendBool("authoritative", 1);
        }

        BSONObj cmd = cmdBuilder.obj();

        LOG(1) << "    setShardVersion  " << s.getName() << " " << conn.getServerAddress()
               << "  " << ns << "  " << cmd
               << (manager ? string(str::stream() << " " << manager->getSequenceNumber()) : "");

        return conn.runCommand("admin", cmd, result, 0);
    }
Beispiel #3
0
    bool initShardVersion( DBClientBase& conn_in, BSONObj& result ){

        WriteBackListener::init( conn_in );

        DBClientBase* conn = getVersionable( &conn_in );
        assert( conn ); // errors thrown above

        BSONObjBuilder cmdBuilder;

        cmdBuilder.append( "setShardVersion" , "" );
        cmdBuilder.appendBool( "init", true );
        cmdBuilder.append( "configdb" , configServer.modelServer() );
        cmdBuilder.appendOID( "serverID" , &serverID );
        cmdBuilder.appendBool( "authoritative" , true );

        BSONObj cmd = cmdBuilder.obj();

        LOG(1) << "initializing shard connection to " << conn->toString() << endl;
        LOG(2) << "initial sharding settings : " << cmd << endl;

        bool ok = conn->runCommand( "admin" , cmd , result );

        LOG(3) << "initial sharding result : " << result << endl;

        return ok;

    }
    bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){

        WriteBackListener::init( *conn_in );

        bool ok;
        DBClientBase* conn = NULL;
        try {
            // May throw if replica set primary is down
            conn = getVersionable( conn_in );
            dassert( conn ); // errors thrown above

            BSONObjBuilder cmdBuilder;

            cmdBuilder.append( "setShardVersion" , "" );
            cmdBuilder.appendBool( "init", true );
            cmdBuilder.append( "configdb" , configServer.modelServer() );
            cmdBuilder.appendOID( "serverID" , &serverID );
            cmdBuilder.appendBool( "authoritative" , true );

            BSONObj cmd = cmdBuilder.obj();

            LOG(1) << "initializing shard connection to " << conn->toString() << endl;
            LOG(2) << "initial sharding settings : " << cmd << endl;

            ok = conn->runCommand("admin", cmd, result, 0);
        }
        catch( const DBException& ex ) {

            bool ignoreFailure = ShardConnection::ignoreInitialVersionFailure
                                 && conn_in->type() == ConnectionString::SET;
            if ( !ignoreFailure )
                throw;

            // Using initShardVersion is not strictly required when talking to replica sets - it is
            // preferred to do so because it registers mongos early with the mongod.  This info is
            // also sent by checkShardVersion before a connection is used for a write or read.

            OCCASIONALLY {
                warning() << "failed to initialize new replica set connection version, "
                          << "will initialize on first use" << endl;
            }

            return true;
        }

        // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
        // Result is false, but will still initialize serverID and configdb
        if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ ||
                                                  result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ ))
        {
            ok = true;
        }

        LOG(3) << "initial sharding result : " << result << endl;

        return ok;

    }
void DocumentSourceOutReplaceColl::initializeWriteNs() {
    DBClientBase* conn = pExpCtx->mongoProcessInterface->directClient();

    const auto& outputNs = getOutputNs();
    _tempNs = NamespaceString(str::stream() << outputNs.db() << ".tmp.agg_out."
                                            << aggOutCounter.addAndFetch(1));

    // Save the original collection options and index specs so we can check they didn't change
    // during computation.
    _originalOutOptions = pExpCtx->mongoProcessInterface->getCollectionOptions(outputNs);
    _originalIndexes = conn->getIndexSpecs(outputNs.ns());

    // Check if it's capped to make sure we have a chance of succeeding before we do all the work.
    // If the collection becomes capped during processing, the collection options will have changed,
    // and the $out will fail.
    uassert(17152,
            str::stream() << "namespace '" << outputNs.ns()
                          << "' is capped so it can't be used for $out",
            _originalOutOptions["capped"].eoo());

    // We will write all results into a temporary collection, then rename the temporary
    // collection to be the target collection once we are done.
    _tempNs = NamespaceString(str::stream() << outputNs.db() << ".tmp.agg_out."
                                            << aggOutCounter.addAndFetch(1));

    // Create temp collection, copying options from the existing output collection if any.
    {
        BSONObjBuilder cmd;
        cmd << "create" << _tempNs.coll();
        cmd << "temp" << true;
        cmd.appendElementsUnique(_originalOutOptions);

        BSONObj info;
        uassert(16994,
                str::stream() << "failed to create temporary $out collection '" << _tempNs.ns()
                              << "': "
                              << info.toString(),
                conn->runCommand(outputNs.db().toString(), cmd.done(), info));
    }

    if (_originalIndexes.empty()) {
        return;
    }

    // Copy the indexes of the output collection to the temp collection.
    std::vector<BSONObj> tempNsIndexes;
    for (const auto& indexSpec : _originalIndexes) {
        // Replace the spec's 'ns' field value, which is the original collection, with the temp
        // collection.
        tempNsIndexes.push_back(indexSpec.addField(BSON("ns" << _tempNs.ns()).firstElement()));
    }
    try {
        conn->createIndexes(_tempNs.ns(), tempNsIndexes);
    } catch (DBException& ex) {
        ex.addContext("Copying indexes for $out failed");
        throw;
    }
};
    boost::optional<Document> DocumentSourceOut::getNext() {
        pExpCtx->checkForInterrupt();

        // make sure we only write out once
        if (_done)
            return boost::none;
        _done = true;

        verify(_mongod);
        DBClientBase* conn = _mongod->directClient();

        prepTempCollection();
        verify(_tempNs.size() != 0);

        vector<BSONObj> bufferedObjects;
        int bufferedBytes = 0;
        while (boost::optional<Document> next = pSource->getNext()) {
            BSONObj toInsert = next->toBson();
            bufferedBytes += toInsert.objsize();
            if (!bufferedObjects.empty() && bufferedBytes > BSONObjMaxUserSize) {
                spill(conn, bufferedObjects);
                bufferedObjects.clear();
                bufferedBytes = toInsert.objsize();
            }
            bufferedObjects.push_back(toInsert);
        }

        if (!bufferedObjects.empty())
            spill(conn, bufferedObjects);

        // Checking again to make sure we didn't become sharded while running.
        uassert(17018, str::stream() << "namespace '" << _outputNs.ns()
                                     << "' became sharded so it can't be used for $out'",
                !_mongod->isSharded(_outputNs));

        BSONObj rename = BSON("renameCollection" << _tempNs.ns()
                           << "to" << _outputNs.ns()
                           << "dropTarget" << true
                           );
        BSONObj info;
        bool ok = conn->runCommand("admin", rename, info);
        uassert(16997,  str::stream() << "renameCollection for $out failed: " << info,
                ok);

        // We don't need to drop the temp collection in our destructor if the rename succeeded.
        _tempNs = NamespaceString("");

        // This "DocumentSource" doesn't produce output documents. This can change in the future
        // if we support using $out in "tee" mode.
        return boost::none;
    }
Beispiel #7
0
    bool setShardVersion( DBClientBase & conn , const string& ns , ShardChunkVersion version , bool authoritative , BSONObj& result ){

        BSONObjBuilder cmdBuilder;
        cmdBuilder.append( "setShardVersion" , ns.c_str() );
        cmdBuilder.append( "configdb" , configServer.modelServer() );
        cmdBuilder.appendTimestamp( "version" , version );
        cmdBuilder.appendOID( "serverID" , &serverID );
        if ( authoritative )
            cmdBuilder.appendBool( "authoritative" , 1 );

        Shard s = Shard::make( conn.getServerAddress() );
        cmdBuilder.append( "shard" , s.getName() );
        cmdBuilder.append( "shardHost" , s.getConnString() );
        BSONObj cmd = cmdBuilder.obj();
        
        log(1) << "    setShardVersion  " << s.getName() << " " << conn.getServerAddress() << "  " << ns << "  " << cmd << " " << &conn << endl;
        
        return conn.runCommand( "admin" , cmd , result );
    }
/* ****************************************************************************
*
* mongoEntityTypes -
*/
HttpStatusCode mongoEntityTypes
(
  EntityTypesResponse*                  responseP,
  const std::string&                    tenant,
  const std::vector<std::string>&       servicePathV,
  std::map<std::string, std::string>&   uriParams
)
{
  unsigned int offset         = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str());
  unsigned int limit          = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str());
  std::string  detailsString  = uriParams[URI_PARAM_PAGINATION_DETAILS];
  bool         details        = (strcasecmp("on", detailsString.c_str()) == 0)? true : false;
  LM_T(LmtMongo, ("Query Entity Types"));
  LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false"));

  reqSemTake(__FUNCTION__, "query types request");

  DBClientBase* connection = getMongoConnection();

  /* Compose query based on this aggregation command:
   *
   * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the
   * aggregation query below is fully correct
   *
   * db.runCommand({aggregate: "entities",
   *                pipeline: [ {$match: { "_id.servicePath": /.../ } },
   *                            {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} },
   *                            {$project: { "attrs"
   *                                  {$cond: [ {$eq: [ "$attrs", [ ] ] }, [null], "$attrs"] }
   *                               }
   *                            },
   *                            {$unwind: "$attrs"},
   *                            {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} },
   *                            {$sort: {_id: 1} }
   *                          ]
   *                })
   *
   * The $cond part is hard... more information at http://stackoverflow.com/questions/27510143/empty-array-prevents-document-to-appear-in-query
   * As a consequence, some "null" values may appear in the resulting attrs vector, which are prunned by the result processing logic.
   *
   * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the
   * following command can be used:
   *
   * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]})
   *
   */

  BSONObj result;

  // Building the projection part of the query that includes types that have no attributes
  // See bug: https://github.com/telefonicaid/fiware-orion/issues/686
  BSONArrayBuilder emptyArrayBuilder;
  BSONArrayBuilder nulledArrayBuilder;
  nulledArrayBuilder.appendNull();

  // We are using the $cond: [ .. ] and not the $cond: { .. } one, as the former is the only one valid in MongoDB 2.4
  BSONObj projection = BSON(
    "$project" << BSON(
      "attrs" << BSON(
        "$cond" << BSON_ARRAY(
          BSON("$eq" << BSON_ARRAY(S_ATTRS << emptyArrayBuilder.arr()) ) <<
          nulledArrayBuilder.arr() <<
          S_ATTRS
        )
      )
    )
  );

  BSONObj cmd = BSON("aggregate" << COL_ENTITIES <<
                     "pipeline" << BSON_ARRAY(
                                              BSON("$match" << BSON(C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) <<
                                              BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) <<
                                              projection <<
                                              BSON("$unwind" << S_ATTRS) <<
                                              BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) <<
                                              BSON("$sort" << BSON("_id" << 1))
                                             )
                     );

  LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str()));

  mongoSemTake(__FUNCTION__, "aggregation command");  
  try
  {

    connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result);
    mongoSemGive(__FUNCTION__, "aggregation command");
    LM_I(("Database Operation Successful (%s)", cmd.toString().c_str()));
  }
  catch (const DBException& e)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + e.what();

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }
  catch (...)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + "generic";

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }

  /* Processing result to build response */
  LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str()));

  std::vector<BSONElement> resultsArray = result.getField("result").Array();

  /* Another strategy to implement pagination is to use the $skip and $limit operators in the
   * aggregation framework. However, doing so, we don't know the total number of results, which can
   * be needed in the case of details=on (using that approach, we need to do two queries: one to get
   * the count and other to get the actual results with $skip and $limit, in the same "transaction" to
   * avoid incoherence between both if some entity type is created or deleted in the process).
   *
   * However, considering that the number of types will be small compared with the number of entities,
   * the current approach seems to be ok
   */
  for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix)
  {
    BSONObj                  resultItem = resultsArray[ix].embeddedObject();
    TypeEntity*              type       = new TypeEntity(resultItem.getStringField("_id"));
    std::vector<BSONElement> attrsArray = resultItem.getField("attrs").Array();

    if (!attrsArray[0].isNull())
    {
      for (unsigned int jx = 0; jx < attrsArray.size(); ++jx)
      {
        /* This is the place in which null elements in the resulting attrs vector are prunned */
        if (attrsArray[jx].isNull())
        {
          continue;
        }
        
        BSONObj jAttr = attrsArray[jx].embeddedObject();
        ContextAttribute* ca = new ContextAttribute(jAttr.getStringField(ENT_ATTRS_NAME), jAttr.getStringField(ENT_ATTRS_TYPE));
        type->contextAttributeVector.push_back(ca);
      }
    }

    responseP->typeEntityVector.push_back(type);
  }

  char detailsMsg[256];
  if (responseP->typeEntityVector.size() > 0)
  {
    if (details)
    {
      snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size());
      responseP->statusCode.fill(SccOk, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccOk);
    }
  }
  else
  {
    if (details)
    {      
      snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset);
      responseP->statusCode.fill(SccContextElementNotFound, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccContextElementNotFound);
    }
  }

  reqSemGive(__FUNCTION__, "query types request");

  return SccOk;

}
/* ****************************************************************************
*
* mongoAttributesForEntityType -
*/
HttpStatusCode mongoAttributesForEntityType
(
  std::string                           entityType,
  EntityTypeAttributesResponse*         responseP,
  const std::string&                    tenant,
  const std::vector<std::string>&       servicePathV,
  std::map<std::string, std::string>&   uriParams
)
{
  unsigned int offset         = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str());
  unsigned int limit          = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str());
  std::string  detailsString  = uriParams[URI_PARAM_PAGINATION_DETAILS];
  bool         details        = (strcasecmp("on", detailsString.c_str()) == 0)? true : false;

  // Setting the name of the entity type for the response
  responseP->entityType.type = entityType;

  LM_T(LmtMongo, ("Query Types Attribute for <%s>", entityType.c_str()));
  LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false"));

  reqSemTake(__FUNCTION__, "query types attributes request");

  DBClientBase* connection = getMongoConnection();

  /* Compose query based on this aggregation command:
   *
   * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the
   * aggregation query below is fully correct
   *
   * db.runCommand({aggregate: "entities",
   *                pipeline: [ {$match: { "_id.type": "TYPE" , "_id.servicePath": /.../ } },
   *                            {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} },
   *                            {$unwind: "$attrs"},
   *                            {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} },
   *                            {$unwind: "$attrs"},
   *                            {$group: {_id: "$attrs" }},
   *                            {$sort: {_id.name: 1, _id.type: 1} }
   *                          ]
   *                })
   *
   */

  BSONObj result;
  BSONObj cmd = BSON("aggregate" << COL_ENTITIES <<
                     "pipeline" << BSON_ARRAY(
                                              BSON("$match" << BSON(C_ID_ENTITY << entityType << C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) <<
                                              BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) <<
                                              BSON("$unwind" << S_ATTRS) <<
                                              BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) <<
                                              BSON("$unwind" << S_ATTRS) <<
                                              BSON("$group" << BSON("_id" << S_ATTRS)) <<
                                              BSON("$sort" << BSON(C_ID_NAME << 1 << C_ID_TYPE << 1))
                                             )
                    );

  LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str()));

  mongoSemTake(__FUNCTION__, "aggregation command"); 
  try
  {

    connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result);
    mongoSemGive(__FUNCTION__, "aggregation command");
    LM_I(("Database Operation Successful (%s)", cmd.toString().c_str()));
  }
  catch (const DBException& e)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + e.what();

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }
  catch (...)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + "generic";

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }

  /* Processing result to build response*/
  LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str()));

  std::vector<BSONElement> resultsArray = result.getField("result").Array();

  /* See comment above in the other method regarding this strategy to implement pagination */
  for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix)
  {
    BSONElement        idField    = resultsArray[ix].embeddedObject().getField("_id");

    //
    // BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub'
    //
    // Now, if 'resultsArray[ix].embeddedObject().getField("_id")' is not found, if we continue,
    // calling embeddedObject() on it, then we get an exception and the broker crashes.
    //
    if (idField.eoo() == true)
    {
      LM_E(("Database Error (error retrieving _id field in doc: %s)", resultsArray[ix].embeddedObject().toString().c_str()));
      continue;
    }

    BSONObj            resultItem = idField.embeddedObject();
    ContextAttribute*  ca         = new ContextAttribute(resultItem.getStringField(ENT_ATTRS_NAME), resultItem.getStringField(ENT_ATTRS_TYPE));
    responseP->entityType.contextAttributeVector.push_back(ca);
  }

  char detailsMsg[256];
  if (responseP->entityType.contextAttributeVector.size() > 0)
  {
    if (details)
    {
      snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size());
      responseP->statusCode.fill(SccOk, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccOk);
    }
  }
  else
  {
    if (details)
    {
      snprintf(detailsMsg, sizeof(detailsMsg), "Number of attributes: %d. Offset is %d", (int) resultsArray.size(), offset);
      responseP->statusCode.fill(SccContextElementNotFound, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccContextElementNotFound);
    }
  }

  reqSemGive(__FUNCTION__, "query types request");

  return SccOk;
}
Beispiel #10
0
void DocumentSourceOut::initialize() {
    DBClientBase* conn = pExpCtx->mongoProcessInterface->directClient();

    // Save the original collection options and index specs so we can check they didn't change
    // during computation.
    _originalOutOptions = pExpCtx->mongoProcessInterface->getCollectionOptions(_outputNs);
    _originalIndexes = conn->getIndexSpecs(_outputNs.ns());

    // Check if it's sharded or capped to make sure we have a chance of succeeding before we do all
    // the work. If the collection becomes capped during processing, the collection options will
    // have changed, and the $out will fail. If it becomes sharded during processing, the final
    // rename will fail.
    uassert(17017,
            str::stream() << "namespace '" << _outputNs.ns()
                          << "' is sharded so it can't be used for $out'",
            !pExpCtx->mongoProcessInterface->isSharded(pExpCtx->opCtx, _outputNs));
    uassert(17152,
            str::stream() << "namespace '" << _outputNs.ns()
                          << "' is capped so it can't be used for $out",
            _originalOutOptions["capped"].eoo());

    // We will write all results into a temporary collection, then rename the temporary collection
    // to be the target collection once we are done.
    _tempNs = NamespaceString(str::stream() << _outputNs.db() << ".tmp.agg_out."
                                            << aggOutCounter.addAndFetch(1));

    // Create output collection, copying options from existing collection if any.
    {
        BSONObjBuilder cmd;
        cmd << "create" << _tempNs.coll();
        cmd << "temp" << true;
        cmd.appendElementsUnique(_originalOutOptions);

        BSONObj info;
        bool ok = conn->runCommand(_outputNs.db().toString(), cmd.done(), info);
        uassert(16994,
                str::stream() << "failed to create temporary $out collection '" << _tempNs.ns()
                              << "': "
                              << info.toString(),
                ok);
    }

    // copy indexes to _tempNs
    for (std::list<BSONObj>::const_iterator it = _originalIndexes.begin();
         it != _originalIndexes.end();
         ++it) {
        MutableDocument index((Document(*it)));
        index.remove("_id");  // indexes shouldn't have _ids but some existing ones do
        index["ns"] = Value(_tempNs.ns());

        BSONObj indexBson = index.freeze().toBson();
        conn->insert(_tempNs.getSystemIndexesCollection(), indexBson);
        BSONObj err = conn->getLastErrorDetailed();
        uassert(16995,
                str::stream() << "copying index for $out failed."
                              << " index: "
                              << indexBson
                              << " error: "
                              << err,
                DBClientBase::getLastErrorString(err).empty());
    }
    _initialized = true;
}
/* ****************************************************************************
*
* mongoEntityTypes -
*/
HttpStatusCode mongoEntityTypes
(
  EntityTypesResponse*                  responseP,
  const std::string&                    tenant,
  const std::vector<std::string>&       servicePathV,
  std::map<std::string, std::string>&   uriParams
)
{
  unsigned int offset         = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str());
  unsigned int limit          = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str());
  std::string  detailsString  = uriParams[URI_PARAM_PAGINATION_DETAILS];
  bool         details        = (strcasecmp("on", detailsString.c_str()) == 0)? true : false;

  LM_T(LmtMongo, ("Query Entity Types"));
  LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false"));

  reqSemTake(__FUNCTION__, "query types request");

  DBClientBase* connection = getMongoConnection();

  /* Compose query based on this aggregation command:
   *
   * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the
   * aggregation query below is fully correct
   *
   * db.runCommand({aggregate: "entities",
   *                pipeline: [ {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} },
   *                            {$unwind: "$attrs"},
   *                            {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} },
   *                            {$sort: {_id: 1} }
   *                          ]
   *                })
   *
   * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the
   * following command can be used:
   *
   * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]})
   *
   */

  BSONObj result;
  BSONObj cmd = BSON("aggregate" << COL_ENTITIES <<
                     "pipeline" << BSON_ARRAY(
                                              BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) <<
                                              BSON("$unwind" << S_ATTRS) <<
                                              BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) <<
                                              BSON("$sort" << BSON("_id" << 1))
                                             )
                    );

  LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str()));

  mongoSemTake(__FUNCTION__, "aggregation command");  
  try
  {

    connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result);
    mongoSemGive(__FUNCTION__, "aggregation command");
    LM_I(("Database Operation Successful (%s)", cmd.toString().c_str()));
  }
  catch (const DBException& e)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + e.what();

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }
  catch (...)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + "generic";

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }

  /* Processing result to build response*/
  LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str()));

  std::vector<BSONElement> resultsArray = result.getField("result").Array();

  /* Another strategy to implement pagination is to use the $skip and $limit operators in the
   * aggregation framework. However, doing so, we don't know the total number of results, which can
   * be needed in the case of details=on (using that approach, we need to do two queries: one to get
   * the count and other to get the actual results with $skip and $limit, in the same "transaction" to
   * avoid incoherence between both if some entity type is created or deleted in the process).
   *
   * However, considering that the number of types will be small compared with the number of entities,
   * the current approach seems to be ok
   */
  for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix)
  {

    BSONObj                  resultItem = resultsArray[ix].embeddedObject();
    TypeEntity*              type       = new TypeEntity(resultItem.getStringField("_id"));
    std::vector<BSONElement> attrsArray = resultItem.getField("attrs").Array();

    for (unsigned int jx = 0; jx < attrsArray.size(); ++jx)
    {
      BSONObj jAttr = attrsArray[jx].embeddedObject();
      ContextAttribute* ca = new ContextAttribute(jAttr.getStringField(ENT_ATTRS_NAME), jAttr.getStringField(ENT_ATTRS_TYPE));
      type->contextAttributeVector.push_back(ca);
    }

    responseP->typeEntityVector.push_back(type);

  }

  char detailsMsg[256];
  if (responseP->typeEntityVector.size() > 0)
  {
    if (details)
    {
      snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size());
      responseP->statusCode.fill(SccOk, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccOk);
    }
  }
  else
  {
    if (details)
    {      
      snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset);
      responseP->statusCode.fill(SccContextElementNotFound, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccContextElementNotFound);
    }
  }

  reqSemGive(__FUNCTION__, "query types request");

  return SccOk;

}
/* ****************************************************************************
*
* mongoConnect -
*
* Default value for writeConcern == 1 (0: unacknowledged, 1: acknowledged)
*/
static DBClientBase* mongoConnect
(
    const char*  host,
    const char*  db,
    const char*  rplSet,
    const char*  username,
    const char*  passwd,
    bool         multitenant,
    int          writeConcern,
    double       timeout
)
{
    std::string   err;
    DBClientBase* connection = NULL;

    LM_T(LmtMongo, ("Connection info: dbName='%s', rplSet='%s', timeout=%f", db, rplSet, timeout));

    bool connected     = false;
    int  retries       = RECONNECT_RETRIES;

    if (strlen(rplSet) == 0)
    {
        /* Setting the first argument to true is to use autoreconnect */
        connection = new DBClientConnection(true);

        /* Not sure of how to generalize the following code, given that DBClientBase class doesn't have a common connect() method (surprisingly) */
        for (int tryNo = 0; tryNo < retries; ++tryNo)
        {
            if ( ((DBClientConnection*)connection)->connect(host, err))
            {
                connected = true;
                break;
            }

            if (tryNo == 0)
            {
                LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)", retries, RECONNECT_DELAY));
            }
            else
            {
                LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo));
            }

            usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds
        }
    }
    else
    {
        LM_T(LmtMongo, ("Using replica set %s", rplSet));

        // autoReconnect is always on for DBClientReplicaSet connections.
        std::vector<std::string>  hostTokens;
        int components = stringSplit(host, ',', hostTokens);

        std::vector<HostAndPort> rplSetHosts;
        for (int ix = 0; ix < components; ix++)
        {
            LM_T(LmtMongo, ("rplSet host <%s>", hostTokens[ix].c_str()));
            rplSetHosts.push_back(HostAndPort(hostTokens[ix]));
        }

        connection = new DBClientReplicaSet(rplSet, rplSetHosts, timeout);

        /* Not sure of to generalize the following code, given that DBClientBase class hasn't a common connect() method (surprisingly) */
        for (int tryNo = 0; tryNo < retries; ++tryNo)
        {
            if ( ((DBClientReplicaSet*)connection)->connect())
            {
                connected = true;
                break;
            }

            if (tryNo == 0)
            {
                LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)", retries, RECONNECT_DELAY));
            }
            else
            {
                LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo));
            }

            usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds
        }
    }

    if (connected == false)
    {
        LM_E(("Database Error (connection failed, after %d retries: '%s')", retries, err.c_str()));
        return NULL;
    }

    LM_I(("Successful connection to database"));

    //
    // WriteConcern
    //
    mongo::WriteConcern writeConcernCheck;

    // In legacy driver writeConcern is no longer an int, but a class. We need a small
    // conversion step here
    mongo::WriteConcern wc = writeConcern == 1 ? mongo::WriteConcern::acknowledged : mongo::WriteConcern::unacknowledged;

    connection->setWriteConcern((mongo::WriteConcern) wc);
    writeConcernCheck = (mongo::WriteConcern) connection->getWriteConcern();

    if (writeConcernCheck.nodes() != wc.nodes())
    {
        LM_E(("Database Error (Write Concern not set as desired)"));
        return NULL;
    }
    LM_T(LmtMongo, ("Active DB Write Concern mode: %d", writeConcern));

    /* Authentication is different depending if multiservice is used or not. In the case of not
     * using multiservice, we authenticate in the single-service database. In the case of using
     * multiservice, it isn't a default database that we know at contextBroker start time (when
     * this connection function is invoked) so we authenticate on the admin database, which provides
     * access to any database */
    if (multitenant)
    {
        if (strlen(username) != 0 && strlen(passwd) != 0)
        {
            if (!connection->auth("admin", std::string(username), std::string(passwd), err))
            {
                LM_E(("Database Startup Error (authentication: db='admin', username='%s', password='*****': %s)", username, err.c_str()));
                return NULL;
            }
        }
    }
    else
    {
        if (strlen(db) != 0 && strlen(username) != 0 && strlen(passwd) != 0)
        {
            if (!connection->auth(std::string(db), std::string(username), std::string(passwd), err))
            {
                LM_E(("Database Startup Error (authentication: db='%s', username='%s', password='*****': %s)", db, username, err.c_str()));
                return NULL;
            }
        }
    }

    /* Get mongo version with the 'buildinfo' command */
    BSONObj result;
    std::string extra;
    connection->runCommand("admin", BSON("buildinfo" << 1), result);
    std::string versionString = std::string(result.getStringField("version"));
    if (!versionParse(versionString, mongoVersionMayor, mongoVersionMinor, extra))
    {
        LM_E(("Database Startup Error (invalid version format: %s)", versionString.c_str()));
        return NULL;
    }
    LM_T(LmtMongo, ("mongo version server: %s (mayor: %d, minor: %d, extra: %s)", versionString.c_str(), mongoVersionMayor, mongoVersionMinor, extra.c_str()));

    return connection;
}
Beispiel #13
0
    bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){

        WriteBackListener::init( *conn_in );

        bool ok;
        DBClientBase* conn = NULL;
        try {
            // May throw if replica set primary is down
            conn = getVersionable( conn_in );
            dassert( conn ); // errors thrown above

            BSONObjBuilder cmdBuilder;

            cmdBuilder.append( "setShardVersion" , "" );
            cmdBuilder.appendBool( "init", true );
            cmdBuilder.append( "configdb" , configServer.modelServer() );
            cmdBuilder.appendOID( "serverID" , &serverID );
            cmdBuilder.appendBool( "authoritative" , true );

            BSONObj cmd = cmdBuilder.obj();

            LOG(1) << "initializing shard connection to " << conn->toString() << endl;
            LOG(2) << "initial sharding settings : " << cmd << endl;

            ok = conn->runCommand("admin", cmd, result, 0);
        }
        catch( const DBException& ) {

            if ( conn_in->type() != ConnectionString::SET ) {
                throw;
            }

            // NOTE: Only old-style cluster operations will talk via DBClientReplicaSets - using
            // checkShardVersion is required (which includes initShardVersion information) if these
            // connections are used.

            OCCASIONALLY {
                warning() << "failed to initialize new replica set connection version, "
                          << "will initialize on first use" << endl;
            }

            return true;
        }

        // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
        // Result is false, but will still initialize serverID and configdb
        if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ ||
                                                  result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ ))
        {
            ok = true;
        }

        // Record the connection wire version if sent in the response, initShardVersion is a
        // handshake for mongos->mongod connections.
        if ( !result["minWireVersion"].eoo() ) {

            int minWireVersion = result["minWireVersion"].numberInt();
            int maxWireVersion = result["maxWireVersion"].numberInt();
            conn->setWireVersions( minWireVersion, maxWireVersion );
        }

        LOG(3) << "initial sharding result : " << result << endl;

        return ok;

    }