DBClientBase* ConnectionString::connect( std::string& errmsg, double socketTimeout ) const {

        switch ( _type ) {
        case MASTER: {
            DBClientConnection * c = new DBClientConnection(true);
            c->setSoTimeout( socketTimeout );
            LOG(1) << "creating new connection to:" << _servers[0];
            if ( ! c->connect( _servers[0] , errmsg ) ) {
                delete c;
                return 0;
            }
            LOG(1) << "connected connection!";
            return c;
        }

        case PAIR:
        case SET: {
            DBClientReplicaSet * set = new DBClientReplicaSet( _setName , _servers , socketTimeout );
            if( ! set->connect() ) {
                delete set;
                errmsg = "connect failed to replica set ";
                errmsg += toString();
                return 0;
            }
            return set;
        }

        case SYNC: {
            // TODO , don't copy
            std::list<HostAndPort> l;
            for ( unsigned i=0; i<_servers.size(); i++ )
                l.push_back( _servers[i] );
            SyncClusterConnection* c = new SyncClusterConnection( l, socketTimeout );
            return c;
        }

        case CUSTOM: {

            // Lock in case other things are modifying this at the same time
            boost::lock_guard<boost::mutex> lk( _connectHookMutex );

            // Allow the replacement of connections with other connections - useful for testing.

            uassert( 16335, "custom connection to " + this->toString() +
                        " specified with no connection hook", _connectHook );

            // Double-checked lock, since this will never be active during normal operation
            DBClientBase* replacementConn = _connectHook->connect( *this, errmsg, socketTimeout );

            log() << "replacing connection to " << this->toString() << " with "
                  << ( replacementConn ? replacementConn->getServerAddress() : "(empty)" );

            return replacementConn;
        }

        case INVALID:
            throw UserException( 13421 , "trying to connect to invalid ConnectionString" );
            break;
        }

        verify( 0 );
        return 0;
    }
Example #2
0
void DocumentSourceOut::initialize() {
    DBClientBase* conn = pExpCtx->mongoProcessInterface->directClient();

    // Save the original collection options and index specs so we can check they didn't change
    // during computation.
    _originalOutOptions = pExpCtx->mongoProcessInterface->getCollectionOptions(_outputNs);
    _originalIndexes = conn->getIndexSpecs(_outputNs.ns());

    // Check if it's sharded or capped to make sure we have a chance of succeeding before we do all
    // the work. If the collection becomes capped during processing, the collection options will
    // have changed, and the $out will fail. If it becomes sharded during processing, the final
    // rename will fail.
    uassert(17017,
            str::stream() << "namespace '" << _outputNs.ns()
                          << "' is sharded so it can't be used for $out'",
            !pExpCtx->mongoProcessInterface->isSharded(pExpCtx->opCtx, _outputNs));
    uassert(17152,
            str::stream() << "namespace '" << _outputNs.ns()
                          << "' is capped so it can't be used for $out",
            _originalOutOptions["capped"].eoo());

    // We will write all results into a temporary collection, then rename the temporary collection
    // to be the target collection once we are done.
    _tempNs = NamespaceString(str::stream() << _outputNs.db() << ".tmp.agg_out."
                                            << aggOutCounter.addAndFetch(1));

    // Create output collection, copying options from existing collection if any.
    {
        BSONObjBuilder cmd;
        cmd << "create" << _tempNs.coll();
        cmd << "temp" << true;
        cmd.appendElementsUnique(_originalOutOptions);

        BSONObj info;
        bool ok = conn->runCommand(_outputNs.db().toString(), cmd.done(), info);
        uassert(16994,
                str::stream() << "failed to create temporary $out collection '" << _tempNs.ns()
                              << "': "
                              << info.toString(),
                ok);
    }

    // copy indexes to _tempNs
    for (std::list<BSONObj>::const_iterator it = _originalIndexes.begin();
         it != _originalIndexes.end();
         ++it) {
        MutableDocument index((Document(*it)));
        index.remove("_id");  // indexes shouldn't have _ids but some existing ones do
        index["ns"] = Value(_tempNs.ns());

        BSONObj indexBson = index.freeze().toBson();
        conn->insert(_tempNs.getSystemIndexesCollection(), indexBson);
        BSONObj err = conn->getLastErrorDetailed();
        uassert(16995,
                str::stream() << "copying index for $out failed."
                              << " index: "
                              << indexBson
                              << " error: "
                              << err,
                DBClientBase::getLastErrorString(err).empty());
    }
    _initialized = true;
}
Example #3
0
/* ****************************************************************************
*
* mongoEntityTypes -
*/
HttpStatusCode mongoEntityTypes
(
  EntityTypesResponse*                  responseP,
  const std::string&                    tenant,
  const std::vector<std::string>&       servicePathV,
  std::map<std::string, std::string>&   uriParams
)
{
  unsigned int offset         = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str());
  unsigned int limit          = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str());
  std::string  detailsString  = uriParams[URI_PARAM_PAGINATION_DETAILS];
  bool         details        = (strcasecmp("on", detailsString.c_str()) == 0)? true : false;
  LM_T(LmtMongo, ("Query Entity Types"));
  LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false"));

  reqSemTake(__FUNCTION__, "query types request");

  DBClientBase* connection = getMongoConnection();

  /* Compose query based on this aggregation command:
   *
   * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the
   * aggregation query below is fully correct
   *
   * db.runCommand({aggregate: "entities",
   *                pipeline: [ {$match: { "_id.servicePath": /.../ } },
   *                            {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} },
   *                            {$project: { "attrs"
   *                                  {$cond: [ {$eq: [ "$attrs", [ ] ] }, [null], "$attrs"] }
   *                               }
   *                            },
   *                            {$unwind: "$attrs"},
   *                            {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} },
   *                            {$sort: {_id: 1} }
   *                          ]
   *                })
   *
   * The $cond part is hard... more information at http://stackoverflow.com/questions/27510143/empty-array-prevents-document-to-appear-in-query
   * As a consequence, some "null" values may appear in the resulting attrs vector, which are prunned by the result processing logic.
   *
   * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the
   * following command can be used:
   *
   * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]})
   *
   */

  BSONObj result;

  // Building the projection part of the query that includes types that have no attributes
  // See bug: https://github.com/telefonicaid/fiware-orion/issues/686
  BSONArrayBuilder emptyArrayBuilder;
  BSONArrayBuilder nulledArrayBuilder;
  nulledArrayBuilder.appendNull();

  // We are using the $cond: [ .. ] and not the $cond: { .. } one, as the former is the only one valid in MongoDB 2.4
  BSONObj projection = BSON(
    "$project" << BSON(
      "attrs" << BSON(
        "$cond" << BSON_ARRAY(
          BSON("$eq" << BSON_ARRAY(S_ATTRS << emptyArrayBuilder.arr()) ) <<
          nulledArrayBuilder.arr() <<
          S_ATTRS
        )
      )
    )
  );

  BSONObj cmd = BSON("aggregate" << COL_ENTITIES <<
                     "pipeline" << BSON_ARRAY(
                                              BSON("$match" << BSON(C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) <<
                                              BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) <<
                                              projection <<
                                              BSON("$unwind" << S_ATTRS) <<
                                              BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) <<
                                              BSON("$sort" << BSON("_id" << 1))
                                             )
                     );

  LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str()));

  mongoSemTake(__FUNCTION__, "aggregation command");  
  try
  {

    connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result);
    mongoSemGive(__FUNCTION__, "aggregation command");
    LM_I(("Database Operation Successful (%s)", cmd.toString().c_str()));
  }
  catch (const DBException& e)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + e.what();

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }
  catch (...)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + "generic";

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }

  /* Processing result to build response */
  LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str()));

  std::vector<BSONElement> resultsArray = result.getField("result").Array();

  /* Another strategy to implement pagination is to use the $skip and $limit operators in the
   * aggregation framework. However, doing so, we don't know the total number of results, which can
   * be needed in the case of details=on (using that approach, we need to do two queries: one to get
   * the count and other to get the actual results with $skip and $limit, in the same "transaction" to
   * avoid incoherence between both if some entity type is created or deleted in the process).
   *
   * However, considering that the number of types will be small compared with the number of entities,
   * the current approach seems to be ok
   */
  for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix)
  {
    BSONObj                  resultItem = resultsArray[ix].embeddedObject();
    TypeEntity*              type       = new TypeEntity(resultItem.getStringField("_id"));
    std::vector<BSONElement> attrsArray = resultItem.getField("attrs").Array();

    if (!attrsArray[0].isNull())
    {
      for (unsigned int jx = 0; jx < attrsArray.size(); ++jx)
      {
        /* This is the place in which null elements in the resulting attrs vector are prunned */
        if (attrsArray[jx].isNull())
        {
          continue;
        }
        
        BSONObj jAttr = attrsArray[jx].embeddedObject();
        ContextAttribute* ca = new ContextAttribute(jAttr.getStringField(ENT_ATTRS_NAME), jAttr.getStringField(ENT_ATTRS_TYPE));
        type->contextAttributeVector.push_back(ca);
      }
    }

    responseP->typeEntityVector.push_back(type);
  }

  char detailsMsg[256];
  if (responseP->typeEntityVector.size() > 0)
  {
    if (details)
    {
      snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size());
      responseP->statusCode.fill(SccOk, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccOk);
    }
  }
  else
  {
    if (details)
    {      
      snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset);
      responseP->statusCode.fill(SccContextElementNotFound, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccContextElementNotFound);
    }
  }

  reqSemGive(__FUNCTION__, "query types request");

  return SccOk;

}
Example #4
0
    void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) {
        verify( candidateChunks );

        //
        // 1. Check whether there is any sharded collection to be balanced by querying
        // the ShardsNS::collections collection
        //

        auto_ptr<DBClientCursor> cursor = conn.query( ShardNS::collection , BSONObj() );
        vector< string > collections;
        while ( cursor->more() ) {
            BSONObj col = cursor->nextSafe();

            // sharded collections will have a shard "key".
            if ( ! col["key"].eoo() && ! col["noBalance"].trueValue() ){
                collections.push_back( col["_id"].String() );
            }
            else if( col["noBalance"].trueValue() ){
                LOG(1) << "not balancing collection " << col["_id"].String() << ", explicitly disabled" << endl;
            }

        }
        cursor.reset();

        if ( collections.empty() ) {
            LOG(1) << "no collections to balance" << endl;
            return;
        }

        //
        // 2. Get a list of all the shards that are participating in this balance round
        // along with any maximum allowed quotas and current utilization. We get the
        // latter by issuing db.serverStatus() (mem.mapped) to all shards.
        //
        // TODO: skip unresponsive shards and mark information as stale.
        //

        vector<Shard> allShards;
        Shard::getAllShards( allShards );
        if ( allShards.size() < 2) {
            LOG(1) << "can't balance without more active shards" << endl;
            return;
        }
        
        ShardInfoMap shardInfo;
        for ( vector<Shard>::const_iterator it = allShards.begin(); it != allShards.end(); ++it ) {
            const Shard& s = *it;
            ShardStatus status = s.getStatus();
            shardInfo[ s.getName() ] = ShardInfo( s.getMaxSize(),
                                                  status.mapped(),
                                                  s.isDraining(),
                                                  status.hasOpsQueued(),
                                                  s.tags()
                                                  );
        }

        //
        // 3. For each collection, check if the balancing policy recommends moving anything around.
        //

        for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) {
            const string& ns = *it;

            map< string,vector<BSONObj> > shardToChunksMap;
            cursor = conn.query( ShardNS::chunk , QUERY( "ns" << ns ).sort( "min" ) );
            while ( cursor->more() ) {
                BSONObj chunk = cursor->nextSafe();
                if ( chunk["jumbo"].trueValue() )
                    continue;
                vector<BSONObj>& chunks = shardToChunksMap[chunk["shard"].String()];
                chunks.push_back( chunk.getOwned() );
            }
            cursor.reset();

            if (shardToChunksMap.empty()) {
                LOG(1) << "skipping empty collection (" << ns << ")";
                continue;
            }
            
            for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ) {
                // this just makes sure there is an entry in shardToChunksMap for every shard
                Shard s = *i;
                shardToChunksMap[s.getName()].size();
            }

            DistributionStatus status( shardInfo, shardToChunksMap );
            
            // load tags
            conn.ensureIndex( ShardNS::tags, BSON( "ns" << 1 << "min" << 1 ), true );
            cursor = conn.query( ShardNS::tags , QUERY( "ns" << ns ).sort( "min" ) );
            while ( cursor->more() ) {
                BSONObj tag = cursor->nextSafe();
                uassert( 16356 , str::stream() << "tag ranges not valid for: " << ns ,
                         status.addTagRange( TagRange( tag["min"].Obj().getOwned(), 
                                                       tag["max"].Obj().getOwned(), 
                                                       tag["tag"].String() ) ) );
                    
            }
            cursor.reset();
            
            CandidateChunk* p = _policy->balance( ns, status, _balancedLastTime );
            if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) );
        }
    }
/* ****************************************************************************
*
* mongoUpdateContextSubscription - 
*/
HttpStatusCode mongoUpdateContextSubscription(UpdateContextSubscriptionRequest* requestP, UpdateContextSubscriptionResponse* responseP, Format inFormat, const std::string& tenant)
{
  reqSemTake(__FUNCTION__, "ngsi10 update subscription request");

  LM_T(LmtMongo, ("Update Context Subscription"));

  DBClientBase* connection = getMongoConnection();

  /* Look for document */
  BSONObj  sub;
  try
  {
      OID id = OID(requestP->subscriptionId.get());

      mongoSemTake(__FUNCTION__, "findOne in SubscribeContextCollection");
      sub = connection->findOne(getSubscribeContextCollectionName(tenant).c_str(), BSON("_id" << id));
      mongoSemGive(__FUNCTION__, "findOne in SubscribeContextCollection");
      LM_I(("Database Operation Successful (findOne _id: %s)", id.toString().c_str()));
  }
  catch (const AssertionException &e)
  {
      /* This happens when OID format is wrong */
      // FIXME P4: this checking should be done at the parsing stage, without progressing to
      // mongoBackend. For the moment we can leave this here, but we should remove it in the future
      // (old issue #95)
      mongoSemGive(__FUNCTION__, "findOne in SubscribeContextCollection (mongo assertion exception)");
      reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo assertion exception)");

      responseP->subscribeError.errorCode.fill(SccContextElementNotFound);
      LM_W(("Bad Input (invalid OID format)"));
      return SccOk;
  }
  catch (const DBException &e)
  {
      mongoSemGive(__FUNCTION__, "findOne in SubscribeContextCollection (mongo db exception)");
      reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo db exception)");

      responseP->subscribeError.errorCode.fill(SccReceiverInternalError,
                                               std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
                                               " - findOne() _id: " + requestP->subscriptionId.get() +
                                               " - exception: " + e.what());
      LM_E(("Database Error (%s)", responseP->subscribeError.errorCode.details.c_str()));
      return SccOk;
  }
  catch (...)
  {
      mongoSemGive(__FUNCTION__, "findOne in SubscribeContextCollection (mongo generic exception)");
      reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo generic exception)");

      responseP->subscribeError.errorCode.fill(SccReceiverInternalError,
                                               std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
                                               " - findOne() _id: " + requestP->subscriptionId.get() +
                                               " - exception: " + "generic");
      LM_E(("Database Error (%s)", responseP->subscribeError.errorCode.details.c_str()));
      return SccOk;
  }


  if (sub.isEmpty()) {
      responseP->subscribeError.errorCode.fill(SccContextElementNotFound);
      reqSemGive(__FUNCTION__, "ngsi10 update subscription request (no subscriptions found)");
      return SccOk;
  }

  /* We start with an empty BSONObjBuilder and process requestP for all the fields that can
   * be updated. I don't like too much this strategy (I would have preferred to start with
   * a copy of the original document, then modify as neded, but this doesn't seem to be easy
   * using the API provide by the Mongo C++ driver)
   *
   * FIXME: a better implementation strategy could be doing an findAndModify() query to do the
   * update, so detecting if the document was not found, instead of using findOne() + update()
   * with $set operation. One operations to MongoDb. vs two operations.
   */
  BSONObjBuilder newSub;

  /* Entities, attribute list and reference are not updatable, so they are appended directly */
  newSub.appendArray(CSUB_ENTITIES, sub.getField(CSUB_ENTITIES).Obj());
  newSub.appendArray(CSUB_ATTRS, sub.getField(CSUB_ATTRS).Obj());
  newSub.append(CSUB_REFERENCE, STR_FIELD(sub, CSUB_REFERENCE));

  /* Duration update */
  if (requestP->duration.isEmpty()) {      
      newSub.append(CSUB_EXPIRATION, sub.getField(CSUB_EXPIRATION).numberLong());
  }
  else {
      long long expiration = getCurrentTime() + requestP->duration.parse();
      newSub.append(CSUB_EXPIRATION, expiration);
      LM_T(LmtMongo, ("New subscription expiration: %l", expiration));
  }

  /* Restriction update */
  // FIXME: Restrictions not implemented yet

  /* Throttling update */
  if (!requestP->throttling.isEmpty()) {
      /* Throttling equal to 0 removes throttling */
      long long throttling = requestP->throttling.parse();
      if (throttling != 0) {
          newSub.append(CSUB_THROTTLING, throttling);
      }
  }
  else {
      /* The hasField check is needed due to Throttling could not be present in the original doc */
      if (sub.hasField(CSUB_THROTTLING)) {
          newSub.append(CSUB_THROTTLING, sub.getField(CSUB_THROTTLING).numberLong());
      }
  }

  /* Notify conditions */
  bool notificationDone = false;
  if (requestP->notifyConditionVector.size() == 0) {
      newSub.appendArray(CSUB_CONDITIONS, sub.getField(CSUB_CONDITIONS).embeddedObject());
  }
  else {
      /* Destroy any previous ONTIMEINTERVAL thread */
      getNotifier()->destroyOntimeIntervalThreads(requestP->subscriptionId.get());

      /* Build conditions array (including side-effect notifications and threads creation)
       * In order to do so, we have to create and EntityIdVector and AttributeList from sub
       * document, given the processConditionVector() signature */
       EntityIdVector enV = subToEntityIdVector(sub);
       AttributeList attrL = subToAttributeList(sub);

       BSONArray conds = processConditionVector(&requestP->notifyConditionVector,
                                                enV,
                                                attrL,
                                                requestP->subscriptionId.get(),
                                                C_STR_FIELD(sub, CSUB_REFERENCE),
                                                &notificationDone,
                                                inFormat,
                                                tenant);
       newSub.appendArray(CSUB_CONDITIONS, conds);

       /* Remove EntityIdVector and AttributeList dynamic memory */
       enV.release();
       attrL.release();
  }

  int count = sub.hasField(CSUB_COUNT) ? sub.getIntField(CSUB_COUNT) : 0;

  /* Last notification */
  if (notificationDone) {
      newSub.append(CSUB_LASTNOTIFICATION, getCurrentTime());
      newSub.append(CSUB_COUNT, count + 1);
  }
  else {
      /* The hasField check is needed due to lastNotification/count could not be present in the original doc */
      if (sub.hasField(CSUB_LASTNOTIFICATION)) {
          newSub.append(CSUB_LASTNOTIFICATION, sub.getIntField(CSUB_LASTNOTIFICATION));
      }
      if (sub.hasField(CSUB_COUNT)) {
          newSub.append(CSUB_COUNT, count);
      }
  }

  /* Adding format to use in notifications */
  newSub.append(CSUB_FORMAT, std::string(formatToString(inFormat)));

  /* Update document in MongoDB */
  BSONObj update = newSub.obj();
  try
  {
      LM_T(LmtMongo, ("update() in '%s' collection _id '%s': %s}", getSubscribeContextCollectionName(tenant).c_str(),
                         requestP->subscriptionId.get().c_str(),
                         update.toString().c_str()));
      mongoSemTake(__FUNCTION__, "update in SubscribeContextCollection");
      connection->update(getSubscribeContextCollectionName(tenant).c_str(), BSON("_id" << OID(requestP->subscriptionId.get())), update);
      mongoSemGive(__FUNCTION__, "update in SubscribeContextCollection");
      LM_I(("Database Operation Successful (update _id: %s, %s)", requestP->subscriptionId.get().c_str(), update.toString().c_str()));
  }
  catch (const DBException &e)
  {
      mongoSemGive(__FUNCTION__, "update in SubscribeContextCollection (mongo db exception)");
      reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo db exception)");
      responseP->subscribeError.errorCode.fill(SccReceiverInternalError,
                                               std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
                                               " - update() _id: " + requestP->subscriptionId.get().c_str() +
                                               " - update() doc: " + update.toString() +
                                               " - exception: " + e.what());

      LM_E(("Database Error (%s)", responseP->subscribeError.errorCode.details.c_str()));
      return SccOk;
  }
  catch (...)
  {
      mongoSemGive(__FUNCTION__, "update in SubscribeContextCollection (mongo generic exception)");
      reqSemGive(__FUNCTION__, "ngsi10 update subscription request (mongo generic exception)");
      responseP->subscribeError.errorCode.fill(SccReceiverInternalError,
                                               std::string("collection: ") + getSubscribeContextCollectionName(tenant).c_str() +
                                               " - update() _id: " + requestP->subscriptionId.get().c_str() +
                                               " - update() doc: " + update.toString() +
                                               " - exception: " + "generic");

      LM_E(("Database Error (%s)", responseP->subscribeError.errorCode.details.c_str()));
      return SccOk;
  }

  /* Duration and throttling are optional parameters, they are only added in the case they
   * was used for update */
  if (!requestP->duration.isEmpty()) {      
      responseP->subscribeResponse.duration = requestP->duration;
  }
  if (!requestP->throttling.isEmpty()) {      
      responseP->subscribeResponse.throttling = requestP->throttling;
  }  
  responseP->subscribeResponse.subscriptionId = requestP->subscriptionId;

  reqSemGive(__FUNCTION__, "ngsi10 update subscription request");
  return SccOk;
}
Example #6
0
    void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) {
        verify( candidateChunks );

        //
        // 1. Check whether there is any sharded collection to be balanced by querying
        // the ShardsNS::collections collection
        //

        auto_ptr<DBClientCursor> cursor = conn.query(CollectionType::ConfigNS, BSONObj());

        if ( NULL == cursor.get() ) {
            warning() << "could not query " << CollectionType::ConfigNS
                      << " while trying to balance" << endl;
            return;
        }

        vector< string > collections;
        while ( cursor->more() ) {
            BSONObj col = cursor->nextSafe();

            // sharded collections will have a shard "key".
            if ( ! col[CollectionType::keyPattern()].eoo() &&
                 ! col[CollectionType::noBalance()].trueValue() ){
                collections.push_back( col[CollectionType::ns()].String() );
            }
            else if( col[CollectionType::noBalance()].trueValue() ){
                LOG(1) << "not balancing collection " << col[CollectionType::ns()].String()
                       << ", explicitly disabled" << endl;
            }

        }
        cursor.reset();

        if ( collections.empty() ) {
            LOG(1) << "no collections to balance" << endl;
            return;
        }

        //
        // 2. Get a list of all the shards that are participating in this balance round
        // along with any maximum allowed quotas and current utilization. We get the
        // latter by issuing db.serverStatus() (mem.mapped) to all shards.
        //
        // TODO: skip unresponsive shards and mark information as stale.
        //

        ShardInfoMap shardInfo;
        Status loadStatus = DistributionStatus::populateShardInfoMap(&shardInfo);

        if (!loadStatus.isOK()) {
            warning() << "failed to load shard metadata" << causedBy(loadStatus) << endl;
            return;
        }

        if (shardInfo.size() < 2) {
            LOG(1) << "can't balance without more active shards" << endl;
            return;
        }

        OCCASIONALLY warnOnMultiVersion( shardInfo );

        //
        // 3. For each collection, check if the balancing policy recommends moving anything around.
        //

        for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) {
            const string& ns = *it;

            OwnedPointerMap<string, OwnedPointerVector<ChunkType> > shardToChunksMap;
            cursor = conn.query(ChunkType::ConfigNS,
                                QUERY(ChunkType::ns(ns)).sort(ChunkType::min()));

            set<BSONObj> allChunkMinimums;

            while ( cursor->more() ) {
                BSONObj chunkDoc = cursor->nextSafe().getOwned();

                auto_ptr<ChunkType> chunk(new ChunkType());
                string errmsg;
                if (!chunk->parseBSON(chunkDoc, &errmsg)) {
                    error() << "bad chunk format for " << chunkDoc
                            << ": " << errmsg << endl;
                    return;
                }

                allChunkMinimums.insert(chunk->getMin().getOwned());
                OwnedPointerVector<ChunkType>*& chunkList =
                        shardToChunksMap.mutableMap()[chunk->getShard()];

                if (chunkList == NULL) {
                    chunkList = new OwnedPointerVector<ChunkType>();
                }

                chunkList->mutableVector().push_back(chunk.release());
            }
            cursor.reset();

            if (shardToChunksMap.map().empty()) {
                LOG(1) << "skipping empty collection (" << ns << ")";
                continue;
            }

            for (ShardInfoMap::const_iterator i = shardInfo.begin(); i != shardInfo.end(); ++i) {
                // this just makes sure there is an entry in shardToChunksMap for every shard
                OwnedPointerVector<ChunkType>*& chunkList =
                        shardToChunksMap.mutableMap()[i->first];

                if (chunkList == NULL) {
                    chunkList = new OwnedPointerVector<ChunkType>();
                }
            }

            DistributionStatus status(shardInfo, shardToChunksMap.map());

            // load tags
            Status result = clusterCreateIndex(TagsType::ConfigNS,
                                               BSON(TagsType::ns() << 1 << TagsType::min() << 1),
                                               true, // unique
                                               WriteConcernOptions::AllConfigs,
                                               NULL);

            if ( !result.isOK() ) {
                warning() << "could not create index tags_1_min_1: " << result.reason() << endl;
                continue;
            }

            cursor = conn.query(TagsType::ConfigNS,
                                QUERY(TagsType::ns(ns)).sort(TagsType::min()));

            vector<TagRange> ranges;

            while ( cursor->more() ) {
                BSONObj tag = cursor->nextSafe();
                TagRange tr(tag[TagsType::min()].Obj().getOwned(),
                            tag[TagsType::max()].Obj().getOwned(),
                            tag[TagsType::tag()].String());
                ranges.push_back(tr);
                uassert(16356,
                        str::stream() << "tag ranges not valid for: " << ns,
                        status.addTagRange(tr) );

            }
            cursor.reset();

            DBConfigPtr cfg = grid.getDBConfig( ns );
            if ( !cfg ) {
                warning() << "could not load db config to balance " << ns << " collection" << endl;
                continue;
            }

            // This line reloads the chunk manager once if this process doesn't know the collection
            // is sharded yet.
            ChunkManagerPtr cm = cfg->getChunkManagerIfExists( ns, true );
            if ( !cm ) {
                warning() << "could not load chunks to balance " << ns << " collection" << endl;
                continue;
            }

            // loop through tags to make sure no chunk spans tags; splits on tag min. for all chunks
            bool didAnySplits = false;
            for ( unsigned i = 0; i < ranges.size(); i++ ) {
                BSONObj min = ranges[i].min;

                min = cm->getShardKey().extendRangeBound( min, false );

                if ( allChunkMinimums.count( min ) > 0 )
                    continue;

                didAnySplits = true;

                log() << "ns: " << ns << " need to split on "
                      << min << " because there is a range there" << endl;

                ChunkPtr c = cm->findIntersectingChunk( min );

                vector<BSONObj> splitPoints;
                splitPoints.push_back( min );

                BSONObj res;
                if ( !c->multiSplit( splitPoints, res ) ) {
                    error() << "split failed: " << res << endl;
                }
                else {
                    LOG(1) << "split worked: " << res << endl;
                }
                break;
            }

            if ( didAnySplits ) {
                // state change, just wait till next round
                continue;
            }

            CandidateChunk* p = _policy->balance( ns, status, _balancedLastTime );
            if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) );
        }
    }
Example #7
0
/**
 * Updates the remote cached version on the remote shard host (primary, in the case of replica
 * sets) if needed with a fully-qualified shard version for the given namespace:
 *   config server(s) + shard name + shard version
 *
 * If no remote cached version has ever been set, an initial shard version is sent.
 *
 * If the namespace is empty and no version has ever been sent, the config server + shard name
 * is sent to the remote shard host to initialize the connection as coming from mongos.
 * NOTE: This initialization is *best-effort only*.  Operations which wish to correctly version
 * must send the namespace.
 *
 * Config servers are special and are not (unless otherwise a shard) kept up to date with this
 * protocol.  This is safe so long as config servers only contain unversioned collections.
 *
 * It is an error to call checkShardVersion with an unversionable connection (isVersionableCB).
 *
 * @return true if we contacted the remote host
 */
bool checkShardVersion(DBClientBase* conn_in,
                       const string& ns,
                       ChunkManagerPtr refManager,
                       bool authoritative,
                       int tryNumber) {
    // TODO: cache, optimize, etc...

    // Empty namespaces are special - we require initialization but not versioning
    if (ns.size() == 0) {
        return initShardVersionEmptyNS(conn_in);
    }

    auto status = grid.catalogCache()->getDatabase(nsToDatabase(ns));
    if (!status.isOK()) {
        return false;
    }

    shared_ptr<DBConfig> conf = status.getValue();

    DBClientBase* conn = getVersionable(conn_in);
    verify(conn);  // errors thrown above

    unsigned long long officialSequenceNumber = 0;

    ShardPtr primary;
    ChunkManagerPtr manager;
    if (authoritative)
        conf->getChunkManagerIfExists(ns, true);

    conf->getChunkManagerOrPrimary(ns, manager, primary);

    if (manager) {
        officialSequenceNumber = manager->getSequenceNumber();
    }

    const auto shard = grid.shardRegistry()->getShard(conn->getServerAddress());
    uassert(ErrorCodes::ShardNotFound,
            str::stream() << conn->getServerAddress() << " is not recognized as a shard",
            shard);

    // Check this manager against the reference manager
    if (manager) {
        if (refManager && !refManager->compatibleWith(*manager, shard->getId())) {
            const ChunkVersion refVersion(refManager->getVersion(shard->getId()));
            const ChunkVersion currentVersion(manager->getVersion(shard->getId()));

            string msg(str::stream()
                       << "manager (" << currentVersion.toString() << " : "
                       << manager->getSequenceNumber() << ") "
                       << "not compatible with reference manager (" << refVersion.toString()
                       << " : " << refManager->getSequenceNumber() << ") "
                       << "on shard " << shard->getId() << " (" << shard->getConnString().toString()
                       << ")");

            throw SendStaleConfigException(ns, msg, refVersion, currentVersion);
        }
    } else if (refManager) {
        string msg(str::stream() << "not sharded ("
                                 << ((manager.get() == 0) ? string("<none>") : str::stream()
                                             << manager->getSequenceNumber())
                                 << ") but has reference manager ("
                                 << refManager->getSequenceNumber() << ") "
                                 << "on conn " << conn->getServerAddress() << " ("
                                 << conn_in->getServerAddress() << ")");

        throw SendStaleConfigException(
            ns, msg, refManager->getVersion(shard->getId()), ChunkVersion::UNSHARDED());
    }

    // Do not send setShardVersion to collections on the config servers - this causes problems
    // when config servers are also shards and get SSV with conflicting names.
    // TODO: Make config servers regular shards
    if (primary && primary->getId() == "config") {
        return false;
    }

    // Has the ChunkManager been reloaded since the last time we updated the shard version over
    // this connection?  If we've never updated the shard version, do so now.
    unsigned long long sequenceNumber = 0;
    if (connectionShardStatus.getSequence(conn, ns, &sequenceNumber)) {
        if (sequenceNumber == officialSequenceNumber) {
            return false;
        }
    }

    ChunkVersion version = ChunkVersion(0, 0, OID());
    if (manager) {
        version = manager->getVersion(shard->getId());
    }

    LOG(1) << "setting shard version of " << version << " for " << ns << " on shard "
           << shard->toString();

    LOG(3) << "last version sent with chunk manager iteration " << sequenceNumber
           << ", current chunk manager iteration is " << officialSequenceNumber;

    BSONObj result;
    if (setShardVersion(*conn,
                        ns,
                        grid.catalogManager()->connectionString().toString(),
                        version,
                        manager.get(),
                        authoritative,
                        result)) {
        LOG(1) << "      setShardVersion success: " << result;
        connectionShardStatus.setSequence(conn, ns, officialSequenceNumber);
        return true;
    }

    LOG(1) << "       setShardVersion failed!\n" << result << endl;

    if (result["need_authoritative"].trueValue())
        massert(10428, "need_authoritative set but in authoritative mode already", !authoritative);

    if (!authoritative) {
        // use the original connection and get a fresh versionable connection
        // since conn can be invalidated (or worse, freed) after the failure
        checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
        return true;
    }

    if (result["reloadConfig"].trueValue()) {
        if (result["version"].timestampTime() == Date_t()) {
            warning() << "reloading full configuration for " << conf->name()
                      << ", connection state indicates significant version changes";

            // reload db
            conf->reload();
        } else {
            // reload config
            conf->getChunkManager(ns, true);
        }
    }

    const int maxNumTries = 7;
    if (tryNumber < maxNumTries) {
        LOG(tryNumber < (maxNumTries / 2) ? 1 : 0)
            << "going to retry checkShardVersion shard: " << shard->toString() << " " << result;
        sleepmillis(10 * tryNumber);
        // use the original connection and get a fresh versionable connection
        // since conn can be invalidated (or worse, freed) after the failure
        checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
        return true;
    }

    string errmsg = str::stream() << "setShardVersion failed shard: " << shard->toString() << " "
                                  << result;
    log() << "     " << errmsg << endl;
    massert(10429, errmsg, 0);
    return true;
}
Example #8
0
    /**
     * @return true if had to do something
     */
    bool checkShardVersion( DBClientBase& conn_in , const string& ns , bool authoritative , int tryNumber ) {
        // TODO: cache, optimize, etc...

        WriteBackListener::init( conn_in );

        DBConfigPtr conf = grid.getDBConfig( ns );
        if ( ! conf )
            return false;

        DBClientBase* conn = getVersionable( &conn_in );
        assert(conn); // errors thrown above

        unsigned long long officialSequenceNumber = 0;

        ChunkManagerPtr manager;
        const bool isSharded = conf->isSharded( ns );
        if ( isSharded ) {
            manager = conf->getChunkManagerIfExists( ns , authoritative );
            // It's possible the chunk manager was reset since we checked whether sharded was true,
            // so must check this here.
            if( manager ) officialSequenceNumber = manager->getSequenceNumber();
        }

        // has the ChunkManager been reloaded since the last time we updated the connection-level version?
        // (ie., last time we issued the setShardVersions below)
        unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
        if ( sequenceNumber == officialSequenceNumber ) {
            return false;
        }


        ShardChunkVersion version = 0;
        if ( isSharded && manager ) {
            version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
        }

        if( version == 0 ){
            LOG(0) << "resetting shard version of " << ns << " on " << conn->getServerAddress() << ", " <<
                      ( ! isSharded ? "no longer sharded" :
                      ( ! manager ? "no chunk manager found" :
                                    "version is zero" ) ) << endl;
        }


        LOG(2) << " have to set shard version for conn: " << conn << " ns:" << ns
               << " my last seq: " << sequenceNumber << "  current: " << officialSequenceNumber
               << " version: " << version << " manager: " << manager.get()
               << endl;

        BSONObj result;
        if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
            // success!
            LOG(1) << "      setShardVersion success: " << result << endl;
            connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
            return true;
        }

        LOG(1) << "       setShardVersion failed!\n" << result << endl;

        if ( result["need_authoritative"].trueValue() )
            massert( 10428 ,  "need_authoritative set but in authoritative mode already" , ! authoritative );

        if ( ! authoritative ) {
            checkShardVersion( *conn , ns , 1 , tryNumber + 1 );
            return true;
        }
        
        if ( result["reloadConfig"].trueValue() ) {
            if( result["version"].timestampTime() == 0 ){
                // reload db
                conf->reload();
            }
            else {
                // reload config
                conf->getChunkManager( ns , true );
            }
        }

        const int maxNumTries = 7;
        if ( tryNumber < maxNumTries ) {
            LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 ) 
                << "going to retry checkShardVersion host: " << conn->getServerAddress() << " " << result << endl;
            sleepmillis( 10 * tryNumber );
            checkShardVersion( *conn , ns , true , tryNumber + 1 );
            return true;
        }
        
        string errmsg = str::stream() << "setShardVersion failed host: " << conn->getServerAddress() << " " << result;
        log() << "     " << errmsg << endl;
        massert( 10429 , errmsg , 0 );
        return true;
    }
Example #9
0
    void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) {
        assert( candidateChunks );

        //
        // 1. Check whether there is any sharded collection to be balanced by querying
        // the ShardsNS::collections collection
        //

        auto_ptr<DBClientCursor> cursor = conn.query( ShardNS::collection , BSONObj() );
        vector< string > collections;
        while ( cursor->more() ) {
            BSONObj col = cursor->nextSafe();

            // sharded collections will have a shard "key".
            if ( ! col["key"].eoo() )
                collections.push_back( col["_id"].String() );
        }
        cursor.reset();

        if ( collections.empty() ) {
            LOG(1) << "no collections to balance" << endl;
            return;
        }

        //
        // 2. Get a list of all the shards that are participating in this balance round
        // along with any maximum allowed quotas and current utilization. We get the
        // latter by issuing db.serverStatus() (mem.mapped) to all shards.
        //
        // TODO: skip unresponsive shards and mark information as stale.
        //

        vector<Shard> allShards;
        Shard::getAllShards( allShards );
        if ( allShards.size() < 2) {
            LOG(1) << "can't balance without more active shards" << endl;
            return;
        }

        map< string, BSONObj > shardLimitsMap;
        for ( vector<Shard>::const_iterator it = allShards.begin(); it != allShards.end(); ++it ) {
            const Shard& s = *it;
            ShardStatus status = s.getStatus();

            BSONObj limitsObj = BSON( ShardFields::maxSize( s.getMaxSize() ) <<
                                      LimitsFields::currSize( status.mapped() ) <<
                                      ShardFields::draining( s.isDraining() )  <<
                                      LimitsFields::hasOpsQueued( status.hasOpsQueued() )
                                    );

            shardLimitsMap[ s.getName() ] = limitsObj;
        }

        //
        // 3. For each collection, check if the balancing policy recommends moving anything around.
        //

        for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) {
            const string& ns = *it;

            map< string,vector<BSONObj> > shardToChunksMap;
            cursor = conn.query( ShardNS::chunk , QUERY( "ns" << ns ).sort( "min" ) );
            while ( cursor->more() ) {
                BSONObj chunk = cursor->nextSafe();
                if ( chunk["jumbo"].trueValue() )
                    continue;
                vector<BSONObj>& chunks = shardToChunksMap[chunk["shard"].String()];
                chunks.push_back( chunk.getOwned() );
            }
            cursor.reset();

            if (shardToChunksMap.empty()) {
                LOG(1) << "skipping empty collection (" << ns << ")";
                continue;
            }

            for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ) {
                // this just makes sure there is an entry in shardToChunksMap for every shard
                Shard s = *i;
                shardToChunksMap[s.getName()].size();
            }

            CandidateChunk* p = _policy->balance( ns , shardLimitsMap , shardToChunksMap , _balancedLastTime );
            if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) );
        }
    }
Example #10
0
    void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) {
        verify( candidateChunks );

        //
        // 1. Check whether there is any sharded collection to be balanced by querying
        // the ShardsNS::collections collection
        //

        auto_ptr<DBClientCursor> cursor = conn.query(CollectionType::ConfigNS, BSONObj());
        vector< string > collections;
        while ( cursor->more() ) {
            BSONObj col = cursor->nextSafe();

            // sharded collections will have a shard "key".
            if ( ! col[CollectionType::keyPattern()].eoo() &&
                 ! col[CollectionType::noBalance()].trueValue() ){
                collections.push_back( col[CollectionType::ns()].String() );
            }
            else if( col[CollectionType::noBalance()].trueValue() ){
                LOG(1) << "not balancing collection " << col[CollectionType::ns()].String()
                       << ", explicitly disabled" << endl;
            }

        }
        cursor.reset();

        if ( collections.empty() ) {
            LOG(1) << "no collections to balance" << endl;
            return;
        }

        //
        // 2. Get a list of all the shards that are participating in this balance round
        // along with any maximum allowed quotas and current utilization. We get the
        // latter by issuing db.serverStatus() (mem.mapped) to all shards.
        //
        // TODO: skip unresponsive shards and mark information as stale.
        //

        vector<Shard> allShards;
        Shard::getAllShards( allShards );
        if ( allShards.size() < 2) {
            LOG(1) << "can't balance without more active shards" << endl;
            return;
        }
        
        ShardInfoMap shardInfo;
        for ( vector<Shard>::const_iterator it = allShards.begin(); it != allShards.end(); ++it ) {
            const Shard& s = *it;
            ShardStatus status = s.getStatus();
            shardInfo[ s.getName() ] = ShardInfo( s.getMaxSize(),
                                                  status.mapped(),
                                                  s.isDraining(),
                                                  status.hasOpsQueued(),
                                                  s.tags()
                                                  );
        }

        //
        // 3. For each collection, check if the balancing policy recommends moving anything around.
        //

        for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) {
            const string& ns = *it;

            map< string,vector<BSONObj> > shardToChunksMap;
            cursor = conn.query(ChunkType::ConfigNS,
                                QUERY(ChunkType::ns(ns)).sort(ChunkType::min()));

            set<BSONObj> allChunkMinimums;

            while ( cursor->more() ) {
                BSONObj chunk = cursor->nextSafe().getOwned();
                vector<BSONObj>& chunks = shardToChunksMap[chunk[ChunkType::shard()].String()];
                allChunkMinimums.insert( chunk[ChunkType::min()].Obj() );
                chunks.push_back( chunk );
            }
            cursor.reset();

            if (shardToChunksMap.empty()) {
                LOG(1) << "skipping empty collection (" << ns << ")";
                continue;
            }

            for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ) {
                // this just makes sure there is an entry in shardToChunksMap for every shard
                Shard s = *i;
                shardToChunksMap[s.getName()].size();
            }

            DistributionStatus status( shardInfo, shardToChunksMap );

            // load tags
            conn.ensureIndex(TagsType::ConfigNS,
                             BSON(TagsType::ns() << 1 << TagsType::min() << 1),
                             true);

            cursor = conn.query(TagsType::ConfigNS,
                                QUERY(TagsType::ns(ns)).sort(TagsType::min()));

            vector<TagRange> ranges;

            while ( cursor->more() ) {
                BSONObj tag = cursor->nextSafe();
                TagRange tr(tag[TagsType::min()].Obj().getOwned(),
                            tag[TagsType::max()].Obj().getOwned(),
                            tag[TagsType::tag()].String());
                ranges.push_back(tr);
                uassert(16356,
                        str::stream() << "tag ranges not valid for: " << ns,
                        status.addTagRange(tr) );

            }
            cursor.reset();

            DBConfigPtr cfg = grid.getDBConfig( ns );
            verify( cfg );
            ChunkManagerPtr cm = cfg->getChunkManager( ns );
            verify( cm );

            // loop through tags to make sure no chunk spans tags; splits on tag min. for all chunks
            bool didAnySplits = false;
            for ( unsigned i = 0; i < ranges.size(); i++ ) {
                BSONObj min = ranges[i].min;

                min = cm->getShardKey().extendRangeBound( min, false );

                if ( allChunkMinimums.count( min ) > 0 )
                    continue;

                didAnySplits = true;

                log() << "ns: " << ns << " need to split on "
                      << min << " because there is a range there" << endl;

                ChunkPtr c = cm->findIntersectingChunk( min );

                vector<BSONObj> splitPoints;
                splitPoints.push_back( min );

                BSONObj res;
                if ( !c->multiSplit( splitPoints, res ) ) {
                    error() << "split failed: " << res << endl;
                }
                else {
                    LOG(1) << "split worked: " << res << endl;
                }
                break;
            }

            if ( didAnySplits ) {
                // state change, just wait till next round
                continue;
            }

            CandidateChunk* p = _policy->balance( ns, status, _balancedLastTime );
            if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) );
        }
    }
/* ****************************************************************************
*
* prepareDatabase -
*
* This function is called before every test, to populate some information in the
* entities collection.
*/
static void prepareDatabase(bool extraEntities = false) {

  /* Set database */
  setupDatabase();

  DBClientBase* connection = getMongoConnection();

  /* We create the following entities:
   *
   * - E1
   *     S: "running"
   *     N: 26.5
   *     D: ISODate("2017-06-17T07:12:25.823Z")
   * - E2
   *     S: "running"
   *     N: 27
   *     D: ISODate("2017-06-17T07:21:24.238Z")
   * - E3
   *     S: "shutdown"
   *     N: 31
   *     D: ISODate("2017-06-17T08:19:12.231Z")
   * - E4
   *     S: "error"
   *     N: 17.8
   *     D: ISODate("2017-06-17T07:22:43.112Z")
   * - E5
   *     S: "shutdown"
   *     N: 24
   *     D: ISODate("2017-06-17T07:10:12.328Z")
   * - C1
   *     colour=black,white
   * - C2
   *     colour=red,blue
   * - C3
   *     colour=red, blue
   */

  // FIXME: D will be set with dates once https://github.com/telefonicaid/fiware-orion/issues/1039 implementation
  // gets addressed

  BSONObj en1 = BSON("_id" << BSON("id" << "E1" << "type" << "T") <<
                     "attrNames" << BSON_ARRAY("S" << "N" << "D") <<
                     "attrs" << BSON(
                        "S" << BSON("type" << "T" << "value" << "running") <<
                        "N" << BSON("type" << "T" << "value" << 26.5) <<
                        "D" << BSON("type" << "T" << "value" << "")
                        )
                    );

  BSONObj en2 = BSON("_id" << BSON("id" << "E2" << "type" << "T") <<
                     "attrNames" << BSON_ARRAY("S" << "N" << "D") <<
                     "attrs" << BSON(
                        "S" << BSON("type" << "T" << "value" << "running") <<
                        "N" << BSON("type" << "T" << "value" << 27) <<
                        "D" << BSON("type" << "T" << "value" << "")
                        )
                    );

  BSONObj en3 = BSON("_id" << BSON("id" << "E3" << "type" << "T") <<
                     "attrNames" << BSON_ARRAY("S" << "N" << "D") <<
                     "attrs" << BSON(
                        "S" << BSON("type" << "T" << "value" << "shutdown") <<
                        "N" << BSON("type" << "T" << "value" << 31) <<
                        "D" << BSON("type" << "T" << "value" << "")
                        )
                    );

  BSONObj en4 = BSON("_id" << BSON("id" << "E4" << "type" << "T") <<
                     "attrNames" << BSON_ARRAY("S" << "N" << "D") <<
                     "attrs" << BSON(
                        "S" << BSON("type" << "T" << "value" << "error") <<
                        "N" << BSON("type" << "T" << "value" << 17.8) <<
                        "D" << BSON("type" << "T" << "value" << "")
                        )
                    );

  BSONObj en5 = BSON("_id" << BSON("id" << "E5" << "type" << "T") <<
                     "attrNames" << BSON_ARRAY("S" << "N" << "D") <<
                     "attrs" << BSON(
                        "S" << BSON("type" << "T" << "value" << "shutdown") <<
                        "N" << BSON("type" << "T" << "value" << 24) <<
                        "D" << BSON("type" << "T" << "value" << "")
                        )
                    );

  BSONObj c1 = BSON("_id" << BSON("id" << "C1" << "type" << "T") <<
                     "attrNames" << BSON_ARRAY("colour") <<
                     "attrs" << BSON(
                        "colour" << BSON("type" << "T" << "value" << "black,white")
                        )
                    );

  BSONObj c2 = BSON("_id" << BSON("id" << "C2" << "type" << "T") <<
                     "attrNames" << BSON_ARRAY("colour") <<
                     "attrs" << BSON(
                        "colour" << BSON("type" << "T" << "value" << "red,blue")
                        )
                    );

  BSONObj c3 = BSON("_id" << BSON("id" << "C3" << "type" << "T") <<
                     "attrNames" << BSON_ARRAY("colour") <<
                     "attrs" << BSON(
                        "colour" << BSON("type" << "T" << "value" << "black, white")
                        )
                    );

  connection->insert(ENTITIES_COLL, en1);
  connection->insert(ENTITIES_COLL, en2);
  connection->insert(ENTITIES_COLL, en3);
  connection->insert(ENTITIES_COLL, en4);
  connection->insert(ENTITIES_COLL, en5);
  connection->insert(ENTITIES_COLL, c1);
  connection->insert(ENTITIES_COLL, c2);
  connection->insert(ENTITIES_COLL, c3);

  if (extraEntities)
  {
    /* Adding additional entities, used in some tests
     * - E6
     *     N: 26.5
     * - E7 (no type, "")
     *     N: 27
     * - E8 (no type, <none>)
     *     N: 27
     */

    // FIXME: D will be set with dates once https://github.com/telefonicaid/fiware-orion/issues/1039 implementation
    // gets addressed

    BSONObj en6 = BSON("_id" << BSON("id" << "E6" << "type" << "T") <<
                       "attrNames" << BSON_ARRAY("S" << "N") <<
                       "attrs" << BSON(
                          "N" << BSON("type" << "T" << "value" << 26.5)
                          )
                      );

    BSONObj en7 = BSON("_id" << BSON("id" << "E7" << "type" << "") <<
                       "attrNames" << BSON_ARRAY("S" << "N") <<
                       "attrs" << BSON(
                          "N" << BSON("type" << "T" << "value" << 27)
                          )
                      );

    BSONObj en8 = BSON("_id" << BSON("id" << "E8") <<
                       "attrNames" << BSON_ARRAY("S" << "N") <<
                       "attrs" << BSON(
                          "N" << BSON("type" << "T" << "value" << 27)
                          )
                      );

    connection->insert(ENTITIES_COLL, en6);
    connection->insert(ENTITIES_COLL, en7);
    connection->insert(ENTITIES_COLL, en8);

  }

}
/* ****************************************************************************
*
* associationsQuery -
*/
static bool associationsQuery
(
  EntityIdVector*                  enV,
  AttributeList*                   attrL,
  const std::string&               scope,
  MetadataVector*                  mdV,
  std::string*                     err,
  const std::string&               tenant,
  int                              offset,
  int                              limit,
  bool                             details,
  const std::vector<std::string>&  servicePathV
)
{
    DBClientBase* connection = getMongoConnection();

    /* Note that SCOPE_VALUE_ASSOC_SOURCE means that the argument is a target (so we use ASSOC_TARGET_ENT and
     * ASSOC_ATTRS_TARGET in the query), while SCOPE_VALUE_ASSOC_TARGET means that the argument is a source (so we
     * use ASSOC_SOURCE_ENT and ASSOC_ATTRS_source in the query) */
    BSONObjBuilder queryB;

    /* Build query (entity part) */
    BSONArrayBuilder enArray;
    for (unsigned int ix = 0; ix < enV->size(); ++ix) {
        enArray.append(BSON(ASSOC_ENT_ID << enV->get(ix)->id << ASSOC_ENT_TYPE << enV->get(ix)->type));
    }
    BSONObj queryEn;
    if (scope == SCOPE_VALUE_ASSOC_SOURCE) {
        queryB.append(ASSOC_TARGET_ENT, BSON("$in" << enArray.arr()));

    }
    else {  // SCOPE_VALUE_ASSOC_TARGET
        queryB.append(ASSOC_SOURCE_ENT, BSON("$in" << enArray.arr()));
    }

    /* Build query (attribute part) */
    BSONArrayBuilder attrArray;
    for (unsigned int ix = 0; ix < attrL->size() ; ++ix) {
        attrArray.append(attrL->get(ix));
    }
    std::string attrField;
    if (scope == SCOPE_VALUE_ASSOC_SOURCE) {
        attrField = ASSOC_ATTRS "." ASSOC_ATTRS_TARGET;
    }
    else {  // SCOPE_VALUE_ASSOC_TARGET
        attrField = ASSOC_ATTRS "." ASSOC_ATTRS_SOURCE;
    }

    // If there are no attributes specified we want them all
    if (attrArray.arrSize() != 0)
    {
      queryB.append(attrField, BSON("$in" << attrArray.arr()));
    }

    /* Do the query in MongoDB */
    auto_ptr<DBClientCursor> cursor;
    Query                    query(queryB.obj());
    Query                    sortCriteria  = query.sort(BSON("_id" << 1));
    
    try
    {
        LM_T(LmtMongo, ("query() in '%s' collection: '%s'", getAssociationsCollectionName(tenant).c_str(), query.toString().c_str()));

        mongoSemTake(__FUNCTION__, "query in AssociationsCollection");
        cursor = connection->query(getAssociationsCollectionName(tenant).c_str(), query, limit, offset);

        /*
         * We have observed that in some cases of DB errors (e.g. the database daemon is down) instead of
         * raising an exception, the query() method sets the cursor to NULL. In this case, we raise the
         * exception ourselves
         */
        if (cursor.get() == NULL) {
            throw DBException("Null cursor from mongo (details on this is found in the source code)", 0);
        }
        mongoSemGive(__FUNCTION__, "query in AssociationsCollection");
        LM_I(("Database Operation Successful (%s)", query.toString().c_str()));
    }
    catch (const DBException &e)
    {
        mongoSemGive(__FUNCTION__, "query in AssociationsCollection (DBException)");
        *err = std::string("collection: ") + getAssociationsCollectionName(tenant).c_str() +
                " - query(): " + query.toString() +
                " - exception: " + e.what();
        LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), err->c_str()));
        return false;
    }
    catch (...)
    {
        mongoSemGive(__FUNCTION__, "query in AssociationsCollection (Generic Exception)");
        *err = std::string("collection: ") + getAssociationsCollectionName(tenant).c_str() +
                " - query(): " + query.toString() +
                " - exception: " + "generic";
        LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), err->c_str()));
        return false;
    }

    /* Process query result */
    while (cursor->more()) {
        BSONObj r = cursor->next();
        LM_T(LmtMongo, ("retrieved document: '%s'", r.toString().c_str()));

        std::string name      = STR_FIELD(r, "_id");
        std::string srcEnId   = STR_FIELD(r.getField(ASSOC_SOURCE_ENT).embeddedObject(), ASSOC_ENT_ID);
        std::string srcEnType = STR_FIELD(r.getField(ASSOC_SOURCE_ENT).embeddedObject(), ASSOC_ENT_TYPE);
        std::string tgtEnId   = STR_FIELD(r.getField(ASSOC_TARGET_ENT).embeddedObject(), ASSOC_ENT_ID);
        std::string tgtEnType = STR_FIELD(r.getField(ASSOC_TARGET_ENT).embeddedObject(), ASSOC_ENT_TYPE);

        Metadata* md = new Metadata(name, "Association");
        md->association.entityAssociation.source.fill(srcEnId, srcEnType, "false");
        md->association.entityAssociation.target.fill(tgtEnId, tgtEnType, "false");
        
        std::vector<BSONElement> attrs = r.getField(ASSOC_ATTRS).Array();
        for (unsigned int ix = 0; ix < attrs.size(); ++ix) {
            std::string srcAttr = STR_FIELD(attrs[ix].embeddedObject(), ASSOC_ATTRS_SOURCE);
            std::string tgtAttr = STR_FIELD(attrs[ix].embeddedObject(), ASSOC_ATTRS_TARGET);
            AttributeAssociation* attrAssoc = new AttributeAssociation();
            attrAssoc->source = srcAttr;
            attrAssoc->target = tgtAttr;
            md->association.attributeAssociationList.push_back(attrAssoc);
        }

        mdV->push_back(md);
    }

    return true;
}
/* ****************************************************************************
*
* mongoConnect -
*
* Default value for writeConcern == 1 (0: unacknowledged, 1: acknowledged)
*/
static DBClientBase* mongoConnect
(
    const char*  host,
    const char*  db,
    const char*  rplSet,
    const char*  username,
    const char*  passwd,
    bool         multitenant,
    int          writeConcern,
    double       timeout
)
{
    std::string   err;
    DBClientBase* connection = NULL;

    LM_T(LmtMongo, ("Connection info: dbName='%s', rplSet='%s', timeout=%f", db, rplSet, timeout));

    bool connected     = false;
    int  retries       = RECONNECT_RETRIES;

    if (strlen(rplSet) == 0)
    {
        /* Setting the first argument to true is to use autoreconnect */
        connection = new DBClientConnection(true);

        /* Not sure of how to generalize the following code, given that DBClientBase class doesn't have a common connect() method (surprisingly) */
        for (int tryNo = 0; tryNo < retries; ++tryNo)
        {
            if ( ((DBClientConnection*)connection)->connect(host, err))
            {
                connected = true;
                break;
            }

            if (tryNo == 0)
            {
                LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)", retries, RECONNECT_DELAY));
            }
            else
            {
                LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo));
            }

            usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds
        }
    }
    else
    {
        LM_T(LmtMongo, ("Using replica set %s", rplSet));

        // autoReconnect is always on for DBClientReplicaSet connections.
        std::vector<std::string>  hostTokens;
        int components = stringSplit(host, ',', hostTokens);

        std::vector<HostAndPort> rplSetHosts;
        for (int ix = 0; ix < components; ix++)
        {
            LM_T(LmtMongo, ("rplSet host <%s>", hostTokens[ix].c_str()));
            rplSetHosts.push_back(HostAndPort(hostTokens[ix]));
        }

        connection = new DBClientReplicaSet(rplSet, rplSetHosts, timeout);

        /* Not sure of to generalize the following code, given that DBClientBase class hasn't a common connect() method (surprisingly) */
        for (int tryNo = 0; tryNo < retries; ++tryNo)
        {
            if ( ((DBClientReplicaSet*)connection)->connect())
            {
                connected = true;
                break;
            }

            if (tryNo == 0)
            {
                LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d microsecond interval)", retries, RECONNECT_DELAY));
            }
            else
            {
                LM_T(LmtMongo, ("Try %d connecting to mongo failed", tryNo));
            }

            usleep(RECONNECT_DELAY * 1000); // usleep accepts microseconds
        }
    }

    if (connected == false)
    {
        LM_E(("Database Error (connection failed, after %d retries: '%s')", retries, err.c_str()));
        return NULL;
    }

    LM_I(("Successful connection to database"));

    //
    // WriteConcern
    //
    mongo::WriteConcern writeConcernCheck;

    // In legacy driver writeConcern is no longer an int, but a class. We need a small
    // conversion step here
    mongo::WriteConcern wc = writeConcern == 1 ? mongo::WriteConcern::acknowledged : mongo::WriteConcern::unacknowledged;

    connection->setWriteConcern((mongo::WriteConcern) wc);
    writeConcernCheck = (mongo::WriteConcern) connection->getWriteConcern();

    if (writeConcernCheck.nodes() != wc.nodes())
    {
        LM_E(("Database Error (Write Concern not set as desired)"));
        return NULL;
    }
    LM_T(LmtMongo, ("Active DB Write Concern mode: %d", writeConcern));

    /* Authentication is different depending if multiservice is used or not. In the case of not
     * using multiservice, we authenticate in the single-service database. In the case of using
     * multiservice, it isn't a default database that we know at contextBroker start time (when
     * this connection function is invoked) so we authenticate on the admin database, which provides
     * access to any database */
    if (multitenant)
    {
        if (strlen(username) != 0 && strlen(passwd) != 0)
        {
            if (!connection->auth("admin", std::string(username), std::string(passwd), err))
            {
                LM_E(("Database Startup Error (authentication: db='admin', username='%s', password='*****': %s)", username, err.c_str()));
                return NULL;
            }
        }
    }
    else
    {
        if (strlen(db) != 0 && strlen(username) != 0 && strlen(passwd) != 0)
        {
            if (!connection->auth(std::string(db), std::string(username), std::string(passwd), err))
            {
                LM_E(("Database Startup Error (authentication: db='%s', username='%s', password='*****': %s)", db, username, err.c_str()));
                return NULL;
            }
        }
    }

    /* Get mongo version with the 'buildinfo' command */
    BSONObj result;
    std::string extra;
    connection->runCommand("admin", BSON("buildinfo" << 1), result);
    std::string versionString = std::string(result.getStringField("version"));
    if (!versionParse(versionString, mongoVersionMayor, mongoVersionMinor, extra))
    {
        LM_E(("Database Startup Error (invalid version format: %s)", versionString.c_str()));
        return NULL;
    }
    LM_T(LmtMongo, ("mongo version server: %s (mayor: %d, minor: %d, extra: %s)", versionString.c_str(), mongoVersionMayor, mongoVersionMinor, extra.c_str()));

    return connection;
}
Example #14
0
    bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){

        WriteBackListener::init( *conn_in );

        bool ok;
        DBClientBase* conn = NULL;
        try {
            // May throw if replica set primary is down
            conn = getVersionable( conn_in );
            dassert( conn ); // errors thrown above

            BSONObjBuilder cmdBuilder;

            cmdBuilder.append( "setShardVersion" , "" );
            cmdBuilder.appendBool( "init", true );
            cmdBuilder.append( "configdb" , configServer.modelServer() );
            cmdBuilder.appendOID( "serverID" , &serverID );
            cmdBuilder.appendBool( "authoritative" , true );

            BSONObj cmd = cmdBuilder.obj();

            LOG(1) << "initializing shard connection to " << conn->toString() << endl;
            LOG(2) << "initial sharding settings : " << cmd << endl;

            ok = conn->runCommand("admin", cmd, result, 0);
        }
        catch( const DBException& ) {

            if ( conn_in->type() != ConnectionString::SET ) {
                throw;
            }

            // NOTE: Only old-style cluster operations will talk via DBClientReplicaSets - using
            // checkShardVersion is required (which includes initShardVersion information) if these
            // connections are used.

            OCCASIONALLY {
                warning() << "failed to initialize new replica set connection version, "
                          << "will initialize on first use" << endl;
            }

            return true;
        }

        // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
        // Result is false, but will still initialize serverID and configdb
        if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ ||
                                                  result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ ))
        {
            ok = true;
        }

        // Record the connection wire version if sent in the response, initShardVersion is a
        // handshake for mongos->mongod connections.
        if ( !result["minWireVersion"].eoo() ) {

            int minWireVersion = result["minWireVersion"].numberInt();
            int maxWireVersion = result["maxWireVersion"].numberInt();
            conn->setWireVersions( minWireVersion, maxWireVersion );
        }

        LOG(3) << "initial sharding result : " << result << endl;

        return ok;

    }
/* ****************************************************************************
*
* prepareDatabase -
*
* This function is called before every test, to populate some information in the
* entities collection.
*/
static void prepareDatabase(void) {

  /* Set database */
  setupDatabase();

  DBClientBase* connection = getMongoConnection();

  /* We create the following entities:
   *
   * - E1:
   *     A1: val1
   *     A2: (no value)
   * - E2
   *     A3: val3
   *     A4: (no value)
   * - E3
   *     A5: val5
   *     A6: (no value)
   * - E1*:
   *     A1: val1bis2
   * - E1**:
   *     A1: val1
   *     A2: (no value)
   *
   * (*) Means that entity/type is using same name but different type. This is included to check that type is
   *     taken into account.
   *
   * (**)same name but without type
   *
   */

  BSONObj en1 = BSON("_id" << BSON("id" << "E1" << "type" << "T1") <<
                     "attrs" << BSON_ARRAY(
                        BSON("name" << "A1" << "type" << "TA1" << "value" << "val1") <<
                        BSON("name" << "A2" << "type" << "TA2")
                        )
                    );

  BSONObj en2 = BSON("_id" << BSON("id" << "E2" << "type" << "T2") <<
                     "attrs" << BSON_ARRAY(
                        BSON("name" << "A3" << "type" << "TA3" << "value" << "val3") <<
                        BSON("name" << "A4" << "type" << "TA4")
                        )
                    );

  BSONObj en3 = BSON("_id" << BSON("id" << "E3" << "type" << "T3") <<
                     "attrs" << BSON_ARRAY(
                        BSON("name" << "A5" << "type" << "TA5" << "value" << "val5") <<
                        BSON("name" << "A6" << "type" << "TA6")
                        )
                    );

  BSONObj en4 = BSON("_id" << BSON("id" << "E1" << "type" << "T1bis") <<
                     "attrs" << BSON_ARRAY(
                        BSON("name" << "A1" << "type" << "TA1" << "value" << "val1bis2")
                        )
                    );

  BSONObj en1nt = BSON("_id" << BSON("id" << "E1") <<
                     "attrs" << BSON_ARRAY(
                        BSON("name" << "A1" << "type" << "TA1" << "value" << "val1-nt") <<
                        BSON("name" << "A2" << "type" << "TA2")
                        )
                    );

  connection->insert(ENTITIES_COLL, en1);
  connection->insert(ENTITIES_COLL, en2);
  connection->insert(ENTITIES_COLL, en3);
  connection->insert(ENTITIES_COLL, en4);
  connection->insert(ENTITIES_COLL, en1nt);

}
Example #16
0
    void ShardChunkManager::_init( const string& configServer , const string& ns , const string& shardName, ShardChunkManagerPtr oldManager ) {

        // have to get a connection to the config db
        // special case if I'm the configdb since I'm locked and if I connect to myself
        // its a deadlock
        scoped_ptr<ScopedDbConnection> scoped;
        scoped_ptr<DBDirectClient> direct;
        DBClientBase * conn;
        if ( configServer.empty() ) {
            direct.reset( new DBDirectClient() );
            conn = direct.get();
        }
        else {
            scoped.reset( ScopedDbConnection::getInternalScopedDbConnection( configServer, 30.0 ) );
            conn = scoped->get();
        }

        // get this collection's sharding key
        BSONObj collectionDoc = conn->findOne( "config.collections", BSON( "_id" << ns ) );

        if( collectionDoc.isEmpty() ){
            warning() << ns << " does not exist as a sharded collection" << endl;
            return;
        }

        if( collectionDoc["dropped"].Bool() ){
            warning() << ns << " was dropped.  Re-shard collection first." << endl;
            return;
        }

        _fillCollectionKey( collectionDoc );

        map<string,ShardChunkVersion> versionMap;
        versionMap[ shardName ] = _version;
        _collVersion = ShardChunkVersion( 0, OID() );

        // Check to see if we have an old ShardChunkManager to use
        if( oldManager && oldManager->_collVersion.isSet() ){

            versionMap[ shardName ] = oldManager->_version;
            _collVersion = oldManager->_collVersion;
            // TODO: This could be made more efficient if copying not required, but not as
            // frequently reloaded as in mongos.
            _chunksMap = oldManager->_chunksMap;

            LOG(2) << "loading new chunks for collection " << ns << " using old chunk manager w/ version " << _collVersion
                   << " and " << _chunksMap.size() << " chunks" << endl;
        }

        // Attach our config diff tracker to our range map and versions
        SCMConfigDiffTracker differ( shardName );
        differ.attach( ns, _chunksMap, _collVersion, versionMap );

        // Need to do the query ourselves, since we may use direct conns to the db
        Query query = differ.configDiffQuery();
        auto_ptr<DBClientCursor> cursor = conn->query( "config.chunks" , query );

        uassert( 16181, str::stream() << "could not initialize cursor to config server chunks collection for ns " << ns, cursor.get() );

        // Diff tracker should *always* find at least one chunk if collection exists
        int diffsApplied = differ.calculateConfigDiff( *cursor );
        if( diffsApplied > 0 ){

            LOG(2) << "loaded " << diffsApplied << " chunks into new chunk manager for " << ns
                   << " with version " << _collVersion << endl;

            // Save the new version of this shard
            _version = versionMap[ shardName ];
            _fillRanges();

        }
        else if( diffsApplied == 0 ){

            // No chunks were found for the ns
            warning() << "no chunks found when reloading " << ns << ", previous version was " << _collVersion << endl;

            _version = ShardChunkVersion( 0, OID() );
            _collVersion = ShardChunkVersion( 0, OID() );
            _chunksMap.clear();
        }
        else{

            // TODO: make this impossible by making sure we don't migrate / split on this shard during the
            // reload
            // No chunks were found for the ns
            warning() << "invalid chunks found when reloading " << ns << ", previous version was " << _collVersion
                      << ", this should be rare" << endl;

            // Handle the same way as a connectivity error, for now
            // TODO: handle inline
            uassert( 16229,
                     str::stream() << "could not initialize cursor to config server chunks collection for ns "
                                   << ns, cursor.get() );
        }

        if ( scoped.get() )
            scoped->done();

        if ( _chunksMap.empty() )
            log() << "no chunk for collection " << ns << " on shard " << shardName << endl;
    }
/* ****************************************************************************
*
* createEntity -
*/
TEST(mongoNotifyContextRequest, createEntity)
{
    HttpStatusCode         ms;
    NotifyContextRequest   req;
    NotifyContextResponse  res;

    /* Prepare database */
    prepareDatabase();

    /* Forge the request */
    ContextElementResponse cer;
    req.subscriptionId.set("51307b66f481db11bf860001");
    req.originator.set("localhost");
    cer.contextElement.entityId.fill("E10", "T10", "false");
    ContextAttribute ca("A1", "TA1", "new_val");
    cer.contextElement.contextAttributeVector.push_back(&ca);
    cer.statusCode.fill(SccOk);
    req.contextElementResponseVector.push_back(&cer);

    /* Prepare mock */
    TimerMock* timerMock = new TimerMock();
    ON_CALL(*timerMock, getCurrentTime())
            .WillByDefault(Return(1360232700));
    setTimer(timerMock);

    /* Invoke the function in mongoBackend library */
    ms = mongoNotifyContext(&req, &res);

    /* Check response is as expected */
    EXPECT_EQ(SccOk, ms);

    EXPECT_EQ(SccOk, res.responseCode.code);
    EXPECT_EQ("OK", res.responseCode.reasonPhrase);
    EXPECT_EQ(0, res.responseCode.details.size());

    /* Check that every involved collection at MongoDB is as expected */
    /* Note we are using EXPECT_STREQ() for some cases, as Mongo Driver returns const char*, not string
     * objects (see http://code.google.com/p/googletest/wiki/Primer#String_Comparison) */

    DBClientBase* connection = getMongoConnection();

    /* entities collection */
    BSONObj ent;
    std::vector<BSONElement> attrs;
    ASSERT_EQ(6, connection->count(ENTITIES_COLL, BSONObj()));

    ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E1" << "_id.type" << "T1"));
    EXPECT_STREQ("E1", C_STR_FIELD(ent.getObjectField("_id"), "id"));
    EXPECT_STREQ("T1", C_STR_FIELD(ent.getObjectField("_id"), "type"));
    EXPECT_FALSE(ent.hasField("modDate"));
    attrs = ent.getField("attrs").Array();
    ASSERT_EQ(2, attrs.size());
    BSONObj a1 = getAttr(attrs, "A1", "TA1");
    BSONObj a2 = getAttr(attrs, "A2", "TA2");
    EXPECT_STREQ("A1", C_STR_FIELD(a1, "name"));
    EXPECT_STREQ("TA1",C_STR_FIELD(a1, "type"));
    EXPECT_STREQ("val1", C_STR_FIELD(a1, "value"));
    EXPECT_FALSE(a1.hasField("modDate"));
    EXPECT_STREQ("A2", C_STR_FIELD(a2, "name"));
    EXPECT_STREQ("TA2", C_STR_FIELD(a2, "type"));
    EXPECT_FALSE(a2.hasField("value"));
    EXPECT_FALSE(a2.hasField("modDate"));

    ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E2" << "_id.type" << "T2"));
    EXPECT_STREQ("E2", C_STR_FIELD(ent.getObjectField("_id"), "id"));
    EXPECT_STREQ("T2", C_STR_FIELD(ent.getObjectField("_id"), "type"));
    EXPECT_FALSE(ent.hasField("modDate"));
    attrs = ent.getField("attrs").Array();
    ASSERT_EQ(2, attrs.size());
    BSONObj a3 = getAttr(attrs, "A3", "TA3");
    BSONObj a4 = getAttr(attrs, "A4", "TA4");
    EXPECT_STREQ("A3", C_STR_FIELD(a3, "name"));
    EXPECT_STREQ("TA3", C_STR_FIELD(a3, "type"));
    EXPECT_STREQ("val3", C_STR_FIELD(a3, "value"));
    EXPECT_FALSE(a3.hasField("modDate"));
    EXPECT_STREQ("A4", C_STR_FIELD(a4, "name"));
    EXPECT_STREQ("TA4", C_STR_FIELD(a4, "type"));
    EXPECT_FALSE(a4.hasField("value"));
    EXPECT_FALSE(a4.hasField("modDate"));

    ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E3" << "_id.type" << "T3"));
    EXPECT_STREQ("E3", C_STR_FIELD(ent.getObjectField("_id"), "id"));
    EXPECT_STREQ("T3", C_STR_FIELD(ent.getObjectField("_id"), "type"));
    EXPECT_FALSE(ent.hasField("modDate"));
    attrs = ent.getField("attrs").Array();
    ASSERT_EQ(2, attrs.size());
    BSONObj a5 = getAttr(attrs, "A5", "TA5");
    BSONObj a6 = getAttr(attrs, "A6", "TA6");
    EXPECT_STREQ("A5", C_STR_FIELD(a5, "name"));
    EXPECT_STREQ("TA5", C_STR_FIELD(a5, "type"));
    EXPECT_STREQ("val5", C_STR_FIELD(a5, "value"));
    EXPECT_FALSE(a5.hasField("modDate"));
    EXPECT_STREQ("A6", C_STR_FIELD(a6, "name"));
    EXPECT_STREQ("TA6", C_STR_FIELD(a6, "type"));
    EXPECT_FALSE(a6.hasField("value"));
    EXPECT_FALSE(a6.hasField("modDate"));

    ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E1" << "_id.type" << "T1bis"));
    EXPECT_STREQ("E1", C_STR_FIELD(ent.getObjectField("_id"), "id"));
    EXPECT_STREQ("T1bis", C_STR_FIELD(ent.getObjectField("_id"), "type"));
    EXPECT_FALSE(ent.hasField("modDate"));
    attrs = ent.getField("attrs").Array();
    ASSERT_EQ(1, attrs.size());
    a1 = getAttr(attrs, "A1", "TA1");
    EXPECT_STREQ("A1", C_STR_FIELD(a1, "name"));
    EXPECT_STREQ("TA1",C_STR_FIELD(a1, "type"));
    EXPECT_STREQ("val1bis2", C_STR_FIELD(a1, "value"));
    EXPECT_FALSE(a1.hasField("modDate"));

    ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E10" << "_id.type" << "T10"));
    EXPECT_STREQ("E10", C_STR_FIELD(ent.getObjectField("_id"), "id"));
    EXPECT_STREQ("T10", C_STR_FIELD(ent.getObjectField("_id"), "type"));
    EXPECT_TRUE(ent.hasField("creDate"));
    EXPECT_TRUE(ent.hasField("modDate"));
    attrs = ent.getField("attrs").Array();
    ASSERT_EQ(1, attrs.size());
    a1 = getAttr(attrs, "A1", "TA1");
    EXPECT_STREQ("A1", C_STR_FIELD(a1, "name"));
    EXPECT_STREQ("TA1",C_STR_FIELD(a1, "type"));
    EXPECT_STREQ("new_val", C_STR_FIELD(a1, "value"));
    EXPECT_TRUE(a1.hasField("creDate"));
    EXPECT_TRUE(a1.hasField("modDate"));

    /* Note "_id.type: {$exists: false}" is a way for querying for entities without type */
    ent = connection->findOne(ENTITIES_COLL, BSON("_id.id" << "E1" << "_id.type" << BSON("$exists" << false)));
    EXPECT_STREQ("E1", C_STR_FIELD(ent.getObjectField("_id"), "id"));
    EXPECT_FALSE(ent.getObjectField("_id").hasField("type"));
    EXPECT_FALSE(ent.hasField("modDate"));
    attrs = ent.getField("attrs").Array();
    ASSERT_EQ(2, attrs.size());
    a1 = getAttr(attrs, "A1", "TA1");
    a2 = getAttr(attrs, "A2", "TA2");
    EXPECT_STREQ("A1", C_STR_FIELD(a1, "name"));
    EXPECT_STREQ("TA1",C_STR_FIELD(a1, "type"));
    EXPECT_STREQ("val1-nt", C_STR_FIELD(a1, "value"));
    EXPECT_FALSE(a1.hasField("modDate"));
    EXPECT_STREQ("A2", C_STR_FIELD(a2, "name"));
    EXPECT_STREQ("TA2", C_STR_FIELD(a2, "type"));
    EXPECT_FALSE(a2.hasField("value"));
    EXPECT_FALSE(a2.hasField("modDate"));

    /* Release connection */
    mongoDisconnect();

    /* Release mock */
    delete timerMock;

}
Example #18
0
/* ****************************************************************************
*
* attributeType -
*
*/
static std::string attributeType
(
  const std::string&                    tenant,
  const std::vector<std::string>&       servicePathV,
  const std::string                     entityType,
  const std::string                     attrName
)
{
  std::string  idType         = std::string("_id.")    + ENT_ENTITY_TYPE;
  std::string  idServicePath  = std::string("_id.")    + ENT_SERVICE_PATH;
  std::string  attributeName  = std::string(ENT_ATTRS) + "." + attrName;

  BSONObj query = BSON(idType        << entityType <<
                       idServicePath << fillQueryServicePath(servicePathV) <<
                       attributeName << BSON("$exists" << true));

  std::auto_ptr<DBClientCursor> cursor;
  DBClientBase*                 connection = NULL;

  LM_T(LmtMongo, ("query() in '%s' collection: '%s'",
                  getEntitiesCollectionName(tenant).c_str(),
                  query.toString().c_str()));

  try
  {
    connection = getMongoConnection();
    cursor     = connection->query(getEntitiesCollectionName(tenant).c_str(), query);

    /*
     * We have observed that in some cases of DB errors (e.g. the database daemon is down) instead of
     * raising an exception, the query() method sets the cursor to NULL. In this case, we raise the
     * exception ourselves
     */
    if (cursor.get() == NULL)
    {
      throw DBException("Null cursor from mongo (details on this is found in the source code)", 0);
    }
    releaseMongoConnection(connection);

    LM_I(("Database Operation Successful (%s)", query.toString().c_str()));
  }
  catch (const DBException &e)
  {
    releaseMongoConnection(connection);
    LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), e.what()));
    return "";
  }
  catch (...)
  {
    releaseMongoConnection(connection);
    LM_E(("Database Error ('%s', '%s')", query.toString().c_str(), "generic exception"));
    return "";
  }

  while (cursor->more())
  {
    BSONObj r = cursor->next();

    LM_T(LmtMongo, ("retrieved document: '%s'", r.toString().c_str()));

    /* It could happen that different entities within the same entity type may have attributes with the same name
     * but different types. In that case, one type (at random) is returned. A list could be returned but the
     * NGSIv2 operations only allow to set one type */
    return r.getField(ENT_ATTRS).embeddedObject().getField(attrName).embeddedObject().getStringField(ENT_ATTRS_TYPE);
  }

  return "";
}
Example #19
0
/**
 * Special internal logic to run reduced version handshake for empty namespace operations to
 * shards.
 *
 * Eventually this should go completely away, but for now many commands rely on unversioned but
 * mongos-specific behavior on mongod (auditing and replication information in commands)
 */
static bool initShardVersionEmptyNS(DBClientBase* conn_in) {
    bool ok;
    BSONObj result;
    DBClientBase* conn = NULL;
    try {
        // May throw if replica set primary is down
        conn = getVersionable(conn_in);
        dassert(conn);  // errors thrown above

        // Check to see if we've already initialized this connection
        if (connectionShardStatus.hasAnySequenceSet(conn))
            return false;

        // Check to see if this is actually a shard and not a single config server
        // NOTE: Config servers are registered only by the name "config" in the shard cache, not
        // by host, so lookup by host will fail unless the host is also a shard.
        const auto shard = grid.shardRegistry()->getShard(conn->getServerAddress());
        if (!shard) {
            return false;
        }

        LOG(1) << "initializing shard connection to " << shard->toString() << endl;

        ok = setShardVersion(*conn,
                             "",
                             grid.catalogManager()->connectionString().toString(),
                             ChunkVersion(),
                             NULL,
                             true,
                             result);
    } catch (const DBException&) {
        // NOTE: Replica sets may fail to initShardVersion because future calls relying on
        // correct versioning must later call checkShardVersion on the primary.
        // Secondary queries and commands may not call checkShardVersion, but secondary ops
        // aren't versioned at all.
        if (conn_in->type() != ConnectionString::SET) {
            throw;
        }

        // NOTE: Only old-style cluster operations will talk via DBClientReplicaSets - using
        // checkShardVersion is required (which includes initShardVersion information) if these
        // connections are used.

        OCCASIONALLY {
            warning() << "failed to initialize new replica set connection version, "
                      << "will initialize on first use" << endl;
        }

        return false;
    }

    // Record the connection wire version if sent in the response, initShardVersion is a
    // handshake for mongos->mongod connections.
    if (!result["minWireVersion"].eoo()) {
        int minWireVersion = result["minWireVersion"].numberInt();
        int maxWireVersion = result["maxWireVersion"].numberInt();
        conn->setWireVersions(minWireVersion, maxWireVersion);
    }

    LOG(3) << "initial sharding result : " << result << endl;

    connectionShardStatus.setSequence(conn, "", 0);
    return ok;
}
/* ****************************************************************************
*
* updateWrongIdNoHex -
*
* FIXME P3: check that we cover this case in the proper place, e.g. check() in the pre-mongoBackend layers, before permanent removal
*/
TEST(DISABLED_mongoRegisterContext_update, updateWrongIdNoHex)
{
    HttpStatusCode           ms;
    RegisterContextRequest   req;
    RegisterContextResponse  res;

    utInit();

    /* Forge the request (from "inside" to "outside") */
    EntityId en("E1", "T1");
    ContextRegistrationAttribute cra("A1", "TA1", "true");
    ContextRegistration cr;
    cr.entityIdVector.push_back(&en);
    cr.contextRegistrationAttributeVector.push_back(&cra);
    cr.providingApplication.set("http://newurl.com");
    req.contextRegistrationVector.push_back(&cr);
    req.registrationId.set("51307b66f481db11bf8600XX");
    req.duration.set("PT1M");

    /* Prepare database */
    prepareDatabase();

    /* Invoke the function in mongoBackend library */
    ms = mongoRegisterContext(&req, &res, uriParams);

    /* Check that every involved collection at MongoDB is as expected */
    /* Note we are using EXPECT_STREQ() for some cases, as Mongo Driver returns const char*, not string
     * objects (see http://code.google.com/p/googletest/wiki/Primer#String_Comparison) */

    DBClientBase* connection = getMongoConnection();

    /* registrations collection: */
    ASSERT_EQ(2, connection->count(REGISTRATIONS_COLL, BSONObj()));

    BSONObj reg, contextRegistration, ent0, attr0, attr1, attr2;
    std::vector<BSONElement> contextRegistrationV, entities, attrs;

    /* reg #1 (untouched) */
    reg = connection->findOne(REGISTRATIONS_COLL, BSON("_id" << OID("51307b66f481db11bf860001")));
    EXPECT_EQ("51307b66f481db11bf860001", reg.getField("_id").OID().toString());
    EXPECT_EQ(10000000, reg.getIntField("expiration"));

    contextRegistrationV = reg.getField("contextRegistration").Array();
    ASSERT_EQ(1, contextRegistrationV.size());
    contextRegistration = contextRegistrationV[0].embeddedObject();

    EXPECT_STREQ("http://cr1.com", C_STR_FIELD(contextRegistration, "providingApplication"));
    entities = contextRegistration.getField("entities").Array();    
    ASSERT_EQ(1, entities.size());
    ent0 = entities[0].embeddedObject();
    EXPECT_STREQ("E1", C_STR_FIELD(ent0, "id"));
    EXPECT_STREQ("T1", C_STR_FIELD(ent0, "type"));

    attrs = contextRegistration.getField("attrs").Array();
    ASSERT_EQ(3, attrs.size());
    attr0 = attrs[0].embeddedObject();
    attr1 = attrs[1].embeddedObject();
    attr2 = attrs[2].embeddedObject();
    EXPECT_STREQ("A1", C_STR_FIELD(attr0, "name"));
    EXPECT_STREQ("TA1", C_STR_FIELD(attr0, "type"));
    EXPECT_STREQ("true", C_STR_FIELD(attr0, "isDomain"));
    EXPECT_STREQ("A2", C_STR_FIELD(attr1, "name"));
    EXPECT_STREQ("TA2", C_STR_FIELD(attr1, "type"));
    EXPECT_STREQ("false", C_STR_FIELD(attr1, "isDomain"));
    EXPECT_STREQ("A3", C_STR_FIELD(attr2, "name"));
    EXPECT_STREQ("TA3", C_STR_FIELD(attr2, "type"));
    EXPECT_STREQ("true", C_STR_FIELD(attr2, "isDomain"));

    /* reg #2 (untouched) */
    reg = connection->findOne(REGISTRATIONS_COLL, BSON("_id" << OID("51307b66f481db11bf860002")));
    EXPECT_EQ("51307b66f481db11bf860002", reg.getField("_id").OID().toString());
    EXPECT_EQ(20000000, reg.getIntField("expiration"));

    contextRegistrationV = reg.getField("contextRegistration").Array();
    ASSERT_EQ(2, contextRegistrationV.size());
    contextRegistration = contextRegistrationV[0].embeddedObject();

    EXPECT_STREQ("http://cr1.com", C_STR_FIELD(contextRegistration, "providingApplication"));
    entities = contextRegistration.getField("entities").Array();    
    ASSERT_EQ(1, entities.size());
    ent0 = entities[0].embeddedObject();
    EXPECT_STREQ("E1", C_STR_FIELD(ent0, "id"));
    EXPECT_STREQ("T1", C_STR_FIELD(ent0, "type"));

    attrs = contextRegistration.getField("attrs").Array();    
    ASSERT_EQ(1, attrs.size());
    attr0 = attrs[0].embeddedObject();
    EXPECT_STREQ("A1", C_STR_FIELD(attr0, "name"));
    EXPECT_STREQ("TA1", C_STR_FIELD(attr0, "type"));
    EXPECT_STREQ("true", C_STR_FIELD(attr0, "isDomain"));

    contextRegistration = contextRegistrationV[1].embeddedObject();

    EXPECT_STREQ("http://cr2.com", C_STR_FIELD(contextRegistration, "providingApplication"));
    entities = contextRegistration.getField("entities").Array();    
    ASSERT_EQ(1, entities.size());
    ent0 = entities[0].embeddedObject();
    EXPECT_STREQ("E1", C_STR_FIELD(ent0, "id"));
    EXPECT_STREQ("T1", C_STR_FIELD(ent0, "type"));

    attrs = contextRegistration.getField("attrs").Array();    
    ASSERT_EQ(1, attrs.size());
    attr0 = attrs[0].embeddedObject();
    EXPECT_STREQ("A1", C_STR_FIELD(attr0, "name"));
    EXPECT_STREQ("TA1", C_STR_FIELD(attr0, "type"));
    EXPECT_STREQ("true", C_STR_FIELD(attr0, "isDomain"));

    /* Check response is as expected */
    EXPECT_EQ(SccOk, ms);
    EXPECT_EQ(0, res.duration.get().size());
    EXPECT_EQ("51307b66f481db11bf8600XX", res.registrationId.get());
    EXPECT_EQ(SccContextElementNotFound, res.errorCode.code);
    EXPECT_EQ("Registration Not Found", res.errorCode.reasonPhrase);
    EXPECT_EQ(0, res.errorCode.details.size());

    utExit();
}
Example #21
0
    void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ){
        assert( candidateChunks );

        //
        // 1. Check whether there is any sharded collection to be balanced by querying
        // the ShardsNS::database collection
        //
        // { "_id" : "test", "partitioned" : true, "primary" : "shard0",
        //   "sharded" : {
        //       "test.images" : { "key" : { "_id" : 1 }, "unique" : false },
        //       ...  
        //   }
        // }
        //

        auto_ptr<DBClientCursor> cursor = conn.query( ShardNS::database , BSON( "partitioned" << true ) );
        vector< string > collections;
        while ( cursor->more() ){
            BSONObj db = cursor->next();

            // A database may be partitioned but not yet have a sharded collection. 
            // 'cursor' will point to docs that do not contain the "sharded" key. Since 
            // there'd be nothing to balance, we want to skip those here.

            BSONElement shardedColls = db["sharded"];
            if ( shardedColls.eoo() ){
                log(2) << "balancer: skipping database with no sharded collection (" 
                      << db["_id"].str() << ")" << endl;
                continue;
            }
            
            BSONObjIterator i( shardedColls.Obj() );
            while ( i.more() ){
                BSONElement e = i.next();
                collections.push_back( e.fieldName() );
            }
        }
        cursor.reset();

        if ( collections.empty() ) {
            log(1) << "balancer: no collections to balance" << endl;
            return;
        }

        //
        // 2. Get a list of all the shards that are participating in this balance round
        // along with any maximum allowed quotas and current utilization. We get the
        // latter by issuing db.serverStatus() (mem.mapped) to all shards.
        //
        // TODO: skip unresponsive shards and mark information as stale.
        //
 
        vector<Shard> allShards;
        Shard::getAllShards( allShards );
        if ( allShards.size() < 2) {
            log(1) << "balancer: can't balance without more active shards" << endl;
            return;
        }

        map< string, BSONObj > shardLimitsMap; 
        for ( vector<Shard>::const_iterator it = allShards.begin(); it != allShards.end(); ++it ){
            const Shard& s = *it;
            ShardStatus status = s.getStatus();

            BSONObj limitsObj = BSON( ShardFields::maxSize( s.getMaxSize() ) << 
                                      ShardFields::currSize( status.mapped() ) <<
                                      ShardFields::draining( s.isDraining()) );

            shardLimitsMap[ s.getName() ] = limitsObj;
        }

        //
        // 3. For each collection, check if the balancing policy recommends moving anything around.
        //

        for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) {
            const string& ns = *it;

            map< string,vector<BSONObj> > shardToChunksMap;
            cursor = conn.query( ShardNS::chunk , QUERY( "ns" << ns ).sort( "min" ) );
            while ( cursor->more() ){
                BSONObj chunk = cursor->next();
                vector<BSONObj>& chunks = shardToChunksMap[chunk["shard"].String()];
                chunks.push_back( chunk.getOwned() );
            }
            cursor.reset();

            if (shardToChunksMap.empty()) {
                log(1) << "balancer: skipping empty collection (" << ns << ")";
                continue;
            }
                
            for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ){
                // this just makes sure there is an entry in shardToChunksMap for every shard
                Shard s = *i;
                shardToChunksMap[s.getName()].size();
            }

            CandidateChunk* p = _policy->balance( ns , shardLimitsMap , shardToChunksMap , _balancedLastTime );
            if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) );
        }
    }
/* ****************************************************************************
*
* MongoDbFindOneFail -
*/
TEST(mongoRegisterContext_update, MongoDbFindOneFail)
{
    HttpStatusCode           ms;
    RegisterContextRequest   req;
    RegisterContextResponse  res;   

    utInit();

    /* Prepare mock */
    const DBException e = DBException("boom!!", 33);
    DBClientConnectionMock* connectionMock = new DBClientConnectionMock();
    ON_CALL(*connectionMock, findOne("utest.registrations",_,_,_))
            .WillByDefault(Throw(e));

    /* Forge the request (from "inside" to "outside") */
    EntityId en("E1", "T1");
    ContextRegistrationAttribute cra("A1", "TA1", "true");
    ContextRegistration cr;
    cr.entityIdVector.push_back(&en);
    cr.contextRegistrationAttributeVector.push_back(&cra);
    cr.providingApplication.set("http://newurl.com");
    req.contextRegistrationVector.push_back(&cr);
    req.registrationId.set("51307b66f481db11bf860001");
    req.duration.set("PT1M");

    /* Prepare database */
    prepareDatabase();

    /* Set MongoDB connection mock (preserving "actual" connection for later use) */
    DBClientBase* connectionDb = getMongoConnection();
    setMongoConnectionForUnitTest(connectionMock);

    /* Invoke the function in mongoBackend library */
    ms = mongoRegisterContext(&req, &res, uriParams);

    /* Check response is as expected */
    EXPECT_EQ(SccOk, ms);
    EXPECT_TRUE(res.duration.isEmpty());
    EXPECT_TRUE(res.registrationId.isEmpty());
    EXPECT_EQ(SccReceiverInternalError, res.errorCode.code);
    EXPECT_EQ("Internal Server Error", res.errorCode.reasonPhrase);
    EXPECT_EQ("Database Error (collection: utest.registrations "
              "- findOne(): { _id: ObjectId('51307b66f481db11bf860001'), servicePath: \"/\" } "
              "- exception: boom!!)", res.errorCode.details);

    /* Restore real DB connection */
    setMongoConnectionForUnitTest(connectionDb);

    /* Release mock */    
    delete connectionMock;

    /* Check that every involved collection at MongoDB is as expected */
    /* Note we are using EXPECT_STREQ() for some cases, as Mongo Driver returns const char*, not string
     * objects (see http://code.google.com/p/googletest/wiki/Primer#String_Comparison) */   

    /* registrations collection: */
    ASSERT_EQ(2, connectionDb->count(REGISTRATIONS_COLL, BSONObj()));

    BSONObj reg, contextRegistration, ent0, attr0, attr1, attr2;
    std::vector<BSONElement> contextRegistrationV, entities, attrs;

    /* reg #1 (untouched) */
    reg = connectionDb->findOne(REGISTRATIONS_COLL, BSON("_id" << OID("51307b66f481db11bf860001")));
    EXPECT_EQ("51307b66f481db11bf860001", reg.getField("_id").OID().toString());
    EXPECT_EQ(10000000, reg.getIntField("expiration"));

    contextRegistrationV = reg.getField("contextRegistration").Array();
    ASSERT_EQ(1, contextRegistrationV.size());
    contextRegistration = contextRegistrationV[0].embeddedObject();

    EXPECT_STREQ("http://cr1.com", C_STR_FIELD(contextRegistration, "providingApplication"));
    entities = contextRegistration.getField("entities").Array();
    ASSERT_EQ(1, entities.size());
    ent0 = entities[0].embeddedObject();
    EXPECT_STREQ("E1", C_STR_FIELD(ent0, "id"));
    EXPECT_STREQ("T1", C_STR_FIELD(ent0, "type"));

    attrs = contextRegistration.getField("attrs").Array();
    ASSERT_EQ(3, attrs.size());
    attr0 = attrs[0].embeddedObject();
    attr1 = attrs[1].embeddedObject();
    attr2 = attrs[2].embeddedObject();
    EXPECT_STREQ("A1", C_STR_FIELD(attr0, "name"));
    EXPECT_STREQ("TA1", C_STR_FIELD(attr0, "type"));
    EXPECT_STREQ("true", C_STR_FIELD(attr0, "isDomain"));
    EXPECT_STREQ("A2", C_STR_FIELD(attr1, "name"));
    EXPECT_STREQ("TA2", C_STR_FIELD(attr1, "type"));
    EXPECT_STREQ("false", C_STR_FIELD(attr1, "isDomain"));
    EXPECT_STREQ("A3", C_STR_FIELD(attr2, "name"));
    EXPECT_STREQ("TA3", C_STR_FIELD(attr2, "type"));
    EXPECT_STREQ("true", C_STR_FIELD(attr2, "isDomain"));

    /* reg #2 (untouched) */
    reg = connectionDb->findOne(REGISTRATIONS_COLL, BSON("_id" << OID("51307b66f481db11bf860002")));
    EXPECT_EQ("51307b66f481db11bf860002", reg.getField("_id").OID().toString());
    EXPECT_EQ(20000000, reg.getIntField("expiration"));

    contextRegistrationV = reg.getField("contextRegistration").Array();
    ASSERT_EQ(2, contextRegistrationV.size());
    contextRegistration = contextRegistrationV[0].embeddedObject();

    EXPECT_STREQ("http://cr1.com", C_STR_FIELD(contextRegistration, "providingApplication"));
    entities = contextRegistration.getField("entities").Array();    
    ASSERT_EQ(1, entities.size());
    ent0 = entities[0].embeddedObject();
    EXPECT_STREQ("E1", C_STR_FIELD(ent0, "id"));
    EXPECT_STREQ("T1", C_STR_FIELD(ent0, "type"));

    attrs = contextRegistration.getField("attrs").Array();    
    ASSERT_EQ(1, attrs.size());
    attr0 = attrs[0].embeddedObject();
    EXPECT_STREQ("A1", C_STR_FIELD(attr0, "name"));
    EXPECT_STREQ("TA1", C_STR_FIELD(attr0, "type"));
    EXPECT_STREQ("true", C_STR_FIELD(attr0, "isDomain"));

    contextRegistration = contextRegistrationV[1].embeddedObject();

    EXPECT_STREQ("http://cr2.com", C_STR_FIELD(contextRegistration, "providingApplication"));
    entities = contextRegistration.getField("entities").Array();    
    ASSERT_EQ(1, entities.size());
    ent0 = entities[0].embeddedObject();
    EXPECT_STREQ("E1", C_STR_FIELD(ent0, "id"));
    EXPECT_STREQ("T1", C_STR_FIELD(ent0, "type"));

    attrs = contextRegistration.getField("attrs").Array();    
    ASSERT_EQ(1, attrs.size());
    attr0 = attrs[0].embeddedObject();
    EXPECT_STREQ("A1", C_STR_FIELD(attr0, "name"));
    EXPECT_STREQ("TA1", C_STR_FIELD(attr0, "type"));
    EXPECT_STREQ("true", C_STR_FIELD(attr0, "isDomain"));

    utExit();
}
Example #23
0
    /**
     * @return true if had to do something
     */
    bool checkShardVersion( DBClientBase& conn , const string& ns , bool authoritative , int tryNumber ) {
        // TODO: cache, optimize, etc...

        WriteBackListener::init( conn );

        DBConfigPtr conf = grid.getDBConfig( ns );
        if ( ! conf )
            return false;

        unsigned long long officialSequenceNumber = 0;

        ChunkManagerPtr manager;
        const bool isSharded = conf->isSharded( ns );
        if ( isSharded ) {
            manager = conf->getChunkManager( ns , authoritative );
            officialSequenceNumber = manager->getSequenceNumber();
        }

        // has the ChunkManager been reloaded since the last time we updated the connection-level version?
        // (ie, last time we issued the setShardVersions below)
        unsigned long long sequenceNumber = connectionShardStatus.getSequence(&conn,ns);
        if ( sequenceNumber == officialSequenceNumber ) {
            return false;
        }


        ShardChunkVersion version = 0;
        if ( isSharded ) {
            version = manager->getVersion( Shard::make( conn.getServerAddress() ) );
        }

        log(2) << " have to set shard version for conn: " << &conn << " ns:" << ns
               << " my last seq: " << sequenceNumber << "  current: " << officialSequenceNumber
               << " version: " << version << " manager: " << manager.get()
               << endl;

        BSONObj result;
        if ( setShardVersion( conn , ns , version , authoritative , result ) ) {
            // success!
            LOG(1) << "      setShardVersion success: " << result << endl;
            connectionShardStatus.setSequence( &conn , ns , officialSequenceNumber );
            return true;
        }

        log(1) << "       setShardVersion failed!\n" << result << endl;

        if ( result.getBoolField( "need_authoritative" ) )
            massert( 10428 ,  "need_authoritative set but in authoritative mode already" , ! authoritative );

        if ( ! authoritative ) {
            checkShardVersion( conn , ns , 1 , tryNumber + 1 );
            return true;
        }

        if ( tryNumber < 4 ) {
            log(1) << "going to retry checkShardVersion" << endl;
            sleepmillis( 10 );
            checkShardVersion( conn , ns , 1 , tryNumber + 1 );
            return true;
        }

        log() << "     setShardVersion failed: " << result << endl;
        massert( 10429 , (string)"setShardVersion failed! " + result.jsonString() , 0 );
        return true;
    }
/* ****************************************************************************
*
* prepareDatabase -
*
* This function is called before every test, to populate some information in the
* registrations collection.
*
*/
static void prepareDatabase(void) {

  /* Set database */
  setupDatabase();

  DBClientBase* connection = getMongoConnection();

  /* We create the following registrations:
   *
   * - Reg1: CR: E1 - (A1,A2,A3) - http://cr1.com
   * - Reg2: CR: E1 - A1 - http://cr1.com
   *         CR: E1 - A1 - http://cr3.com
   */

  BSONObj cr1 = BSON("providingApplication" << "http://cr1.com" <<
                     "entities" << BSON_ARRAY(
                         BSON("id" << "E1" << "type" << "T1")
                         ) <<
                     "attrs" << BSON_ARRAY(
                         BSON("name" << "A1" << "type" << "TA1" << "isDomain" << "true") <<
                         BSON("name" << "A2" << "type" << "TA2" << "isDomain" << "false") <<
                         BSON("name" << "A3" << "type" << "TA3" << "isDomain" << "true")
                         )
                     );
  BSONObj cr2 = BSON("providingApplication" << "http://cr1.com" <<
                     "entities" << BSON_ARRAY(
                         BSON("id" << "E1" << "type" << "T1")
                         ) <<
                     "attrs" << BSON_ARRAY(
                         BSON("name" << "A1" << "type" << "TA1" << "isDomain" << "true")
                         )
                     );
  BSONObj cr3 = BSON("providingApplication" << "http://cr2.com" <<
                     "entities" << BSON_ARRAY(
                         BSON("id" << "E1" << "type" << "T1")
                         ) <<
                     "attrs" << BSON_ARRAY(
                         BSON("name" << "A1" << "type" << "TA1" << "isDomain" << "true")
                         )
                     );

  BSONObjBuilder reg1;
  reg1.appendElements(BSON(
                        "_id" << OID("51307b66f481db11bf860001") <<
                        "expiration" << 10000000 <<
                        "contextRegistration" << BSON_ARRAY(cr1) <<
                        "servicePath" << "/"
                        ));

  BSONObjBuilder reg2;
  reg2.appendElements(BSON(
                        "_id" << OID("51307b66f481db11bf860002") <<
                        "expiration" << 20000000 <<
                        "contextRegistration" << BSON_ARRAY(cr2 << cr3) <<
                        "servicePath" << "/"
                        ));

  /* 1879048191 corresponds to year 2029 so we avoid any expiration problem in the next 16 years :) */
  BSONObj sub1 = BSON("_id" << OID("51307b66f481db11bf860010") <<
                      "expiration" << 1879048191 <<
                      "reference" << "http://notify1.me" <<
                      "entities" << BSON_ARRAY(BSON("id" << "E5" << "type" << "T5" << "isPattern" << "false")) <<
                      "attrs" << BSONArray());

  BSONObj sub2 = BSON("_id" << OID("51307b66f481db11bf860020") <<
                      "expiration" << 1879048191 <<
                      "reference" << "http://notify2.me" <<
                      "entities" << BSON_ARRAY(BSON("id" << "E5" << "type" << "T5" << "isPattern" << "false")) <<
                      "attrs" << BSON_ARRAY("A1"));

  connection->insert(REGISTRATIONS_COLL, reg1.obj());
  connection->insert(REGISTRATIONS_COLL, reg2.obj());

  connection->insert(SUBSCRIBECONTEXTAVAIL_COLL, sub1);
  connection->insert(SUBSCRIBECONTEXTAVAIL_COLL, sub2);

}
Example #25
0
    /**
     * @return true if had to do something
     */
    bool checkShardVersion( DBClientBase& conn_in , const string& ns , bool authoritative , int tryNumber ) {
        // TODO: cache, optimize, etc...

        WriteBackListener::init( conn_in );

        DBConfigPtr conf = grid.getDBConfig( ns );
        if ( ! conf )
            return false;

        DBClientBase* conn = 0;

        switch ( conn_in.type() ) {
        case ConnectionString::INVALID:
            assert(0);
            break;
        case ConnectionString::MASTER:
            // great
            conn = &conn_in;
            break;
        case ConnectionString::PAIR:
            assert( ! "pair not support for sharding" );
            break;
        case ConnectionString::SYNC:
            // TODO: we should check later that we aren't actually sharded on this
            conn = &conn_in;
            break;
        case ConnectionString::SET:
            DBClientReplicaSet* set = (DBClientReplicaSet*)&conn_in;
            conn = &(set->masterConn());
            break;
        }

        assert(conn);

        unsigned long long officialSequenceNumber = 0;

        ChunkManagerPtr manager;
        const bool isSharded = conf->isSharded( ns );
        if ( isSharded ) {
            manager = conf->getChunkManagerIfExists( ns , authoritative );
            // It's possible the chunk manager was reset since we checked whether sharded was true,
            // so must check this here.
            if( manager ) officialSequenceNumber = manager->getSequenceNumber();
        }

        // has the ChunkManager been reloaded since the last time we updated the connection-level version?
        // (ie., last time we issued the setShardVersions below)
        unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
        if ( sequenceNumber == officialSequenceNumber ) {
            return false;
        }


        ShardChunkVersion version = 0;
        if ( isSharded && manager ) {
            version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
        }

        LOG(2) << " have to set shard version for conn: " << conn << " ns:" << ns
               << " my last seq: " << sequenceNumber << "  current: " << officialSequenceNumber
               << " version: " << version << " manager: " << manager.get()
               << endl;

        BSONObj result;
        if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
            // success!
            LOG(1) << "      setShardVersion success: " << result << endl;
            connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
            return true;
        }

        LOG(1) << "       setShardVersion failed!\n" << result << endl;

        if ( result["need_authoritative"].trueValue() )
            massert( 10428 ,  "need_authoritative set but in authoritative mode already" , ! authoritative );

        if ( ! authoritative ) {
            checkShardVersion( *conn , ns , 1 , tryNumber + 1 );
            return true;
        }
        
        if ( result["reloadConfig"].trueValue() ) {
            if( result["version"].timestampTime() == 0 ){
                // reload db
                conf->reload();
            }
            else {
                // reload config
                conf->getChunkManager( ns , true );
            }
        }

        const int maxNumTries = 7;
        if ( tryNumber < maxNumTries ) {
            LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 ) 
                << "going to retry checkShardVersion host: " << conn->getServerAddress() << " " << result << endl;
            sleepmillis( 10 * tryNumber );
            checkShardVersion( *conn , ns , true , tryNumber + 1 );
            return true;
        }
        
        string errmsg = str::stream() << "setShardVersion failed host: " << conn->getServerAddress() << " " << result;
        log() << "     " << errmsg << endl;
        massert( 10429 , errmsg , 0 );
        return true;
    }
/* ****************************************************************************
*
* prepareDatabase -
*
* This function is called before every test, to populate some information in the
* registrations collection.
*/
static void prepareDatabase(void)
{

  /* Set database */
  setupDatabase();

  DBClientBase* connection = getMongoConnection();

  /* We create the following registrations:
   *
   * - Reg1: CR: (E1,E2,E3) (A1,A2,A3)
   *         CR: (E1)       (A1,A4)
   * - Reg2: CR: (E2)       (A2, A3)
   * - Reg3: CR: (E1*)      (A1*)
   * - Reg4: CR: (E1**)     (A1)
   *
   * (*) same name but different types. This is included to check that type is taken into account,
   *     so Reg3 is not returned never (except noPatternNoType). You can try to change types in Reg3
   *     to make them equal to the ones in Reg1 and Reg2 and check that some tests are failing.
   * (**)same name but without type
   */

  BSONObj cr1 = BSON("providingApplication" << "http://cr1.com" <<
                     "entities" << BSON_ARRAY(
                       BSON("id" << "E1" << "type" << "T1") <<
                       BSON("id" << "E2" << "type" << "T2") <<
                       BSON("id" << "E3" << "type" << "T3")
                       ) <<
                     "attrs" << BSON_ARRAY(
                       BSON("name" << "A1" << "type" << "TA1" << "isDomain" << "true") <<
                       BSON("name" << "A2" << "type" << "TA2" << "isDomain" << "false") <<
                       BSON("name" << "A3" << "type" << "TA3" << "isDomain" << "true")
                       )
                     );
  BSONObj cr2 = BSON("providingApplication" << "http://cr2.com" <<
                     "entities" << BSON_ARRAY(
                       BSON("id" << "E1" << "type" << "T1")
                       ) <<
                     "attrs" << BSON_ARRAY(
                       BSON("name" << "A1" << "type" << "TA1" << "isDomain" << "true") <<
                       BSON("name" << "A4" << "type" << "TA4" << "isDomain" << "false")
                       )
                     );
  BSONObj cr3 = BSON("providingApplication" << "http://cr3.com" <<
                     "entities" << BSON_ARRAY(
                       BSON("id" << "E2" << "type" << "T2")
                       ) <<
                     "attrs" << BSON_ARRAY(
                       BSON("name" << "A2" << "type" << "TA2" << "isDomain" << "false") <<
                       BSON("name" << "A3" << "type" << "TA3" << "isDomain" << "true")
                       )
                     );

  BSONObj cr4 = BSON("providingApplication" << "http://cr4.com" <<
                     "entities" << BSON_ARRAY(
                       BSON("id" << "E1" << "type" << "T1bis")
                       ) <<
                     "attrs" << BSON_ARRAY(
                       BSON("name" << "A1" << "type" << "TA1bis" << "isDomain" << "false")
                       )
                     );

  BSONObj cr5 = BSON("providingApplication" << "http://cr5.com" <<
                     "entities" << BSON_ARRAY(
                       BSON("id" << "E1")
                       ) <<
                     "attrs" << BSON_ARRAY(
                       BSON("name" << "A1" << "type" << "TA1" << "isDomain" << "true")
                       )
                     );

  /* 1879048191 corresponds to year 2029 so we avoid any expiration problem in the next 16 years :) */
  BSONObj reg1 = BSON(
        "_id" << OID("51307b66f481db11bf860001") <<
        "expiration" << 1879048191 <<
        "contextRegistration" << BSON_ARRAY(cr1 << cr2)
        );

  BSONObj reg2 = BSON(
        "_id" << OID("51307b66f481db11bf860002") <<
        "expiration" << 1879048191 <<
        "contextRegistration" << BSON_ARRAY(cr3)
        );

  BSONObj reg3 = BSON(
        "_id" << OID("51307b66f481db11bf860003") <<
        "expiration" << 1879048191 <<
        "contextRegistration" << BSON_ARRAY(cr4)
        );

  BSONObj reg4 = BSON(
        "_id" << OID("51307b66f481db11bf860004") <<
        "expiration" << 1879048191 <<
        "contextRegistration" << BSON_ARRAY(cr5)
        );

  connection->insert(REGISTRATIONS_COLL, reg1);
  connection->insert(REGISTRATIONS_COLL, reg2);
  connection->insert(REGISTRATIONS_COLL, reg3);
  connection->insert(REGISTRATIONS_COLL, reg4);


}
Example #27
0
/* ****************************************************************************
*
* mongoAttributesForEntityType -
*/
HttpStatusCode mongoAttributesForEntityType
(
  std::string                           entityType,
  EntityTypeAttributesResponse*         responseP,
  const std::string&                    tenant,
  const std::vector<std::string>&       servicePathV,
  std::map<std::string, std::string>&   uriParams
)
{
  unsigned int offset         = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str());
  unsigned int limit          = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str());
  std::string  detailsString  = uriParams[URI_PARAM_PAGINATION_DETAILS];
  bool         details        = (strcasecmp("on", detailsString.c_str()) == 0)? true : false;

  // Setting the name of the entity type for the response
  responseP->entityType.type = entityType;

  LM_T(LmtMongo, ("Query Types Attribute for <%s>", entityType.c_str()));
  LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false"));

  reqSemTake(__FUNCTION__, "query types attributes request");

  DBClientBase* connection = getMongoConnection();

  /* Compose query based on this aggregation command:
   *
   * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the
   * aggregation query below is fully correct
   *
   * db.runCommand({aggregate: "entities",
   *                pipeline: [ {$match: { "_id.type": "TYPE" , "_id.servicePath": /.../ } },
   *                            {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} },
   *                            {$unwind: "$attrs"},
   *                            {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} },
   *                            {$unwind: "$attrs"},
   *                            {$group: {_id: "$attrs" }},
   *                            {$sort: {_id.name: 1, _id.type: 1} }
   *                          ]
   *                })
   *
   */

  BSONObj result;
  BSONObj cmd = BSON("aggregate" << COL_ENTITIES <<
                     "pipeline" << BSON_ARRAY(
                                              BSON("$match" << BSON(C_ID_ENTITY << entityType << C_ID_SERVICEPATH << fillQueryServicePath(servicePathV))) <<
                                              BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) <<
                                              BSON("$unwind" << S_ATTRS) <<
                                              BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) <<
                                              BSON("$unwind" << S_ATTRS) <<
                                              BSON("$group" << BSON("_id" << S_ATTRS)) <<
                                              BSON("$sort" << BSON(C_ID_NAME << 1 << C_ID_TYPE << 1))
                                             )
                    );

  LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str()));

  mongoSemTake(__FUNCTION__, "aggregation command"); 
  try
  {

    connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result);
    mongoSemGive(__FUNCTION__, "aggregation command");
    LM_I(("Database Operation Successful (%s)", cmd.toString().c_str()));
  }
  catch (const DBException& e)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + e.what();

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }
  catch (...)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + "generic";

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }

  /* Processing result to build response*/
  LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str()));

  std::vector<BSONElement> resultsArray = result.getField("result").Array();

  /* See comment above in the other method regarding this strategy to implement pagination */
  for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix)
  {
    BSONElement        idField    = resultsArray[ix].embeddedObject().getField("_id");

    //
    // BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub'
    //
    // Now, if 'resultsArray[ix].embeddedObject().getField("_id")' is not found, if we continue,
    // calling embeddedObject() on it, then we get an exception and the broker crashes.
    //
    if (idField.eoo() == true)
    {
      LM_E(("Database Error (error retrieving _id field in doc: %s)", resultsArray[ix].embeddedObject().toString().c_str()));
      continue;
    }

    BSONObj            resultItem = idField.embeddedObject();
    ContextAttribute*  ca         = new ContextAttribute(resultItem.getStringField(ENT_ATTRS_NAME), resultItem.getStringField(ENT_ATTRS_TYPE));
    responseP->entityType.contextAttributeVector.push_back(ca);
  }

  char detailsMsg[256];
  if (responseP->entityType.contextAttributeVector.size() > 0)
  {
    if (details)
    {
      snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size());
      responseP->statusCode.fill(SccOk, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccOk);
    }
  }
  else
  {
    if (details)
    {
      snprintf(detailsMsg, sizeof(detailsMsg), "Number of attributes: %d. Offset is %d", (int) resultsArray.size(), offset);
      responseP->statusCode.fill(SccContextElementNotFound, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccContextElementNotFound);
    }
  }

  reqSemGive(__FUNCTION__, "query types request");

  return SccOk;
}
/* ****************************************************************************
*
* mongoEntityTypes -
*/
HttpStatusCode mongoEntityTypes
(
  EntityTypesResponse*                  responseP,
  const std::string&                    tenant,
  const std::vector<std::string>&       servicePathV,
  std::map<std::string, std::string>&   uriParams
)
{
  unsigned int offset         = atoi(uriParams[URI_PARAM_PAGINATION_OFFSET].c_str());
  unsigned int limit          = atoi(uriParams[URI_PARAM_PAGINATION_LIMIT].c_str());
  std::string  detailsString  = uriParams[URI_PARAM_PAGINATION_DETAILS];
  bool         details        = (strcasecmp("on", detailsString.c_str()) == 0)? true : false;

  LM_T(LmtMongo, ("Query Entity Types"));
  LM_T(LmtPagination, ("Offset: %d, Limit: %d, Details: %s", offset, limit, (details == true)? "true" : "false"));

  reqSemTake(__FUNCTION__, "query types request");

  DBClientBase* connection = getMongoConnection();

  /* Compose query based on this aggregation command:
   *
   * FIXME P9: taking into account that type is no longer used as part of the attribute "key", not sure if the
   * aggregation query below is fully correct
   *
   * db.runCommand({aggregate: "entities",
   *                pipeline: [ {$project: {_id: 1, "attrs.name": 1, "attrs.type": 1} },
   *                            {$unwind: "$attrs"},
   *                            {$group: {_id: "$_id.type", attrs: {$addToSet: "$attrs"}} },
   *                            {$sort: {_id: 1} }
   *                          ]
   *                })
   *
   * FIXME P6: in the future, we can interpret the collapse parameter at this layer. If collapse=true so we don't need attributes, the
   * following command can be used:
   *
   * db.runCommand({aggregate: "entities", pipeline: [ {$group: {_id: "$_id.type"} }]})
   *
   */

  BSONObj result;
  BSONObj cmd = BSON("aggregate" << COL_ENTITIES <<
                     "pipeline" << BSON_ARRAY(
                                              BSON("$project" << BSON("_id" << 1 << C_ATTR_NAME << 1 << C_ATTR_TYPE << 1)) <<
                                              BSON("$unwind" << S_ATTRS) <<
                                              BSON("$group" << BSON("_id" << CS_ID_ENTITY << "attrs" << BSON("$addToSet" << S_ATTRS))) <<
                                              BSON("$sort" << BSON("_id" << 1))
                                             )
                    );

  LM_T(LmtMongo, ("runCommand() in '%s' database: '%s'", composeDatabaseName(tenant).c_str(), cmd.toString().c_str()));

  mongoSemTake(__FUNCTION__, "aggregation command");  
  try
  {

    connection->runCommand(composeDatabaseName(tenant).c_str(), cmd, result);
    mongoSemGive(__FUNCTION__, "aggregation command");
    LM_I(("Database Operation Successful (%s)", cmd.toString().c_str()));
  }
  catch (const DBException& e)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + e.what();

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }
  catch (...)
  {
      mongoSemGive(__FUNCTION__, "aggregation command");
      std::string err = std::string("database: ") + composeDatabaseName(tenant).c_str() +
              " - command: " + cmd.toString() +
              " - exception: " + "generic";

      LM_E(("Database Error (%s)", err.c_str()));
      responseP->statusCode.fill(SccReceiverInternalError, err);
      reqSemGive(__FUNCTION__, "query types request");
      return SccOk;
  }

  /* Processing result to build response*/
  LM_T(LmtMongo, ("aggregation result: %s", result.toString().c_str()));

  std::vector<BSONElement> resultsArray = result.getField("result").Array();

  /* Another strategy to implement pagination is to use the $skip and $limit operators in the
   * aggregation framework. However, doing so, we don't know the total number of results, which can
   * be needed in the case of details=on (using that approach, we need to do two queries: one to get
   * the count and other to get the actual results with $skip and $limit, in the same "transaction" to
   * avoid incoherence between both if some entity type is created or deleted in the process).
   *
   * However, considering that the number of types will be small compared with the number of entities,
   * the current approach seems to be ok
   */
  for (unsigned int ix = offset; ix < MIN(resultsArray.size(), offset + limit); ++ix)
  {

    BSONObj                  resultItem = resultsArray[ix].embeddedObject();
    TypeEntity*              type       = new TypeEntity(resultItem.getStringField("_id"));
    std::vector<BSONElement> attrsArray = resultItem.getField("attrs").Array();

    for (unsigned int jx = 0; jx < attrsArray.size(); ++jx)
    {
      BSONObj jAttr = attrsArray[jx].embeddedObject();
      ContextAttribute* ca = new ContextAttribute(jAttr.getStringField(ENT_ATTRS_NAME), jAttr.getStringField(ENT_ATTRS_TYPE));
      type->contextAttributeVector.push_back(ca);
    }

    responseP->typeEntityVector.push_back(type);

  }

  char detailsMsg[256];
  if (responseP->typeEntityVector.size() > 0)
  {
    if (details)
    {
      snprintf(detailsMsg, sizeof(detailsMsg), "Count: %d", (int) resultsArray.size());
      responseP->statusCode.fill(SccOk, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccOk);
    }
  }
  else
  {
    if (details)
    {      
      snprintf(detailsMsg, sizeof(detailsMsg), "Number of types: %d. Offset is %d", (int) resultsArray.size(), offset);
      responseP->statusCode.fill(SccContextElementNotFound, detailsMsg);
    }
    else
    {
      responseP->statusCode.fill(SccContextElementNotFound);
    }
  }

  reqSemGive(__FUNCTION__, "query types request");

  return SccOk;

}
Example #29
0
    /**
     * @return true if had to do something
     */
    bool checkShardVersion( DBClientBase * conn_in , const string& ns , ChunkManagerPtr refManager, bool authoritative , int tryNumber ) {
        // TODO: cache, optimize, etc...

        WriteBackListener::init( *conn_in );

        DBConfigPtr conf = grid.getDBConfig( ns );
        if ( ! conf )
            return false;

        DBClientBase* conn = getVersionable( conn_in );
        verify(conn); // errors thrown above

        unsigned long long officialSequenceNumber = 0;

        ChunkManagerPtr manager;
        const bool isSharded = conf->isSharded( ns );
        if ( isSharded ) {
            manager = conf->getChunkManagerIfExists( ns , authoritative );
            // It's possible the chunk manager was reset since we checked whether sharded was true,
            // so must check this here.
            if( manager ) officialSequenceNumber = manager->getSequenceNumber();
        }

        // Check this manager against the reference manager
        if( isSharded && manager ){

            Shard shard = Shard::make( conn->getServerAddress() );
            if( refManager && ! refManager->compatibleWith( manager, shard ) ){
                throw SendStaleConfigException( ns, str::stream() << "manager (" << manager->getVersion( shard ).toString()  << " : " << manager->getSequenceNumber() << ") "
                                                                      << "not compatible with reference manager (" << refManager->getVersion( shard ).toString()  << " : " << refManager->getSequenceNumber() << ") "
                                                                      << "on shard " << shard.getName() << " (" << shard.getAddress().toString() << ")",
                                                refManager->getVersion( shard ), manager->getVersion( shard ) );
            }
        }
        else if( refManager ){
            Shard shard = Shard::make( conn->getServerAddress() );
            string msg( str::stream() << "not sharded ("
                        << ( (manager.get() == 0) ? string( "<none>" ) :
                                str::stream() << manager->getSequenceNumber() )
                        << ") but has reference manager ("
                        << refManager->getSequenceNumber() << ") "
                        << "on conn " << conn->getServerAddress() << " ("
                        << conn_in->getServerAddress() << ")" );

            throw SendStaleConfigException( ns, msg,
                    refManager->getVersion( shard ), ShardChunkVersion( 0, OID() ));
        }

        // has the ChunkManager been reloaded since the last time we updated the connection-level version?
        // (ie., last time we issued the setShardVersions below)
        unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
        if ( sequenceNumber == officialSequenceNumber ) {
            return false;
        }


        ShardChunkVersion version = ShardChunkVersion( 0, OID() );
        if ( isSharded && manager ) {
            version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
        }

        if( ! version.isSet() ){
            LOG(0) << "resetting shard version of " << ns << " on " << conn->getServerAddress() << ", " <<
                      ( ! isSharded ? "no longer sharded" :
                      ( ! manager ? "no chunk manager found" :
                                    "version is zero" ) ) << endl;
        }

        LOG(2) << " have to set shard version for conn: " << conn->getServerAddress() << " ns:" << ns
               << " my last seq: " << sequenceNumber << "  current: " << officialSequenceNumber
               << " version: " << version << " manager: " << manager.get()
               << endl;

        const string versionableServerAddress(conn->getServerAddress());

        BSONObj result;
        if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
            // success!
            LOG(1) << "      setShardVersion success: " << result << endl;
            connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
            return true;
        }

        LOG(1) << "       setShardVersion failed!\n" << result << endl;

        if ( result["need_authoritative"].trueValue() )
            massert( 10428 ,  "need_authoritative set but in authoritative mode already" , ! authoritative );

        if ( ! authoritative ) {
            // use the original connection and get a fresh versionable connection
            // since conn can be invalidated (or worse, freed) after the failure
            checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
            return true;
        }
        
        if ( result["reloadConfig"].trueValue() ) {
            if( result["version"].timestampTime() == 0 ){

                warning() << "reloading full configuration for " << conf->getName()
                          << ", connection state indicates significant version changes" << endl;

                // reload db
                conf->reload();
            }
            else {
                // reload config
                conf->getChunkManager( ns , true );
            }
        }

        const int maxNumTries = 7;
        if ( tryNumber < maxNumTries ) {
            LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 ) 
                << "going to retry checkShardVersion host: " << versionableServerAddress << " " << result << endl;
            sleepmillis( 10 * tryNumber );
            // use the original connection and get a fresh versionable connection
            // since conn can be invalidated (or worse, freed) after the failure
            checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
            return true;
        }
        
        string errmsg = str::stream() << "setShardVersion failed host: " << versionableServerAddress << " " << result;
        log() << "     " << errmsg << endl;
        massert( 10429 , errmsg , 0 );
        return true;
    }
Example #30
0
    ChunkMatcherPtr ShardingState::getChunkMatcher( const string& ns ){
        if ( ! _enabled )
            return ChunkMatcherPtr();
        
        if ( ! ShardedConnectionInfo::get( false ) )
            return ChunkMatcherPtr();
        
        ConfigVersion version;
        { 
            // check cache
            scoped_lock lk( _mutex );
            version = _versions[ns];
            
            if ( ! version )
                return ChunkMatcherPtr();
            
            ChunkMatcherPtr p = _chunks[ns];
            if ( p && p->_version >= version ){
                // our cached version is good, so just return
                return p;                
            }
        }

        BSONObj q;
        {
            BSONObjBuilder b;
            b.append( "ns" , ns.c_str() );
            b.append( "shard" , BSON( "$in" << BSON_ARRAY( _shardHost << _shardName ) ) );
            q = b.obj();
        }

        // have to get a connection to the config db
        // special case if i'm the congigdb since i'm locked and if i connect to myself
        // its a deadlock
        auto_ptr<ScopedDbConnection> scoped;
        auto_ptr<DBDirectClient> direct;
        
        DBClientBase * conn;

        if ( _configServer == _shardHost ){
            direct.reset( new DBDirectClient() );
            conn = direct.get();
        }
        else {
            scoped.reset( new ScopedDbConnection( _configServer ) );
            conn = scoped->get();
        }

        // actually query all the chunks
        // sorting so we can efficiently bucket them
        auto_ptr<DBClientCursor> cursor = conn->query( "config.chunks" , Query(q).sort( "min" ) );
        assert( cursor.get() );
        if ( ! cursor->more() ){
            // TODO: should we update the local version or cache this result?
            if ( scoped.get() )
                scoped->done();
            return ChunkMatcherPtr();
        }
        
        ChunkMatcherPtr p( new ChunkMatcher( version ) );
        
        BSONObj min,max;
        while ( cursor->more() ){
            BSONObj d = cursor->next();
            
            if ( min.isEmpty() ){
                min = d["min"].Obj().getOwned();
                max = d["max"].Obj().getOwned();
                continue;
            }

            if ( max == d["min"].Obj() ){
                max = d["max"].Obj().getOwned();
                continue;
            }

            p->gotRange( min.getOwned() , max.getOwned() );
            min = d["min"].Obj().getOwned();
            max = d["max"].Obj().getOwned();
        }
        assert( ! min.isEmpty() );
        p->gotRange( min.getOwned() , max.getOwned() );
        
        if ( scoped.get() )
            scoped->done();

        { 
            scoped_lock lk( _mutex );
            _chunks[ns] = p;
        }

        return p;
    }