Ejemplo n.º 1
0
void ListenerManager::handleListenerInsert(const idgs::actor::ActorMessagePtr& msg) {
  pb::InsertRequest* request = dynamic_cast<pb::InsertRequest*>(msg->getPayload().get());
  const std::string& storeName = request->store_name();
  std::shared_ptr<StoreConfigWrapper> config;
  auto code = idgs::util::singleton<DataStore>::getInstance().getStoreConfigWrappers().getStoreConfig(storeName, config);
  if (code != RC_SUCCESS) {
    LOG(ERROR) << "store " << storeName << " is not found.";
    return;
  }

  pb::StoreListenerInfo* listenerInfo = new pb::StoreListenerInfo;
  if (!msg->parseAttachment(STORE_ATTACH_LISTENER, listenerInfo)) {
    LOG(ERROR) << "store " << storeName << " is not found.";
    return;
  }

  uint32_t index = listenerInfo->listener_index();
  auto& listeners = config->getStoreListener();
  for (int32_t i = index; i < listeners.size(); ++ i) {
    StoreListener* listener = listeners[i];
    auto code = listener->insert(msg);
    switch (code) {
      case LRC_CONTINUE:
        continue;
      case LRC_BREAK:
      case LRC_END:
        break;
      case LRC_ERROR:
        LOG(ERROR) << "error when execute listener " << listener->getName() << ", index " << i;
        break;
    }
  }

  delete listenerInfo;
}
void MigrationVerifyActor::handleVerifyRequest(const idgs::actor::ActorMessagePtr& msg) {
  pb::MigrationVerifyRequest* request = NULL;
  if (msg->getPayload()) {
    request = dynamic_cast<pb::MigrationVerifyRequest*>(msg->getPayload().get());
  }

  if (request && request->has_schema_name() && request->has_store_name()) {
    auto datastore = idgs::store::idgs_store_module()->getDataStore();
    auto store = datastore->getStore(request->schema_name(), request->store_name());
    if (!store) {
      LOG(ERROR) << "store " << request->schema_name() << "." << request->store_name() << " is not found.";
      auto payload = std::make_shared<pb::MigrationVerifyResponse>();
      payload->set_result_code(static_cast<int32_t>(RC_STORE_NOT_FOUND));
      auto respMsg = msg->createResponse();
      respMsg->setOperationName("VERIFY_RESPONSE");
      respMsg->setPayload(payload);
      idgs::actor::sendMessage(respMsg);
      return;
    }

    if (store->getStoreConfig()->getStoreConfig().partition_type() != idgs::store::pb::PARTITION_TABLE) {
      LOG(ERROR) << "store " << request->schema_name() << "." << request->store_name() << " is not a partition store.";
      auto payload = std::make_shared<pb::MigrationVerifyResponse>();
      payload->set_result_code(static_cast<int32_t>(RC_STORE_NOT_FOUND));
      auto respMsg = msg->createResponse();
      respMsg->setOperationName("VERIFY_RESPONSE");
      respMsg->setPayload(payload);
      idgs::actor::sendMessage(respMsg);
      return;
    }
  }

  clientMsg = msg;
  auto& members = idgs_application()->getMemberManager()->getMemberTable();
  auto it = members.begin();
  for (; it != members.end(); ++ it) {
    if (it->isLocalStore() && (it->getState() == idgs::pb::MS_PREPARED || it->getState() == idgs::pb::MS_ACTIVE)) {
      auto reqMsg = createActorMessage();
      reqMsg->setOperationName("LOCAL_VERIFY_REQUEST");
      reqMsg->setDestMemberId(it->getId());
      reqMsg->setDestActorId(actorId);
      reqMsg->setPayload(msg->getPayload());
      idgs::actor::sendMessage(reqMsg);
      ++ memberSize;
    }
  }
}
Ejemplo n.º 3
0
RddResultCode SumAction::action(const idgs::actor::ActorMessagePtr& msg, const BaseRddPartition* input,
    std::vector<PbVariant>& output) {
  ActionRequest* payload = dynamic_cast<ActionRequest*>(msg->getPayload().get());
  if (!payload->has_expression()) {
    LOG(ERROR)<< "RDD \"" << input->getRddName() << " partition[" << input->getPartition() << "]" << "sum action error, lack sum's field";
    return RRC_INVALID_ACTION_PARAM;
  }
  Expression* field_expr = NULL;
  auto rc = ExpressionFactory::build(&field_expr, payload->expression(), input->getKeyTemplate(),
      input->getValueTemplate());
  if (idgs::RC_SUCCESS != rc) {
    LOG(ERROR)<< "RDD \"" << input->getRddName() << " partition[" << input->getPartition() << "]" << "parse sum's field expression error, caused by " << idgs::getErrorDescription(rc);
    return RRC_NOT_SUPPORT;
  }
  double sum = 0;
  if (payload->has_filter()) { /// with filter
    Expression* filter_expr = NULL;
    rc = ExpressionFactory::build(&filter_expr, payload->filter(), input->getKeyTemplate(), input->getValueTemplate());
    if (idgs::RC_SUCCESS != rc) {
      LOG(ERROR)<< "RDD \"" << input->getRddName() << " partition[" << input->getPartition() << "]" << "parse filter expression error, caused by " << idgs::getErrorDescription(rc);
      return RRC_NOT_SUPPORT;
    }
    ExpressionContext ctx;
    input->foreach([filter_expr, field_expr, &sum, &ctx] (const PbMessagePtr& key, const PbMessagePtr& value) {
      ctx.setKeyValue(&key, &value);
      bool flag = (bool) filter_expr->evaluate(&ctx); /// filter expression's value is true or false
        if(flag) { /// meet filter condition
          PbVariant field_value = field_expr->evaluate(&ctx);
          sum += (double) field_value;
        }
      });
    output.push_back(sum);
  } /// end with filter
  else { /// without filter
    ExpressionContext ctx;
    input->foreach([field_expr, &sum, &ctx] (const PbMessagePtr& key, const PbMessagePtr& value) {
      ctx.setKeyValue(&key, &value);
      PbVariant field_value = field_expr->evaluate(&ctx);
      sum += (double) field_value;
    });
    output.push_back(sum);
  } /// end without filter
  return RRC_SUCCESS;
}
void PairStoreDelegateRddPartition::handleActionRequest(const idgs::actor::ActorMessagePtr& msg) {
  ActionRequest* request = dynamic_cast<ActionRequest*>(msg->getPayload().get());
  DVLOG(3) << getPartitionName() << " run " << request->action_op_name() << " action";

  shared_ptr<ActionResponse> response = make_shared<ActionResponse>();
  response->set_action_id(request->action_id());
  response->set_partition(partition);
  ActionMgr& actionMgr = *idgs_rdd_module()->getActionManager();
  const ActionPtr& action = actionMgr.get(request->action_op_name());

  RddResultCode code = RRC_SUCCESS;
  if (action.get()) {
    if (rddLocal->isReplicatedRdd() && partition > 0) {
      code = RRC_SUCCESS;
    } else {
      action::ActionContext ctx;
      if (ctx.initContext(&msg, &rddLocal->getKeyTemplate(), &rddLocal->getValueTemplate())) {
        code = action->action(&ctx, this);
        if (code == RRC_SUCCESS) {
          auto& result = ctx.getPartitionResult();
          for (int32_t i = 0; i < result.size(); ++i) {
            response->add_action_value(result[i].toString());
          }
        }
      } else {
        code = RRC_INVALID_ACTION_PARAM;
      }
    }
  } else {
    LOG(ERROR)<< "action " << request->action_op_name() << " is not registered.";
    code = RRC_ACTION_NOT_FOUND;
  }

  response->set_result_code(code);

  VLOG(2) << getPartitionName() << " process " << request->action_op_name() << " action done.";
  auto respMsg = msg->createResponse();
  respMsg->setOperationName(RDD_ACTION_RESPONSE);
  respMsg->setPayload(response);
  idgs::actor::sendMessage(respMsg);
}
void MigrationVerifyActor::handleVerifyResponse(const idgs::actor::ActorMessagePtr& msg) {
  auto response = dynamic_cast<pb::MigrationVerifyResponse*>(msg->getPayload().get());
  memberData.insert(std::pair<int32_t, pb::MemberMigrationData>(msg->getSourceMemberId(), response->member_data(0)));

  if (memberData.size() == memberSize) {
    auto payload = std::make_shared<pb::MigrationVerifyResponse>();
    payload->set_result_code(static_cast<int32_t>(RC_SUCCESS));
    auto it = memberData.begin();
    for (; it != memberData.end(); ++ it) {
      payload->add_member_data()->CopyFrom(it->second);
    }

    auto respMsg = clientMsg->createResponse();
    respMsg->setOperationName("VERIFY_RESPONSE");
    respMsg->setPayload(payload);

    idgs::actor::sendMessage(respMsg);

    memberData.clear();
    memberSize = 0;
  }
}
void DataAggregatorActor::handleCountResponse(const idgs::actor::ActorMessagePtr& msg) {
  DVLOG_FIRST_N(2, 20) << "receive store size of member " << msg->getSourceActorId();
  idgs::store::pb::SizeResponse* response = dynamic_cast<idgs::store::pb::SizeResponse*>(msg->getPayload().get());

  if (response->result_code() != idgs::store::pb::SRC_SUCCESS) {
    // error
    DVLOG_FIRST_N(2, 20) << "error at member " << msg->getSourceActorId();
    resultCode = response->result_code();
  }

  // collect response from all members and calculate size and then response to client.
  shared_ptr<idgs::store::pb::SizeResponse> globalResponse = make_shared<idgs::store::pb::SizeResponse>();
  dataSizeMap[msg->getSourceMemberId()] = response->size();
  if (dataSizeMap.size() == requestCount) {
    DVLOG_FIRST_N(2, 20) << "aggregate done.";
    if (resultCode != idgs::store::pb::SRC_SUCCESS) {
      globalResponse->set_result_code(resultCode);
    } else {
      // calculate size
      size_t size = 0;
      map<int32_t, size_t>::iterator it = dataSizeMap.begin();
      for (; it != dataSizeMap.end(); ++ it) {
        size += (* it).second;
      }

      globalResponse->set_result_code(idgs::store::pb::SRC_SUCCESS);
      globalResponse->set_size(size);
    }

    // response result
    DVLOG_FIRST_N(2, 20) << "response client.";
    sendResponse(OP_COUNT_RESPONSE, globalResponse);

    terminate();
  }
}
void DataAggregatorActor::handleTruncateResponse(const idgs::actor::ActorMessagePtr& msg) {
  DVLOG_FIRST_N(2, 20) << "aggregator handle truncate store response";
  idgs::store::pb::TruncateResponse* response = dynamic_cast<idgs::store::pb::TruncateResponse*>(msg->getPayload().get());

  if (response->result_code() != idgs::store::pb::SRC_SUCCESS) {
    LOG(ERROR)<< "Truncate store error at member " << msg->getSourceMemberId();
    resultCode = response->result_code();
  }

  ++ responseCount;
  DVLOG_FIRST_N(2, 20) << "member " << msg->getSourceMemberId() << " response, left " << requestCount - responseCount << " members.";
  if (requestCount == responseCount) {
    DVLOG_FIRST_N(2, 20) << "all member responsed, send response to client.";

    shared_ptr<idgs::store::pb::TruncateResponse> globalResponse = make_shared<idgs::store::pb::TruncateResponse>();
    globalResponse->set_result_code(resultCode);
    sendResponse(OP_TRUNCATE_RESPONSE, globalResponse);

    terminate();
  }
}
void MigrationVerifyActor::handleLocalVerifyRequest(const idgs::actor::ActorMessagePtr& msg) {
  pb::MigrationVerifyRequest* request = NULL;
  if (msg->getPayload()) {
    request = dynamic_cast<pb::MigrationVerifyRequest*>(msg->getPayload().get());
  }

  auto datastore = idgs::store::idgs_store_module()->getDataStore();
  std::vector<idgs::store::StorePtr> stores;
  if (request && request->has_schema_name() && request->has_store_name()) {
    auto store = datastore->getStore(request->schema_name(), request->store_name());
    stores.push_back(store);
  } else {
    datastore->getStores(stores);
  }

  auto app = idgs_application();
  auto cluster = app->getClusterFramework();
  auto pcnt = cluster->getPartitionCount();
  auto bkcnt = cluster->getClusterConfig()->max_replica_count() - 1;
  auto local = cluster->getLocalMember()->getId();
  auto partitionMgr = app->getPartitionManager();

  auto payload = std::make_shared<pb::MigrationVerifyResponse>();
  payload->set_result_code(static_cast<int32_t>(RC_SUCCESS));

  auto memberData = payload->add_member_data();
  memberData->set_member_id(local);

  for (int32_t i = 0; i < stores.size(); ++ i) {
    auto& store = stores.at(i);
    auto& storeConfigWrapper = store->getStoreConfig();
    if (storeConfigWrapper->getStoreConfig().partition_type() == idgs::store::pb::PARTITION_TABLE) {
      auto pstore = dynamic_cast<idgs::store::PartitionedStore*>(store.get());

      auto& schemaName = store->getStoreConfig()->getSchema();
      auto& storeName = store->getStoreConfig()->getStoreConfig().name();
      auto storeData = memberData->add_store_data();
      storeData->set_schema_name(schemaName);
      storeData->set_store_name(storeName);

      for (int32_t p = 0; p < pcnt; ++ p) {
        auto partition = partitionMgr->getPartition(p);
        for (int32_t pos = 0; pos < bkcnt + 1; ++ pos) {
          if (partition->getMemberId(pos) == local) {
            auto partitionData = storeData->add_partition_data();
            partitionData->set_partition_id(p);
            partitionData->set_position(pos);
            partitionData->set_member_id(local);
            partitionData->set_size(pstore->dataSize(p));
            VLOG(0) << schemaName << "." << storeName << " partition " << p << "(" << pos << ") data size " << partitionData->size() << " on member " << local;

            std::shared_ptr<idgs::store::StoreMap> map;
            pstore->snapshotStore(p, map);
            auto it = map->iterator();
            while (it->hasNext()) {
              idgs::store::StoreOption ps;
              storeConfigWrapper->calculatePartitionInfo(it->key(), &ps);
              ps.memberId = partitionMgr->getPartition(ps.partitionId)->getMemberId(pos);

              auto keyPartition = partitionData->add_key_partition();
              keyPartition->set_key_partition_id(ps.partitionId);
              keyPartition->set_key_member_id(ps.memberId);

              it->next();
            }
          }
        }
      }
    }
  }

  auto respMsg = msg->createResponse();
  respMsg->setOperationName("VERIFY_RESPONSE");
  respMsg->setPayload(payload);
  idgs::actor::sendMessage(respMsg);
}