Example #1
0
bool ActorMessage::parseAttachment1(const std::string& name, PbMessagePtr& msg) {
  if (!msg) {
    LOG(ERROR)<< "message is NULL";
    return false;
  }

  auto it = attachments.find(name);
  if (it != attachments.end()) {
    msg->CopyFrom(*it->second);
    return true;
  }

  protobuf::SerdesMode mode = static_cast<protobuf::SerdesMode>(getRpcMessage()->serdes_type());
  auto range = rawAttachments.equal_range(name);
  if (range.first == range.second) {
    LOG(ERROR) << "No raw attachment";
    return false;
  }
  for (auto it = range.first; it != range.second; ++it) {
    PbMessagePtr v(msg->New());
    bool result = protobuf::ProtoSerdesHelper::deserialize(mode, it->second, v.get());
    if(!result) {
      LOG(ERROR) << "Failed to deserialize raw attachment: " << mode << ", data: " << it->second;
    } else {
      attachments.insert(std::pair<std::string, PbMessagePtr>(name, v));
    }
  }

  return true;
}
RddResultCode TopNAction::action(ActionContext* ctx, const BaseRddPartition* input) {
  TopNActionRequest request;
  if (!ctx->getActionParam(ACTION_PARAM, &request)) {
    LOG(ERROR)<< input->getPartitionName() << " parse top N expression error";
    return RRC_INVALID_ACTION_PARAM;
  }

  int32_t fieldSize = request.order_field_size();
  vector<FieldInfo> fields;
  vector<bool> fieldDesc;
  fields.resize(fieldSize);
  fieldDesc.resize(fieldSize);
  string typeName = input->getRddName();
  for (int32_t i = 0; i < fieldSize; ++ i) {
    auto& field = request.order_field(i);
    auto rc = ExpressionFactory::build(&fields[i].expr, field.expr(), ctx->getKeyTemplate(), ctx->getValueTemplate());
    if (idgs::RC_SUCCESS != rc) {
      LOG(ERROR)<< input->getPartitionName() << " parsed filter expression error, caused by " << idgs::getErrorDescription(rc);
      return RRC_NOT_SUPPORT;
    }

    if (field.has_field_name()) {
      fields[i].name = field.field_name();
    } else if (field.expr().name() == "FIELD") {
      fields[i].name = field.expr().value();
    } else {
      fields[i].name = "column" + to_string(i);
    }

    if (field.has_field_type()) {
      fields[i].type = field.field_type();
    } else if (field.expr().name() == "FIELD") {
      FieldExtractor* fieldExt = dynamic_cast<FieldExtractor*>(fields[i].expr);
      auto fldType = fieldExt->getFieldType();
      fields[i].type = static_cast<idgs::pb::DataType>(fldType);
    } else {
      fields[i].type = idgs::pb::STRING;
    }

    fieldDesc[i] = field.desc();
    typeName.append("_").append(fields[i].name);
  }

  string fullTypeName = "idgs.rdd.pb." + typeName;

  auto helper = idgs_rdd_module()->getMessageHelper();
  if (!helper->isMessageRegistered(fullTypeName)) {
    std::lock_guard<std::mutex> lockGuard(lock);
    if (!helper->isMessageRegistered(fullTypeName)) {
      registerTempKeyMessage(input->getPartition(), typeName, fields);
    }
  }

  if (input->empty()) {
    return RRC_SUCCESS;
  }

  PbMessagePtr tmpKeyTemp = helper->createMessage(fullTypeName);
  if (!tmpKeyTemp) {
    LOG(ERROR)<< "RDD \"" << input->getRddName() << "\" partition[" << input->getPartition() << "] generate dynamic key error.";
    return RRC_INVALID_KEY;
  }

  orderless compare(fieldDesc);
  multimap<PbMessagePtr, KeyValueMessagePair, orderless> tmpMap(compare);
  auto tmpDescriptor = tmpKeyTemp->GetDescriptor();

  uint64_t start = 1, dataSize = 0;
  bool hasTop = request.has_top_n();
  if (request.has_start()) {
    start = request.start();
  }

  if (request.has_top_n()) {
    auto topN = request.top_n();
    dataSize = topN + start - 1;
  }

  input->foreach(
    [&ctx, &tmpMap, fields, tmpKeyTemp, tmpDescriptor, dataSize, hasTop, helper]
     (const PbMessagePtr& key, const PbMessagePtr& value) {
      ctx->setKeyValue(&key, &value);
      if (!ctx->evaluateFilterExpr()) {
        return;
      }

      PbMessagePtr tmpKey(tmpKeyTemp->New());
      for (int32_t i = 0; i < tmpDescriptor->field_count(); ++ i) {
        auto field = tmpDescriptor->field(i);
        PbVariant var = fields[i].expr->evaluate(ctx->getExpressionContext());
        helper->setMessageValue(tmpKey.get(), field, var);
      }

      KeyValueMessagePair pair;
      pair.key = &key;
      pair.value = &value;

      tmpMap.insert(std::pair<PbMessagePtr, KeyValueMessagePair>(tmpKey, pair));

      if (hasTop && tmpMap.size() > dataSize) {
        auto itTmp = tmpMap.end();
        -- itTmp;
        tmpMap.erase(itTmp);
      }
    });

  if (!tmpMap.empty()) {
    TopNActionData data;
    data.set_key_name(fullTypeName);

    auto serdesMode = ctx->getSerdesMode();
    auto itfst = tmpMap.begin();
    auto itlst = tmpMap.end();
    for (; itfst != itlst; ++ itfst) {
      auto pair = data.add_data();
      ProtoSerdesHelper::serialize(serdesMode, itfst->first.get(), pair->mutable_key());
      ProtoSerdesHelper::serialize(serdesMode, itfst->second.key->get(), pair->mutable_pair()->mutable_key());
      ProtoSerdesHelper::serialize(serdesMode, itfst->second.value->get(), pair->mutable_pair()->mutable_value());
    }

    string var;
    ProtoSerdesHelper::serialize(serdesMode, &data, &var);
    ctx->addPartitionResult(var);
  }
  return RRC_SUCCESS;
}