void TopNAction::registerTempKeyMessage(const uint32_t& partition, const std::string& typeName,
const std::vector<FieldInfo>& fields) {
DynamicTypeComposer composer;
composer.setPackage("idgs.rdd.pb");
composer.setName(typeName);
DynamicMessage dmKey;
dmKey.name = typeName;
auto it = fields.begin();
for (; it != fields.end(); ++ it) {
DynamicField dyField = DynamicField("required", idgs::str::toLower(DataType_Name(it->type)), it->name);
dmKey.addField(dyField);
}
composer.addMessage(dmKey);
string fileName = RDD_DYNAMIC_PROTO_PATH + typeName + "_" + to_string(partition) + ".proto";
composer.saveFile(fileName);
auto helper = idgs_rdd_module()->getMessageHelper();
auto code = helper->registerDynamicMessage(fileName);
if (code != RC_SUCCESS) {
LOG(ERROR)<< "Invalid message type, caused by " << getErrorDescription(code);
}
remove(fileName.c_str());
}
int TpcModule::stop(void){
function_footprint();
LOG(INFO) << "stop module tpc_svc";
TransformerMgr& transformerMgr = idgs_rdd_module()->getTransformManager();
transformerMgr.remove(TPCH_Q6_TRANSFORMER);
transformerMgr.remove(SSB_Q1_1_TRANSFORMER);
ActionMgr& actionMgr = idgs_rdd_module()->getActionManager();
actionMgr.remove(TPCH_Q6_ACTION);
actionMgr.remove(SSB_Q1_1_ACTION);
DVLOG(3) << "begin unregister actor";
app->getActorframework()->unRegisterStatelessActor("linecrud_actor"); /// unregister actor
DVLOG(3) << "end unregister actor";
DVLOG(3) << "begin unregister module descriptor";
::idgs::util::singleton<idgs::actor::ActorDescriptorMgr>::getInstance().unRegisterModuleDescriptor(TPC_MODULE_DESCRIPTOR_NAME);/// unregister module descriptor
DVLOG(3) << "end unregister module actor";
return RC_OK;
}
int TpcModule::init(const char* config_path, idgs::Application* theApp){
function_footprint();
LOG(INFO) << "initialize module tpc_svc";
app = theApp;
/// Create LineCrud Actor
LineCrudActor* actor = new LineCrudActor;
actor->init();
app->getActorframework()->Register(actor->getActorId(), actor); /// Register Actor
/// Register module descriptor
std::shared_ptr<ModuleDescriptorWrapper> module_descriptor(new ModuleDescriptorWrapper);
module_descriptor->setName(TPC_MODULE_DESCRIPTOR_NAME);
module_descriptor->setDescription("tpc module descriptor");
module_descriptor->addActorDescriptor(actor->getDescriptor());
::idgs::util::singleton<idgs::actor::ActorDescriptorMgr>::getInstance().registerModuleDescriptor(module_descriptor->getName(), module_descriptor);
TransformerMgr& transformerMgr = idgs_rdd_module()->getTransformManager();
TransformerPtr tpchQ6Transformer(new TpchQ6Transformer);
transformerMgr.put(TPCH_Q6_TRANSFORMER, tpchQ6Transformer);
TransformerPtr ssbQ1Transformer(new SsbQ1_1Transformer);
transformerMgr.put(SSB_Q1_1_TRANSFORMER, ssbQ1Transformer);
ActionMgr& actionMgr = idgs_rdd_module()->getActionManager();
ActionPtr tpchQ6Action(new TpchQ6Action);
actionMgr.put(TPCH_Q6_ACTION, tpchQ6Action);
ActionPtr ssbQ1Action(new SsbQ1_1Action);
actionMgr.put(SSB_Q1_1_ACTION, ssbQ1Action);
ActionPtr partition_count_action(new PartitionCountAction);
actionMgr.put(PARTITION_COUNT_ACTION, partition_count_action);
return RC_OK;
}
void PairStoreDelegateRddPartition::handleActionRequest(const idgs::actor::ActorMessagePtr& msg) {
ActionRequest* request = dynamic_cast<ActionRequest*>(msg->getPayload().get());
DVLOG(3) << getPartitionName() << " run " << request->action_op_name() << " action";
shared_ptr<ActionResponse> response = make_shared<ActionResponse>();
response->set_action_id(request->action_id());
response->set_partition(partition);
ActionMgr& actionMgr = *idgs_rdd_module()->getActionManager();
const ActionPtr& action = actionMgr.get(request->action_op_name());
RddResultCode code = RRC_SUCCESS;
if (action.get()) {
if (rddLocal->isReplicatedRdd() && partition > 0) {
code = RRC_SUCCESS;
} else {
action::ActionContext ctx;
if (ctx.initContext(&msg, &rddLocal->getKeyTemplate(), &rddLocal->getValueTemplate())) {
code = action->action(&ctx, this);
if (code == RRC_SUCCESS) {
auto& result = ctx.getPartitionResult();
for (int32_t i = 0; i < result.size(); ++i) {
response->add_action_value(result[i].toString());
}
}
} else {
code = RRC_INVALID_ACTION_PARAM;
}
}
} else {
LOG(ERROR)<< "action " << request->action_op_name() << " is not registered.";
code = RRC_ACTION_NOT_FOUND;
}
response->set_result_code(code);
VLOG(2) << getPartitionName() << " process " << request->action_op_name() << " action done.";
auto respMsg = msg->createResponse();
respMsg->setOperationName(RDD_ACTION_RESPONSE);
respMsg->setPayload(response);
idgs::actor::sendMessage(respMsg);
}
RddResultCode TopNAction::action(ActionContext* ctx, const BaseRddPartition* input) {
TopNActionRequest request;
if (!ctx->getActionParam(ACTION_PARAM, &request)) {
LOG(ERROR)<< input->getPartitionName() << " parse top N expression error";
return RRC_INVALID_ACTION_PARAM;
}
int32_t fieldSize = request.order_field_size();
vector<FieldInfo> fields;
vector<bool> fieldDesc;
fields.resize(fieldSize);
fieldDesc.resize(fieldSize);
string typeName = input->getRddName();
for (int32_t i = 0; i < fieldSize; ++ i) {
auto& field = request.order_field(i);
auto rc = ExpressionFactory::build(&fields[i].expr, field.expr(), ctx->getKeyTemplate(), ctx->getValueTemplate());
if (idgs::RC_SUCCESS != rc) {
LOG(ERROR)<< input->getPartitionName() << " parsed filter expression error, caused by " << idgs::getErrorDescription(rc);
return RRC_NOT_SUPPORT;
}
if (field.has_field_name()) {
fields[i].name = field.field_name();
} else if (field.expr().name() == "FIELD") {
fields[i].name = field.expr().value();
} else {
fields[i].name = "column" + to_string(i);
}
if (field.has_field_type()) {
fields[i].type = field.field_type();
} else if (field.expr().name() == "FIELD") {
FieldExtractor* fieldExt = dynamic_cast<FieldExtractor*>(fields[i].expr);
auto fldType = fieldExt->getFieldType();
fields[i].type = static_cast<idgs::pb::DataType>(fldType);
} else {
fields[i].type = idgs::pb::STRING;
}
fieldDesc[i] = field.desc();
typeName.append("_").append(fields[i].name);
}
string fullTypeName = "idgs.rdd.pb." + typeName;
auto helper = idgs_rdd_module()->getMessageHelper();
if (!helper->isMessageRegistered(fullTypeName)) {
std::lock_guard<std::mutex> lockGuard(lock);
if (!helper->isMessageRegistered(fullTypeName)) {
registerTempKeyMessage(input->getPartition(), typeName, fields);
}
}
if (input->empty()) {
return RRC_SUCCESS;
}
PbMessagePtr tmpKeyTemp = helper->createMessage(fullTypeName);
if (!tmpKeyTemp) {
LOG(ERROR)<< "RDD \"" << input->getRddName() << "\" partition[" << input->getPartition() << "] generate dynamic key error.";
return RRC_INVALID_KEY;
}
orderless compare(fieldDesc);
multimap<PbMessagePtr, KeyValueMessagePair, orderless> tmpMap(compare);
auto tmpDescriptor = tmpKeyTemp->GetDescriptor();
uint64_t start = 1, dataSize = 0;
bool hasTop = request.has_top_n();
if (request.has_start()) {
start = request.start();
}
if (request.has_top_n()) {
auto topN = request.top_n();
dataSize = topN + start - 1;
}
input->foreach(
[&ctx, &tmpMap, fields, tmpKeyTemp, tmpDescriptor, dataSize, hasTop, helper]
(const PbMessagePtr& key, const PbMessagePtr& value) {
ctx->setKeyValue(&key, &value);
if (!ctx->evaluateFilterExpr()) {
return;
}
PbMessagePtr tmpKey(tmpKeyTemp->New());
for (int32_t i = 0; i < tmpDescriptor->field_count(); ++ i) {
auto field = tmpDescriptor->field(i);
PbVariant var = fields[i].expr->evaluate(ctx->getExpressionContext());
helper->setMessageValue(tmpKey.get(), field, var);
}
KeyValueMessagePair pair;
pair.key = &key;
pair.value = &value;
tmpMap.insert(std::pair<PbMessagePtr, KeyValueMessagePair>(tmpKey, pair));
if (hasTop && tmpMap.size() > dataSize) {
auto itTmp = tmpMap.end();
-- itTmp;
tmpMap.erase(itTmp);
}
});
if (!tmpMap.empty()) {
TopNActionData data;
data.set_key_name(fullTypeName);
auto serdesMode = ctx->getSerdesMode();
auto itfst = tmpMap.begin();
auto itlst = tmpMap.end();
for (; itfst != itlst; ++ itfst) {
auto pair = data.add_data();
ProtoSerdesHelper::serialize(serdesMode, itfst->first.get(), pair->mutable_key());
ProtoSerdesHelper::serialize(serdesMode, itfst->second.key->get(), pair->mutable_pair()->mutable_key());
ProtoSerdesHelper::serialize(serdesMode, itfst->second.value->get(), pair->mutable_pair()->mutable_value());
}
string var;
ProtoSerdesHelper::serialize(serdesMode, &data, &var);
ctx->addPartitionResult(var);
}
return RRC_SUCCESS;
}
RddResultCode TopNAction::aggregate(ActionContext* ctx) {
TopNActionRequest request;
if (!ctx->getActionParam(ACTION_PARAM, &request)) {
return RRC_INVALID_ACTION_PARAM;
}
int32_t fieldSize = request.order_field_size();
vector<bool> fieldDesc;
fieldDesc.resize(fieldSize);
for (int32_t i = 0; i < fieldSize; ++ i) {
auto& field = request.order_field(i);
fieldDesc[i] = field.desc();
}
uint64_t start = (request.has_start()) ? request.start() : 1;
bool hasTop = request.has_top_n();
uint64_t dataSize = (hasTop) ? request.top_n() + start - 1 : 0;
orderless compare(fieldDesc);
multimap<PbMessagePtr, KeyValuePair, orderless> tmpMap(compare);
auto helper = idgs_rdd_module()->getMessageHelper();
TopNActionData data;
auto& input = ctx->getAggregateResult();
auto it = input.begin();
auto serdesMode = ctx->getSerdesMode();
for (; it != input.end(); ++ it) {
if (!it->empty()) {
ProtoSerdesHelper::deserialize(serdesMode, (*it)[0], &data);
for (int32_t i = 0; i < data.data_size(); ++i) {
auto tmpKey = helper->createMessage(data.key_name());
auto pair = data.data(i);
ProtoSerdesHelper::deserialize(serdesMode, pair.key(), tmpKey.get());
KeyValuePair kvPair = pair.pair();
tmpMap.insert(std::pair<PbMessagePtr, KeyValuePair>(tmpKey, kvPair));
if (hasTop && tmpMap.size() > dataSize) {
auto itTmp = tmpMap.end();
-- itTmp;
tmpMap.erase(itTmp);
}
}
}
}
shared_ptr<TopNActionResult> result = make_shared<TopNActionResult>();
if (!tmpMap.empty()) {
auto itfst = tmpMap.begin();
auto itlst = tmpMap.end();
uint64_t index = 1;
for (; itfst != itlst; ++ itfst, ++ index) {
if (index >= start) {
result->add_pair()->CopyFrom(itfst->second);
}
}
}
ctx->setActionResult(result);
return RRC_SUCCESS;
}
