void BaseRddPartition::transform() { auto& downstreamPartition = rddLocal->getDownstreamPartition(partition); if (!downstreamPartition.empty()) { pb::RddResultCode code = pb::RRC_SUCCESS; for (int32_t i = 0; i < downstreamPartition.size(); ++ i) { auto& rddlocal = downstreamPartition[i]->getRddLocal(); auto& ctx = transformerContexts.at(i); if (!rddlocal->isUpstreamSync()) { auto upstreamPartition = rddlocal->getUpstreamPartition(partition); if (upstreamPartition[0] == this) { if (ctx.getParamRdds().empty()) { std::vector<BaseRddPartition*> params; for (int32_t i = 1; i < upstreamPartition.size(); ++ i) { params.push_back(upstreamPartition[i]); } ctx.setParamRdds(params); } } else { continue; } } code = rddlocal->getTransformer()->transform(&ctx, this, downstreamPartition[i]); if (code != idgs::rdd::pb::RRC_SUCCESS) { LOG(ERROR) << getPartitionName() << " transform error, caused by " << idgs::rdd::pb::RddResultCode_Name(code); } } } }
void BaseRddPartition::transform(const idgs::actor::PbMessagePtr& key, const idgs::actor::PbMessagePtr& value) { if (!key.get()) { LOG(ERROR)<< getPartitionName() << " transform data, key is null"; return; } if(!value) { LOG(ERROR)<< getPartitionName() << " transform data, value is null"; return; } auto& downstreamPartition = rddLocal->getDownstreamPartition(partition); if (!downstreamPartition.empty()) { idgs::rdd::pb::RddResultCode code = idgs::rdd::pb::RRC_SUCCESS; for (int32_t i = 0; i < downstreamPartition.size(); ++ i) { auto& rddlocal = downstreamPartition[i]->getRddLocal(); auto& ctx = transformerContexts.at(i); idgs::actor::PbMessagePtr outkey, outvalue; ctx.getExpressionContext()->setKeyValue(&key, &value); ctx.getExpressionContext()->setOutputKeyValue(&outkey, &outvalue); if (!rddlocal->isUpstreamSync()) { auto upstreamPartition = rddlocal->getUpstreamPartition(partition); if (upstreamPartition[0] == this) { if (ctx.getParamRdds().empty()) { std::vector<BaseRddPartition*> params; for (int32_t i = 1; i < upstreamPartition.size(); ++ i) { params.push_back(upstreamPartition[i]); } ctx.setParamRdds(params); } } else { continue; } } code = rddlocal->getTransformer()->transform(&ctx, downstreamPartition[i]); if (code != idgs::rdd::pb::RRC_SUCCESS) { LOG(ERROR) << getPartitionName() << " transform error, caused by " << idgs::rdd::pb::RddResultCode_Name(code); } } } }
void PairStoreDelegateRddPartition::handleActionRequest(const idgs::actor::ActorMessagePtr& msg) { ActionRequest* request = dynamic_cast<ActionRequest*>(msg->getPayload().get()); DVLOG(3) << getPartitionName() << " run " << request->action_op_name() << " action"; shared_ptr<ActionResponse> response = make_shared<ActionResponse>(); response->set_action_id(request->action_id()); response->set_partition(partition); ActionMgr& actionMgr = *idgs_rdd_module()->getActionManager(); const ActionPtr& action = actionMgr.get(request->action_op_name()); RddResultCode code = RRC_SUCCESS; if (action.get()) { if (rddLocal->isReplicatedRdd() && partition > 0) { code = RRC_SUCCESS; } else { action::ActionContext ctx; if (ctx.initContext(&msg, &rddLocal->getKeyTemplate(), &rddLocal->getValueTemplate())) { code = action->action(&ctx, this); if (code == RRC_SUCCESS) { auto& result = ctx.getPartitionResult(); for (int32_t i = 0; i < result.size(); ++i) { response->add_action_value(result[i].toString()); } } } else { code = RRC_INVALID_ACTION_PARAM; } } } else { LOG(ERROR)<< "action " << request->action_op_name() << " is not registered."; code = RRC_ACTION_NOT_FOUND; } response->set_result_code(code); VLOG(2) << getPartitionName() << " process " << request->action_op_name() << " action done."; auto respMsg = msg->createResponse(); respMsg->setOperationName(RDD_ACTION_RESPONSE); respMsg->setPayload(response); idgs::actor::sendMessage(respMsg); }
const NAString ElemDDLPartitionSystem::displayLabel3() const { if (NOT getLocationName().isNull()) { return NAString("Partition name: ") + getPartitionName(); } else { return NAString("Partition name not specified."); } }
void PairStoreDelegateRddPartition::initPartitionStore(const idgs::store::StorePtr& store) { schemaName = store->getStoreConfig()->getSchema(); storeName = store->getStoreConfig()->getStoreConfig().name(); auto& storeConfigWrapper = store->getStoreConfig(); switch (storeConfigWrapper->getStoreConfig().partition_type()) { case idgs::store::pb::PARTITION_TABLE: { PartitionedStore* pstore = dynamic_cast<PartitionedStore*>(store.get()); pstore->scanPartitionData(partition, dataMap); break; } case idgs::store::pb::REPLICATED: { ReplicatedStore* rstore = dynamic_cast<ReplicatedStore*>(store.get()); rstore->scanData(dataMap); break; } default: { break; } } DVLOG(2) << getPartitionName() << " catch store " << schemaName << "." << storeName << " " << dataMap->size() << " data"; }