InnerTcpConnection::~InnerTcpConnection() {
function_footprint();
if (socket.is_open()) {
socket.close();
}
}
void MembershipTableMgr::handleNormalJoinEvent(const MemberWrapper& member) {
function_footprint();
queue.push(member);
if(joining) {
DVLOG(0) << "$$$$$$$$$$$$$$$$$$$$$$$$$$";
DVLOG(0) << "$$$$$$$$$$$$$$$$$$$$$$$$$$";
DVLOG(0) << "current joining member queue size: " << queue.unsafe_size();
DVLOG(0) << "$$$$$$$$$$$$$$$$$$$$$$$$$$";
DVLOG(0) << "$$$$$$$$$$$$$$$$$$$$$$$$$$";
return;
}
while(!queue.empty()) {
joining.store(true);
MemberWrapper joined_member;
queue.try_pop(joined_member);
// leading generate whole member table and send to joining member.
auto wholeMemberTable = std::make_shared<WholeMembershipTableEvent>();
genMembershipTable(*wholeMemberTable->mutable_table());
ResultCode rs;
rs = multicastMemberMessage(WHOLE_MEMBERSHIP_TABLE, wholeMemberTable);
if (rs != RC_SUCCESS) {
LOG(ERROR)<< getErrorDescription(rs);
return;
}
// leading send delta member to all members.
auto deltaMember = std::make_shared<DeltaMemberEvent>();
deltaMember->set_position(findAddPos());
deltaMember->mutable_member()->CopyFrom(joined_member.getMember());
rs = multicastMemberMessage(DELTA_MEMBER_AND_JOIN_POSITION, deltaMember);
if (rs != RC_SUCCESS) {
LOG(ERROR)<< getErrorDescription(rs);
return;
}
}
}
void MembershipTableMgr::handleLeadingJoinEvent(const MemberWrapper& joined_member) {
function_footprint();
auto leading = addMember(joined_member);
// set local member index
localIndex = leading->getId();
// joined
leading->setStatus(JOINED);
setMemberStatus(leading, JOINED);
}
void MembershipTableMgr::handleJoinedEvent(const std::shared_ptr<ActorMessage>& actor_msg_ptr) {
function_footprint();
Member* payload = dynamic_cast<Member*>(actor_msg_ptr->getPayload().get());
MemberWrapper joined_member;
joined_member.setMember(*payload);
if (joined_member.isLeading()) { // handle leading member join
handleLeadingJoinEvent(joined_member);
} else if (getLocalMember() && getLocalMember()->isLeading()) { // only leading handle normal member join
handleNormalJoinEvent(joined_member);
}
}
void MembershipTableMgr::handleWholeTableEvent(const std::shared_ptr<ActorMessage>& actor_msg_ptr) {
function_footprint();
if(!members.empty()) {
return;
}
auto evt = dynamic_cast<WholeMembershipTableEvent*>(actor_msg_ptr->getPayload().get());
for (int i = 0, size = evt->table().member_size(); i < size; ++i) {
MemberWrapper member;
member.setMember(evt->table().member(i));
members.push_back(member);
}
LOG(INFO)<< toSimpleString();
}
void ActorDescriptorMgr::unRegisterModuleDescriptor(const std::string& module_name) {
function_footprint();
auto itr = moduleDescriptors.find(module_name);
if (itr == moduleDescriptors.end()) {
return;
}
const std::map<std::string, idgs::actor::ActorDescriptorPtr>& map = itr->second->getActorDescriptors();
for (auto it = map.begin(); it != map.end(); ++it) {
removeActorDescriptor(it->first);
}
moduleDescriptors.erase(module_name);
}
void MembershipTableMgr::handleDeltaMemberEvent(const std::shared_ptr<ActorMessage>& msg) {
function_footprint();
if(members.empty()){
return;
}
auto payload = dynamic_cast<DeltaMemberEvent*>(msg->getPayload().get());
MemberWrapper member;
member.setMember(payload->member());
const uint32_t pos = payload->position();
auto joined_member = addMember(member, pos);
if(isLocalMember(*joined_member)) {
localIndex = pos;
}
setMemberStatus(joined_member, JOINED);
}
void MembershipTableMgr::handleMemberStatusEvent(const std::shared_ptr<ActorMessage>& actor_msg_ptr) {
function_footprint();
if (members.empty() || !getLocalMember()) {
return;
}
auto payload = dynamic_cast<MemberStatusEvent*>(actor_msg_ptr->getPayload().get());
uint32_t pos = payload->member_id();
auto status = payload->status();
auto member = getMember(pos);
if (!member) {
return;
}
setMemberStatus(member, status);
if(member->isPrepared()) {
DVLOG(0) << toSimpleString();
joining.store(false);
}
}
int TpcModule::stop(void){
function_footprint();
LOG(INFO) << "stop module tpc_svc";
TransformerMgr& transformerMgr = idgs_rdd_module()->getTransformManager();
transformerMgr.remove(TPCH_Q6_TRANSFORMER);
transformerMgr.remove(SSB_Q1_1_TRANSFORMER);
ActionMgr& actionMgr = idgs_rdd_module()->getActionManager();
actionMgr.remove(TPCH_Q6_ACTION);
actionMgr.remove(SSB_Q1_1_ACTION);
DVLOG(3) << "begin unregister actor";
app->getActorframework()->unRegisterStatelessActor("linecrud_actor"); /// unregister actor
DVLOG(3) << "end unregister actor";
DVLOG(3) << "begin unregister module descriptor";
::idgs::util::singleton<idgs::actor::ActorDescriptorMgr>::getInstance().unRegisterModuleDescriptor(TPC_MODULE_DESCRIPTOR_NAME);/// unregister module descriptor
DVLOG(3) << "end unregister module actor";
return RC_OK;
}
int TpcModule::init(const char* config_path, idgs::Application* theApp){
function_footprint();
LOG(INFO) << "initialize module tpc_svc";
app = theApp;
/// Create LineCrud Actor
LineCrudActor* actor = new LineCrudActor;
actor->init();
app->getActorframework()->Register(actor->getActorId(), actor); /// Register Actor
/// Register module descriptor
std::shared_ptr<ModuleDescriptorWrapper> module_descriptor(new ModuleDescriptorWrapper);
module_descriptor->setName(TPC_MODULE_DESCRIPTOR_NAME);
module_descriptor->setDescription("tpc module descriptor");
module_descriptor->addActorDescriptor(actor->getDescriptor());
::idgs::util::singleton<idgs::actor::ActorDescriptorMgr>::getInstance().registerModuleDescriptor(module_descriptor->getName(), module_descriptor);
TransformerMgr& transformerMgr = idgs_rdd_module()->getTransformManager();
TransformerPtr tpchQ6Transformer(new TpchQ6Transformer);
transformerMgr.put(TPCH_Q6_TRANSFORMER, tpchQ6Transformer);
TransformerPtr ssbQ1Transformer(new SsbQ1_1Transformer);
transformerMgr.put(SSB_Q1_1_TRANSFORMER, ssbQ1Transformer);
ActionMgr& actionMgr = idgs_rdd_module()->getActionManager();
ActionPtr tpchQ6Action(new TpchQ6Action);
actionMgr.put(TPCH_Q6_ACTION, tpchQ6Action);
ActionPtr ssbQ1Action(new SsbQ1_1Action);
actionMgr.put(SSB_Q1_1_ACTION, ssbQ1Action);
ActionPtr partition_count_action(new PartitionCountAction);
actionMgr.put(PARTITION_COUNT_ACTION, partition_count_action);
return RC_OK;
}
idgs::ResultCode MembershipTableMgr::memberLeft(const std::vector<MemberWrapper*>& leftMembers) {
function_footprint();
if (!getLocalMember()) {
return RC_OK;
}
for(auto it = leftMembers.begin(); it != leftMembers.end(); ++it) { // local member exists in left member list, quit, do nothing
if(getLocalMemberId() == (*it)->getId()) { ///left member itself, do nothing
return RC_OK;
}
}
MemberWrapper* leaveLeadingMember = NULL;
for(auto it = leftMembers.begin(); it != leftMembers.end(); ++it) { // check if exists leading leave
if ((*it)->isLeading()) {
leaveLeadingMember = *it;
break;
}
}
if(leaveLeadingMember) { /// leading member leave
MemberWrapper* newLeadingMember = selectLeading(leftMembers);
if(!newLeadingMember) { // no member can be selected as leading,quit!
return RC_OK;
}
DVLOG(0) << "Member " << *newLeadingMember << " is selected as leading";
newLeadingMember->setIsleading(true);
leaveLeadingMember->setIsleading(false);
for(auto it = leftMembers.begin(); it != leftMembers.end(); ++it) {
setMemberStatus(*it, INACTIVE);
}
} /// end if
else { /// normal member leave
for(auto it = leftMembers.begin(); it != leftMembers.end(); ++it) { // local member exists in left member list, quit, do nothing
setMemberStatus(*it, INACTIVE);
}
}
LOG(INFO) << toSimpleString();
return RC_OK;
}
ActorDescriptorMgr::~ActorDescriptorMgr() {
function_footprint();
}
int TpcModule::start(void) {
function_footprint();
return RC_OK;
}
void InnerTcpConnection::realSendMessage() {
function_footprint();
static uint32_t BATCH_TCP_MESSAGES = InnerTcpConnection::innerTcpServer->getTcpBatchSize();
InnerTcpConnectionState expectedState = READY;
if(!state.compare_exchange_strong(expectedState, WRITING)) {
VLOG(3) << "Connection is not ready: " << state.load();
return;
}
auto conn = shared_from_this();
if(unlikely(!socket.is_open())) {
terminate();
return;
}
idgs::actor::ActorMessagePtr msg;
std::vector<idgs::actor::ActorMessagePtr>* msgs = new std::vector<idgs::actor::ActorMessagePtr>();
msgs->reserve(BATCH_TCP_MESSAGES);
std::vector<asio::const_buffer> outBuffers;
outBuffers.reserve(BATCH_TCP_MESSAGES * 2 + 1);
uint32_t sendLength;
sendLength = 0;
outBuffers.push_back(asio::buffer(reinterpret_cast<void*>(&sendLength), sizeof(sendLength)));
while(getQueue().try_pop(msg)) {
DVLOG(2) << "send message: " << msg->toString();
DVLOG(3) << "Send head size: " << sizeof(idgs::net::TcpHeader) << ", content: " << dumpBinaryBuffer(std::string((char*)msg->getRpcBuffer()->getHeader(), sizeof(idgs::net::TcpHeader)));
DVLOG(3) << "Send body size: " << msg->getRpcBuffer()->getBodyLength() << ", content: " << dumpBinaryBuffer(std::string(msg->getRpcBuffer()->getBody(), msg->getRpcBuffer()->getBodyLength()));
msgs->push_back(msg);
outBuffers.push_back(asio::buffer(reinterpret_cast<void*>(msg->getRpcBuffer()->getHeader()), sizeof(idgs::net::TcpHeader)));
outBuffers.push_back(asio::buffer(reinterpret_cast<void*>(msg->getRpcBuffer()->getBody()), msg->getRpcBuffer()->getBodyLength()));
sendLength += sizeof(idgs::net::TcpHeader) + msg->getRpcBuffer()->getBodyLength();
// if(msgs->size() >= BATCH_TCP_MESSAGES || sendLength > (1024 * 50)) {
if(msgs->size() >= BATCH_TCP_MESSAGES) {
break;
}
}
if(msgs->empty()) {
DVLOG(2) << "No availabe message to send.";
state.store(READY);
delete msgs;
return;
}
try {
DVLOG(2) << "Send " << msgs->size() << " messages";
if(unlikely(!socket.is_open())) {
// @fixme messagea lost
delete msgs;
terminate();
return;
}
asio::async_write(
socket,
outBuffers,
asio::transfer_all(),
[msgs, conn] (const asio::error_code& error, const std::size_t& bytes_transferred) {
DVLOG(2) << "Sent " << msgs->size() << " messages";
conn->handleSendMessage(error);
static NetworkStatistics* stats = InnerTcpConnection::innerTcpServer->network->getNetworkStatistics();
stats->innerTcpBytesSent.fetch_add(bytes_transferred);
stats->innerTcpPacketSent.fetch_add(msgs->size());
delete msgs;
}
);
} catch (std::exception& e) {
LOG(ERROR) << "send message error, exception: " << e.what() << ", messages: " << msgs->size();
delete msgs;
terminate();
} catch (...) {
LOG(ERROR) << "send message error " << ", messages: " << msgs->size();
catchUnknownException();
delete msgs;
terminate();
}
}
void MembershipTableMgr::onDestroy() {
list<MemberEventListener*>().swap(this->listeners);
function_footprint();
}
MembershipTableMgr::~MembershipTableMgr() {
function_footprint();
}
ScheduledMessageService::~ScheduledMessageService() {
function_footprint();
}
int client_run() {
function_footprint();
try {
ResultCode resultCode;
auto client = getTcpClientPool().getTcpClient(resultCode);
if (resultCode != RC_SUCCESS) {
cerr << "Get Client error: " << getErrorDescription(resultCode) << endl;
return RC_ERROR;
}
// step 1, 2
idgs::client::ClientActorMessagePtr clientActorMsg = make_shared<idgs::client::ClientActorMessage>();
clientActorMsg->setDestActorId(test_server_id);
clientActorMsg->setDestMemberId(ANY_MEMBER);
clientActorMsg->setSourceActorId("client_actor_id");
clientActorMsg->setSourceMemberId(CLIENT_MEMBER);
clientActorMsg->setOperationName(start_work_operation);
clientActorMsg->getRpcMessage()->set_payload("payload");
ResultCode errorCode;
idgs::client::ClientActorMessagePtr response;
errorCode = client->sendRecv(clientActorMsg, response);
if (errorCode != RC_SUCCESS) {
cerr << "execute the command error: " << getErrorDescription(errorCode) << endl;
client->close();
return RC_ERROR;
}
DLOG(INFO) << "write message to server" << clientActorMsg->toString();
// step 3, 4
errorCode = client->receive(response);
if (errorCode != RC_SUCCESS) {
DLOG(INFO) << "get response error " << errorCode;
return errorCode;
}
if (response->getOperationName() == sending_reponse_succ) {
response->setOperationName(new_client_message_comes);
response->setDestActorId(test_server_id);
response->setSourceActorId("client_actor_id");
response->setDestMemberId(response->getSourceMemberId());
response->setChannel(TC_TCP);
response->setSourceMemberId(CLIENT_MEMBER);
ResultCode errorCode;
idgs::client::ClientActorMessagePtr tcpResponse;
errorCode = client->sendRecv(response, tcpResponse);
DVLOG(2) << "after sendRecv";
if (errorCode != RC_SUCCESS) {
cerr << "execute the command error: " << getErrorDescription(errorCode) << endl;
client->close();
return RC_ERROR;
}
if (tcpResponse->getOperationName() == sending_total) {
idgs::pb::Long l;
tcpResponse->parsePayload(&l);
test_server_count = l.value();
}
++count;
} else {
std::cerr <<"Test client error, receive error server response" ;
return RC_ERROR;
}
//while (1) {}
//socket.close();
} catch (std::exception& e) {
std::cerr <<"Test client error" << e.what() << std::endl;
return RC_ERROR;
}
return RC_SUCCESS;
}