int clsConnWorker::AddListen(bool bIsInternal, const Endpoint_t &tLocalAddr)
{
int iRet;
int iBacklog = 8096;
int iFD = CreateSocketFD(&tLocalAddr, true);
if (iFD == -1)
{
CertainLogError("CreateSocketFD ret -1");
return -1;
}
iRet = listen(iFD, iBacklog);
if (iRet == -1)
{
CertainLogError("listen ret - 1 errno %d", errno);
return -2;
}
clsListenContext *poContext = new clsListenContext(
iFD, this, bIsInternal, tLocalAddr);
iRet = m_poEpollIO->Add(poContext);
if (iRet != 0)
{
delete poContext, poContext = NULL;
CertainLogError("m_poEpollIO->Add ret %d", iRet);
return -3;
}
return 0;
}
int clsConnWorker::AddAllListen()
{
int iRet;
uint32_t iLocalAcceptorID = m_poConf->GetLocalAcceptorID();
vector<Endpoint_t> vecAcceptorEndpoint = m_poConf->GetAcceptorEndpoints();
AssertLess(iLocalAcceptorID, vecAcceptorEndpoint.size());
Endpoint_t tInternal = vecAcceptorEndpoint[iLocalAcceptorID];
Endpoint_t tExteranl = m_poConf->GetExtpoint();
iRet = AddListen(true, tInternal);
if (iRet != 0)
{
CertainLogError("AddListen ret %d", iRet);
return -1;
}
iRet = AddListen(false, tExteranl);
if (iRet != 0)
{
CertainLogError("AddListen ret %d", iRet);
return -2;
}
return 0;
}
int clsEntityWorker::DoWithWaitingList(list<clsPaxosCmd *> *ptWaitingList)
{
uint32_t iFailCnt = 0;
while (ptWaitingList->size() > 0)
{
clsPaxosCmd *poPaxosCmd = ptWaitingList->front();
ptWaitingList->pop_front();
int iRet = DoWithIOReq(dynamic_cast<clsCmdBase *>(poPaxosCmd));
if (iRet < 0)
{
iFailCnt++;
CertainLogError("DoWithIOReq ret %d cmd %s",
iRet, poPaxosCmd->GetTextCmd().c_str());
}
// iRet = 1: reuse the ptr, use clsRefPtr in future
if (iRet != 1)
{
delete poPaxosCmd, poPaxosCmd = NULL;
}
}
if (iFailCnt > 0)
{
CertainLogError("iFailCnt %u", iFailCnt);
return -1;
}
return 0;
}
int clsConnWorker::HandleListening(clsFDContext *poFDCtx)
{
clsListenContext *poContext = dynamic_cast<clsListenContext *>(poFDCtx);
ConnInfo_t tConnInfo;
int iRet = AcceptOneFD(poContext, tConnInfo);
if (iRet < 0)
{
CertainLogError("Accept ret %d", iRet);
return -1;
}
else if (iRet == 1)
{
return 1;
}
if (poFDCtx->IsInternal())
{
clsNegoContext *poNegoCtx = new clsNegoContext(this, tConnInfo);
m_poEpollIO->Add(poNegoCtx);
}
else
{
iRet = clsConnInfoMng::GetInstance()->PutByOneThread(
INVALID_ACCEPTOR_ID, tConnInfo);
if (iRet != 0)
{
CertainLogError("clsConnInfoMng PutByOneThread ret %d", iRet);
}
}
return 0;
}
void clsEntityWorker::Run()
{
SetThreadTitle("entity_%u_%u", m_iLocalAcceptorID, m_iWorkerID);
CertainLogDebug("worker_id %u run", m_iWorkerID);
while (1)
{
bool bHasWork = false;
// 1.Do with IO request.
clsCmdBase *poCmd = NULL;
int iRet = m_poIOReqQueue->TakeByOneThread(&poCmd);
if (iRet == 0)
{
assert(poCmd != NULL);
bHasWork = true;
iRet = DoWithIOReq(poCmd);
if (iRet < 0)
{
CertainLogError("DoWithIOReq ret %d cmd %s",
iRet, poCmd->GetTextCmd().c_str());
}
// iRet = 1: reuse the ptr, use clsRefPtr in future
if (iRet != 1)
{
delete poCmd, poCmd = NULL;
}
}
// 2.Do with PLog response.
clsPaxosCmd *poPaxosCmd;
iRet = m_poPLogRspQueue->TakeByOneThread(&poPaxosCmd);
if (iRet == 0)
{
bHasWork = true;
iRet = DoWithPLogRsp(poPaxosCmd);
if (iRet < 0)
{
const EntryRecord_t &tSrcRecord = poPaxosCmd->GetSrcRecord();
CertainLogError("DoWithPLogRsp ret %d record %s",
iRet, EntryRecordToString(tSrcRecord).c_str());
}
delete poPaxosCmd, poPaxosCmd = NULL;
}
// 3.Do with Timeout EntryState.
iRet = DoWithTimeout();
AssertEqual(iRet, 0);
if (!bHasWork)
{
usleep(1);
}
}
}
void clsConnWorker::HandleIOEvent(clsFDBase *poFD)
{
int iRet;
clsFDContext *poFDCtx = dynamic_cast<clsFDContext *>(poFD);
if (poFDCtx->IsListening())
{
while (1)
{
iRet = HandleListening(poFDCtx);
if (iRet == 0)
{
continue;
}
else if (iRet < 0)
{
// (TODO)rock: remove and close
CertainLogError("HandleListening ret %d", iRet);
break;
}
AssertEqual(iRet, 1);
break;
}
}
else
{
clsNegoContext *poNegoCtx = dynamic_cast<clsNegoContext *>(poFD);
clsAutoDelete<clsNegoContext> oAuto(poNegoCtx);
iRet = RecvNegoMsg(poNegoCtx);
if (iRet != 0)
{
CertainLogError("RecvNegoMsg ret %d", iRet);
m_poEpollIO->RemoveAndCloseFD(dynamic_cast<clsFDBase *>(poNegoCtx));
return;
}
m_poEpollIO->Remove(dynamic_cast<clsFDBase *>(poNegoCtx));
iRet = clsConnInfoMng::GetInstance()->PutByOneThread(
poNegoCtx->GetAcceptorID(), poNegoCtx->GetConnInfo());
if (iRet != 0)
{
CertainLogError("clsConnInfoMng PutByOneThread ret %d", iRet);
m_poEpollIO->RemoveAndCloseFD(dynamic_cast<clsFDBase *>(poNegoCtx));
}
}
}
void clsEntityWorker::CheckForCatchUp(EntityInfo_t *ptEntityInfo,
uint32_t iGlobalMaxChosenEntry)
{
uint32_t iEntityID = ptEntityInfo->iEntityID;
uint32_t iLocalMaxChosenEntry = ptEntityInfo->iMaxChosenEntry;
CertainLogError("iEntityID %u iLocalMax %u iGlobalMax %u",
iEntityID, iLocalMaxChosenEntry, iGlobalMaxChosenEntry);
for (uint32_t iEntry = iLocalMaxChosenEntry + 1;
iEntry <= iGlobalMaxChosenEntry; ++iEntry)
{
EntryInfo_t *ptInfo = m_poEntryMng->FindEntryInfo(iEntityID, iEntry);
if (ptInfo == NULL)
{
ptInfo = m_poEntryMng->CreateEntryInfo(iEntityID, iEntry);
assert(ptInfo != NULL);
BroadcastToRemote(iEntityID, iEntry, NULL);
}
else
{
CertainLogDebug("EntryInfo Found: iEntityID %u iEntry %u",
iEntityID, iEntry);
}
}
}
void clsEntityWorker::ActivateEntry(EntryInfo_t *ptInfo)
{
uint32_t iEntityID = ptInfo->iEntityID;
uint32_t iEntry = ptInfo->iEntry;
CertainLogError("iEntityID %u iEntry %u TODO", iEntityID, iEntry);
}
int clsConnWorker::RecvNegoMsg(clsNegoContext *poNegoCtx)
{
int iRet;
const ConnInfo_t &tConnInfo = poNegoCtx->GetConnInfo();
int iFD = tConnInfo.iFD;
uint8_t acNego[1];
while (1)
{
iRet = read(iFD, acNego, 1);
if (iRet == -1)
{
if (errno == EAGAIN || errno == EINTR)
{
continue;
}
CertainLogError("fd %d errno %d", iFD, errno);
return -1;
}
else if (iRet == 0)
{
Endpoint_t tPeerAddr = tConnInfo.tPeerAddr;
CertainLogError("closed by peer %s:%hu fd %d",
tPeerAddr.sIP, tPeerAddr.hPort, iFD);
return -2;
}
else
{
AssertEqual(iRet, 1);
break;
}
}
// (TODO)rock: uncomment this
//uint32_t iAcceptorID = uint32_t(acNego[0]);
uint32_t iAcceptorID = uint32_t(acNego[0] - '0');
poNegoCtx->SetAcceptorID(iAcceptorID);
CertainLogDebug("iFD %d local %s:%hu peer %s:%hu iAcceptorID %u",
iFD, tConnInfo.tLocalAddr.sIP, tConnInfo.tLocalAddr.hPort,
tConnInfo.tPeerAddr.sIP, tConnInfo.tPeerAddr.hPort, iAcceptorID);
return 0;
}
int clsAsyncPipeMng::SyncWaitByPipeIdx(uint32_t iIdx, uintptr_t iPtr)
{
CertainLogDebug("iIdx %u iPtr %lu", iIdx, iPtr);
int iInFD = m_aaAsyncPipe[iIdx][0];
struct pollfd PollFd;
PollFd.events = POLLIN | POLLERR;
PollFd.fd = iInFD;
int iEventCnt = 0;
int iTimeout = 1000;
int iTimeoutCnt = 0;
while (true)
{
errno = 0;
iEventCnt = poll(&PollFd, 1, iTimeout);
if (errno == EINTR)
{
continue;
}
else if (iEventCnt > 0)
{
break;
}
else
{
iTimeoutCnt++;
OSS::ReportPollTimeout();
// It won't be timeout forever.
CertainLogError("iPtr %lu PipeIdx %u iTimeoutCnt %d",
iPtr, iIdx, iTimeoutCnt);
}
}
if (iEventCnt <= 0)
{
return -1;
}
uintptr_t iRetPtr;
int iRet = read(iInFD, &iRetPtr, sizeof(iRetPtr));
AssertEqual(iRet, sizeof(iRetPtr));
if (iRetPtr != iPtr)
{
CertainLogFatal("BUG %lu %lu", iRetPtr, iPtr);
return eRetCodePipePtrErr;
}
return 0;
}
void clsConnInfoMng::RemoveInvalidConn()
{
uint32_t iInvalidCnt = 0;
uint32_t iConnNotToken = 0;
for (uint32_t i = 0; i < m_vecIntConnQueue.size(); ++i)
{
while (!m_vecIntConnQueue[i].empty())
{
iConnNotToken++;
ConnInfo_t tConnInfo = m_vecIntConnQueue[i].front();
if (CheckFD(tConnInfo.iFD))
{
break;
}
iInvalidCnt++;
int iRet = close(tConnInfo.iFD);
if (iRet != 0)
{
CertainLogError("close fail fd %d errno %d",
tConnInfo.iFD, errno);
}
m_vecIntConnQueue[i].pop();
}
}
if (iInvalidCnt > 0)
{
CertainLogError("iInvalidCnt %u", iInvalidCnt);
}
else if (iConnNotToken > 0)
{
CertainLogError("iConnNotToken %u", iConnNotToken);
}
}
void clsEntityWorker::InvalidClientCmd(clsClientCmd *poCmd,
string strResult)
{
if (poCmd == NULL)
{
return;
}
poCmd->SetResult(strResult);
CertainLogError("InvalidClientCmd cmd %u uuid %lu result: %s",
poCmd->GetCmdID(), poCmd->GetUUID(), strResult.c_str());
m_poIOWorkerRouter->GoAndDeleteIfFailed(poCmd);
}
int clsEntityWorker::DoWithTimeout()
{
int iExpiredCnt = 0;
EntryInfo_t *ptInfo;
while ((ptInfo = m_poEntryMng->TakeTimeout()) != NULL)
{
uint32_t iEntityID = ptInfo->iEntityID;
uint32_t iEntry = ptInfo->iEntry;
if (ptInfo->bUncertain)
{
m_poEntryMng->AddTimeout(ptInfo, 10);
CertainLogDebug("iEntityID %u iEntry %u wait 10ms for uncertain",
iEntityID, iEntry);
continue;
}
// (TODO)rock: it must retry, when it's uncertain
EntityInfo_t *ptEntityInfo = m_poEntityMng->GetEntityInfo(iEntityID);
if (ptInfo->bDBPending || ptEntityInfo->bDBPending)
{
assert(ptInfo->bDBPending && ptEntityInfo->bDBPending);
ptInfo->bDBPending = false;
ptEntityInfo->bDBPending = false;
}
if (ptEntityInfo->poClientCmd != NULL
&& ptEntityInfo->poClientCmd->GetEntry() == iEntry)
{
InvalidClientCmd(ptEntityInfo, " : Timeout");
}
iExpiredCnt++;
ActivateEntry(ptInfo);
}
if (iExpiredCnt > 0)
{
CertainLogError("iExpiredCnt %d", iExpiredCnt);
}
return 0;
}
void clsConnWorker::Run()
{
uint32_t iLocalAcceptorID = m_poConf->GetLocalAcceptorID();
SetThreadTitle("conn_%u", iLocalAcceptorID);
CertainLogDebug("run");
int iRet = AddAllListen();
if (iRet != 0)
{
CertainLogError("AddAllListen ret %d", iRet);
exit(-1);
}
while (1)
{
m_poEpollIO->RunOnce(-1);
}
}
int clsConnWorker::AcceptOneFD(clsListenContext *poContext,
ConnInfo_t &tConnInfo)
{
int iListenFD = poContext->GetFD();
struct sockaddr_in tSockAddr;
socklen_t tLen = sizeof(tSockAddr);
int iFD = accept(iListenFD, (struct sockaddr *)(&tSockAddr), &tLen);
if (iFD == -1)
{
if (errno == EAGAIN)
{
return 1;
}
// (TODO)rock: close it
CertainLogError("accept ret -1 errno %d", errno);
return -1;
}
int iFlags = fcntl(iFD, F_GETFL, 0);
int iRet = fcntl(iFD, F_SETFL, iFlags | O_NONBLOCK);
AssertEqual(iRet, 0);
Endpoint_t tPeerAddr;
strcpy(tPeerAddr.sIP, inet_ntoa(tSockAddr.sin_addr));
tPeerAddr.hPort = ntohs(tSockAddr.sin_port);
tConnInfo.tLocalAddr = poContext->GetLocalAddr();
tConnInfo.tPeerAddr = tPeerAddr;
tConnInfo.iFD = iFD;
CertainLogDebug("iFD %d local %s:%hu peer %s:%hu",
iFD, tConnInfo.tLocalAddr.sIP, tConnInfo.tLocalAddr.hPort,
tConnInfo.tPeerAddr.sIP, tConnInfo.tPeerAddr.hPort);
return 0;
}
int clsEntityWorker::DoWithIOReq(clsCmdBase *poCmd)
{
CertainLogDebug("cmd: %s", poCmd->GetTextCmd().c_str());
clsPaxosCmd *poPaxosCmd = NULL;
clsClientCmd *poClientCmd = NULL;
switch (poCmd->GetCmdID())
{
case kNoopCmd:
case kSimpleCmd:
poClientCmd = dynamic_cast<clsClientCmd *>(poCmd);
return DoWithClientCmd(poClientCmd);
case kPaxosCmd:
poPaxosCmd = dynamic_cast<clsPaxosCmd *>(poCmd);
return DoWithPaxosCmd(poPaxosCmd);
default:
CertainLogError("cmd: %s", poCmd->GetTextCmd().c_str());
assert(false);
}
return 0;
}
int clsEntityWorker::UpdateRecord(clsPaxosCmd *poPaxosCmd)
{
uint32_t iAcceptorID = poPaxosCmd->GetSrcAcceptorID();
uint32_t iEntityID = poPaxosCmd->GetEntityID();
uint32_t iEntry = poPaxosCmd->GetEntry();
CertainLogDebug("iAcceptorID %u iEntityID %u iEntry %u",
iAcceptorID, iEntityID, iEntry);
EntityInfo_t *ptEntityInfo = m_poEntityMng->GetEntityInfo(iEntityID);
assert(ptEntityInfo != NULL);
EntryInfo_t *ptInfo = m_poEntryMng->FindEntryInfo(iEntityID, iEntry);
assert(ptInfo != NULL);
clsEntryStateMachine *poMachine = ptInfo->poMachine;
const EntryRecord_t tOldRecord = poMachine->GetRecord(m_iLocalAcceptorID);
CertainLogDebug("before update cmd: %s state %u",
poPaxosCmd->GetTextCmd().c_str(), poMachine->GetEntryState());
const EntryRecord_t &tSrcRecord = poPaxosCmd->GetSrcRecord();
const EntryRecord_t &tDestRecord = poPaxosCmd->GetDestRecord();
poMachine->Update(m_iLocalAcceptorID, iAcceptorID, tSrcRecord);
CertainLogDebug("after update cmd: %s state %u",
poPaxosCmd->GetTextCmd().c_str(), poMachine->GetEntryState());
if (poMachine->GetEntryState() == kEntryStateMajorityPromise)
{
string strValue;
if (ptEntityInfo->poClientCmd != NULL
&& ptEntityInfo->poClientCmd->GetEntry() == iEntry)
{
strValue = MakePaxosValue(ptEntityInfo->poClientCmd, true);
}
else
{
clsNoopCmd oNoopCmd(iEntityID, iEntry);
strValue = MakePaxosValue(&oNoopCmd, false);
}
if (poMachine->AcceptOnMajorityPromise(m_iLocalAcceptorID, strValue))
{
ptInfo->bBroadcast = true;
}
else
{
InvalidClientCmd(ptEntityInfo);
}
}
if (poMachine->GetEntryState() == kEntryStateChosen)
{
if (!tSrcRecord.bChosen)
{
// only one proposer do broadcast for a proposal
if (poMachine->IsLocalChosen(m_iLocalAcceptorID))
{
ptInfo->bBroadcast = true;
}
}
}
const EntryRecord_t &tNewRecord = poMachine->GetRecord(m_iLocalAcceptorID);
ptInfo->bRemoteUpdated = IsEntryRecordUpdated(tDestRecord, tNewRecord);
bool bLocalUpdated = IsEntryRecordUpdated(tOldRecord, tNewRecord);
clsPaxosCmd *po = new clsPaxosCmd(m_iLocalAcceptorID,
iEntityID, iEntry, &tNewRecord);
if (!ptInfo->bBroadcast)
{
po->SetDestAcceptorID(iAcceptorID);
}
if (bLocalUpdated)
{
int iRet = m_poPLogReqQueue->PushByOneThread(po);
if (iRet != 0)
{
CertainLogError("PushByOneThread ret %d", iRet);
delete po, po = NULL;
return 0;
}
ptInfo->bUncertain = true;
}
else
{
clsAutoDelete<clsPaxosCmd> oAuto(po);
if (poMachine->IsLocalEmpty())
{
poMachine->SetCheckedEmpty(poPaxosCmd->GetSrcAcceptorID());
}
if (ptEntityInfo->poClientCmd != NULL
&& ptEntityInfo->poClientCmd->IsReadOnly())
{
if (poMachine->IsReadOK())
{
m_poIOWorkerRouter->GoAndDeleteIfFailed(
ptEntityInfo->poClientCmd);
ptEntityInfo->poClientCmd = NULL;
if (poMachine->IsLocalEmpty())
{
CleanUpEntry(ptEntityInfo, ptInfo);
}
return 0;
}
else if (!poMachine->IsLocalEmpty())
{
InvalidClientCmd(ptEntityInfo, " : ReadFail");
}
}
SyncEntryRecord(ptInfo, po);
}
return 0;
}
int clsEntityWorker::DoWithPaxosCmd(clsPaxosCmd *poPaxosCmd)
{
int iRet;
uint32_t iAcceptorID = poPaxosCmd->GetSrcAcceptorID();
uint32_t iEntityID = poPaxosCmd->GetEntityID();
uint32_t iEntry = poPaxosCmd->GetEntry();
AssertNotEqual(iAcceptorID, m_iLocalAcceptorID);
EntityInfo_t *ptEntityInfo = m_poEntityMng->GetEntityInfo(iEntityID);
assert(ptEntityInfo != NULL);
CertainLogDebug("iAcceptorID %u iEntityID %u iEntry %u iMaxChosenEntry %u",
iAcceptorID, iEntityID, iEntry, ptEntityInfo->iMaxChosenEntry);
if (ptEntityInfo->iMaxChosenEntry >= iEntry)
{
EntryInfo_t *ptInfo = m_poEntryMng->FindEntryInfo(iEntityID, iEntry);
const EntryRecord_t &tRecord = poPaxosCmd->GetSrcRecord();
// ptInfo == NULL means chosen, as ptInfo is cleanup once it's chosen.
if (ptInfo == NULL)
{
if (tRecord.bChosen)
{
return 0;
}
clsPaxosCmd *po = new clsPaxosCmd(m_iLocalAcceptorID,
iEntityID, iEntry);
po->SetDestAcceptorID(iAcceptorID);
po->SetForCatchUp(true);
iRet = m_poPLogReqQueue->PushByOneThread(po);
if (iRet != 0)
{
CertainLogError("PushByOneThread ret %d", iRet);
delete po, po = NULL;
return -1;
}
return 0;
}
CertainLogError("iEntityID %u iEntry %u iMaxChosenEntry %u",
iEntityID, iEntry, ptEntityInfo->iMaxChosenEntry);
}
uint32_t iGlobalMaxChosenEntry = iEntry - 1;
if (ptEntityInfo->iMaxChosenEntry < iGlobalMaxChosenEntry)
{
CheckForCatchUp(ptEntityInfo, iGlobalMaxChosenEntry);
ptEntityInfo->iMaxChosenEntry = iGlobalMaxChosenEntry;
}
EntryInfo_t *ptInfo = m_poEntryMng->FindEntryInfo(iEntityID, iEntry);
if (ptInfo == NULL)
{
ptInfo = m_poEntryMng->CreateEntryInfo(iEntityID, iEntry);
assert(ptInfo != NULL);
}
if (ptInfo->bUncertain)
{
uint32_t iWaitingListSize = ptInfo->ptWaitingList->size();
if (iWaitingListSize > 10)
{
CertainLogError("iEntityID %u iEntry %u iWaitingListSize %u",
iEntityID, iEntry, iWaitingListSize);
return -2;
}
ptInfo->ptWaitingList->push_back(poPaxosCmd);
return 1;
}
assert(ptInfo->poMachine->GetEntryState() != kEntryStateChosen);
// It should have created the ptInfo for the entry not more than
// iMaxChosenEntry, and it won't be cleanup until chosen.
assert(ptInfo->poMachine->GetEntryState() != kEntryStateChosen);
iRet = UpdateRecord(poPaxosCmd);
if (iRet != 0)
{
CertainLogError("UpdateRecord ret %d", iRet);
}
return 0;
}
int clsEntityWorker::DoWithPLogRsp(clsPaxosCmd *poPaxosCmd)
{
int iRet;
uint32_t iEntityID = poPaxosCmd->GetEntityID();
uint32_t iEntry = poPaxosCmd->GetEntry();
CertainLogDebug("iEntityID %u iEntry %u", iEntityID, iEntry);
EntityInfo_t *ptEntityInfo = m_poEntityMng->GetEntityInfo(iEntityID);
assert(ptEntityInfo != NULL);
EntryInfo_t *ptInfo = m_poEntryMng->FindEntryInfo(iEntityID, iEntry);
assert(ptInfo != NULL);
assert(ptInfo->bUncertain);
ptInfo->bUncertain = false;
SyncEntryRecord(ptInfo, poPaxosCmd);
clsEntryStateMachine *poMachine = ptInfo->poMachine;
const EntryRecord_t &tSrcRecord = poMachine->GetRecord(m_iLocalAcceptorID);
if (CERTAIN_DEBUG)
{
string strInfoRecord = EntryRecordToString(poPaxosCmd->GetSrcRecord());
string strMachineRecord = EntryRecordToString(tSrcRecord);
assert(strInfoRecord == strMachineRecord);
}
list<clsPaxosCmd *> tWaitingList;
swap(*ptInfo->ptWaitingList, tWaitingList);
if (tWaitingList.size() > 0)
{
CertainLogDebug("iEntityID %u iEntry %u tWaitingList.size() %lu",
iEntityID, iEntry, tWaitingList.size());
}
if (poMachine->GetEntryState() == kEntryStateChosen)
{
clsCmdFactory *poCmdFactory = clsCmdFactory::GetInstance();
// (TODO)rock: reduce PB once
clsCmdBase *poCmd = poCmdFactory->CreateCmd(
tSrcRecord.strValue.c_str(), tSrcRecord.strValue.size());
clsClientCmd *poClientCmd = ptEntityInfo->poClientCmd;
clsClientCmd *poChosenCmd = dynamic_cast<clsClientCmd *>(poCmd);
if (poClientCmd != NULL && poClientCmd->GetEntry() == iEntry)
{
CertainLogDebug("cli_cmd: %s chosen: %s",
poClientCmd->GetTextCmd().c_str(),
EntryRecordToString(tSrcRecord).c_str());
if (poClientCmd->GetUUID() == poChosenCmd->GetUUID())
{
delete poChosenCmd, poChosenCmd = NULL;
poChosenCmd = ptEntityInfo->poClientCmd;
ptEntityInfo->poClientCmd = NULL;
poChosenCmd->SetNeedRsp(true);
if (ptEntityInfo->iMaxEntryChosenByLocal < iEntry)
{
ptEntityInfo->iMaxEntryChosenByLocal = iEntry;
}
}
else
{
CertainLogError("uuid (%lu %lu) cmd %s",
poClientCmd->GetUUID(), poChosenCmd->GetUUID(),
poClientCmd->GetTextCmd().c_str());
InvalidClientCmd(ptEntityInfo);
}
}
// (TODO)rock: Push Rsp here instead of in DB thread
poChosenCmd->SetEntry(iEntry);
iRet = m_poDBReqQueue->PushByMultiThread(poChosenCmd);
AssertEqual(iRet, 0); // (TODO)rock: dbpending
if (ptEntityInfo->iMaxChosenEntry < iEntry)
{
ptEntityInfo->iMaxChosenEntry = iEntry;
}
CleanUpEntry(ptEntityInfo, ptInfo);
}
iRet = DoWithWaitingList(&tWaitingList);
if (iRet != 0)
{
CertainLogError("DoWithWaitingList ret %d", iRet);
}
return 0;
}
int clsEntityWorker::DoWithClientCmd(clsClientCmd *poCmd)
{
int iRet;
uint32_t iEntityID = poCmd->GetEntityID();
EntityInfo_t *ptEntityInfo = m_poEntityMng->GetEntityInfo(iEntityID);
if (ptEntityInfo->poClientCmd != NULL
|| (!poCmd->IsEvalOnly() && ptEntityInfo->bDBPending))
{
InvalidClientCmd(poCmd, " : TODO");
//iRet = AddWaitingCmd(poCmd);
//if (iRet < 0)
//{
// CertainLogError("poWaitingCmd->Add ret %d", iRet);
// return -1;
//}
return 1;
}
if (poCmd->IsEvalOnly())
{
if (poCmd->GetEntry() != ptEntityInfo->iMaxChosenEntry + 1)
{
InvalidClientCmd(poCmd, " : DISCARD");
return 1;
}
}
assert(ptEntityInfo->poClientCmd == NULL);
uint32_t iEntry = ptEntityInfo->iMaxChosenEntry + 1;
EntryInfo_t *ptInfo = m_poEntryMng->FindEntryInfo(iEntityID, iEntry);
if (ptInfo == NULL)
{
ptInfo = m_poEntryMng->CreateEntryInfo(iEntityID, iEntry);
assert(ptInfo != NULL);
}
if (ptInfo->bUncertain)
{
InvalidClientCmd(poCmd, " : Uncertain");
return 1;
}
clsEntryStateMachine *poMachine = ptInfo->poMachine;
int iEntryState = poMachine->GetEntryState();
CertainLogDebug("iEntityID %u iEntry %u state %d",
iEntityID, iEntry, iEntryState);
assert(!ptInfo->bRemoteUpdated);
assert(!ptInfo->bBroadcast);
// give in if has accepted remote
if (poMachine->GetEntryState() == kEntryStateAcceptRemote)
{
InvalidClientCmd(poCmd, " : GiveIn");
return 1;
}
poCmd->SetEntry(iEntry);
m_poEntryMng->AddTimeout(ptInfo, 50);
if (m_poConf->GetPLogType() == kPLogTypeRes)
{
if (!poCmd->IsEvalOnly())
{
assert(!poCmd->IsNeedRsp());
poCmd->SetEvalOnly(true);
iRet = m_poDBReqQueue->PushByMultiThread(poCmd);
if (iRet != 0)
{
CertainLogError("PushByMultiThread ret %d", iRet);
return -1;
}
ptInfo->bDBPending = true;
ptEntityInfo->bDBPending = true;
return 1;
}
poCmd->SetEvalOnly(false);
ptInfo->bDBPending = false;
ptEntityInfo->bDBPending = false;
}
ptEntityInfo->poClientCmd = poCmd;
if (m_poConf->IsEnableReadOpt() && poCmd->IsReadOnly()
&& poMachine->IsLocalEmpty())
{
poMachine->ResetAllCheckedEmpty();
poMachine->SetCheckedEmpty(m_iLocalAcceptorID);
BroadcastToRemote(iEntityID, iEntry, NULL, true);
return 1;
}
uint32_t iProposalNum = poMachine->GetNextPreparedNum(m_iLocalAcceptorID);
EntryRecord_t tTempRecord;
InitEntryRecord(&tTempRecord);
if (m_poConf->GetPaxosType() == kPaxosTypeMulti)
{
if (iProposalNum == m_iLocalAcceptorID + 1
&& iEntry == ptEntityInfo->iMaxEntryChosenByLocal + 1)
{
tTempRecord.iAcceptedNum = iProposalNum;
tTempRecord.iValueID = iProposalNum;
tTempRecord.strValue = MakePaxosValue(poCmd, true);
}
else
{
iProposalNum = poMachine->GetNextPreparedNum(m_iLocalAcceptorID);
}
}
tTempRecord.iPreparedNum = iProposalNum;
tTempRecord.iPromisedNum = iProposalNum;
iEntryState = poMachine->Update(m_iLocalAcceptorID,
m_iLocalAcceptorID, tTempRecord);
const EntryRecord_t &tUpdatedRecord = poMachine->GetRecord(
m_iLocalAcceptorID);
clsPaxosCmd *poPaxosCmd = new clsPaxosCmd(m_iLocalAcceptorID,
iEntityID, iEntry, &tUpdatedRecord);
CertainLogDebug("record: %s state %d",
EntryRecordToString(tUpdatedRecord).c_str(), iEntryState);
iRet = m_poPLogReqQueue->PushByOneThread(poPaxosCmd);
if (iRet != 0)
{
CertainLogError("PushByOneThread ret %d", iRet);
delete poPaxosCmd, poPaxosCmd = NULL;
return -1;
}
// It's uncertain even if fail to push to PLogReqQueue.
// It will retry when timeout, as this update has happened actually.
ptInfo->bUncertain = true;
ptInfo->bBroadcast = true;
return 1;
}