int Database :: PutToLevelDB(const bool bSync, const uint64_t llInstanceID, const std::string & sValue)
{
string sKey = GenKey(llInstanceID);
leveldb::WriteOptions oLevelDBWriteOptions;
oLevelDBWriteOptions.sync = bSync;
m_oTimeStat.Point();
leveldb::Status oStatus = m_poLevelDB->Put(oLevelDBWriteOptions, sKey, sValue);
if (!oStatus.ok())
{
BP->GetLogStorageBP()->LevelDBPutFail();
PLG1Err("LevelDB.Put fail, instanceid %lu valuelen %zu", llInstanceID, sValue.size());
return -1;
}
BP->GetLogStorageBP()->LevelDBPutOK(m_oTimeStat.Point());
return 0;
}
int Database :: GetFromLevelDB(const uint64_t llInstanceID, std::string & sValue)
{
string sKey = GenKey(llInstanceID);
leveldb::Status oStatus = m_poLevelDB->Get(leveldb::ReadOptions(), sKey, &sValue);
if (!oStatus.ok())
{
if (oStatus.IsNotFound())
{
BP->GetLogStorageBP()->LevelDBGetNotExist();
PLG1Debug("LevelDB.Get not found, instanceid %lu", llInstanceID);
return 1;
}
BP->GetLogStorageBP()->LevelDBGetFail();
PLG1Err("LevelDB.Get fail, instanceid %lu", llInstanceID);
return -1;
}
return 0;
}
int Database :: SetMinChosenInstanceID(const WriteOptions & oWriteOptions, const uint64_t llMinInstanceID)
{
if (!m_bHasInit)
{
PLG1Err("no init yet");
return -1;
}
static uint64_t llMinKey = MINCHOSEN_KEY;
char sValue[sizeof(uint64_t)] = {0};
memcpy(sValue, &llMinInstanceID, sizeof(uint64_t));
int ret = PutToLevelDB(true, llMinKey, string(sValue, sizeof(uint64_t)));
if (ret != 0)
{
return ret;
}
PLG1Imp("ok, min chosen instanceid %lu", llMinInstanceID);
return 0;
}
void MasterDamon :: TryBeMaster(const int iLeaseTime)
{
nodeid_t iMasterNodeID = nullnode;
uint64_t llMasterVersion = 0;
//step 1 check exist master and get version
m_oDefaultMasterSM.SafeGetMaster(iMasterNodeID, llMasterVersion);
if (iMasterNodeID != nullnode && (iMasterNodeID != m_poPaxosNode->GetMyNodeID()))
{
PLG1Imp("Ohter as master, can't try be master, masterid %lu myid %lu",
iMasterNodeID, m_poPaxosNode->GetMyNodeID());
return;
}
//step 2 try be master
std::string sPaxosValue;
if (!MasterStateMachine::MakeOpValue(
m_poPaxosNode->GetMyNodeID(),
llMasterVersion,
iLeaseTime,
MasterOperatorType_Complete,
sPaxosValue))
{
PLG1Err("Make paxos value fail");
return;
}
const int iMasterLeaseTimeout = iLeaseTime - 100;
uint64_t llAbsMasterTimeout = Time::GetTimestampMS() + iMasterLeaseTimeout;
uint64_t llCommitInstanceID = 0;
SMCtx oCtx;
oCtx.m_iSMID = MASTER_V_SMID;
oCtx.m_pCtx = (void *)&llAbsMasterTimeout;
m_poPaxosNode->Propose(m_iMyGroupIdx, sPaxosValue, llCommitInstanceID, &oCtx);
}
void SystemVSM :: AddNodeIDList(const NodeInfoList & vecNodeInfoList)
{
if (m_oSystemVariables.gid() != 0)
{
PLG1Err("No need to add, i already have membership info.");
return;
}
m_setNodeID.clear();
m_oSystemVariables.clear_membership();
for (auto & tNodeInfo : vecNodeInfoList)
{
PaxosNodeInfo * poNodeInfo = m_oSystemVariables.add_membership();
//to do, what rid?
poNodeInfo->set_rid(0);
poNodeInfo->set_nodeid(tNodeInfo.GetNodeID());
NodeInfo tTmpNode(poNodeInfo->nodeid());
}
RefleshNodeID();
}
int SystemVSM :: CreateGid_OPValue(const uint64_t llGid, std::string & sOpValue)
{
SystemVariables oVariables = m_oSystemVariables;
oVariables.set_gid(llGid);
/*
** only founder need to check this. but now all is founder.
if (oVariables.membership_size() == 0)
{
PLG1Err("no membership, can't create gid");
return -1;
}
*/
bool sSucc = oVariables.SerializeToString(&sOpValue);
if (!sSucc)
{
PLG1Err("Variables.Serialize fail");
return -1;
}
return 0;
}
int MasterStateMachine :: GetCheckpointBuffer(std::string & sCPBuffer)
{
std::lock_guard<std::mutex> oLockGuard(m_oMutex);
if (m_llMasterVersion == (uint64_t)-1)
{
return 0;
}
MasterVariables oVariables;
oVariables.set_masternodeid(m_iMasterNodeID);
oVariables.set_version(m_llMasterVersion);
oVariables.set_leasetime(m_iLeaseTime);
bool sSucc = oVariables.SerializeToString(&sCPBuffer);
if (!sSucc)
{
PLG1Err("Variables.Serialize fail");
return -1;
}
return 0;
}
int SystemVSM :: Membership_OPValue(const NodeInfoList & vecNodeInfoList, const uint64_t llVersion, std::string & sOpValue)
{
SystemVariables oVariables;
//must must set version first!
oVariables.set_version(llVersion);
oVariables.set_gid(m_oSystemVariables.gid());
for (auto & tNodeInfo : vecNodeInfoList)
{
PaxosNodeInfo * poNodeInfo = oVariables.add_membership();
//to do, what rid?
poNodeInfo->set_rid(0);
poNodeInfo->set_nodeid(tNodeInfo.GetNodeID());
}
bool sSucc = oVariables.SerializeToString(&sOpValue);
if (!sSucc)
{
PLG1Err("Variables.Serialize fail");
return -1;
}
return 0;
}
int Database :: GetMaxInstanceIDFileID(std::string & sFileID, uint64_t & llInstanceID)
{
uint64_t llMaxInstanceID = 0;
int ret = GetMaxInstanceID(llMaxInstanceID);
if (ret != 0 && ret != 1)
{
return ret;
}
if (ret == 1)
{
sFileID = "";
return 0;
}
string sKey = GenKey(llMaxInstanceID);
leveldb::Status oStatus = m_poLevelDB->Get(leveldb::ReadOptions(), sKey, &sFileID);
if (!oStatus.ok())
{
if (oStatus.IsNotFound())
{
BP->GetLogStorageBP()->LevelDBGetNotExist();
//PLG1Err("LevelDB.Get not found %s", sKey.c_str());
return 1;
}
BP->GetLogStorageBP()->LevelDBGetFail();
PLG1Err("LevelDB.Get fail");
return -1;
}
llInstanceID = llMaxInstanceID;
return 0;
}
int Database :: Put(const WriteOptions & oWriteOptions, const uint64_t llInstanceID, const std::string & sValue)
{
if (!m_bHasInit)
{
PLG1Err("no init yet");
return -1;
}
std::string sFileID;
// 将值写入到文件之中。
int ret = ValueToFileID(oWriteOptions, llInstanceID, sValue, sFileID);
if (ret != 0)
{
return ret;
}
// 有点迷糊，为啥还要写一遍到 levelDB里呢?
//04-23 : 已经搞清楚，这里的levelDB其实只存索引，充分利用levelDB的写优势。
// 这样写索引会非常快速，读真实的值由于通过索引查找也会非常快速，而且真实
// 的值存储是顺序存储，按照InstanceID的顺序，删除会非常方便。
ret = PutToLevelDB(false, llInstanceID, sFileID);
return ret;
}
int Database :: ClearAllLog()
{
string sSystemVariablesBuffer;
int ret = GetSystemVariables(sSystemVariablesBuffer);
if (ret != 0 && ret != 1)
{
PLG1Err("GetSystemVariables fail, ret %d", ret);
return ret;
}
string sMasterVariablesBuffer;
ret = GetMasterVariables(sMasterVariablesBuffer);
if (ret != 0 && ret != 1)
{
PLG1Err("GetMasterVariables fail, ret %d", ret);
return ret;
}
m_bHasInit = false;
delete m_poLevelDB;
m_poLevelDB = nullptr;
delete m_poValueStore;
m_poValueStore = nullptr;
string sBakPath = m_sDBPath + ".bak";
ret = FileUtils::DeleteDir(sBakPath);
if (ret != 0)
{
PLG1Err("Delete bak dir fail, dir %s", sBakPath.c_str());
return -1;
}
ret = rename(m_sDBPath.c_str(), sBakPath.c_str());
assert(ret == 0);
ret = Init(m_sDBPath, m_iMyGroupIdx);
if (ret != 0)
{
PLG1Err("Init again fail, ret %d", ret);
return ret;
}
WriteOptions oWriteOptions;
oWriteOptions.bSync = true;
if (sSystemVariablesBuffer.size() > 0)
{
ret = SetSystemVariables(oWriteOptions, sSystemVariablesBuffer);
if (ret != 0)
{
PLG1Err("SetSystemVariables fail, ret %d", ret);
return ret;
}
}
if (sMasterVariablesBuffer.size() > 0)
{
ret = SetMasterVariables(oWriteOptions, sMasterVariablesBuffer);
if (ret != 0)
{
PLG1Err("SetMasterVariables fail, ret %d", ret);
return ret;
}
}
return 0;
}
int MasterStateMachine :: UpdateByCheckpoint(const std::string & sCPBuffer, bool & bChange)
{
if (sCPBuffer.size() == 0)
{
return 0;
}
MasterVariables oVariables;
bool bSucc = oVariables.ParseFromArray(sCPBuffer.data(), sCPBuffer.size());
if (!bSucc)
{
PLG1Err("Variables.ParseFromArray fail, bufferlen %zu", sCPBuffer.size());
return -1;
}
std::lock_guard<std::mutex> oLockGuard(m_oMutex);
if (oVariables.version() <= m_llMasterVersion
&& m_llMasterVersion != (uint64_t)-1)
{
PLG1Imp("lag checkpoint, no need update, cp.version %lu now.version %lu",
oVariables.version(), m_llMasterVersion);
return 0;
}
int ret = UpdateMasterToStore(oVariables.masternodeid(), oVariables.version(), oVariables.leasetime());
if (ret != 0)
{
return -1;
}
PLG1Head("ok, cp.version %lu cp.masternodeid %lu old.version %lu old.masternodeid %lu",
oVariables.version(), oVariables.masternodeid(),
m_llMasterVersion, m_iMasterNodeID);
bool bMasterChange = false;
m_llMasterVersion = oVariables.version();
if (oVariables.masternodeid() == m_iMyNodeID)
{
m_iMasterNodeID = nullnode;
m_llAbsExpireTime = 0;
}
else
{
if (m_iMasterNodeID != oVariables.masternodeid())
{
bMasterChange = true;
}
m_iMasterNodeID = oVariables.masternodeid();
m_llAbsExpireTime = Time::GetSteadyClockMS() + oVariables.leasetime();
}
if (bMasterChange)
{
if (m_pMasterChangeCallback != nullptr)
{
m_pMasterChangeCallback(m_iMyGroupIdx, NodeInfo(m_iMasterNodeID), m_llMasterVersion);
}
}
return 0;
}
int MasterStateMachine :: LearnMaster(
const uint64_t llInstanceID,
const MasterOperator & oMasterOper,
const uint64_t llAbsMasterTimeout)
{
std::lock_guard<std::mutex> oLockGuard(m_oMutex);
PLG1Debug("my last version %lu other last version %lu this version %lu instanceid %lu",
m_llMasterVersion, oMasterOper.lastversion(), oMasterOper.version(), llInstanceID);
if (oMasterOper.lastversion() != 0
&& llInstanceID > m_llMasterVersion
&& oMasterOper.lastversion() != m_llMasterVersion)
{
BP->GetMasterBP()->MasterSMInconsistent();
PLG1Err("other last version %lu not same to my last version %lu, instanceid %lu",
oMasterOper.lastversion(), m_llMasterVersion, llInstanceID);
PLG1Err("try to fix, set my master version %lu as other last version %lu, instanceid %lu",
m_llMasterVersion, oMasterOper.lastversion(), llInstanceID);
m_llMasterVersion = oMasterOper.lastversion();
}
if (oMasterOper.version() != m_llMasterVersion)
{
PLG1Debug("version conflit, op version %lu now master version %lu",
oMasterOper.version(), m_llMasterVersion);
return 0;
}
int ret = UpdateMasterToStore(oMasterOper.nodeid(), llInstanceID, oMasterOper.timeout());
if (ret != 0)
{
PLG1Err("UpdateMasterToStore fail, ret %d", ret);
return -1;
}
bool bMasterChange = false;
if (m_iMasterNodeID != oMasterOper.nodeid())
{
bMasterChange = true;
}
m_iMasterNodeID = oMasterOper.nodeid();
if (m_iMasterNodeID == m_iMyNodeID)
{
//self be master
//use local abstimeout
m_llAbsExpireTime = llAbsMasterTimeout;
BP->GetMasterBP()->SuccessBeMaster();
PLG1Head("Be master success, absexpiretime %lu", m_llAbsExpireTime);
}
else
{
//other be master
//use new start timeout
m_llAbsExpireTime = Time::GetSteadyClockMS() + oMasterOper.timeout();
BP->GetMasterBP()->OtherBeMaster();
PLG1Head("Ohter be master, absexpiretime %lu", m_llAbsExpireTime);
}
m_iLeaseTime = oMasterOper.timeout();
m_llMasterVersion = llInstanceID;
if (bMasterChange)
{
if (m_pMasterChangeCallback != nullptr)
{
m_pMasterChangeCallback(m_iMyGroupIdx, NodeInfo(m_iMasterNodeID), m_llMasterVersion);
}
}
PLG1Imp("OK, masternodeid %lu version %lu abstimeout %lu",
m_iMasterNodeID, m_llMasterVersion, m_llAbsExpireTime);
return 0;
}