void ApplicationCluster::close () {
if (! enabled()) {
return;
}
if (_heartbeat != 0) {
_heartbeat->stop();
}
// change into shutdown state
ServerState::instance()->setState(ServerState::STATE_SHUTDOWN);
AgencyComm comm;
comm.sendServerState(0.0);
}
void ApplicationCluster::stop () {
if (! enabled()) {
return;
}
// change into shutdown state
ServerState::instance()->setState(ServerState::STATE_SHUTDOWN);
AgencyComm comm;
comm.sendServerState(0.0);
if (_heartbeat != 0) {
_heartbeat->stop();
}
{
AgencyCommLocker locker("Current", "WRITE");
if (locker.successful()) {
// unregister ourselves
ServerState::RoleEnum role = ServerState::instance()->getRole();
if (role == ServerState::ROLE_PRIMARY) {
comm.removeValues("Current/DBServers/" + _myId, false);
}
else if (role == ServerState::ROLE_COORDINATOR) {
comm.removeValues("Current/Coordinators/" + _myId, false);
}
// unregister ourselves
comm.removeValues("Current/ServersRegistered/" + _myId, false);
}
}
ClusterComm::cleanup();
ClusterInfo::cleanup();
AgencyComm::cleanup();
}
void ClusterFeature::unprepare() {
if (_enableCluster) {
if (_heartbeatThread != nullptr) {
_heartbeatThread->beginShutdown();
}
// change into shutdown state
ServerState::instance()->setState(ServerState::STATE_SHUTDOWN);
AgencyComm comm;
comm.sendServerState(0.0);
if (_heartbeatThread != nullptr) {
int counter = 0;
while (_heartbeatThread->isRunning()) {
usleep(100000);
// emit warning after 5 seconds
if (++counter == 10 * 5) {
LOG(WARN) << "waiting for heartbeat thread to finish";
}
}
}
if (_unregisterOnShutdown) {
ServerState::instance()->unregister();
}
}
if (!_enableCluster) {
ClusterComm::cleanup();
return;
}
// change into shutdown state
ServerState::instance()->setState(ServerState::STATE_SHUTDOWN);
AgencyComm comm;
comm.sendServerState(0.0);
// Try only once to unregister because maybe the agencycomm
// is shutting down as well...
ServerState::RoleEnum role = ServerState::instance()->getRole();
AgencyWriteTransaction unreg;
// Remove from role
if (role == ServerState::ROLE_PRIMARY) {
unreg.operations.push_back(AgencyOperation(
"Current/DBServers/" + _myId, AgencySimpleOperationType::DELETE_OP));
} else if (role == ServerState::ROLE_COORDINATOR) {
unreg.operations.push_back(AgencyOperation(
"Current/Coordinators/" + _myId, AgencySimpleOperationType::DELETE_OP));
}
// Unregister
unreg.operations.push_back(
AgencyOperation("Current/ServersRegistered/" + _myId,
AgencySimpleOperationType::DELETE_OP));
comm.sendTransactionWithFailover(unreg, 120.0);
while (_heartbeatThread->isRunning()) {
usleep(50000);
}
AgencyComm::cleanup();
ClusterComm::cleanup();
}
void ClusterFeature::start() {
// return if cluster is disabled
if (!_enableCluster) {
return;
}
ServerState::instance()->setState(ServerState::STATE_STARTUP);
// the agency about our state
AgencyComm comm;
comm.sendServerState(0.0);
std::string const version = comm.getVersion();
ServerState::instance()->setInitialized();
std::string const endpoints = AgencyComm::getEndpointsString();
ServerState::RoleEnum role = ServerState::instance()->getRole();
LOG(INFO) << "Cluster feature is turned on. Agency version: " << version
<< ", Agency endpoints: " << endpoints << ", server id: '" << _myId
<< "', internal address: " << _myAddress
<< ", role: " << ServerState::roleToString(role);
if (!_disableHeartbeat) {
AgencyCommResult result = comm.getValues("Sync/HeartbeatIntervalMs");
if (result.successful()) {
velocypack::Slice HeartbeatIntervalMs =
result.slice()[0].get(std::vector<std::string>(
{AgencyComm::prefix(), "Sync", "HeartbeatIntervalMs"}));
if (HeartbeatIntervalMs.isInteger()) {
try {
_heartbeatInterval = HeartbeatIntervalMs.getUInt();
LOG(INFO) << "using heartbeat interval value '" << _heartbeatInterval
<< " ms' from agency";
} catch (...) {
// Ignore if it is not a small int or uint
}
}
}
// no value set in agency. use default
if (_heartbeatInterval == 0) {
_heartbeatInterval = 5000; // 1/s
LOG(WARN) << "unable to read heartbeat interval from agency. Using "
<< "default value '" << _heartbeatInterval << " ms'";
}
// start heartbeat thread
_heartbeatThread = std::make_shared<HeartbeatThread>(
_agencyCallbackRegistry.get(), _heartbeatInterval * 1000, 5,
SchedulerFeature::SCHEDULER->ioService());
if (!_heartbeatThread->init() || !_heartbeatThread->start()) {
LOG(FATAL) << "heartbeat could not connect to agency endpoints ("
<< endpoints << ")";
FATAL_ERROR_EXIT();
}
while (!_heartbeatThread->isReady()) {
// wait until heartbeat is ready
usleep(10000);
}
}
AgencyCommResult result;
while (true) {
VPackBuilder builder;
try {
VPackObjectBuilder b(&builder);
builder.add("endpoint", VPackValue(_myAddress));
} catch (...) {
LOG(FATAL) << "out of memory";
FATAL_ERROR_EXIT();
}
result = comm.setValue("Current/ServersRegistered/" + _myId,
builder.slice(), 0.0);
if (!result.successful()) {
LOG(FATAL) << "unable to register server in agency: http code: "
<< result.httpCode() << ", body: " << result.body();
FATAL_ERROR_EXIT();
} else {
break;
}
sleep(1);
}
if (role == ServerState::ROLE_COORDINATOR) {
ServerState::instance()->setState(ServerState::STATE_SERVING);
} else if (role == ServerState::ROLE_PRIMARY) {
ServerState::instance()->setState(ServerState::STATE_SERVINGASYNC);
} else if (role == ServerState::ROLE_SECONDARY) {
ServerState::instance()->setState(ServerState::STATE_SYNCING);
}
}
bool ApplicationCluster::start () {
// set authentication data
ServerState::instance()->setAuthentication(_username, _password);
// overwrite memory area
_username = _password = "******";
ServerState::instance()->setDataPath(_dataPath);
ServerState::instance()->setLogPath(_logPath);
ServerState::instance()->setAgentPath(_agentPath);
ServerState::instance()->setArangodPath(_arangodPath);
ServerState::instance()->setDBserverConfig(_dbserverConfig);
ServerState::instance()->setCoordinatorConfig(_coordinatorConfig);
ServerState::instance()->setDisableDispatcherFrontend(_disableDispatcherFrontend);
ServerState::instance()->setDisableDispatcherKickstarter(_disableDispatcherKickstarter);
if (! enabled()) {
return true;
}
ServerState::instance()->setId(_myId);
// perfom an initial connect to the agency
const std::string endpoints = AgencyComm::getEndpointsString();
if (! AgencyComm::tryConnect()) {
LOG_FATAL_AND_EXIT("Could not connect to agency endpoints (%s)",
endpoints.c_str());
}
ServerState::RoleEnum role = ServerState::instance()->getRole();
if (role == ServerState::ROLE_UNDEFINED) {
// no role found
LOG_FATAL_AND_EXIT("unable to determine unambiguous role for server '%s'. No role configured in agency (%s)",
_myId.c_str(),
endpoints.c_str());
}
// check if my-address is set
if (_myAddress.empty()) {
// no address given, now ask the agency for out address
_myAddress = ServerState::instance()->getAddress();
}
else {
// register our own address
ServerState::instance()->setAddress(_myAddress);
}
if (_myAddress.empty()) {
LOG_FATAL_AND_EXIT("unable to determine internal address for server '%s'. "
"Please specify --cluster.my-address or configure the address for this server in the agency.",
_myId.c_str());
}
// now we can validate --cluster.my-address
const string unified = triagens::rest::Endpoint::getUnifiedForm(_myAddress);
if (unified.empty()) {
LOG_FATAL_AND_EXIT("invalid endpoint '%s' specified for --cluster.my-address",
_myAddress.c_str());
}
ServerState::instance()->setState(ServerState::STATE_STARTUP);
// initialise ConnectionManager library
httpclient::ConnectionManager::instance()->initialise();
// the agency about our state
AgencyComm comm;
comm.sendServerState(0.0);
const std::string version = comm.getVersion();
ServerState::instance()->setInitialised();
LOG_INFO("Cluster feature is turned on. "
"Agency version: %s, Agency endpoints: %s, "
"server id: '%s', internal address: %s, role: %s",
version.c_str(),
endpoints.c_str(),
_myId.c_str(),
_myAddress.c_str(),
ServerState::roleToString(role).c_str());
if (! _disableHeartbeat) {
AgencyCommResult result = comm.getValues("Sync/HeartbeatIntervalMs", false);
if (result.successful()) {
result.parse("", false);
std::map<std::string, AgencyCommResultEntry>::const_iterator it = result._values.begin();
if (it != result._values.end()) {
_heartbeatInterval = triagens::basics::JsonHelper::stringUInt64((*it).second._json);
LOG_INFO("using heartbeat interval value '%llu ms' from agency",
(unsigned long long) _heartbeatInterval);
}
}
// no value set in agency. use default
if (_heartbeatInterval == 0) {
_heartbeatInterval = 1000; // 1/s
LOG_WARNING("unable to read heartbeat interval from agency. Using default value '%llu ms'",
(unsigned long long) _heartbeatInterval);
}
// start heartbeat thread
_heartbeat = new HeartbeatThread(_server, _dispatcher, _applicationV8, _heartbeatInterval * 1000, 5);
if (_heartbeat == 0) {
LOG_FATAL_AND_EXIT("unable to start cluster heartbeat thread");
}
if (! _heartbeat->init() || ! _heartbeat->start()) {
LOG_FATAL_AND_EXIT("heartbeat could not connect to agency endpoints (%s)",
endpoints.c_str());
}
while (! _heartbeat->ready()) {
// wait until heartbeat is ready
usleep(10000);
}
}
return true;
}