BOOST_FOREACH(const Zone::Ptr& zone, ConfigType::GetObjectsByType<Zone>()) {
/* don't connect to global zones */
if (zone->GetGlobal())
continue;
/* only connect to endpoints in a) the same zone b) our parent zone c) immediate child zones */
if (my_zone != zone && my_zone != zone->GetParent() && zone != my_zone->GetParent()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Zone '" << zone->GetName()
<< "' because it's not in the same zone, a parent or a child zone.";
continue;
}
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
/* don't connect to ourselves */
if (endpoint == GetLocalEndpoint()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName() << "' because that's us.";
continue;
}
/* don't try to connect to endpoints which don't have a host and port */
if (endpoint->GetHost().IsEmpty() || endpoint->GetPort().IsEmpty()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because the host/port attributes are missing.";
continue;
}
/* don't try to connect if there's already a connection attempt */
if (endpoint->GetConnecting()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because we're already trying to connect to it.";
continue;
}
/* don't try to connect if we're already connected */
if (endpoint->GetConnected()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because we're already connected to it.";
continue;
}
boost::thread thread(boost::bind(&ApiListener::AddConnection, this, endpoint));
thread.detach();
}
}
Endpoint::Ptr ApiListener::GetMaster(void) const
{
Zone::Ptr zone = Zone::GetLocalZone();
if (!zone)
return Endpoint::Ptr();
std::vector<String> names;
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints())
if (endpoint->IsConnected() || endpoint->GetName() == GetIdentity())
names.push_back(endpoint->GetName());
std::sort(names.begin(), names.end());
return Endpoint::GetByName(*names.begin());
}
bool ApiListener::RelayMessageOne(const Zone::Ptr& targetZone, const MessageOrigin::Ptr& origin, const Dictionary::Ptr& message, const Endpoint::Ptr& currentMaster)
{
ASSERT(targetZone);
Zone::Ptr myZone = Zone::GetLocalZone();
/* only relay the message to a) the same zone, b) the parent zone and c) direct child zones. Exception is a global zone. */
if (!targetZone->GetGlobal() &&
targetZone != myZone &&
targetZone != myZone->GetParent() &&
targetZone->GetParent() != myZone) {
return true;
}
Endpoint::Ptr myEndpoint = GetLocalEndpoint();
std::vector<Endpoint::Ptr> skippedEndpoints;
bool relayed = false, log_needed = false, log_done = false;
std::set<Endpoint::Ptr> targetEndpoints;
if (targetZone->GetGlobal()) {
targetEndpoints = myZone->GetEndpoints();
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
/* Fetch immediate child zone members */
if (zone->GetParent() == myZone) {
std::set<Endpoint::Ptr> endpoints = zone->GetEndpoints();
targetEndpoints.insert(endpoints.begin(), endpoints.end());
}
}
} else {
targetEndpoints = targetZone->GetEndpoints();
}
for (const Endpoint::Ptr& endpoint : targetEndpoints) {
/* don't relay messages to ourselves */
if (endpoint == GetLocalEndpoint())
continue;
log_needed = true;
/* don't relay messages to disconnected endpoints */
if (!endpoint->GetConnected()) {
if (targetZone == myZone)
log_done = false;
continue;
}
log_done = true;
/* don't relay the message to the zone through more than one endpoint unless this is our own zone */
if (relayed && targetZone != myZone) {
skippedEndpoints.push_back(endpoint);
continue;
}
/* don't relay messages back to the endpoint which we got the message from */
if (origin && origin->FromClient && endpoint == origin->FromClient->GetEndpoint()) {
skippedEndpoints.push_back(endpoint);
continue;
}
/* don't relay messages back to the zone which we got the message from */
if (origin && origin->FromZone && targetZone == origin->FromZone) {
skippedEndpoints.push_back(endpoint);
continue;
}
/* only relay message to the master if we're not currently the master */
if (currentMaster != myEndpoint && currentMaster != endpoint) {
skippedEndpoints.push_back(endpoint);
continue;
}
relayed = true;
SyncSendMessage(endpoint, message);
}
if (!skippedEndpoints.empty()) {
double ts = message->Get("ts");
for (const Endpoint::Ptr& endpoint : skippedEndpoints)
endpoint->SetLocalLogPosition(ts);
}
return !log_needed || log_done;
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
REQUIRE_NOT_NULL(checkable);
REQUIRE_NOT_NULL(cr);
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.emplace_back("service", service);
resolvers.emplace_back("host", host);
resolvers.emplace_back("command", commandObj);
resolvers.emplace_back("icinga", IcingaApplication::GetInstance());
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(),
nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
String missingLagWarning;
String missingLagCritical;
double lagWarning = MacroProcessor::ResolveMacros("$cluster_lag_warning$", resolvers, checkable->GetLastCheckResult(),
&missingLagWarning, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
double lagCritical = MacroProcessor::ResolveMacros("$cluster_lag_critical$", resolvers, checkable->GetLastCheckResult(),
&missingLagCritical, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
double zoneLag = 0;
double lastMessageSent = 0;
double lastMessageReceived = 0;
double messagesSentPerSecond = 0;
double messagesReceivedPerSecond = 0;
double bytesSentPerSecond = 0;
double bytesReceivedPerSecond = 0;
for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
if (endpoint->GetConnected())
connected = true;
double eplag = ApiListener::CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
if (endpoint->GetLastMessageSent() > lastMessageSent)
lastMessageSent = endpoint->GetLastMessageSent();
if (endpoint->GetLastMessageReceived() > lastMessageReceived)
lastMessageReceived = endpoint->GetLastMessageReceived();
messagesSentPerSecond += endpoint->GetMessagesSentPerSecond();
messagesReceivedPerSecond += endpoint->GetMessagesReceivedPerSecond();
bytesSentPerSecond += endpoint->GetBytesSentPerSecond();
bytesReceivedPerSecond += endpoint->GetBytesReceivedPerSecond();
}
if (connected) {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));
/* Check whether the thresholds have been resolved and compare them */
if (missingLagCritical.IsEmpty() && zoneLag > lagCritical) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
+ " greater than critical threshold: " + Utility::FormatDuration(lagCritical));
} else if (missingLagWarning.IsEmpty() && zoneLag > lagWarning) {
cr->SetState(ServiceWarning);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
+ " greater than warning threshold: " + Utility::FormatDuration(lagWarning));
}
} else {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
}
cr->SetPerformanceData(new Array({
new PerfdataValue("slave_lag", zoneLag, false, "s", lagWarning, lagCritical),
new PerfdataValue("last_messages_sent", lastMessageSent),
new PerfdataValue("last_messages_received", lastMessageReceived),
new PerfdataValue("sum_messages_sent_per_second", messagesSentPerSecond),
new PerfdataValue("sum_messages_received_per_second", messagesReceivedPerSecond),
new PerfdataValue("sum_bytes_sent_per_second", bytesSentPerSecond),
new PerfdataValue("sum_bytes_received_per_second", bytesReceivedPerSecond)
}));
checkable->ProcessCheckResult(cr);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.push_back(std::make_pair("service", service));
resolvers.push_back(std::make_pair("host", host));
resolvers.push_back(std::make_pair("command", commandObj));
resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(),
NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
double zoneLag = 0;
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
if (endpoint->GetConnected())
connected = true;
double eplag = ApiListener::CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
}
if (!connected) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
} else {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));
}
Array::Ptr perfdata = new Array();
perfdata->Add(new PerfdataValue("slave_lag", zoneLag));
cr->SetPerformanceData(perfdata);
checkable->ProcessCheckResult(cr);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.push_back(std::make_pair("service", service));
resolvers.push_back(std::make_pair("host", host));
resolvers.push_back(std::make_pair("command", commandObj));
resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult());
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
if (endpoint->IsConnected()) {
connected = true;
break;
}
}
if (!connected) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected.");
} else {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected.");
}
checkable->ProcessCheckResult(cr);
}