void Zone::OnAllConfigLoaded()
{
ObjectImpl<Zone>::OnAllConfigLoaded();
m_Parent = Zone::GetByName(GetParentRaw());
if (m_Parent && m_Parent->IsGlobal())
BOOST_THROW_EXCEPTION(ScriptError("Zone '" + GetName() + "' can not have a global zone as parent.", GetDebugInfo()));
Zone::Ptr zone = m_Parent;
int levels = 0;
Array::Ptr endpoints = GetEndpointsRaw();
if (endpoints) {
ObjectLock olock(endpoints);
for (const String& endpoint : endpoints) {
Endpoint::Ptr ep = Endpoint::GetByName(endpoint);
if (ep)
ep->SetCachedZone(this);
}
}
while (zone) {
m_AllParents.push_back(zone);
zone = Zone::GetByName(zone->GetParentRaw());
levels++;
if (levels > 32)
BOOST_THROW_EXCEPTION(ScriptError("Infinite recursion detected while resolving zone graph. Check your zone hierarchy.", GetDebugInfo()));
}
}
bool ApiListener::IsHACluster(void)
{
Zone::Ptr zone = Zone::GetLocalZone();
if (!zone)
return false;
return zone->IsSingleInstance();
}
Dictionary::Ptr ZoneDbObject::GetConfigFields() const
{
Zone::Ptr zone = static_pointer_cast<Zone>(GetObject());
return new Dictionary({
{ "is_global", zone->IsGlobal() ? 1 : 0 },
{ "parent_zone_object_id", zone->GetParent() }
});
}
Dictionary::Ptr ZoneDbObject::GetStatusFields() const
{
Zone::Ptr zone = static_pointer_cast<Zone>(GetObject());
Log(LogDebug, "ZoneDbObject")
<< "update status for zone '" << zone->GetName() << "'";
return new Dictionary({
{ "parent_zone_object_id", zone->GetParent() }
});
}
bool Zone::IsChildOf(const Zone::Ptr& zone)
{
Zone::Ptr azone = this;
while (azone) {
if (azone == zone)
return true;
azone = azone->GetParent();
}
return false;
}
Value EndpointsTable::ZoneAccessor(const Value& row)
{
Endpoint::Ptr endpoint = static_cast<Endpoint::Ptr>(row);
if (!endpoint)
return Empty;
Zone::Ptr zone = endpoint->GetZone();
if (!zone)
return Empty;
return zone->GetName();
}
BOOST_FOREACH(const Zone::Ptr& zone, ConfigType::GetObjectsByType<Zone>()) {
/* don't connect to global zones */
if (zone->GetGlobal())
continue;
/* only connect to endpoints in a) the same zone b) our parent zone c) immediate child zones */
if (my_zone != zone && my_zone != zone->GetParent() && zone != my_zone->GetParent()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Zone '" << zone->GetName()
<< "' because it's not in the same zone, a parent or a child zone.";
continue;
}
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
/* don't connect to ourselves */
if (endpoint == GetLocalEndpoint()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName() << "' because that's us.";
continue;
}
/* don't try to connect to endpoints which don't have a host and port */
if (endpoint->GetHost().IsEmpty() || endpoint->GetPort().IsEmpty()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because the host/port attributes are missing.";
continue;
}
/* don't try to connect if there's already a connection attempt */
if (endpoint->GetConnecting()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because we're already trying to connect to it.";
continue;
}
/* don't try to connect if we're already connected */
if (endpoint->GetConnected()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because we're already connected to it.";
continue;
}
boost::thread thread(boost::bind(&ApiListener::AddConnection, this, endpoint));
thread.detach();
}
}
/* Provide a helper function for zone origin name. */
String ApiListener::GetFromZoneName(const Zone::Ptr& fromZone)
{
String fromZoneName;
if (fromZone) {
fromZoneName = fromZone->GetName();
} else {
Zone::Ptr lzone = Zone::GetLocalZone();
if (lzone)
fromZoneName = lzone->GetName();
}
return fromZoneName;
}
void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient)
{
Endpoint::Ptr endpoint = aclient->GetEndpoint();
ASSERT(endpoint);
Zone::Ptr azone = endpoint->GetZone();
Zone::Ptr lzone = Zone::GetLocalZone();
/* don't try to send config updates to our master */
if (!azone->IsChildOf(lzone))
return;
Dictionary::Ptr configUpdateV1 = new Dictionary();
Dictionary::Ptr configUpdateV2 = new Dictionary();
String zonesDir = Configuration::DataDir + "/api/zones";
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
String zoneDir = zonesDir + "/" + zone->GetName();
if (!zone->IsChildOf(azone) && !zone->IsGlobal())
continue;
if (!Utility::PathExists(zoneDir))
continue;
Log(LogInformation, "ApiListener")
<< "Syncing configuration files for " << (zone->IsGlobal() ? "global " : "")
<< "zone '" << zone->GetName() << "' to endpoint '" << endpoint->GetName() << "'.";
ConfigDirInformation config = LoadConfigDir(zonesDir + "/" + zone->GetName());
configUpdateV1->Set(zone->GetName(), config.UpdateV1);
configUpdateV2->Set(zone->GetName(), config.UpdateV2);
}
Dictionary::Ptr message = new Dictionary({
{ "jsonrpc", "2.0" },
{ "method", "config::Update" },
{ "params", new Dictionary({
{ "update", configUpdateV1 },
{ "update_v2", configUpdateV2 }
}) }
});
aclient->SendMessage(message);
}
Endpoint::Ptr ApiListener::GetMaster(void) const
{
Zone::Ptr zone = Zone::GetLocalZone();
if (!zone)
return Endpoint::Ptr();
std::vector<String> names;
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints())
if (endpoint->IsConnected() || endpoint->GetName() == GetIdentity())
names.push_back(endpoint->GetName());
std::sort(names.begin(), names.end());
return Endpoint::GetByName(*names.begin());
}
void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const
{
Dictionary::Ptr newConfig = new Dictionary();
BOOST_FOREACH(const ZoneFragment& zf, ConfigCompiler::GetZoneDirs(zone->GetName())) {
Dictionary::Ptr newConfigPart = LoadConfigDir(zf.Path);
ObjectLock olock(newConfigPart);
BOOST_FOREACH(const Dictionary::Pair& kv, newConfigPart) {
newConfig->Set("/" + zf.Tag + kv.first, kv.second);
}
}
void Checkable::OnAllConfigLoaded(void)
{
ObjectImpl<Checkable>::OnAllConfigLoaded();
Endpoint::Ptr endpoint = GetCommandEndpoint();
if (endpoint) {
Zone::Ptr checkableZone = static_pointer_cast<Zone>(GetZone());
if (!checkableZone)
checkableZone = Zone::GetLocalZone();
Zone::Ptr cmdZone = endpoint->GetZone();
if (checkableZone && cmdZone != checkableZone && cmdZone->GetParent() != checkableZone) {
BOOST_THROW_EXCEPTION(ValidationError(this, boost::assign::list_of("command_endpoint"),
"Command endpoint must be in zone '" + checkableZone->GetName() + "' or in a direct child zone thereof."));
}
}
}
void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const
{
ConfigDirInformation newConfigInfo;
newConfigInfo.UpdateV1 = new Dictionary();
newConfigInfo.UpdateV2 = new Dictionary();
for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zone->GetName())) {
ConfigDirInformation newConfigPart = LoadConfigDir(zf.Path);
{
ObjectLock olock(newConfigPart.UpdateV1);
for (const Dictionary::Pair& kv : newConfigPart.UpdateV1) {
newConfigInfo.UpdateV1->Set("/" + zf.Tag + kv.first, kv.second);
}
}
{
ObjectLock olock(newConfigPart.UpdateV2);
for (const Dictionary::Pair& kv : newConfigPart.UpdateV2) {
newConfigInfo.UpdateV2->Set("/" + zf.Tag + kv.first, kv.second);
}
}
}
int sumUpdates = newConfigInfo.UpdateV1->GetLength() + newConfigInfo.UpdateV2->GetLength();
if (sumUpdates == 0)
return;
String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName();
Log(LogInformation, "ApiListener")
<< "Copying " << sumUpdates << " zone configuration files for zone '" << zone->GetName() << "' to '" << oldDir << "'.";
Utility::MkDirP(oldDir, 0700);
ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir);
UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, true);
}
void Checkable::OnAllConfigLoaded()
{
ObjectImpl<Checkable>::OnAllConfigLoaded();
Endpoint::Ptr endpoint = GetCommandEndpoint();
if (endpoint) {
Zone::Ptr checkableZone = static_pointer_cast<Zone>(GetZone());
if (checkableZone) {
Zone::Ptr cmdZone = endpoint->GetZone();
if (cmdZone != checkableZone && cmdZone->GetParent() != checkableZone) {
BOOST_THROW_EXCEPTION(ValidationError(this, { "command_endpoint" },
"Command endpoint must be in zone '" + checkableZone->GetName() + "' or in a direct child zone thereof."));
}
} else {
BOOST_THROW_EXCEPTION(ValidationError(this, { "command_endpoint" },
"Command endpoint must not be set."));
}
}
}
void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const
{
String newDir = Application::GetZonesDir() + "/" + zone->GetName();
String oldDir = Application::GetLocalStateDir() + "/lib/icinga2/api/zones/" + zone->GetName();
Log(LogInformation, "ApiListener")
<< "Copying zone configuration files from '" << newDir << "' to '" << oldDir << "'.";
if (!Utility::MkDir(oldDir, 0700)) {
Log(LogCritical, "ApiListener")
<< "mkdir() for path '" << oldDir << "'failed with error code " << errno << ", \"" << Utility::FormatErrorNumber(errno) << "\"";
BOOST_THROW_EXCEPTION(posix_error()
<< boost::errinfo_api_function("mkdir")
<< boost::errinfo_errno(errno)
<< boost::errinfo_file_name(oldDir));
}
Dictionary::Ptr newConfig = LoadConfigDir(newDir);
Dictionary::Ptr oldConfig = LoadConfigDir(oldDir);
UpdateConfigDir(oldConfig, newConfig, oldDir);
}
void ApiListener::SendConfigUpdate(const ApiClient::Ptr& aclient)
{
Endpoint::Ptr endpoint = aclient->GetEndpoint();
ASSERT(endpoint);
Zone::Ptr azone = endpoint->GetZone();
Zone::Ptr lzone = Zone::GetLocalZone();
/* don't try to send config updates to our master */
if (!azone->IsChildOf(lzone))
return;
Dictionary::Ptr configUpdate = make_shared<Dictionary>();
String zonesDir = Application::GetLocalStateDir() + "/lib/icinga2/api/zones";
BOOST_FOREACH(const Zone::Ptr& zone, DynamicType::GetObjectsByType<Zone>()) {
String zoneDir = zonesDir + "/" + zone->GetName();
if (!zone->IsChildOf(azone) && !zone->IsGlobal()) {
Log(LogNotice, "ApiListener")
<< "Skipping sync for '" << zone->GetName() << "'. Not a child of zone '" << azone->GetName() << "'.";
continue;
}
if (!Utility::PathExists(zoneDir)) {
Log(LogNotice, "ApiListener")
<< "Ignoring sync for '" << zone->GetName() << "'. Zone directory '" << zoneDir << "' does not exist.";
continue;
}
if (zone->IsGlobal())
Log(LogInformation, "ApiListener")
<< "Syncing global zone '" << zone->GetName() << "'.";
configUpdate->Set(zone->GetName(), LoadConfigDir(zonesDir + "/" + zone->GetName()));
}
bool ApiListener::IsConfigMaster(const Zone::Ptr& zone)
{
String path = Application::GetZonesDir() + "/" + zone->GetName();
return Utility::PathExists(path);
}
bool ApiListener::RelayMessageOne(const Zone::Ptr& targetZone, const MessageOrigin::Ptr& origin, const Dictionary::Ptr& message, const Endpoint::Ptr& currentMaster)
{
ASSERT(targetZone);
Zone::Ptr myZone = Zone::GetLocalZone();
/* only relay the message to a) the same zone, b) the parent zone and c) direct child zones. Exception is a global zone. */
if (!targetZone->GetGlobal() &&
targetZone != myZone &&
targetZone != myZone->GetParent() &&
targetZone->GetParent() != myZone) {
return true;
}
Endpoint::Ptr myEndpoint = GetLocalEndpoint();
std::vector<Endpoint::Ptr> skippedEndpoints;
bool relayed = false, log_needed = false, log_done = false;
std::set<Endpoint::Ptr> targetEndpoints;
if (targetZone->GetGlobal()) {
targetEndpoints = myZone->GetEndpoints();
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
/* Fetch immediate child zone members */
if (zone->GetParent() == myZone) {
std::set<Endpoint::Ptr> endpoints = zone->GetEndpoints();
targetEndpoints.insert(endpoints.begin(), endpoints.end());
}
}
} else {
targetEndpoints = targetZone->GetEndpoints();
}
for (const Endpoint::Ptr& endpoint : targetEndpoints) {
/* don't relay messages to ourselves */
if (endpoint == GetLocalEndpoint())
continue;
log_needed = true;
/* don't relay messages to disconnected endpoints */
if (!endpoint->GetConnected()) {
if (targetZone == myZone)
log_done = false;
continue;
}
log_done = true;
/* don't relay the message to the zone through more than one endpoint unless this is our own zone */
if (relayed && targetZone != myZone) {
skippedEndpoints.push_back(endpoint);
continue;
}
/* don't relay messages back to the endpoint which we got the message from */
if (origin && origin->FromClient && endpoint == origin->FromClient->GetEndpoint()) {
skippedEndpoints.push_back(endpoint);
continue;
}
/* don't relay messages back to the zone which we got the message from */
if (origin && origin->FromZone && targetZone == origin->FromZone) {
skippedEndpoints.push_back(endpoint);
continue;
}
/* only relay message to the master if we're not currently the master */
if (currentMaster != myEndpoint && currentMaster != endpoint) {
skippedEndpoints.push_back(endpoint);
continue;
}
relayed = true;
SyncSendMessage(endpoint, message);
}
if (!skippedEndpoints.empty()) {
double ts = message->Get("ts");
for (const Endpoint::Ptr& endpoint : skippedEndpoints)
endpoint->SetLocalLogPosition(ts);
}
return !log_needed || log_done;
}
std::pair<Dictionary::Ptr, Dictionary::Ptr> ApiListener::GetStatus()
{
Dictionary::Ptr perfdata = new Dictionary();
/* cluster stats */
double allEndpoints = 0;
Array::Ptr allNotConnectedEndpoints = new Array();
Array::Ptr allConnectedEndpoints = new Array();
Zone::Ptr my_zone = Zone::GetLocalZone();
Dictionary::Ptr connectedZones = new Dictionary();
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
/* only check endpoints in a) the same zone b) our parent zone c) immediate child zones */
if (my_zone != zone && my_zone != zone->GetParent() && zone != my_zone->GetParent()) {
Log(LogDebug, "ApiListener")
<< "Not checking connection to Zone '" << zone->GetName() << "' because it's not in the same zone, a parent or a child zone.";
continue;
}
bool zoneConnected = false;
int countZoneEndpoints = 0;
double zoneLag = 0;
ArrayData zoneEndpoints;
for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
zoneEndpoints.emplace_back(endpoint->GetName());
if (endpoint->GetName() == GetIdentity())
continue;
double eplag = CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
allEndpoints++;
countZoneEndpoints++;
if (!endpoint->GetConnected()) {
allNotConnectedEndpoints->Add(endpoint->GetName());
} else {
allConnectedEndpoints->Add(endpoint->GetName());
zoneConnected = true;
}
}
/* if there's only one endpoint inside the zone, we're not connected - that's us, fake it */
if (zone->GetEndpoints().size() == 1 && countZoneEndpoints == 0)
zoneConnected = true;
String parentZoneName;
Zone::Ptr parentZone = zone->GetParent();
if (parentZone)
parentZoneName = parentZone->GetName();
Dictionary::Ptr zoneStats = new Dictionary({
{ "connected", zoneConnected },
{ "client_log_lag", zoneLag },
{ "endpoints", new Array(std::move(zoneEndpoints)) },
{ "parent_zone", parentZoneName }
});
connectedZones->Set(zone->GetName(), zoneStats);
}
/* connection stats */
size_t jsonRpcClients = GetAnonymousClients().size();
size_t httpClients = GetHttpClients().size();
size_t workQueueItems = JsonRpcConnection::GetWorkQueueLength();
size_t workQueueCount = JsonRpcConnection::GetWorkQueueCount();
size_t syncQueueItems = m_SyncQueue.GetLength();
size_t relayQueueItems = m_RelayQueue.GetLength();
double workQueueItemRate = JsonRpcConnection::GetWorkQueueRate();
double syncQueueItemRate = m_SyncQueue.GetTaskCount(60) / 60.0;
double relayQueueItemRate = m_RelayQueue.GetTaskCount(60) / 60.0;
Dictionary::Ptr status = new Dictionary({
{ "identity", GetIdentity() },
{ "num_endpoints", allEndpoints },
{ "num_conn_endpoints", allConnectedEndpoints->GetLength() },
{ "num_not_conn_endpoints", allNotConnectedEndpoints->GetLength() },
{ "conn_endpoints", allConnectedEndpoints },
{ "not_conn_endpoints", allNotConnectedEndpoints },
{ "zones", connectedZones },
{ "json_rpc", new Dictionary({
{ "clients", jsonRpcClients },
{ "work_queue_items", workQueueItems },
{ "work_queue_count", workQueueCount },
{ "sync_queue_items", syncQueueItems },
{ "relay_queue_items", relayQueueItems },
{ "work_queue_item_rate", workQueueItemRate },
{ "sync_queue_item_rate", syncQueueItemRate },
{ "relay_queue_item_rate", relayQueueItemRate }
}) },
{ "http", new Dictionary({
{ "clients", httpClients }
}) }
});
/* performance data */
perfdata->Set("num_endpoints", allEndpoints);
perfdata->Set("num_conn_endpoints", Convert::ToDouble(allConnectedEndpoints->GetLength()));
perfdata->Set("num_not_conn_endpoints", Convert::ToDouble(allNotConnectedEndpoints->GetLength()));
perfdata->Set("num_json_rpc_clients", jsonRpcClients);
perfdata->Set("num_http_clients", httpClients);
perfdata->Set("num_json_rpc_work_queue_items", workQueueItems);
perfdata->Set("num_json_rpc_work_queue_count", workQueueCount);
perfdata->Set("num_json_rpc_sync_queue_items", syncQueueItems);
perfdata->Set("num_json_rpc_relay_queue_items", relayQueueItems);
perfdata->Set("num_json_rpc_work_queue_item_rate", workQueueItemRate);
perfdata->Set("num_json_rpc_sync_queue_item_rate", syncQueueItemRate);
perfdata->Set("num_json_rpc_relay_queue_item_rate", relayQueueItemRate);
return std::make_pair(status, perfdata);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
REQUIRE_NOT_NULL(checkable);
REQUIRE_NOT_NULL(cr);
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.emplace_back("service", service);
resolvers.emplace_back("host", host);
resolvers.emplace_back("command", commandObj);
resolvers.emplace_back("icinga", IcingaApplication::GetInstance());
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(),
nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
String missingLagWarning;
String missingLagCritical;
double lagWarning = MacroProcessor::ResolveMacros("$cluster_lag_warning$", resolvers, checkable->GetLastCheckResult(),
&missingLagWarning, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
double lagCritical = MacroProcessor::ResolveMacros("$cluster_lag_critical$", resolvers, checkable->GetLastCheckResult(),
&missingLagCritical, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
double zoneLag = 0;
double lastMessageSent = 0;
double lastMessageReceived = 0;
double messagesSentPerSecond = 0;
double messagesReceivedPerSecond = 0;
double bytesSentPerSecond = 0;
double bytesReceivedPerSecond = 0;
for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
if (endpoint->GetConnected())
connected = true;
double eplag = ApiListener::CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
if (endpoint->GetLastMessageSent() > lastMessageSent)
lastMessageSent = endpoint->GetLastMessageSent();
if (endpoint->GetLastMessageReceived() > lastMessageReceived)
lastMessageReceived = endpoint->GetLastMessageReceived();
messagesSentPerSecond += endpoint->GetMessagesSentPerSecond();
messagesReceivedPerSecond += endpoint->GetMessagesReceivedPerSecond();
bytesSentPerSecond += endpoint->GetBytesSentPerSecond();
bytesReceivedPerSecond += endpoint->GetBytesReceivedPerSecond();
}
if (connected) {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));
/* Check whether the thresholds have been resolved and compare them */
if (missingLagCritical.IsEmpty() && zoneLag > lagCritical) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
+ " greater than critical threshold: " + Utility::FormatDuration(lagCritical));
} else if (missingLagWarning.IsEmpty() && zoneLag > lagWarning) {
cr->SetState(ServiceWarning);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
+ " greater than warning threshold: " + Utility::FormatDuration(lagWarning));
}
} else {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
}
cr->SetPerformanceData(new Array({
new PerfdataValue("slave_lag", zoneLag, false, "s", lagWarning, lagCritical),
new PerfdataValue("last_messages_sent", lastMessageSent),
new PerfdataValue("last_messages_received", lastMessageReceived),
new PerfdataValue("sum_messages_sent_per_second", messagesSentPerSecond),
new PerfdataValue("sum_messages_received_per_second", messagesReceivedPerSecond),
new PerfdataValue("sum_bytes_sent_per_second", bytesSentPerSecond),
new PerfdataValue("sum_bytes_received_per_second", bytesReceivedPerSecond)
}));
checkable->ProcessCheckResult(cr);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.push_back(std::make_pair("service", service));
resolvers.push_back(std::make_pair("host", host));
resolvers.push_back(std::make_pair("command", commandObj));
resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult());
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
if (endpoint->IsConnected()) {
connected = true;
break;
}
}
if (!connected) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected.");
} else {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected.");
}
checkable->ProcessCheckResult(cr);
}
Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
{
if (!origin->FromClient->GetEndpoint() || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone)))
return Empty;
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
Log(LogCritical, "ApiListener", "No instance available.");
return Empty;
}
if (!listener->GetAcceptConfig()) {
Log(LogWarning, "ApiListener")
<< "Ignoring config update. '" << listener->GetName() << "' does not accept config.";
return Empty;
}
Log(LogInformation, "ApiListener")
<< "Applying config update from endpoint '" << origin->FromClient->GetEndpoint()->GetName()
<< "' of zone '" << GetFromZoneName(origin->FromZone) << "'.";
Dictionary::Ptr updateV1 = params->Get("update");
Dictionary::Ptr updateV2 = params->Get("update_v2");
bool configChange = false;
ObjectLock olock(updateV1);
for (const Dictionary::Pair& kv : updateV1) {
Zone::Ptr zone = Zone::GetByName(kv.first);
if (!zone) {
Log(LogWarning, "ApiListener")
<< "Ignoring config update for unknown zone '" << kv.first << "'.";
continue;
}
if (ConfigCompiler::HasZoneConfigAuthority(kv.first)) {
Log(LogWarning, "ApiListener")
<< "Ignoring config update for zone '" << kv.first << "' because we have an authoritative version of the zone's config.";
continue;
}
String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName();
Utility::MkDirP(oldDir, 0700);
ConfigDirInformation newConfigInfo;
newConfigInfo.UpdateV1 = kv.second;
if (updateV2)
newConfigInfo.UpdateV2 = updateV2->Get(kv.first);
Dictionary::Ptr newConfig = kv.second;
ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir);
if (UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, false))
configChange = true;
}
if (configChange) {
Log(LogInformation, "ApiListener", "Restarting after configuration change.");
Application::RequestRestart();
}
return Empty;
}
void ApiListener::SyncClient(const JsonRpcConnection::Ptr& aclient, const Endpoint::Ptr& endpoint, bool needSync)
{
Zone::Ptr eZone = endpoint->GetZone();
try {
{
ObjectLock olock(endpoint);
endpoint->SetSyncing(true);
}
Zone::Ptr myZone = Zone::GetLocalZone();
if (myZone->GetParent() == eZone) {
Log(LogInformation, "ApiListener")
<< "Requesting new certificate for this Icinga instance from endpoint '" << endpoint->GetName() << "'.";
JsonRpcConnection::SendCertificateRequest(aclient, nullptr, String());
if (Utility::PathExists(ApiListener::GetCertificateRequestsDir()))
Utility::Glob(ApiListener::GetCertificateRequestsDir() + "/*.json", std::bind(&JsonRpcConnection::SendCertificateRequest, aclient, nullptr, _1), GlobFile);
}
/* Make sure that the config updates are synced
* before the logs are replayed.
*/
Log(LogInformation, "ApiListener")
<< "Sending config updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
/* sync zone file config */
SendConfigUpdate(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending config file updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
/* sync runtime config */
SendRuntimeConfigObjects(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending runtime config updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
if (!needSync) {
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
return;
}
Log(LogInformation, "ApiListener")
<< "Sending replay log for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
ReplayLog(aclient);
if (eZone == Zone::GetLocalZone())
UpdateObjectAuthority();
Log(LogInformation, "ApiListener")
<< "Finished sending replay log for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
} catch (const std::exception& ex) {
{
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
}
Log(LogCritical, "ApiListener")
<< "Error while syncing endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex, false);
Log(LogDebug, "ApiListener")
<< "Error while syncing endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex);
}
Log(LogInformation, "ApiListener")
<< "Finished syncing endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
}
std::pair<Dictionary::Ptr, Dictionary::Ptr> ApiListener::GetStatus(void)
{
Dictionary::Ptr status = new Dictionary();
Dictionary::Ptr perfdata = new Dictionary();
/* cluster stats */
status->Set("identity", GetIdentity());
double allEndpoints = 0;
Array::Ptr allNotConnectedEndpoints = new Array();
Array::Ptr allConnectedEndpoints = new Array();
Zone::Ptr my_zone = Zone::GetLocalZone();
Dictionary::Ptr connectedZones = new Dictionary();
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
/* only check endpoints in a) the same zone b) our parent zone c) immediate child zones */
if (my_zone != zone && my_zone != zone->GetParent() && zone != my_zone->GetParent()) {
Log(LogDebug, "ApiListener")
<< "Not checking connection to Zone '" << zone->GetName() << "' because it's not in the same zone, a parent or a child zone.";
continue;
}
bool zoneConnected = false;
int countZoneEndpoints = 0;
double zoneLag = 0;
Array::Ptr zoneEndpoints = new Array();
for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
zoneEndpoints->Add(endpoint->GetName());
if (endpoint->GetName() == GetIdentity())
continue;
double eplag = CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
allEndpoints++;
countZoneEndpoints++;
if (!endpoint->GetConnected()) {
allNotConnectedEndpoints->Add(endpoint->GetName());
} else {
allConnectedEndpoints->Add(endpoint->GetName());
zoneConnected = true;
}
}
/* if there's only one endpoint inside the zone, we're not connected - that's us, fake it */
if (zone->GetEndpoints().size() == 1 && countZoneEndpoints == 0)
zoneConnected = true;
Dictionary::Ptr zoneStats = new Dictionary();
zoneStats->Set("connected", zoneConnected);
zoneStats->Set("client_log_lag", zoneLag);
zoneStats->Set("endpoints", zoneEndpoints);
String parentZoneName;
Zone::Ptr parentZone = zone->GetParent();
if (parentZone)
parentZoneName = parentZone->GetName();
zoneStats->Set("parent_zone", parentZoneName);
connectedZones->Set(zone->GetName(), zoneStats);
}
status->Set("num_endpoints", allEndpoints);
status->Set("num_conn_endpoints", allConnectedEndpoints->GetLength());
status->Set("num_not_conn_endpoints", allNotConnectedEndpoints->GetLength());
status->Set("conn_endpoints", allConnectedEndpoints);
status->Set("not_conn_endpoints", allNotConnectedEndpoints);
status->Set("zones", connectedZones);
perfdata->Set("num_endpoints", allEndpoints);
perfdata->Set("num_conn_endpoints", Convert::ToDouble(allConnectedEndpoints->GetLength()));
perfdata->Set("num_not_conn_endpoints", Convert::ToDouble(allNotConnectedEndpoints->GetLength()));
return std::make_pair(status, perfdata);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.push_back(std::make_pair("service", service));
resolvers.push_back(std::make_pair("host", host));
resolvers.push_back(std::make_pair("command", commandObj));
resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(),
NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
double zoneLag = 0;
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
if (endpoint->GetConnected())
connected = true;
double eplag = ApiListener::CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
}
if (!connected) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
} else {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));
}
Array::Ptr perfdata = new Array();
perfdata->Add(new PerfdataValue("slave_lag", zoneLag));
cr->SetPerformanceData(perfdata);
checkable->ProcessCheckResult(cr);
}
