void ClusterCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
std::pair<Dictionary::Ptr, Dictionary::Ptr> stats = listener->GetStatus();
Dictionary::Ptr status = stats.first;
/* use feature stats perfdata */
std::pair<Dictionary::Ptr, Dictionary::Ptr> feature_stats = CIB::GetFeatureStats();
cr->SetPerformanceData(feature_stats.second);
String connected_endpoints = FormatArray(status->Get("conn_endpoints"));
String not_connected_endpoints = FormatArray(status->Get("not_conn_endpoints"));
if (status->Get("num_not_conn_endpoints") > 0) {
cr->SetState(ServiceCritical);
cr->SetOutput("Icinga 2 Cluster Problem: " + Convert::ToString(status->Get("num_not_conn_endpoints")) +
" Endpoints (" + not_connected_endpoints + ") not connected.");
} else {
cr->SetState(ServiceOK);
cr->SetOutput("Icinga 2 Cluster is running: Connected Endpoints: "+ Convert::ToString(status->Get("num_conn_endpoints")) +
" (" + connected_endpoints + ").");
}
checkable->ProcessCheckResult(cr);
}
void RandomCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
REQUIRE_NOT_NULL(checkable);
REQUIRE_NOT_NULL(cr);
if (resolvedMacros && !useResolvedMacros)
return;
double now = Utility::GetTime();
double uptime = now - Application::GetStartTime();
String output = "Hello from " + IcingaApplication::GetInstance()->GetNodeName()
+ ". Icinga 2 has been running for " + Utility::FormatDuration(uptime)
+ ". Version: " + Application::GetAppVersion();
cr->SetOutput(output);
double random = Utility::Random() % 1000;
cr->SetPerformanceData(new Array({
new PerfdataValue("time", now),
new PerfdataValue("value", random),
new PerfdataValue("value_1m", random * 0.9),
new PerfdataValue("value_5m", random * 0.8),
new PerfdataValue("uptime", uptime),
}));
cr->SetState(static_cast<ServiceState>(Utility::Random() % 4));
checkable->ProcessCheckResult(cr);
}
static void CheckableProcessCheckResult(const CheckResult::Ptr& cr)
{
ScriptFrame *vframe = ScriptFrame::GetCurrentFrame();
Checkable::Ptr self = vframe->Self;
REQUIRE_NOT_NULL(self);
self->ProcessCheckResult(cr);
}
void PluginCheckTask::ProcessFinishedHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, const Value& commandLine, const ProcessResult& pr)
{
Checkable::DecreasePendingChecks();
if (pr.ExitStatus > 3) {
Process::Arguments parguments = Process::PrepareCommand(commandLine);
Log(LogWarning, "PluginCheckTask")
<< "Check command for object '" << checkable->GetName() << "' (PID: " << pr.PID
<< ", arguments: " << Process::PrettyPrintArguments(parguments) << ") terminated with exit code "
<< pr.ExitStatus << ", output: " << pr.Output;
}
String output = pr.Output.Trim();
std::pair<String, String> co = PluginUtility::ParseCheckOutput(output);
cr->SetCommand(commandLine);
cr->SetOutput(co.first);
cr->SetPerformanceData(PluginUtility::SplitPerfdata(co.second));
cr->SetState(PluginUtility::ExitStatusToState(pr.ExitStatus));
cr->SetExitStatus(pr.ExitStatus);
cr->SetExecutionStart(pr.ExecutionStart);
cr->SetExecutionEnd(pr.ExecutionEnd);
checkable->ProcessCheckResult(cr);
}
Dictionary::Ptr ApiActions::ProcessCheckResult(const ConfigObject::Ptr& object,
const Dictionary::Ptr& params)
{
Checkable::Ptr checkable = static_pointer_cast<Checkable>(object);
if (!checkable)
return ApiActions::CreateResult(404,
"Cannot process passive check result for non-existent object.");
if (!checkable->GetEnablePassiveChecks())
return ApiActions::CreateResult(403, "Passive checks are disabled for object '" + checkable->GetName() + "'.");
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
if (!params->Contains("exit_status"))
return ApiActions::CreateResult(403, "Parameter 'exit_status' is required.");
int exitStatus = HttpUtility::GetLastParameter(params, "exit_status");
ServiceState state;
if (!service) {
if (exitStatus == 0)
state = ServiceOK;
else if (exitStatus == 1)
state = ServiceCritical;
else
return ApiActions::CreateResult(403, "Invalid 'exit_status' for Host "
+ checkable->GetName() + ".");
} else {
state = PluginUtility::ExitStatusToState(exitStatus);
}
if (!params->Contains("plugin_output"))
return ApiActions::CreateResult(403, "Parameter 'plugin_output' is required");
CheckResult::Ptr cr = new CheckResult();
cr->SetOutput(HttpUtility::GetLastParameter(params, "plugin_output"));
cr->SetState(state);
cr->SetCheckSource(HttpUtility::GetLastParameter(params, "check_source"));
cr->SetPerformanceData(params->Get("performance_data"));
cr->SetCommand(params->Get("check_command"));
checkable->ProcessCheckResult(cr);
/* Reschedule the next check. The side effect of this is that for as long
* as we receive passive results for a service we won't execute any
* active checks. */
checkable->SetNextCheck(Utility::GetTime() + checkable->GetCheckInterval());
return ApiActions::CreateResult(200, "Successfully processed check result for object '" + checkable->GetName() + "'.");
}
void PluginCheckTask::ProcessFinishedHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, const Value& commandLine, const ProcessResult& pr)
{
String output = pr.Output;
output.Trim();
std::pair<String, Value> co = PluginUtility::ParseCheckOutput(output);
cr->SetCommand(commandLine);
cr->SetOutput(co.first);
cr->SetPerformanceData(co.second);
cr->SetState(PluginUtility::ExitStatusToState(pr.ExitStatus));
cr->SetExitStatus(pr.ExitStatus);
cr->SetExecutionStart(pr.ExecutionStart);
cr->SetExecutionEnd(pr.ExecutionEnd);
checkable->ProcessCheckResult(cr);
}
void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable)
{
try {
checkable->ExecuteCheck();
} catch (const std::exception& ex) {
CheckResult::Ptr cr = new CheckResult();
cr->SetState(ServiceUnknown);
String output = "Exception occurred while checking '" + checkable->GetName() + "': " + DiagnosticInformation(ex);
cr->SetOutput(output);
double now = Utility::GetTime();
cr->SetScheduleStart(now);
cr->SetScheduleEnd(now);
cr->SetExecutionStart(now);
cr->SetExecutionEnd(now);
checkable->ProcessCheckResult(cr);
Log(LogCritical, "checker", output);
}
Checkable::DecreasePendingChecks();
{
boost::mutex::scoped_lock lock(m_Mutex);
/* remove the object from the list of pending objects; if it's not in the
* list this was a manual (i.e. forced) check and we must not re-add the
* object to the list because it's already there. */
auto it = m_PendingCheckables.find(checkable);
if (it != m_PendingCheckables.end()) {
m_PendingCheckables.erase(it);
if (checkable->IsActive())
m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
m_CV.notify_all();
}
}
Log(LogDebug, "CheckerComponent")
<< "Check finished for object '" << checkable->GetName() << "'";
}
void RandomCheckTask::ScriptFunc(const Checkable::Ptr& service, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
if (resolvedMacros && !useResolvedMacros)
return;
String output = "Hello from ";
output += IcingaApplication::GetInstance()->GetNodeName();
Array::Ptr perfdata = new Array();
perfdata->Add(new PerfdataValue("time", Convert::ToDouble(Utility::GetTime())));
cr->SetOutput(output);
cr->SetPerformanceData(perfdata);
cr->SetState(static_cast<ServiceState>(Utility::Random() % 4));
service->ProcessCheckResult(cr);
}
void NullCheckTask::ScriptFunc(const Checkable::Ptr& service, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
if (resolvedMacros && !useResolvedMacros)
return;
String output = "Hello from ";
output += Utility::GetFQDN();
Array::Ptr perfdata = new Array();
perfdata->Add(new PerfdataValue("time", Convert::ToDouble(Utility::GetTime())));
cr->SetOutput(output);
cr->SetPerformanceData(perfdata);
cr->SetState(ServiceOK);
service->ProcessCheckResult(cr);
}
void DummyCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
REQUIRE_NOT_NULL(checkable);
REQUIRE_NOT_NULL(cr);
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.emplace_back("service", service);
resolvers.emplace_back("host", host);
resolvers.emplace_back("command", commandObj);
resolvers.emplace_back("icinga", IcingaApplication::GetInstance());
int dummyState = MacroProcessor::ResolveMacros("$dummy_state$", resolvers, checkable->GetLastCheckResult(),
nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
String dummyText = MacroProcessor::ResolveMacros("$dummy_text$", resolvers, checkable->GetLastCheckResult(),
nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
/* Parse output and performance data. */
std::pair<String, String> co = PluginUtility::ParseCheckOutput(dummyText);
double now = Utility::GetTime();
cr->SetOutput(co.first);
cr->SetPerformanceData(PluginUtility::SplitPerfdata(co.second));
cr->SetState(PluginUtility::ExitStatusToState(dummyState));
cr->SetExitStatus(dummyState);
cr->SetExecutionStart(now);
cr->SetExecutionEnd(now);
checkable->ProcessCheckResult(cr);
}
void NullCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
REQUIRE_NOT_NULL(checkable);
REQUIRE_NOT_NULL(cr);
if (resolvedMacros && !useResolvedMacros)
return;
String output = "Hello from ";
output += IcingaApplication::GetInstance()->GetNodeName();
cr->SetOutput(output);
cr->SetPerformanceData(new Array({
new PerfdataValue("time", Convert::ToDouble(Utility::GetTime()))
}));
cr->SetState(ServiceOK);
checkable->ProcessCheckResult(cr);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.push_back(std::make_pair("service", service));
resolvers.push_back(std::make_pair("host", host));
resolvers.push_back(std::make_pair("command", commandObj));
resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult());
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
if (endpoint->IsConnected()) {
connected = true;
break;
}
}
if (!connected) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected.");
} else {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected.");
}
checkable->ProcessCheckResult(cr);
}
void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.push_back(std::make_pair("service", service));
resolvers.push_back(std::make_pair("host", host));
resolvers.push_back(std::make_pair("command", commandObj));
resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
String idoType = MacroProcessor::ResolveMacros("$ido_type$", resolvers, checkable->GetLastCheckResult(),
NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
if (idoType.IsEmpty()) {
cr->SetOutput("Macro 'ido_type' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
String idoName = MacroProcessor::ResolveMacros("$ido_name$", resolvers, checkable->GetLastCheckResult(),
NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
if (idoName.IsEmpty()) {
cr->SetOutput("Macro 'ido_name' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Type::Ptr type = Type::GetByName(idoType);
if (!type || !DbConnection::TypeInstance->IsAssignableFrom(type)) {
cr->SetOutput("IDO type '" + idoType + "' is invalid.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
ConfigType::Ptr dtype = ConfigType::GetByName(idoType);
VERIFY(dtype);
DbConnection::Ptr conn = static_pointer_cast<DbConnection>(dtype->GetObject(idoName));
double qps = conn->GetQueryCount(60) / 60.0;
if (!conn->GetConnected()) {
if (conn->GetShouldConnect()) {
cr->SetOutput("Could not connect to the database server.");
cr->SetState(ServiceCritical);
} else {
cr->SetOutput("Not currently enabled: Another cluster instance is responsible for the IDO database.");
cr->SetState(ServiceOK);
}
} else {
String schema_version = conn->GetSchemaVersion();
if (Utility::CompareVersion(IDO_CURRENT_SCHEMA_VERSION, schema_version) < 0) {
cr->SetOutput("Outdated schema version: " + schema_version + "; Latest version: " IDO_CURRENT_SCHEMA_VERSION);
cr->SetState(ServiceWarning);
} else {
std::ostringstream msgbuf;
msgbuf << "Connected to the database server; queries per second: " << std::fixed << std::setprecision(3) << qps;
cr->SetOutput(msgbuf.str());
cr->SetState(ServiceOK);
}
}
Array::Ptr perfdata = new Array();
perfdata->Add(new PerfdataValue("queries", qps));
perfdata->Add(new PerfdataValue("queries_1min", conn->GetQueryCount(60)));
perfdata->Add(new PerfdataValue("queries_5mins", conn->GetQueryCount(5 * 60)));
perfdata->Add(new PerfdataValue("queries_15mins", conn->GetQueryCount(15 * 60)));
perfdata->Add(new PerfdataValue("pending_queries", conn->GetPendingQueryCount()));
cr->SetPerformanceData(perfdata);
checkable->ProcessCheckResult(cr);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.push_back(std::make_pair("service", service));
resolvers.push_back(std::make_pair("host", host));
resolvers.push_back(std::make_pair("command", commandObj));
resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(),
NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
double zoneLag = 0;
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
if (endpoint->GetConnected())
connected = true;
double eplag = ApiListener::CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
}
if (!connected) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
} else {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));
}
Array::Ptr perfdata = new Array();
perfdata->Add(new PerfdataValue("slave_lag", zoneLag));
cr->SetPerformanceData(perfdata);
checkable->ProcessCheckResult(cr);
}
void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& service, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
if (resolvedMacros && !useResolvedMacros)
return;
double interval = Utility::GetTime() - Application::GetStartTime();
if (interval > 60)
interval = 60;
Array::Ptr perfdata = new Array();
perfdata->Add(new PerfdataValue("active_host_checks", CIB::GetActiveHostChecksStatistics(interval) / interval));
perfdata->Add(new PerfdataValue("passive_host_checks", CIB::GetPassiveHostChecksStatistics(interval) / interval));
perfdata->Add(new PerfdataValue("active_host_checks_1min", CIB::GetActiveHostChecksStatistics(60)));
perfdata->Add(new PerfdataValue("passive_host_checks_1min", CIB::GetPassiveHostChecksStatistics(60)));
perfdata->Add(new PerfdataValue("active_host_checks_5min", CIB::GetActiveHostChecksStatistics(60 * 5)));
perfdata->Add(new PerfdataValue("passive_host_checks_5min", CIB::GetPassiveHostChecksStatistics(60 * 5)));
perfdata->Add(new PerfdataValue("active_host_checks_15min", CIB::GetActiveHostChecksStatistics(60 * 15)));
perfdata->Add(new PerfdataValue("passive_host_checks_15min", CIB::GetPassiveHostChecksStatistics(60 * 15)));
perfdata->Add(new PerfdataValue("active_service_checks", CIB::GetActiveServiceChecksStatistics(interval) / interval));
perfdata->Add(new PerfdataValue("passive_service_checks", CIB::GetPassiveServiceChecksStatistics(interval) / interval));
perfdata->Add(new PerfdataValue("active_service_checks_1min", CIB::GetActiveServiceChecksStatistics(60)));
perfdata->Add(new PerfdataValue("passive_service_checks_1min", CIB::GetPassiveServiceChecksStatistics(60)));
perfdata->Add(new PerfdataValue("active_service_checks_5min", CIB::GetActiveServiceChecksStatistics(60 * 5)));
perfdata->Add(new PerfdataValue("passive_service_checks_5min", CIB::GetPassiveServiceChecksStatistics(60 * 5)));
perfdata->Add(new PerfdataValue("active_service_checks_15min", CIB::GetActiveServiceChecksStatistics(60 * 15)));
perfdata->Add(new PerfdataValue("passive_service_checks_15min", CIB::GetPassiveServiceChecksStatistics(60 * 15)));
CheckableCheckStatistics scs = CIB::CalculateServiceCheckStats();
perfdata->Add(new PerfdataValue("min_latency", scs.min_latency));
perfdata->Add(new PerfdataValue("max_latency", scs.max_latency));
perfdata->Add(new PerfdataValue("avg_latency", scs.avg_latency));
perfdata->Add(new PerfdataValue("min_execution_time", scs.min_execution_time));
perfdata->Add(new PerfdataValue("max_execution_time", scs.max_execution_time));
perfdata->Add(new PerfdataValue("avg_execution_time", scs.avg_execution_time));
ServiceStatistics ss = CIB::CalculateServiceStats();
perfdata->Add(new PerfdataValue("num_services_ok", ss.services_ok));
perfdata->Add(new PerfdataValue("num_services_warning", ss.services_warning));
perfdata->Add(new PerfdataValue("num_services_critical", ss.services_critical));
perfdata->Add(new PerfdataValue("num_services_unknown", ss.services_unknown));
perfdata->Add(new PerfdataValue("num_services_pending", ss.services_pending));
perfdata->Add(new PerfdataValue("num_services_unreachable", ss.services_unreachable));
perfdata->Add(new PerfdataValue("num_services_flapping", ss.services_flapping));
perfdata->Add(new PerfdataValue("num_services_in_downtime", ss.services_in_downtime));
perfdata->Add(new PerfdataValue("num_services_acknowledged", ss.services_acknowledged));
double uptime = Utility::GetTime() - Application::GetStartTime();
perfdata->Add(new PerfdataValue("uptime", uptime));
HostStatistics hs = CIB::CalculateHostStats();
perfdata->Add(new PerfdataValue("num_hosts_up", hs.hosts_up));
perfdata->Add(new PerfdataValue("num_hosts_down", hs.hosts_down));
perfdata->Add(new PerfdataValue("num_hosts_pending", hs.hosts_pending));
perfdata->Add(new PerfdataValue("num_hosts_unreachable", hs.hosts_unreachable));
perfdata->Add(new PerfdataValue("num_hosts_flapping", hs.hosts_flapping));
perfdata->Add(new PerfdataValue("num_hosts_in_downtime", hs.hosts_in_downtime));
perfdata->Add(new PerfdataValue("num_hosts_acknowledged", hs.hosts_acknowledged));
cr->SetOutput("Icinga 2 has been running for " + Utility::FormatDuration(uptime) +
". Version: " + Application::GetAppVersion());
cr->SetPerformanceData(perfdata);
double lastReloadFailed = Application::GetLastReloadFailed();
if (lastReloadFailed > 0) {
cr->SetOutput(cr->GetOutput() + "; Last reload attempt failed at " + Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", lastReloadFailed));
cr->SetState(ServiceWarning);
} else
cr->SetState(ServiceOK);
service->ProcessCheckResult(cr);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
REQUIRE_NOT_NULL(checkable);
REQUIRE_NOT_NULL(cr);
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
cr->SetOutput("No API listener is configured for this instance.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
Value raw_command = commandObj->GetCommandLine();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.emplace_back("service", service);
resolvers.emplace_back("host", host);
resolvers.emplace_back("command", commandObj);
resolvers.emplace_back("icinga", IcingaApplication::GetInstance());
String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(),
nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
String missingLagWarning;
String missingLagCritical;
double lagWarning = MacroProcessor::ResolveMacros("$cluster_lag_warning$", resolvers, checkable->GetLastCheckResult(),
&missingLagWarning, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
double lagCritical = MacroProcessor::ResolveMacros("$cluster_lag_critical$", resolvers, checkable->GetLastCheckResult(),
&missingLagCritical, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
if (resolvedMacros && !useResolvedMacros)
return;
if (zoneName.IsEmpty()) {
cr->SetOutput("Macro 'cluster_zone' must be set.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
Zone::Ptr zone = Zone::GetByName(zoneName);
if (!zone) {
cr->SetOutput("Zone '" + zoneName + "' does not exist.");
cr->SetState(ServiceUnknown);
checkable->ProcessCheckResult(cr);
return;
}
bool connected = false;
double zoneLag = 0;
double lastMessageSent = 0;
double lastMessageReceived = 0;
double messagesSentPerSecond = 0;
double messagesReceivedPerSecond = 0;
double bytesSentPerSecond = 0;
double bytesReceivedPerSecond = 0;
for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
if (endpoint->GetConnected())
connected = true;
double eplag = ApiListener::CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
if (endpoint->GetLastMessageSent() > lastMessageSent)
lastMessageSent = endpoint->GetLastMessageSent();
if (endpoint->GetLastMessageReceived() > lastMessageReceived)
lastMessageReceived = endpoint->GetLastMessageReceived();
messagesSentPerSecond += endpoint->GetMessagesSentPerSecond();
messagesReceivedPerSecond += endpoint->GetMessagesReceivedPerSecond();
bytesSentPerSecond += endpoint->GetBytesSentPerSecond();
bytesReceivedPerSecond += endpoint->GetBytesReceivedPerSecond();
}
if (connected) {
cr->SetState(ServiceOK);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));
/* Check whether the thresholds have been resolved and compare them */
if (missingLagCritical.IsEmpty() && zoneLag > lagCritical) {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
+ " greater than critical threshold: " + Utility::FormatDuration(lagCritical));
} else if (missingLagWarning.IsEmpty() && zoneLag > lagWarning) {
cr->SetState(ServiceWarning);
cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
+ " greater than warning threshold: " + Utility::FormatDuration(lagWarning));
}
} else {
cr->SetState(ServiceCritical);
cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
}
cr->SetPerformanceData(new Array({
new PerfdataValue("slave_lag", zoneLag, false, "s", lagWarning, lagCritical),
new PerfdataValue("last_messages_sent", lastMessageSent),
new PerfdataValue("last_messages_received", lastMessageReceived),
new PerfdataValue("sum_messages_sent_per_second", messagesSentPerSecond),
new PerfdataValue("sum_messages_received_per_second", messagesReceivedPerSecond),
new PerfdataValue("sum_bytes_sent_per_second", bytesSentPerSecond),
new PerfdataValue("sum_bytes_received_per_second", bytesReceivedPerSecond)
}));
checkable->ProcessCheckResult(cr);
}
Value ClusterEvents::CheckResultAPIHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
{
Endpoint::Ptr endpoint = origin->FromClient->GetEndpoint();
if (!endpoint) {
Log(LogNotice, "ClusterEvents")
<< "Discarding 'check result' message from '" << origin->FromClient->GetIdentity() << "': Invalid endpoint origin (client not allowed).";
return Empty;
}
CheckResult::Ptr cr;
Array::Ptr vperf;
if (params->Contains("cr")) {
cr = new CheckResult();
Dictionary::Ptr vcr = params->Get("cr");
if (vcr && vcr->Contains("performance_data")) {
vperf = vcr->Get("performance_data");
if (vperf)
vcr->Remove("performance_data");
Deserialize(cr, vcr, true);
}
}
if (!cr)
return Empty;
ArrayData rperf;
if (vperf) {
ObjectLock olock(vperf);
for (const Value& vp : vperf) {
Value p;
if (vp.IsObjectType<Dictionary>()) {
PerfdataValue::Ptr val = new PerfdataValue();
Deserialize(val, vp, true);
rperf.push_back(val);
} else
rperf.push_back(vp);
}
}
cr->SetPerformanceData(new Array(std::move(rperf)));
Host::Ptr host = Host::GetByName(params->Get("host"));
if (!host)
return Empty;
Checkable::Ptr checkable;
if (params->Contains("service"))
checkable = host->GetServiceByShortName(params->Get("service"));
else
checkable = host;
if (!checkable)
return Empty;
if (origin->FromZone && !origin->FromZone->CanAccessObject(checkable) && endpoint != checkable->GetCommandEndpoint()) {
Log(LogNotice, "ClusterEvents")
<< "Discarding 'check result' message for checkable '" << checkable->GetName()
<< "' from '" << origin->FromClient->GetIdentity() << "': Unauthorized access.";
return Empty;
}
if (!checkable->IsPaused() && Zone::GetLocalZone() == checkable->GetZone() && endpoint == checkable->GetCommandEndpoint())
checkable->ProcessCheckResult(cr);
else
checkable->ProcessCheckResult(cr, origin);
return Empty;
}
void CheckResultReader::ProcessCheckResultFile(const String& path) const
{
CONTEXT("Processing check result file '" + path + "'");
String crfile = String(path.Begin(), path.End() - 3); /* Remove the ".ok" extension. */
std::ifstream fp;
fp.exceptions(std::ifstream::badbit);
fp.open(crfile.CStr());
std::map<String, String> attrs;
while (fp.good()) {
std::string line;
std::getline(fp, line);
if (line.empty() || line[0] == '#')
continue; /* Ignore comments and empty lines. */
size_t pos = line.find_first_of('=');
if (pos == std::string::npos)
continue; /* Ignore invalid lines. */
String key = line.substr(0, pos);
String value = line.substr(pos + 1);
attrs[key] = value;
}
/* Remove the checkresult files. */
if (unlink(path.CStr()) < 0)
BOOST_THROW_EXCEPTION(posix_error()
<< boost::errinfo_api_function("unlink")
<< boost::errinfo_errno(errno)
<< boost::errinfo_file_name(path));
if (unlink(crfile.CStr()) < 0)
BOOST_THROW_EXCEPTION(posix_error()
<< boost::errinfo_api_function("unlink")
<< boost::errinfo_errno(errno)
<< boost::errinfo_file_name(crfile));
Checkable::Ptr checkable;
Host::Ptr host = Host::GetByName(attrs["host_name"]);
if (!host) {
Log(LogWarning, "CheckResultReader")
<< "Ignoring checkresult file for host '" << attrs["host_name"] << "': Host does not exist.";
return;
}
if (attrs.find("service_description") != attrs.end()) {
Service::Ptr service = host->GetServiceByShortName(attrs["service_description"]);
if (!service) {
Log(LogWarning, "CheckResultReader")
<< "Ignoring checkresult file for host '" << attrs["host_name"]
<< "', service '" << attrs["service_description"] << "': Service does not exist.";
return;
}
checkable = service;
} else
checkable = host;
CheckResult::Ptr result = new CheckResult();
String output = CompatUtility::UnEscapeString(attrs["output"]);
std::pair<String, Value> co = PluginUtility::ParseCheckOutput(output);
result->SetOutput(co.first);
result->SetPerformanceData(PluginUtility::SplitPerfdata(co.second));
result->SetState(PluginUtility::ExitStatusToState(Convert::ToLong(attrs["return_code"])));
if (attrs.find("start_time") != attrs.end())
result->SetExecutionStart(Convert::ToDouble(attrs["start_time"]));
else
result->SetExecutionStart(Utility::GetTime());
if (attrs.find("finish_time") != attrs.end())
result->SetExecutionEnd(Convert::ToDouble(attrs["finish_time"]));
else
result->SetExecutionEnd(result->GetExecutionStart());
checkable->ProcessCheckResult(result);
Log(LogDebug, "CheckResultReader")
<< "Processed checkresult file for object '" << checkable->GetName() << "'";
/* Reschedule the next check. The side effect of this is that for as long
* as we receive check result files for a host/service we won't execute any
* active checks. */
checkable->SetNextCheck(Utility::GetTime() + checkable->GetCheckInterval());
}