String CompatUtility::GetCheckResultLongOutput(const CheckResult::Ptr& cr)
{
if (!cr)
return Empty;
String long_output;
String output;
String raw_output = cr->GetOutput();
/*
* replace semi-colons with colons in output
* semi-colon is used as delimiter in various interfaces
*/
boost::algorithm::replace_all(raw_output, ";", ":");
size_t line_end = raw_output.Find("\n");
if (line_end > 0 && line_end != String::NPos) {
long_output = raw_output.SubStr(line_end+1, raw_output.GetLength());
return EscapeString(long_output);
}
return Empty;
}
void LogstashWriter::NotificationToUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
const User::Ptr& user, NotificationType notification_type, CheckResult::Ptr const& cr,
const String& author, const String& comment_text, const String& command_name)
{
CONTEXT("Logstash Processing notification to all users '" + checkable->GetName() + "'");
Log(LogDebug, "LogstashWriter")
<< "Processing notification for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
String notification_type_str = Notification::NotificationTypeToString(notification_type);
String author_comment = "";
if (notification_type == NotificationCustom || notification_type == NotificationAcknowledgement) {
author_comment = author + ";" + comment_text;
}
double ts = Utility::GetTime();
Dictionary::Ptr fields = new Dictionary();
if (service) {
fields->Set("type", "SERVICE NOTIFICATION");
fields->Set("service_name", service->GetShortName());
} else {
fields->Set("type", "HOST NOTIFICATION");
}
if (cr) {
fields->Set("plugin_output", cr->GetOutput());
ts = cr->GetExecutionEnd();
}
fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
fields->Set("host_name", host->GetName());
fields->Set("command", command_name);
fields->Set("notification_type", notification_type_str);
fields->Set("comment", author_comment);
SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
String CompatUtility::GetCheckResultOutput(const CheckResult::Ptr& cr)
{
if (!cr)
return Empty;
String output;
String raw_output = cr->GetOutput();
/*
* replace semi-colons with colons in output
* semi-colon is used as delimiter in various interfaces
*/
boost::algorithm::replace_all(raw_output, ";", ":");
size_t line_end = raw_output.Find("\n");
return raw_output.SubStr(0, line_end);
}
void LogstashWriter::StateChangeHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type)
{
CONTEXT("Logstash Processing state change '" + checkable->GetName() + "'");
Log(LogDebug, "LogstashWriter")
<< "Processing state change for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr fields = new Dictionary();
fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
fields->Set("type", "StateChange");
fields->Set("current_check_attempt", checkable->GetCheckAttempt());
fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
fields->Set("hostname", host->GetName());
if (service) {
fields->Set("service_name", service->GetShortName());
fields->Set("service_state", Service::StateToString(service->GetState()));
fields->Set("last_state", service->GetLastState());
fields->Set("last_hard_state", service->GetLastHardState());
} else {
fields->Set("last_state", host->GetLastState());
fields->Set("last_hard_state", host->GetLastHardState());
}
double ts = Utility::GetTime();
if (cr) {
fields->Set("plugin_output", cr->GetOutput());
fields->Set("check_source", cr->GetCheckSource());
ts = cr->GetExecutionEnd();
}
SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
void IcingaCheckTask::ScriptFunc(const Checkable::Ptr& service, const CheckResult::Ptr& cr,
const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
if (resolvedMacros && !useResolvedMacros)
return;
double interval = Utility::GetTime() - Application::GetStartTime();
if (interval > 60)
interval = 60;
Array::Ptr perfdata = new Array();
perfdata->Add(new PerfdataValue("active_host_checks", CIB::GetActiveHostChecksStatistics(interval) / interval));
perfdata->Add(new PerfdataValue("passive_host_checks", CIB::GetPassiveHostChecksStatistics(interval) / interval));
perfdata->Add(new PerfdataValue("active_host_checks_1min", CIB::GetActiveHostChecksStatistics(60)));
perfdata->Add(new PerfdataValue("passive_host_checks_1min", CIB::GetPassiveHostChecksStatistics(60)));
perfdata->Add(new PerfdataValue("active_host_checks_5min", CIB::GetActiveHostChecksStatistics(60 * 5)));
perfdata->Add(new PerfdataValue("passive_host_checks_5min", CIB::GetPassiveHostChecksStatistics(60 * 5)));
perfdata->Add(new PerfdataValue("active_host_checks_15min", CIB::GetActiveHostChecksStatistics(60 * 15)));
perfdata->Add(new PerfdataValue("passive_host_checks_15min", CIB::GetPassiveHostChecksStatistics(60 * 15)));
perfdata->Add(new PerfdataValue("active_service_checks", CIB::GetActiveServiceChecksStatistics(interval) / interval));
perfdata->Add(new PerfdataValue("passive_service_checks", CIB::GetPassiveServiceChecksStatistics(interval) / interval));
perfdata->Add(new PerfdataValue("active_service_checks_1min", CIB::GetActiveServiceChecksStatistics(60)));
perfdata->Add(new PerfdataValue("passive_service_checks_1min", CIB::GetPassiveServiceChecksStatistics(60)));
perfdata->Add(new PerfdataValue("active_service_checks_5min", CIB::GetActiveServiceChecksStatistics(60 * 5)));
perfdata->Add(new PerfdataValue("passive_service_checks_5min", CIB::GetPassiveServiceChecksStatistics(60 * 5)));
perfdata->Add(new PerfdataValue("active_service_checks_15min", CIB::GetActiveServiceChecksStatistics(60 * 15)));
perfdata->Add(new PerfdataValue("passive_service_checks_15min", CIB::GetPassiveServiceChecksStatistics(60 * 15)));
CheckableCheckStatistics scs = CIB::CalculateServiceCheckStats();
perfdata->Add(new PerfdataValue("min_latency", scs.min_latency));
perfdata->Add(new PerfdataValue("max_latency", scs.max_latency));
perfdata->Add(new PerfdataValue("avg_latency", scs.avg_latency));
perfdata->Add(new PerfdataValue("min_execution_time", scs.min_execution_time));
perfdata->Add(new PerfdataValue("max_execution_time", scs.max_execution_time));
perfdata->Add(new PerfdataValue("avg_execution_time", scs.avg_execution_time));
ServiceStatistics ss = CIB::CalculateServiceStats();
perfdata->Add(new PerfdataValue("num_services_ok", ss.services_ok));
perfdata->Add(new PerfdataValue("num_services_warning", ss.services_warning));
perfdata->Add(new PerfdataValue("num_services_critical", ss.services_critical));
perfdata->Add(new PerfdataValue("num_services_unknown", ss.services_unknown));
perfdata->Add(new PerfdataValue("num_services_pending", ss.services_pending));
perfdata->Add(new PerfdataValue("num_services_unreachable", ss.services_unreachable));
perfdata->Add(new PerfdataValue("num_services_flapping", ss.services_flapping));
perfdata->Add(new PerfdataValue("num_services_in_downtime", ss.services_in_downtime));
perfdata->Add(new PerfdataValue("num_services_acknowledged", ss.services_acknowledged));
double uptime = Utility::GetTime() - Application::GetStartTime();
perfdata->Add(new PerfdataValue("uptime", uptime));
HostStatistics hs = CIB::CalculateHostStats();
perfdata->Add(new PerfdataValue("num_hosts_up", hs.hosts_up));
perfdata->Add(new PerfdataValue("num_hosts_down", hs.hosts_down));
perfdata->Add(new PerfdataValue("num_hosts_pending", hs.hosts_pending));
perfdata->Add(new PerfdataValue("num_hosts_unreachable", hs.hosts_unreachable));
perfdata->Add(new PerfdataValue("num_hosts_flapping", hs.hosts_flapping));
perfdata->Add(new PerfdataValue("num_hosts_in_downtime", hs.hosts_in_downtime));
perfdata->Add(new PerfdataValue("num_hosts_acknowledged", hs.hosts_acknowledged));
cr->SetOutput("Icinga 2 has been running for " + Utility::FormatDuration(uptime) +
". Version: " + Application::GetAppVersion());
cr->SetPerformanceData(perfdata);
double lastReloadFailed = Application::GetLastReloadFailed();
if (lastReloadFailed > 0) {
cr->SetOutput(cr->GetOutput() + "; Last reload attempt failed at " + Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", lastReloadFailed));
cr->SetState(ServiceWarning);
} else
cr->SetState(ServiceOK);
service->ProcessCheckResult(cr);
}
void LogstashWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
CONTEXT("LOGSTASH Processing check result for '" + checkable->GetName() + "'");
Log(LogDebug, "LogstashWriter")
<< "Processing check result for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr fields = new Dictionary();
if (service) {
fields->Set("service_name", service->GetShortName());
fields->Set("service_state", Service::StateToString(service->GetState()));
fields->Set("last_state", service->GetLastState());
fields->Set("last_hard_state", service->GetLastHardState());
} else {
fields->Set("last_state", host->GetLastState());
fields->Set("last_hard_state", host->GetLastHardState());
}
fields->Set("host_name", host->GetName());
fields->Set("type", "CheckResult");
fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
fields->Set("current_check_attempt", checkable->GetCheckAttempt());
fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
fields->Set("latency", cr->CalculateLatency());
fields->Set("execution_time", cr->CalculateExecutionTime());
fields->Set("reachable", checkable->IsReachable());
double ts = Utility::GetTime();
if (cr) {
fields->Set("plugin_output", cr->GetOutput());
fields->Set("check_source", cr->GetCheckSource());
ts = cr->GetExecutionEnd();
}
Array::Ptr perfdata = cr->GetPerformanceData();
if (perfdata) {
Dictionary::Ptr perfdataItems = new Dictionary();
ObjectLock olock(perfdata);
for (const Value& val : perfdata) {
PerfdataValue::Ptr pdv;
if (val.IsObjectType<PerfdataValue>())
pdv = val;
else {
try {
pdv = PerfdataValue::Parse(val);
} catch (const std::exception&) {
Log(LogWarning, "LogstashWriter")
<< "Ignoring invalid perfdata value: '" << val << "' for object '"
<< checkable->GetName() << "'.";
continue;
}
}
Dictionary::Ptr perfdataItem = new Dictionary();
perfdataItem->Set("value", pdv->GetValue());
if (pdv->GetMin())
perfdataItem->Set("min", pdv->GetMin());
if (pdv->GetMax())
perfdataItem->Set("max", pdv->GetMax());
if (pdv->GetWarn())
perfdataItem->Set("warn", pdv->GetWarn());
if (pdv->GetCrit())
perfdataItem->Set("crit", pdv->GetCrit());
String escaped_key = EscapeMetricLabel(pdv->GetLabel());
perfdataItems->Set(escaped_key, perfdataItem);
}
fields->Set("performance_data", perfdataItems);
}
SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
void ElasticsearchWriter::AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
String prefix = "check_result.";
fields->Set(prefix + "output", cr->GetOutput());
fields->Set(prefix + "check_source", cr->GetCheckSource());
fields->Set(prefix + "exit_status", cr->GetExitStatus());
fields->Set(prefix + "command", cr->GetCommand());
fields->Set(prefix + "state", cr->GetState());
fields->Set(prefix + "vars_before", cr->GetVarsBefore());
fields->Set(prefix + "vars_after", cr->GetVarsAfter());
fields->Set(prefix + "execution_start", FormatTimestamp(cr->GetExecutionStart()));
fields->Set(prefix + "execution_end", FormatTimestamp(cr->GetExecutionEnd()));
fields->Set(prefix + "schedule_start", FormatTimestamp(cr->GetScheduleStart()));
fields->Set(prefix + "schedule_end", FormatTimestamp(cr->GetScheduleEnd()));
/* Add extra calculated field. */
fields->Set(prefix + "latency", cr->CalculateLatency());
fields->Set(prefix + "execution_time", cr->CalculateExecutionTime());
if (!GetEnableSendPerfdata())
return;
Array::Ptr perfdata = cr->GetPerformanceData();
CheckCommand::Ptr checkCommand = checkable->GetCheckCommand();
if (perfdata) {
ObjectLock olock(perfdata);
for (const Value& val : perfdata) {
PerfdataValue::Ptr pdv;
if (val.IsObjectType<PerfdataValue>())
pdv = val;
else {
try {
pdv = PerfdataValue::Parse(val);
} catch (const std::exception&) {
Log(LogWarning, "ElasticsearchWriter")
<< "Ignoring invalid perfdata for checkable '"
<< checkable->GetName() << "' and command '"
<< checkCommand->GetName() << "' with value: " << val;
continue;
}
}
String escapedKey = pdv->GetLabel();
boost::replace_all(escapedKey, " ", "_");
boost::replace_all(escapedKey, ".", "_");
boost::replace_all(escapedKey, "\\", "_");
boost::algorithm::replace_all(escapedKey, "::", ".");
String perfdataPrefix = prefix + "perfdata." + escapedKey;
fields->Set(perfdataPrefix + ".value", pdv->GetValue());
if (pdv->GetMin())
fields->Set(perfdataPrefix + ".min", pdv->GetMin());
if (pdv->GetMax())
fields->Set(perfdataPrefix + ".max", pdv->GetMax());
if (pdv->GetWarn())
fields->Set(perfdataPrefix + ".warn", pdv->GetWarn());
if (pdv->GetCrit())
fields->Set(perfdataPrefix + ".crit", pdv->GetCrit());
if (!pdv->GetUnit().IsEmpty())
fields->Set(perfdataPrefix + ".unit", pdv->GetUnit());
}
}
}