/* host */
int CompatUtility::GetHostCurrentState(const Host::Ptr& host)
{
if (host->GetState() != HostUp && !host->IsReachable())
return 2; /* hardcoded compat state */
return host->GetState();
}
String CompatUtility::GetHostStateString(const Host::Ptr& host)
{
if (host->GetState() != HostUp && !host->IsReachable())
return "UNREACHABLE"; /* hardcoded compat state */
return Host::StateToString(host->GetState());
}
void InfluxdbWriter::SendPerfdata(const Dictionary::Ptr& tmpl, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, double ts)
{
Array::Ptr perfdata = cr->GetPerformanceData();
if (!perfdata)
return;
ObjectLock olock(perfdata);
for (const Value& val : perfdata) {
PerfdataValue::Ptr pdv;
if (val.IsObjectType<PerfdataValue>())
pdv = val;
else {
try {
pdv = PerfdataValue::Parse(val);
} catch (const std::exception&) {
Log(LogWarning, "InfluxdbWriter")
<< "Ignoring invalid perfdata value: " << val;
continue;
}
}
Dictionary::Ptr fields = new Dictionary();
fields->Set(String("value"), pdv->GetValue());
if (GetEnableSendThresholds()) {
if (pdv->GetCrit())
fields->Set(String("crit"), pdv->GetCrit());
if (pdv->GetWarn())
fields->Set(String("warn"), pdv->GetWarn());
if (pdv->GetMin())
fields->Set(String("min"), pdv->GetMin());
if (pdv->GetMax())
fields->Set(String("max"), pdv->GetMax());
}
if (GetEnableSendMetadata()) {
Host::Ptr host;
Service::Ptr service;
boost::tie(host, service) = GetHostService(checkable);
if (service)
fields->Set(String("state"), FormatInteger(service->GetState()));
else
fields->Set(String("state"), FormatInteger(host->GetState()));
fields->Set(String("current_attempt"), FormatInteger(checkable->GetCheckAttempt()));
fields->Set(String("max_check_attempts"), FormatInteger(checkable->GetMaxCheckAttempts()));
fields->Set(String("state_type"), FormatInteger(checkable->GetStateType()));
fields->Set(String("reachable"), FormatBoolean(checkable->IsReachable()));
fields->Set(String("downtime_depth"), FormatInteger(checkable->GetDowntimeDepth()));
fields->Set(String("acknowledgement"), FormatInteger(checkable->GetAcknowledgement()));
fields->Set(String("latency"), cr->CalculateLatency());
fields->Set(String("execution_time"), cr->CalculateExecutionTime());
}
SendMetric(tmpl, pdv->GetLabel(), fields, ts);
}
}
void ElasticsearchWriter::NotificationSentToAllUsersHandlerInternal(const Notification::Ptr& notification,
const Checkable::Ptr& checkable, const std::set<User::Ptr>& users, NotificationType type,
const CheckResult::Ptr& cr, const String& author, const String& text)
{
AssertOnWorkQueue();
CONTEXT("Elasticwriter processing notification to all users '" + checkable->GetName() + "'");
Log(LogDebug, "ElasticsearchWriter")
<< "Processing notification for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
String notificationTypeString = Notification::NotificationTypeToString(type);
Dictionary::Ptr fields = new Dictionary();
if (service) {
fields->Set("service", service->GetShortName());
fields->Set("state", service->GetState());
fields->Set("last_state", service->GetLastState());
fields->Set("last_hard_state", service->GetLastHardState());
} else {
fields->Set("state", host->GetState());
fields->Set("last_state", host->GetLastState());
fields->Set("last_hard_state", host->GetLastHardState());
}
fields->Set("host", host->GetName());
ArrayData userNames;
for (const User::Ptr& user : users) {
userNames.push_back(user->GetName());
}
fields->Set("users", new Array(std::move(userNames)));
fields->Set("notification_type", notificationTypeString);
fields->Set("author", author);
fields->Set("text", text);
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
if (commandObj)
fields->Set("check_command", commandObj->GetName());
double ts = Utility::GetTime();
if (cr) {
AddCheckResult(fields, checkable, cr);
ts = cr->GetExecutionEnd();
}
Enqueue(checkable, "notification", fields, ts);
}
/**
* Periodically sends notifications.
*
* @param - Event arguments for the timer.
*/
void NotificationComponent::NotificationTimerHandler(void)
{
double now = Utility::GetTime();
for (const Notification::Ptr& notification : ConfigType::GetObjectsByType<Notification>()) {
if (!notification->IsActive())
continue;
if (notification->IsPaused() && GetEnableHA())
continue;
Checkable::Ptr checkable = notification->GetCheckable();
if (!IcingaApplication::GetInstance()->GetEnableNotifications() || !checkable->GetEnableNotifications())
continue;
if (notification->GetInterval() <= 0 && notification->GetNoMoreNotifications())
continue;
if (notification->GetNextNotification() > now)
continue;
bool reachable = checkable->IsReachable(DependencyNotification);
{
ObjectLock olock(notification);
notification->SetNextNotification(Utility::GetTime() + notification->GetInterval());
}
{
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
ObjectLock olock(checkable);
if (checkable->GetStateType() == StateTypeSoft)
continue;
if ((service && service->GetState() == ServiceOK) || (!service && host->GetState() == HostUp))
continue;
if (!reachable || checkable->IsInDowntime() || checkable->IsAcknowledged())
continue;
}
try {
Log(LogNotice, "NotificationComponent")
<< "Attempting to send reminder notification '" << notification->GetName() << "'";
notification->BeginExecuteNotification(NotificationProblem, checkable->GetLastCheckResult(), false, true);
} catch (const std::exception& ex) {
Log(LogWarning, "NotificationComponent")
<< "Exception occured during notification for object '"
<< GetName() << "': " << DiagnosticInformation(ex);
}
}
}
void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
CONTEXT("Processing check result for '" + checkable->GetName() + "'");
if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata())
return;
Service::Ptr service = dynamic_pointer_cast<Service>(checkable);
Host::Ptr host;
if (service)
host = service->GetHost();
else
host = static_pointer_cast<Host>(checkable);
String metric;
std::map<String, String> tags;
String escaped_hostName = EscapeTag(host->GetName());
tags["host"] = escaped_hostName;
double ts = cr->GetExecutionEnd();
if (service) {
String serviceName = service->GetShortName();
String escaped_serviceName = EscapeMetric(serviceName);
metric = "icinga.service." + escaped_serviceName;
SendMetric(metric + ".state", tags, service->GetState(), ts);
} else {
metric = "icinga.host";
SendMetric(metric + ".state", tags, host->GetState(), ts);
}
SendMetric(metric + ".state_type", tags, checkable->GetStateType(), ts);
SendMetric(metric + ".reachable", tags, checkable->IsReachable(), ts);
SendMetric(metric + ".downtime_depth", tags, checkable->GetDowntimeDepth(), ts);
SendMetric(metric + ".acknowledgement", tags, checkable->GetAcknowledgement(), ts);
SendPerfdata(metric, tags, cr, ts);
metric = "icinga.check";
if (service) {
tags["type"] = "service";
String serviceName = service->GetShortName();
String escaped_serviceName = EscapeTag(serviceName);
tags["service"] = escaped_serviceName;
} else {
tags["type"] = "host";
}
SendMetric(metric + ".current_attempt", tags, checkable->GetCheckAttempt(), ts);
SendMetric(metric + ".max_check_attempts", tags, checkable->GetMaxCheckAttempts(), ts);
SendMetric(metric + ".latency", tags, cr->CalculateLatency(), ts);
SendMetric(metric + ".execution_time", tags, cr->CalculateExecutionTime(), ts);
}
void ElasticsearchWriter::InternalCheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
AssertOnWorkQueue();
CONTEXT("Elasticwriter processing check result for '" + checkable->GetName() + "'");
if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata())
return;
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr fields = new Dictionary();
if (service) {
fields->Set("service", service->GetShortName());
fields->Set("state", service->GetState());
fields->Set("last_state", service->GetLastState());
fields->Set("last_hard_state", service->GetLastHardState());
} else {
fields->Set("state", host->GetState());
fields->Set("last_state", host->GetLastState());
fields->Set("last_hard_state", host->GetLastHardState());
}
fields->Set("host", host->GetName());
fields->Set("state_type", checkable->GetStateType());
fields->Set("current_check_attempt", checkable->GetCheckAttempt());
fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
fields->Set("reachable", checkable->IsReachable());
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
if (commandObj)
fields->Set("check_command", commandObj->GetName());
double ts = Utility::GetTime();
if (cr) {
AddCheckResult(fields, checkable, cr);
ts = cr->GetExecutionEnd();
}
Enqueue(checkable, "checkresult", fields, ts);
}
Dictionary::Ptr ApiActions::AcknowledgeProblem(const ConfigObject::Ptr& object,
const Dictionary::Ptr& params)
{
Checkable::Ptr checkable = static_pointer_cast<Checkable>(object);
if (!checkable)
return ApiActions::CreateResult(404, "Cannot acknowledge problem for non-existent object.");
if (!params->Contains("author") || !params->Contains("comment"))
return ApiActions::CreateResult(403, "Acknowledgements require author and comment.");
AcknowledgementType sticky = AcknowledgementNormal;
bool notify = false;
double timestamp = 0.0;
if (params->Contains("sticky"))
sticky = AcknowledgementSticky;
if (params->Contains("notify"))
notify = true;
if (params->Contains("expiry"))
timestamp = HttpUtility::GetLastParameter(params, "expiry");
else
timestamp = 0;
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
if (!service) {
if (host->GetState() == HostUp)
return ApiActions::CreateResult(409, "Host " + checkable->GetName() + " is UP.");
} else {
if (service->GetState() == ServiceOK)
return ApiActions::CreateResult(409, "Service " + checkable->GetName() + " is OK.");
}
Comment::AddComment(checkable, CommentAcknowledgement, HttpUtility::GetLastParameter(params, "author"),
HttpUtility::GetLastParameter(params, "comment"), timestamp);
checkable->AcknowledgeProblem(HttpUtility::GetLastParameter(params, "author"),
HttpUtility::GetLastParameter(params, "comment"), sticky, notify, timestamp);
return ApiActions::CreateResult(200, "Successfully acknowledged problem for object '" + checkable->GetName() + "'.");
}
void ElasticsearchWriter::StateChangeHandlerInternal(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type)
{
AssertOnWorkQueue();
CONTEXT("Elasticwriter processing state change '" + checkable->GetName() + "'");
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr fields = new Dictionary();
fields->Set("current_check_attempt", checkable->GetCheckAttempt());
fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
fields->Set("host", host->GetName());
if (service) {
fields->Set("service", service->GetShortName());
fields->Set("state", service->GetState());
fields->Set("last_state", service->GetLastState());
fields->Set("last_hard_state", service->GetLastHardState());
} else {
fields->Set("state", host->GetState());
fields->Set("last_state", host->GetLastState());
fields->Set("last_hard_state", host->GetLastHardState());
}
CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
if (commandObj)
fields->Set("check_command", commandObj->GetName());
double ts = Utility::GetTime();
if (cr) {
AddCheckResult(fields, checkable, cr);
ts = cr->GetExecutionEnd();
}
Enqueue(checkable, "statechange", fields, ts);
}
bool Dependency::IsAvailable(DependencyType dt) const
{
Checkable::Ptr parent = GetParent();
Host::Ptr parentHost;
Service::Ptr parentService;
tie(parentHost, parentService) = GetHostService(parent);
/* ignore if it's the same checkable object */
if (parent == GetChild()) {
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' passed: Parent and child " << (parentService ? "service" : "host") << " are identical.";
return true;
}
/* ignore pending */
if (!parent->GetLastCheckResult()) {
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' passed: Parent " << (parentService ? "service" : "host") << " '" << parent->GetName() << "' hasn't been checked yet.";
return true;
}
if (GetIgnoreSoftStates()) {
/* ignore soft states */
if (parent->GetStateType() == StateTypeSoft) {
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' passed: Parent " << (parentService ? "service" : "host") << " '" << parent->GetName() << "' is in a soft state.";
return true;
}
} else {
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' failed: Parent " << (parentService ? "service" : "host") << " '" << parent->GetName() << "' is in a soft state.";
}
int state;
if (parentService)
state = ServiceStateToFilter(parentService->GetState());
else
state = HostStateToFilter(parentHost->GetState());
/* check state */
if (state & GetStateFilter()) {
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' passed: Parent " << (parentService ? "service" : "host") << " '" << parent->GetName() << "' matches state filter.";
return true;
}
/* ignore if not in time period */
TimePeriod::Ptr tp = GetPeriod();
if (tp && !tp->IsInside(Utility::GetTime())) {
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' passed: Outside time period.";
return true;
}
if (dt == DependencyCheckExecution && !GetDisableChecks()) {
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' passed: Checks are not disabled.";
return true;
} else if (dt == DependencyNotification && !GetDisableNotifications()) {
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' passed: Notifications are not disabled";
return true;
}
Log(LogNotice, "Dependency")
<< "Dependency '" << GetName() << "' failed. Parent "
<< (parentService ? "service" : "host") << " '" << parent->GetName() << "' is "
<< (parentService ? Service::StateToString(parentService->GetState()) : Host::StateToString(parentHost->GetState()));
return false;
}
void Notification::BeginExecuteNotification(NotificationType type, const CheckResult::Ptr& cr, bool force, bool reminder, const String& author, const String& text)
{
Log(LogNotice, "Notification")
<< "Attempting to send " << (reminder ? "reminder " : " ") << "notifications for notification object '" << GetName() << "'.";
Checkable::Ptr checkable = GetCheckable();
if (!force) {
TimePeriod::Ptr tp = GetPeriod();
if (tp && !tp->IsInside(Utility::GetTime())) {
Log(LogNotice, "Notification")
<< "Not sending " << (reminder ? "reminder " : " ") << "notifications for notification object '" << GetName()
<< "': not in timeperiod '" << tp->GetName() << "'";
return;
}
double now = Utility::GetTime();
Dictionary::Ptr times = GetTimes();
if (times && type == NotificationProblem) {
Value timesBegin = times->Get("begin");
Value timesEnd = times->Get("end");
if (timesBegin != Empty && timesBegin >= 0 && now < checkable->GetLastHardStateChange() + timesBegin) {
Log(LogNotice, "Notification")
<< "Not sending " << (reminder ? "reminder " : " ") << "notifications for notification object '" << GetName()
<< "': before specified begin time (" << Utility::FormatDuration(timesBegin) << ")";
/* we need to adjust the next notification time
* to now + begin delaying the first notification
*/
double nextProposedNotification = now + timesBegin + 1.0;
if (GetNextNotification() > nextProposedNotification)
SetNextNotification(nextProposedNotification);
return;
}
if (timesEnd != Empty && timesEnd >= 0 && now > checkable->GetLastHardStateChange() + timesEnd) {
Log(LogNotice, "Notification")
<< "Not sending " << (reminder ? "reminder " : " ") << "notifications for notification object '" << GetName()
<< "': after specified end time (" << Utility::FormatDuration(timesEnd) << ")";
return;
}
}
unsigned long ftype = type;
Log(LogDebug, "Notification")
<< "Type '" << NotificationTypeToStringInternal(type)
<< "', TypeFilter: " << NotificationFilterToString(GetTypeFilter(), GetTypeFilterMap())
<< " (FType=" << ftype << ", TypeFilter=" << GetTypeFilter() << ")";
if (!(ftype & GetTypeFilter())) {
Log(LogNotice, "Notification")
<< "Not sending " << (reminder ? "reminder " : " ") << "notifications for notification object '" << GetName() << "': type '"
<< NotificationTypeToStringInternal(type) << "' does not match type filter: "
<< NotificationFilterToString(GetTypeFilter(), GetTypeFilterMap()) << ".";
return;
}
/* ensure that recovery notifications are always sent, no state filter checks necessary */
if (type != NotificationRecovery) {
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
unsigned long fstate;
String stateStr;
if (service) {
fstate = ServiceStateToFilter(service->GetState());
stateStr = NotificationServiceStateToString(service->GetState());
} else {
fstate = HostStateToFilter(host->GetState());
stateStr = NotificationHostStateToString(host->GetState());
}
Log(LogDebug, "Notification")
<< "State '" << stateStr << "', StateFilter: " << NotificationFilterToString(GetStateFilter(), GetStateFilterMap())
<< " (FState=" << fstate << ", StateFilter=" << GetStateFilter() << ")";
if (!(fstate & GetStateFilter())) {
Log(LogNotice, "Notification")
<< "Not sending " << (reminder ? "reminder " : " ") << "notifications for notification object '" << GetName() << "': state '" << stateStr
<< "' does not match state filter: " << NotificationFilterToString(GetStateFilter(), GetStateFilterMap()) << ".";
return;
}
}
} else {
Log(LogNotice, "Notification")
<< "Not checking " << (reminder ? "reminder " : " ") << "notification filters for notification object '" << GetName() << "': Notification was forced.";
}
{
ObjectLock olock(this);
UpdateNotificationNumber();
double now = Utility::GetTime();
SetLastNotification(now);
if (type == NotificationProblem && GetInterval() <= 0)
SetNoMoreNotifications(true);
else
SetNoMoreNotifications(false);
if (type == NotificationProblem && GetInterval() > 0)
SetNextNotification(now + GetInterval());
if (type == NotificationProblem)
SetLastProblemNotification(now);
}
std::set<User::Ptr> allUsers;
std::set<User::Ptr> users = GetUsers();
std::copy(users.begin(), users.end(), std::inserter(allUsers, allUsers.begin()));
for (const UserGroup::Ptr& ug : GetUserGroups()) {
std::set<User::Ptr> members = ug->GetMembers();
std::copy(members.begin(), members.end(), std::inserter(allUsers, allUsers.begin()));
}
std::set<User::Ptr> allNotifiedUsers;
Array::Ptr notifiedProblemUsers = GetNotifiedProblemUsers();
for (const User::Ptr& user : allUsers) {
String userName = user->GetName();
if (!user->GetEnableNotifications()) {
Log(LogNotice, "Notification")
<< "Disabled notifications for user '" << userName << "'. Not sending notification.";
continue;
}
if (!CheckNotificationUserFilters(type, user, force, reminder)) {
Log(LogNotice, "Notification")
<< "Notification filters for user '" << userName << "' not matched. Not sending notification.";
continue;
}
/* on recovery, check if user was notified before */
if (type == NotificationRecovery) {
if (!notifiedProblemUsers->Contains(userName)) {
Log(LogNotice, "Notification")
<< "We did not notify user '" << userName << "' for a problem before. Not sending recovery notification.";
continue;
}
}
Log(LogInformation, "Notification")
<< "Sending " << (reminder ? "reminder " : "") << "'" << NotificationTypeToStringInternal(type) << "' notification '"
<< GetName() << " for user '" << userName << "'";
Utility::QueueAsyncCallback(boost::bind(&Notification::ExecuteNotificationHelper, this, type, user, cr, force, author, text));
/* collect all notified users */
allNotifiedUsers.insert(user);
/* store all notified users for later recovery checks */
if (type == NotificationProblem && !notifiedProblemUsers->Contains(userName))
notifiedProblemUsers->Add(userName);
}
/* if this was a recovery notification, reset all notified users */
if (type == NotificationRecovery)
notifiedProblemUsers->Clear();
/* used in db_ido for notification history */
Service::OnNotificationSentToAllUsers(this, checkable, allNotifiedUsers, type, cr, author, text, MessageOrigin::Ptr());
}
bool Notification::CheckNotificationUserFilters(NotificationType type, const User::Ptr& user, bool force, bool reminder)
{
if (!force) {
TimePeriod::Ptr tp = user->GetPeriod();
if (tp && !tp->IsInside(Utility::GetTime())) {
Log(LogNotice, "Notification")
<< "Not sending " << (reminder ? "reminder " : " ") << "notifications for notification object '"
<< GetName() << " and user '" << user->GetName()
<< "': user period not in timeperiod '" << tp->GetName() << "'";
return false;
}
unsigned long ftype = type;
Log(LogDebug, "Notification")
<< "User notification, Type '" << NotificationTypeToStringInternal(type)
<< "', TypeFilter: " << NotificationFilterToString(user->GetTypeFilter(), GetTypeFilterMap())
<< " (FType=" << ftype << ", TypeFilter=" << GetTypeFilter() << ")";
if (!(ftype & user->GetTypeFilter())) {
Log(LogNotice, "Notification")
<< "Not sending " << (reminder ? "reminder " : " ") << "notifications for notification object '"
<< GetName() << " and user '" << user->GetName() << "': type '"
<< NotificationTypeToStringInternal(type) << "' does not match type filter: "
<< NotificationFilterToString(user->GetTypeFilter(), GetTypeFilterMap()) << ".";
return false;
}
/* check state filters it this is not a recovery notification */
if (type != NotificationRecovery) {
Checkable::Ptr checkable = GetCheckable();
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
unsigned long fstate;
String stateStr;
if (service) {
fstate = ServiceStateToFilter(service->GetState());
stateStr = NotificationServiceStateToString(service->GetState());
} else {
fstate = HostStateToFilter(host->GetState());
stateStr = NotificationHostStateToString(host->GetState());
}
Log(LogDebug, "Notification")
<< "User notification, State '" << stateStr << "', StateFilter: "
<< NotificationFilterToString(user->GetStateFilter(), GetStateFilterMap())
<< " (FState=" << fstate << ", StateFilter=" << user->GetStateFilter() << ")";
if (!(fstate & user->GetStateFilter())) {
Log(LogNotice, "Notification")
<< "Not " << (reminder ? "reminder " : " ") << "sending notifications for notification object '"
<< GetName() << " and user '" << user->GetName() << "': state '" << stateStr
<< "' does not match state filter: " << NotificationFilterToString(user->GetStateFilter(), GetStateFilterMap()) << ".";
return false;
}
}
} else {
Log(LogNotice, "Notification")
<< "Not checking " << (reminder ? "reminder " : " ") << "notification filters for notification object '"
<< GetName() << "' and user '" << user->GetName() << "': Notification was forced.";
}
return true;
}
void LogstashWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
CONTEXT("LOGSTASH Processing check result for '" + checkable->GetName() + "'");
Log(LogDebug, "LogstashWriter")
<< "Processing check result for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr fields = new Dictionary();
if (service) {
fields->Set("service_name", service->GetShortName());
fields->Set("service_state", Service::StateToString(service->GetState()));
fields->Set("last_state", service->GetLastState());
fields->Set("last_hard_state", service->GetLastHardState());
} else {
fields->Set("last_state", host->GetLastState());
fields->Set("last_hard_state", host->GetLastHardState());
}
fields->Set("host_name", host->GetName());
fields->Set("type", "CheckResult");
fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
fields->Set("current_check_attempt", checkable->GetCheckAttempt());
fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
fields->Set("latency", cr->CalculateLatency());
fields->Set("execution_time", cr->CalculateExecutionTime());
fields->Set("reachable", checkable->IsReachable());
double ts = Utility::GetTime();
if (cr) {
fields->Set("plugin_output", cr->GetOutput());
fields->Set("check_source", cr->GetCheckSource());
ts = cr->GetExecutionEnd();
}
Array::Ptr perfdata = cr->GetPerformanceData();
if (perfdata) {
Dictionary::Ptr perfdataItems = new Dictionary();
ObjectLock olock(perfdata);
for (const Value& val : perfdata) {
PerfdataValue::Ptr pdv;
if (val.IsObjectType<PerfdataValue>())
pdv = val;
else {
try {
pdv = PerfdataValue::Parse(val);
} catch (const std::exception&) {
Log(LogWarning, "LogstashWriter")
<< "Ignoring invalid perfdata value: '" << val << "' for object '"
<< checkable->GetName() << "'.";
continue;
}
}
Dictionary::Ptr perfdataItem = new Dictionary();
perfdataItem->Set("value", pdv->GetValue());
if (pdv->GetMin())
perfdataItem->Set("min", pdv->GetMin());
if (pdv->GetMax())
perfdataItem->Set("max", pdv->GetMax());
if (pdv->GetWarn())
perfdataItem->Set("warn", pdv->GetWarn());
if (pdv->GetCrit())
perfdataItem->Set("crit", pdv->GetCrit());
String escaped_key = EscapeMetricLabel(pdv->GetLabel());
perfdataItems->Set(escaped_key, perfdataItem);
}
fields->Set("performance_data", perfdataItems);
}
SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
void LogstashWriter::StateChangeHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type)
{
CONTEXT("Logstash Processing state change '" + checkable->GetName() + "'");
Log(LogDebug, "LogstashWriter")
<< "Processing state change for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr fields = new Dictionary();
fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
fields->Set("type", "StateChange");
fields->Set("current_check_attempt", checkable->GetCheckAttempt());
fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
fields->Set("hostname", host->GetName());
if (service) {
fields->Set("service_name", service->GetShortName());
fields->Set("service_state", Service::StateToString(service->GetState()));
fields->Set("last_state", service->GetLastState());
fields->Set("last_hard_state", service->GetLastHardState());
} else {
fields->Set("last_state", host->GetLastState());
fields->Set("last_hard_state", host->GetLastHardState());
}
double ts = Utility::GetTime();
if (cr) {
fields->Set("plugin_output", cr->GetOutput());
fields->Set("check_source", cr->GetCheckSource());
ts = cr->GetExecutionEnd();
}
SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
void LogstashWriter::NotificationToUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
const User::Ptr& user, NotificationType notification_type, CheckResult::Ptr const& cr,
const String& author, const String& comment_text, const String& command_name)
{
CONTEXT("Logstash Processing notification to all users '" + checkable->GetName() + "'");
Log(LogDebug, "LogstashWriter")
<< "Processing notification for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
String notification_type_str = Notification::NotificationTypeToString(notification_type);
String author_comment = "";
if (notification_type == NotificationCustom || notification_type == NotificationAcknowledgement) {
author_comment = author + ";" + comment_text;
}
double ts = Utility::GetTime();
Dictionary::Ptr fields = new Dictionary();
if (service) {
fields->Set("type", "SERVICE NOTIFICATION");
fields->Set("service_name", service->GetShortName());
} else {
fields->Set("type", "HOST NOTIFICATION");
}
if (cr) {
fields->Set("plugin_output", cr->GetOutput());
ts = cr->GetExecutionEnd();
}
fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
fields->Set("host_name", host->GetName());
fields->Set("command", command_name);
fields->Set("notification_type", notification_type_str);
fields->Set("comment", author_comment);
SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
void GelfWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
CONTEXT("GELF Processing check result for '" + checkable->GetName() + "'");
Log(LogDebug, "GelfWriter")
<< "GELF Processing check result for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr fields = new Dictionary();
if (service) {
fields->Set("_service_name", service->GetShortName());
fields->Set("_service_state", Service::StateToString(service->GetState()));
fields->Set("_last_state", service->GetLastState());
fields->Set("_last_hard_state", service->GetLastHardState());
} else {
fields->Set("_last_state", host->GetLastState());
fields->Set("_last_hard_state", host->GetLastHardState());
}
fields->Set("_hostname", host->GetName());
fields->Set("_type", "CHECK RESULT");
fields->Set("_state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
fields->Set("_current_check_attempt", checkable->GetCheckAttempt());
fields->Set("_max_check_attempts", checkable->GetMaxCheckAttempts());
if (cr) {
fields->Set("short_message", CompatUtility::GetCheckResultOutput(cr));
fields->Set("full_message", CompatUtility::GetCheckResultLongOutput(cr));
fields->Set("_check_source", cr->GetCheckSource());
}
SendLogMessage(ComposeGelfMessage(fields, GetSource()));
}
void GelfWriter::NotificationToUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
const User::Ptr& user, NotificationType notification_type, CheckResult::Ptr const& cr,
const String& author, const String& comment_text, const String& command_name)
{
CONTEXT("GELF Processing notification to all users '" + checkable->GetName() + "'");
Log(LogDebug, "GelfWriter")
<< "GELF Processing notification for '" << checkable->GetName() << "'";
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
String notification_type_str = Notification::NotificationTypeToString(notification_type);
String author_comment = "";
if (notification_type == NotificationCustom || notification_type == NotificationAcknowledgement) {
author_comment = author + ";" + comment_text;
}
String output;
if (cr)
output = CompatUtility::GetCheckResultOutput(cr);
Dictionary::Ptr fields = new Dictionary();
if (service) {
fields->Set("_type", "SERVICE NOTIFICATION");
fields->Set("_service", service->GetShortName());
fields->Set("short_message", output);
} else {
fields->Set("_type", "HOST NOTIFICATION");
fields->Set("short_message", "(" + (host->IsReachable() ? Host::StateToString(host->GetState()) : String("UNREACHABLE")) + ")");
}
fields->Set("_state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
fields->Set("_hostname", host->GetName());
fields->Set("_command", command_name);
fields->Set("_notification_type", notification_type_str);
fields->Set("_comment", author_comment);
SendLogMessage(ComposeGelfMessage(fields, GetSource()));
}
void StatusDataWriter::DumpCheckableStatusAttrs(std::ostream& fp, const Checkable::Ptr& checkable)
{
CheckResult::Ptr cr = checkable->GetLastCheckResult();
EventCommand::Ptr eventcommand = checkable->GetEventCommand();
CheckCommand::Ptr checkcommand = checkable->GetCheckCommand();
fp << "\t" << "check_command=" << CompatUtility::GetCommandName(checkcommand) << "!" << CompatUtility::GetCheckableCommandArgs(checkable) << "\n"
"\t" "event_handler=" << CompatUtility::GetCommandName(eventcommand) << "\n"
"\t" "check_interval=" << (checkable->GetCheckInterval() / 60.0) << "\n"
"\t" "retry_interval=" << (checkable->GetRetryInterval() / 60.0) << "\n"
"\t" "has_been_checked=" << Convert::ToLong(checkable->HasBeenChecked()) << "\n"
"\t" "should_be_scheduled=" << checkable->GetEnableActiveChecks() << "\n"
"\t" "event_handler_enabled=" << Convert::ToLong(checkable->GetEnableEventHandler()) << "\n";
TimePeriod::Ptr checkPeriod = checkable->GetCheckPeriod();
if (checkPeriod)
fp << "\t" "check_period" "\t" << checkPeriod->GetName() << "\n";
if (cr) {
fp << "\t" << "check_execution_time=" << Convert::ToString(cr->CalculateExecutionTime()) << "\n"
"\t" "check_latency=" << Convert::ToString(cr->CalculateLatency()) << "\n";
}
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
if (service) {
fp << "\t" "current_state=" << service->GetState() << "\n"
"\t" "last_hard_state=" << service->GetLastHardState() << "\n"
"\t" "last_time_ok=" << static_cast<int>(service->GetLastStateOK()) << "\n"
"\t" "last_time_warn=" << static_cast<int>(service->GetLastStateWarning()) << "\n"
"\t" "last_time_critical=" << static_cast<int>(service->GetLastStateCritical()) << "\n"
"\t" "last_time_unknown=" << static_cast<int>(service->GetLastStateUnknown()) << "\n";
} else {
int currentState = host->GetState();
if (currentState != HostUp && !host->IsReachable())
currentState = 2; /* hardcoded compat state */
fp << "\t" "current_state=" << currentState << "\n"
"\t" "last_hard_state=" << host->GetLastHardState() << "\n"
"\t" "last_time_up=" << static_cast<int>(host->GetLastStateUp()) << "\n"
"\t" "last_time_down=" << static_cast<int>(host->GetLastStateDown()) << "\n";
}
fp << "\t" "state_type=" << checkable->GetStateType() << "\n"
"\t" "last_check=" << static_cast<long>(host->GetLastCheck()) << "\n";
if (cr) {
fp << "\t" "plugin_output=" << CompatUtility::GetCheckResultOutput(cr) << "\n"
"\t" "long_plugin_output=" << CompatUtility::GetCheckResultLongOutput(cr) << "\n"
"\t" "performance_data=" << PluginUtility::FormatPerfdata(cr->GetPerformanceData()) << "\n";
}
fp << "\t" << "next_check=" << static_cast<long>(checkable->GetNextCheck()) << "\n"
"\t" "current_attempt=" << checkable->GetCheckAttempt() << "\n"
"\t" "max_attempts=" << checkable->GetMaxCheckAttempts() << "\n"
"\t" "last_state_change=" << static_cast<long>(checkable->GetLastStateChange()) << "\n"
"\t" "last_hard_state_change=" << static_cast<long>(checkable->GetLastHardStateChange()) << "\n"
"\t" "last_update=" << static_cast<long>(Utility::GetTime()) << "\n"
"\t" "notifications_enabled=" << Convert::ToLong(checkable->GetEnableNotifications()) << "\n"
"\t" "active_checks_enabled=" << Convert::ToLong(checkable->GetEnableActiveChecks()) << "\n"
"\t" "passive_checks_enabled=" << Convert::ToLong(checkable->GetEnablePassiveChecks()) << "\n"
"\t" "flap_detection_enabled=" << Convert::ToLong(checkable->GetEnableFlapping()) << "\n"
"\t" "is_flapping=" << Convert::ToLong(checkable->IsFlapping()) << "\n"
"\t" "percent_state_change=" << checkable->GetFlappingCurrent() << "\n"
"\t" "problem_has_been_acknowledged=" << (checkable->GetAcknowledgement() != AcknowledgementNone ? 1 : 0) << "\n"
"\t" "acknowledgement_type=" << checkable->GetAcknowledgement() << "\n"
"\t" "acknowledgement_end_time=" << checkable->GetAcknowledgementExpiry() << "\n"
"\t" "scheduled_downtime_depth=" << checkable->GetDowntimeDepth() << "\n"
"\t" "last_notification=" << CompatUtility::GetCheckableNotificationLastNotification(checkable) << "\n"
"\t" "next_notification=" << CompatUtility::GetCheckableNotificationNextNotification(checkable) << "\n"
"\t" "current_notification_number=" << CompatUtility::GetCheckableNotificationNotificationNumber(checkable) << "\n"
"\t" "is_reachable=" << Convert::ToLong(checkable->IsReachable()) << "\n";
}
void InfluxdbWriter::CheckResultHandlerWQ(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
AssertOnWorkQueue();
CONTEXT("Processing check result for '" + checkable->GetName() + "'");
if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata())
return;
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
MacroProcessor::ResolverList resolvers;
if (service)
resolvers.emplace_back("service", service);
resolvers.emplace_back("host", host);
resolvers.emplace_back("icinga", IcingaApplication::GetInstance());
String prefix;
double ts = cr->GetExecutionEnd();
// Clone the template and perform an in-place macro expansion of measurement and tag values
Dictionary::Ptr tmpl_clean = service ? GetServiceTemplate() : GetHostTemplate();
Dictionary::Ptr tmpl = static_pointer_cast<Dictionary>(tmpl_clean->Clone());
tmpl->Set("measurement", MacroProcessor::ResolveMacros(tmpl->Get("measurement"), resolvers, cr));
Dictionary::Ptr tags = tmpl->Get("tags");
if (tags) {
ObjectLock olock(tags);
for (const Dictionary::Pair& pair : tags) {
String missing_macro;
Value value = MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missing_macro);
if (!missing_macro.IsEmpty())
continue;
tags->Set(pair.first, value);
}
}
Array::Ptr perfdata = cr->GetPerformanceData();
if (perfdata) {
ObjectLock olock(perfdata);
for (const Value& val : perfdata) {
PerfdataValue::Ptr pdv;
if (val.IsObjectType<PerfdataValue>())
pdv = val;
else {
try {
pdv = PerfdataValue::Parse(val);
} catch (const std::exception&) {
Log(LogWarning, "InfluxdbWriter")
<< "Ignoring invalid perfdata value: " << val;
continue;
}
}
Dictionary::Ptr fields = new Dictionary();
fields->Set("value", pdv->GetValue());
if (GetEnableSendThresholds()) {
if (pdv->GetCrit())
fields->Set("crit", pdv->GetCrit());
if (pdv->GetWarn())
fields->Set("warn", pdv->GetWarn());
if (pdv->GetMin())
fields->Set("min", pdv->GetMin());
if (pdv->GetMax())
fields->Set("max", pdv->GetMax());
}
if (!pdv->GetUnit().IsEmpty()) {
fields->Set("unit", pdv->GetUnit());
}
SendMetric(tmpl, pdv->GetLabel(), fields, ts);
}
}
if (GetEnableSendMetadata()) {
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(checkable);
Dictionary::Ptr fields = new Dictionary();
if (service)
fields->Set("state", new InfluxdbInteger(service->GetState()));
else
fields->Set("state", new InfluxdbInteger(host->GetState()));
fields->Set("current_attempt", new InfluxdbInteger(checkable->GetCheckAttempt()));
fields->Set("max_check_attempts", new InfluxdbInteger(checkable->GetMaxCheckAttempts()));
fields->Set("state_type", new InfluxdbInteger(checkable->GetStateType()));
fields->Set("reachable", checkable->IsReachable());
fields->Set("downtime_depth", new InfluxdbInteger(checkable->GetDowntimeDepth()));
fields->Set("acknowledgement", new InfluxdbInteger(checkable->GetAcknowledgement()));
fields->Set("latency", cr->CalculateLatency());
fields->Set("execution_time", cr->CalculateExecutionTime());
SendMetric(tmpl, Empty, fields, ts);
}
}
Dictionary::Ptr HostDbObject::GetStatusFields(void) const
{
Dictionary::Ptr fields = new Dictionary();
Host::Ptr host = static_pointer_cast<Host>(GetObject());
CheckResult::Ptr cr = host->GetLastCheckResult();
if (cr) {
fields->Set("output", CompatUtility::GetCheckResultOutput(cr));
fields->Set("long_output", CompatUtility::GetCheckResultLongOutput(cr));
fields->Set("perfdata", CompatUtility::GetCheckResultPerfdata(cr));
fields->Set("check_source", cr->GetCheckSource());
}
fields->Set("current_state", host->IsReachable() ? host->GetState() : 2);
fields->Set("has_been_checked", CompatUtility::GetCheckableHasBeenChecked(host));
fields->Set("should_be_scheduled", host->GetEnableActiveChecks());
fields->Set("current_check_attempt", host->GetCheckAttempt());
fields->Set("max_check_attempts", host->GetMaxCheckAttempts());
if (cr)
fields->Set("last_check", DbValue::FromTimestamp(cr->GetScheduleEnd()));
fields->Set("next_check", DbValue::FromTimestamp(host->GetNextCheck()));
fields->Set("check_type", CompatUtility::GetCheckableCheckType(host));
fields->Set("last_state_change", DbValue::FromTimestamp(host->GetLastStateChange()));
fields->Set("last_hard_state_change", DbValue::FromTimestamp(host->GetLastHardStateChange()));
fields->Set("last_time_up", DbValue::FromTimestamp(static_cast<int>(host->GetLastStateUp())));
fields->Set("last_time_down", DbValue::FromTimestamp(static_cast<int>(host->GetLastStateDown())));
fields->Set("last_time_unreachable", DbValue::FromTimestamp(static_cast<int>(host->GetLastStateUnreachable())));
fields->Set("state_type", host->GetStateType());
fields->Set("last_notification", DbValue::FromTimestamp(CompatUtility::GetCheckableNotificationLastNotification(host)));
fields->Set("next_notification", DbValue::FromTimestamp(CompatUtility::GetCheckableNotificationNextNotification(host)));
fields->Set("no_more_notifications", Empty);
fields->Set("notifications_enabled", CompatUtility::GetCheckableNotificationsEnabled(host));
{
ObjectLock olock(host);
fields->Set("problem_has_been_acknowledged", CompatUtility::GetCheckableProblemHasBeenAcknowledged(host));
fields->Set("acknowledgement_type", CompatUtility::GetCheckableAcknowledgementType(host));
}
fields->Set("current_notification_number", CompatUtility::GetCheckableNotificationNotificationNumber(host));
fields->Set("passive_checks_enabled", CompatUtility::GetCheckablePassiveChecksEnabled(host));
fields->Set("active_checks_enabled", CompatUtility::GetCheckableActiveChecksEnabled(host));
fields->Set("event_handler_enabled", CompatUtility::GetCheckableEventHandlerEnabled(host));
fields->Set("flap_detection_enabled", CompatUtility::GetCheckableFlapDetectionEnabled(host));
fields->Set("is_flapping", CompatUtility::GetCheckableIsFlapping(host));
fields->Set("percent_state_change", CompatUtility::GetCheckablePercentStateChange(host));
if (cr) {
fields->Set("latency", Convert::ToString(Service::CalculateLatency(cr)));
fields->Set("execution_time", Convert::ToString(Service::CalculateExecutionTime(cr)));
}
fields->Set("scheduled_downtime_depth", host->GetDowntimeDepth());
fields->Set("failure_prediction_enabled", Empty);
fields->Set("process_performance_data", 0); /* this is a host which does not process any perf data */
fields->Set("obsess_over_host", Empty);
fields->Set("modified_host_attributes", host->GetModifiedAttributes());
fields->Set("event_handler", CompatUtility::GetCheckableEventHandler(host));
fields->Set("check_command", CompatUtility::GetCheckableCheckCommand(host));
fields->Set("normal_check_interval", CompatUtility::GetCheckableCheckInterval(host));
fields->Set("retry_check_interval", CompatUtility::GetCheckableRetryInterval(host));
fields->Set("check_timeperiod_object_id", host->GetCheckPeriod());
fields->Set("is_reachable", CompatUtility::GetCheckableIsReachable(host));
return fields;
}