void Checkable::UpdateStatistics(const CheckResult::Ptr& cr, CheckableType type)
{
time_t ts = cr->GetScheduleEnd();
if (type == CheckableHost) {
if (cr->GetActive())
CIB::UpdateActiveHostChecksStatistics(ts, 1);
else
CIB::UpdatePassiveHostChecksStatistics(ts, 1);
} else if (type == CheckableService) {
if (cr->GetActive())
CIB::UpdateActiveServiceChecksStatistics(ts, 1);
else
CIB::UpdatePassiveServiceChecksStatistics(ts, 1);
} else {
Log(LogWarning, "Checkable", "Unknown checkable type for statistic update.");
}
}
void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrigin::Ptr& origin)
{
{
ObjectLock olock(this);
m_CheckRunning = false;
}
double now = Utility::GetTime();
if (cr->GetScheduleStart() == 0)
cr->SetScheduleStart(now);
if (cr->GetScheduleEnd() == 0)
cr->SetScheduleEnd(now);
if (cr->GetExecutionStart() == 0)
cr->SetExecutionStart(now);
if (cr->GetExecutionEnd() == 0)
cr->SetExecutionEnd(now);
if (!origin || origin->IsLocal())
cr->SetCheckSource(IcingaApplication::GetInstance()->GetNodeName());
Endpoint::Ptr command_endpoint = GetCommandEndpoint();
/* override check source if command_endpoint was defined */
if (command_endpoint && !GetExtension("agent_check"))
cr->SetCheckSource(command_endpoint->GetName());
/* agent checks go through the api */
if (command_endpoint && GetExtension("agent_check")) {
ApiListener::Ptr listener = ApiListener::GetInstance();
if (listener) {
/* send message back to its origin */
Dictionary::Ptr message = ClusterEvents::MakeCheckResultMessage(this, cr);
listener->SyncSendMessage(command_endpoint, message);
}
return;
}
bool reachable = IsReachable();
bool notification_reachable = IsReachable(DependencyNotification);
ObjectLock olock(this);
CheckResult::Ptr old_cr = GetLastCheckResult();
ServiceState old_state = GetStateRaw();
StateType old_stateType = GetStateType();
long old_attempt = GetCheckAttempt();
bool recovery = false;
/* Ignore check results older than the current one. */
if (old_cr && cr->GetExecutionStart() < old_cr->GetExecutionStart())
return;
/* The ExecuteCheck function already sets the old state, but we need to do it again
* in case this was a passive check result. */
SetLastStateRaw(old_state);
SetLastStateType(old_stateType);
SetLastReachable(reachable);
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(this);
CheckableType checkableType = CheckableHost;
if (service)
checkableType = CheckableService;
long attempt = 1;
std::set<Checkable::Ptr> children = GetChildren();
if (!old_cr) {
SetStateType(StateTypeHard);
} else if (IsStateOK(cr->GetState())) {
SetStateType(StateTypeHard); // NOT-OK -> HARD OK
if (!IsStateOK(old_state))
recovery = true;
ResetNotificationNumbers();
SaveLastState(ServiceOK, Utility::GetTime());
/* update reachability for child objects in OK state */
if (!children.empty())
OnReachabilityChanged(this, cr, children, origin);
} else {
if (old_attempt >= GetMaxCheckAttempts()) {
SetStateType(StateTypeHard);
} else if (old_stateType == StateTypeSoft && !IsStateOK(old_state)) {
SetStateType(StateTypeSoft);
attempt = old_attempt + 1; // NOT-OK -> NOT-OK counter
} else if (IsStateOK(old_state)) {
SetStateType(StateTypeSoft);
attempt = 1; // OK -> NOT-OK transition, reset the counter
} else {
attempt = old_attempt;
}
if (!IsStateOK(cr->GetState())) {
SaveLastState(cr->GetState(), Utility::GetTime());
}
/* update reachability for child objects in NOT-OK state */
if (!children.empty())
OnReachabilityChanged(this, cr, children, origin);
}
if (!reachable)
SetLastStateUnreachable(Utility::GetTime());
SetCheckAttempt(attempt);
ServiceState new_state = cr->GetState();
SetStateRaw(new_state);
bool stateChange;
/* Exception on state change calculation for hosts. */
if (checkableType == CheckableService)
stateChange = (old_state != new_state);
else
stateChange = (Host::CalculateState(old_state) != Host::CalculateState(new_state));
if (stateChange) {
SetLastStateChange(now);
/* remove acknowledgements */
if (GetAcknowledgement() == AcknowledgementNormal ||
(GetAcknowledgement() == AcknowledgementSticky && IsStateOK(new_state))) {
ClearAcknowledgement();
}
/* reschedule direct parents */
BOOST_FOREACH(const Checkable::Ptr& parent, GetParents()) {
if (parent.get() == this)
continue;
ObjectLock olock(parent);
parent->SetNextCheck(Utility::GetTime());
}
}
bool remove_acknowledgement_comments = false;
if (GetAcknowledgement() == AcknowledgementNone)
remove_acknowledgement_comments = true;
bool hardChange = (GetStateType() == StateTypeHard && old_stateType == StateTypeSoft);
if (stateChange && old_stateType == StateTypeHard && GetStateType() == StateTypeHard)
hardChange = true;
bool is_volatile = GetVolatile();
if (hardChange || is_volatile) {
SetLastHardStateRaw(new_state);
SetLastHardStateChange(now);
}
if (!IsStateOK(new_state))
TriggerDowntimes();
/* statistics for external tools */
Checkable::UpdateStatistics(cr, checkableType);
bool in_downtime = IsInDowntime();
bool send_notification = false;
if (notification_reachable && !in_downtime && !IsAcknowledged()) {
/* Send notifications whether when a hard state change occured. */
if (hardChange)
send_notification = true;
/* Or if the checkable is volatile and in a HARD state. */
else if (is_volatile && GetStateType() == StateTypeHard)
send_notification = true;
}
if (!old_cr)
send_notification = false; /* Don't send notifications for the initial state change */
if (IsStateOK(old_state) && old_stateType == StateTypeSoft)
send_notification = false; /* Don't send notifications for SOFT-OK -> HARD-OK. */
if (is_volatile && IsStateOK(old_state) && IsStateOK(new_state))
send_notification = false; /* Don't send notifications for volatile OK -> OK changes. */
bool send_downtime_notification = (GetLastInDowntime() != in_downtime);
SetLastInDowntime(in_downtime);
olock.Unlock();
if (remove_acknowledgement_comments)
RemoveCommentsByType(CommentAcknowledgement);
Dictionary::Ptr vars_after = new Dictionary();
vars_after->Set("state", new_state);
vars_after->Set("state_type", GetStateType());
vars_after->Set("attempt", GetCheckAttempt());
vars_after->Set("reachable", reachable);
if (old_cr)
cr->SetVarsBefore(old_cr->GetVarsAfter());
cr->SetVarsAfter(vars_after);
olock.Lock();
SetLastCheckResult(cr);
bool was_flapping, is_flapping;
was_flapping = IsFlapping();
if (GetStateType() == StateTypeHard)
UpdateFlappingStatus(stateChange);
is_flapping = IsFlapping();
if (cr->GetActive()) {
UpdateNextCheck(origin);
} else {
/* Reschedule the next check for passive check results. The side effect of
* this is that for as long as we receive passive results for a service we
* won't execute any active checks. */
SetNextCheck(Utility::GetTime() + GetCheckInterval(), false, origin);
}
olock.Unlock();
// Log(LogDebug, "Checkable")
// << "Flapping: Checkable " << GetName()
// << " was: " << (was_flapping)
// << " is: " << is_flapping)
// << " threshold: " << GetFlappingThreshold()
// << "% current: " + GetFlappingCurrent()) << "%.";
OnNewCheckResult(this, cr, origin);
/* signal status updates to for example db_ido */
OnStateChanged(this);
String old_state_str = (service ? Service::StateToString(old_state) : Host::StateToString(Host::CalculateState(old_state)));
String new_state_str = (service ? Service::StateToString(new_state) : Host::StateToString(Host::CalculateState(new_state)));
/* Whether a hard state change or a volatile state change except OK -> OK happened. */
if (hardChange || (is_volatile && !(IsStateOK(old_state) && IsStateOK(new_state)))) {
OnStateChange(this, cr, StateTypeHard, origin);
Log(LogNotice, "Checkable")
<< "State Change: Checkable " << GetName() << " hard state change from " << old_state_str << " to " << new_state_str << " detected." << (is_volatile ? " Checkable is volatile." : "");
}
/* Whether a state change happened or the state type is SOFT (must be logged too). */
else if (stateChange || GetStateType() == StateTypeSoft) {
OnStateChange(this, cr, StateTypeSoft, origin);
Log(LogNotice, "Checkable")
<< "State Change: Checkable " << GetName() << " soft state change from " << old_state_str << " to " << new_state_str << " detected.";
}
if (GetStateType() == StateTypeSoft || hardChange || recovery ||
(is_volatile && !(IsStateOK(old_state) && IsStateOK(new_state))))
ExecuteEventHandler();
if (send_downtime_notification && IsActive())
OnNotificationsRequested(this, in_downtime ? NotificationDowntimeStart : NotificationDowntimeEnd, cr, "", "", MessageOrigin::Ptr());
/* Flapping start/end notifications */
if (!was_flapping && is_flapping) {
if (!IsPaused())
OnNotificationsRequested(this, NotificationFlappingStart, cr, "", "", MessageOrigin::Ptr());
Log(LogNotice, "Checkable")
<< "Flapping: Checkable " << GetName() << " started flapping (" << GetFlappingThreshold() << "% < " << GetFlappingCurrent() << "%).";
NotifyFlapping(origin);
} else if (was_flapping && !is_flapping) {
if (!IsPaused())
OnNotificationsRequested(this, NotificationFlappingEnd, cr, "", "", MessageOrigin::Ptr());
Log(LogNotice, "Checkable")
<< "Flapping: Checkable " << GetName() << " stopped flapping (" << GetFlappingThreshold() << "% >= " << GetFlappingCurrent() << "%).";
NotifyFlapping(origin);
}
/* Problem notifications */
if (send_notification && !is_flapping) {
if (!IsPaused())
OnNotificationsRequested(this, recovery ? NotificationRecovery : NotificationProblem, cr, "", "", MessageOrigin::Ptr());
}
}