void Checkable::UpdateStatistics(const CheckResult::Ptr& cr, CheckableType type) { time_t ts = cr->GetScheduleEnd(); if (type == CheckableHost) { if (cr->GetActive()) CIB::UpdateActiveHostChecksStatistics(ts, 1); else CIB::UpdatePassiveHostChecksStatistics(ts, 1); } else if (type == CheckableService) { if (cr->GetActive()) CIB::UpdateActiveServiceChecksStatistics(ts, 1); else CIB::UpdatePassiveServiceChecksStatistics(ts, 1); } else { Log(LogWarning, "Checkable", "Unknown checkable type for statistic update."); } }
void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrigin::Ptr& origin) { { ObjectLock olock(this); m_CheckRunning = false; } double now = Utility::GetTime(); if (cr->GetScheduleStart() == 0) cr->SetScheduleStart(now); if (cr->GetScheduleEnd() == 0) cr->SetScheduleEnd(now); if (cr->GetExecutionStart() == 0) cr->SetExecutionStart(now); if (cr->GetExecutionEnd() == 0) cr->SetExecutionEnd(now); if (!origin || origin->IsLocal()) cr->SetCheckSource(IcingaApplication::GetInstance()->GetNodeName()); Endpoint::Ptr command_endpoint = GetCommandEndpoint(); /* override check source if command_endpoint was defined */ if (command_endpoint && !GetExtension("agent_check")) cr->SetCheckSource(command_endpoint->GetName()); /* agent checks go through the api */ if (command_endpoint && GetExtension("agent_check")) { ApiListener::Ptr listener = ApiListener::GetInstance(); if (listener) { /* send message back to its origin */ Dictionary::Ptr message = ClusterEvents::MakeCheckResultMessage(this, cr); listener->SyncSendMessage(command_endpoint, message); } return; } bool reachable = IsReachable(); bool notification_reachable = IsReachable(DependencyNotification); ObjectLock olock(this); CheckResult::Ptr old_cr = GetLastCheckResult(); ServiceState old_state = GetStateRaw(); StateType old_stateType = GetStateType(); long old_attempt = GetCheckAttempt(); bool recovery = false; /* Ignore check results older than the current one. */ if (old_cr && cr->GetExecutionStart() < old_cr->GetExecutionStart()) return; /* The ExecuteCheck function already sets the old state, but we need to do it again * in case this was a passive check result. */ SetLastStateRaw(old_state); SetLastStateType(old_stateType); SetLastReachable(reachable); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(this); CheckableType checkableType = CheckableHost; if (service) checkableType = CheckableService; long attempt = 1; std::set<Checkable::Ptr> children = GetChildren(); if (!old_cr) { SetStateType(StateTypeHard); } else if (IsStateOK(cr->GetState())) { SetStateType(StateTypeHard); // NOT-OK -> HARD OK if (!IsStateOK(old_state)) recovery = true; ResetNotificationNumbers(); SaveLastState(ServiceOK, Utility::GetTime()); /* update reachability for child objects in OK state */ if (!children.empty()) OnReachabilityChanged(this, cr, children, origin); } else { if (old_attempt >= GetMaxCheckAttempts()) { SetStateType(StateTypeHard); } else if (old_stateType == StateTypeSoft && !IsStateOK(old_state)) { SetStateType(StateTypeSoft); attempt = old_attempt + 1; // NOT-OK -> NOT-OK counter } else if (IsStateOK(old_state)) { SetStateType(StateTypeSoft); attempt = 1; // OK -> NOT-OK transition, reset the counter } else { attempt = old_attempt; } if (!IsStateOK(cr->GetState())) { SaveLastState(cr->GetState(), Utility::GetTime()); } /* update reachability for child objects in NOT-OK state */ if (!children.empty()) OnReachabilityChanged(this, cr, children, origin); } if (!reachable) SetLastStateUnreachable(Utility::GetTime()); SetCheckAttempt(attempt); ServiceState new_state = cr->GetState(); SetStateRaw(new_state); bool stateChange; /* Exception on state change calculation for hosts. */ if (checkableType == CheckableService) stateChange = (old_state != new_state); else stateChange = (Host::CalculateState(old_state) != Host::CalculateState(new_state)); if (stateChange) { SetLastStateChange(now); /* remove acknowledgements */ if (GetAcknowledgement() == AcknowledgementNormal || (GetAcknowledgement() == AcknowledgementSticky && IsStateOK(new_state))) { ClearAcknowledgement(); } /* reschedule direct parents */ BOOST_FOREACH(const Checkable::Ptr& parent, GetParents()) { if (parent.get() == this) continue; ObjectLock olock(parent); parent->SetNextCheck(Utility::GetTime()); } } bool remove_acknowledgement_comments = false; if (GetAcknowledgement() == AcknowledgementNone) remove_acknowledgement_comments = true; bool hardChange = (GetStateType() == StateTypeHard && old_stateType == StateTypeSoft); if (stateChange && old_stateType == StateTypeHard && GetStateType() == StateTypeHard) hardChange = true; bool is_volatile = GetVolatile(); if (hardChange || is_volatile) { SetLastHardStateRaw(new_state); SetLastHardStateChange(now); } if (!IsStateOK(new_state)) TriggerDowntimes(); /* statistics for external tools */ Checkable::UpdateStatistics(cr, checkableType); bool in_downtime = IsInDowntime(); bool send_notification = false; if (notification_reachable && !in_downtime && !IsAcknowledged()) { /* Send notifications whether when a hard state change occured. */ if (hardChange) send_notification = true; /* Or if the checkable is volatile and in a HARD state. */ else if (is_volatile && GetStateType() == StateTypeHard) send_notification = true; } if (!old_cr) send_notification = false; /* Don't send notifications for the initial state change */ if (IsStateOK(old_state) && old_stateType == StateTypeSoft) send_notification = false; /* Don't send notifications for SOFT-OK -> HARD-OK. */ if (is_volatile && IsStateOK(old_state) && IsStateOK(new_state)) send_notification = false; /* Don't send notifications for volatile OK -> OK changes. */ bool send_downtime_notification = (GetLastInDowntime() != in_downtime); SetLastInDowntime(in_downtime); olock.Unlock(); if (remove_acknowledgement_comments) RemoveCommentsByType(CommentAcknowledgement); Dictionary::Ptr vars_after = new Dictionary(); vars_after->Set("state", new_state); vars_after->Set("state_type", GetStateType()); vars_after->Set("attempt", GetCheckAttempt()); vars_after->Set("reachable", reachable); if (old_cr) cr->SetVarsBefore(old_cr->GetVarsAfter()); cr->SetVarsAfter(vars_after); olock.Lock(); SetLastCheckResult(cr); bool was_flapping, is_flapping; was_flapping = IsFlapping(); if (GetStateType() == StateTypeHard) UpdateFlappingStatus(stateChange); is_flapping = IsFlapping(); if (cr->GetActive()) { UpdateNextCheck(origin); } else { /* Reschedule the next check for passive check results. The side effect of * this is that for as long as we receive passive results for a service we * won't execute any active checks. */ SetNextCheck(Utility::GetTime() + GetCheckInterval(), false, origin); } olock.Unlock(); // Log(LogDebug, "Checkable") // << "Flapping: Checkable " << GetName() // << " was: " << (was_flapping) // << " is: " << is_flapping) // << " threshold: " << GetFlappingThreshold() // << "% current: " + GetFlappingCurrent()) << "%."; OnNewCheckResult(this, cr, origin); /* signal status updates to for example db_ido */ OnStateChanged(this); String old_state_str = (service ? Service::StateToString(old_state) : Host::StateToString(Host::CalculateState(old_state))); String new_state_str = (service ? Service::StateToString(new_state) : Host::StateToString(Host::CalculateState(new_state))); /* Whether a hard state change or a volatile state change except OK -> OK happened. */ if (hardChange || (is_volatile && !(IsStateOK(old_state) && IsStateOK(new_state)))) { OnStateChange(this, cr, StateTypeHard, origin); Log(LogNotice, "Checkable") << "State Change: Checkable " << GetName() << " hard state change from " << old_state_str << " to " << new_state_str << " detected." << (is_volatile ? " Checkable is volatile." : ""); } /* Whether a state change happened or the state type is SOFT (must be logged too). */ else if (stateChange || GetStateType() == StateTypeSoft) { OnStateChange(this, cr, StateTypeSoft, origin); Log(LogNotice, "Checkable") << "State Change: Checkable " << GetName() << " soft state change from " << old_state_str << " to " << new_state_str << " detected."; } if (GetStateType() == StateTypeSoft || hardChange || recovery || (is_volatile && !(IsStateOK(old_state) && IsStateOK(new_state)))) ExecuteEventHandler(); if (send_downtime_notification && IsActive()) OnNotificationsRequested(this, in_downtime ? NotificationDowntimeStart : NotificationDowntimeEnd, cr, "", "", MessageOrigin::Ptr()); /* Flapping start/end notifications */ if (!was_flapping && is_flapping) { if (!IsPaused()) OnNotificationsRequested(this, NotificationFlappingStart, cr, "", "", MessageOrigin::Ptr()); Log(LogNotice, "Checkable") << "Flapping: Checkable " << GetName() << " started flapping (" << GetFlappingThreshold() << "% < " << GetFlappingCurrent() << "%)."; NotifyFlapping(origin); } else if (was_flapping && !is_flapping) { if (!IsPaused()) OnNotificationsRequested(this, NotificationFlappingEnd, cr, "", "", MessageOrigin::Ptr()); Log(LogNotice, "Checkable") << "Flapping: Checkable " << GetName() << " stopped flapping (" << GetFlappingThreshold() << "% >= " << GetFlappingCurrent() << "%)."; NotifyFlapping(origin); } /* Problem notifications */ if (send_notification && !is_flapping) { if (!IsPaused()) OnNotificationsRequested(this, recovery ? NotificationRecovery : NotificationProblem, cr, "", "", MessageOrigin::Ptr()); } }