Exemple #1
0
void Checkable::NotifyDowntimeInternal(const Downtime::Ptr& downtime)
{
	Checkable::Ptr checkable = downtime->GetCheckable();

	if (!checkable->IsPaused())
		OnNotificationsRequested(checkable, NotificationDowntimeStart, checkable->GetLastCheckResult(), downtime->GetAuthor(), downtime->GetComment(), nullptr);
}
/**
 * Periodically sends notifications.
 *
 * @param - Event arguments for the timer.
 */
void NotificationComponent::NotificationTimerHandler(void)
{
	double now = Utility::GetTime();

	for (const Notification::Ptr& notification : ConfigType::GetObjectsByType<Notification>()) {
		if (!notification->IsActive())
			continue;

		if (notification->IsPaused() && GetEnableHA())
			continue;

		Checkable::Ptr checkable = notification->GetCheckable();

		if (!IcingaApplication::GetInstance()->GetEnableNotifications() || !checkable->GetEnableNotifications())
			continue;

		if (notification->GetInterval() <= 0 && notification->GetNoMoreNotifications())
			continue;

		if (notification->GetNextNotification() > now)
			continue;

		bool reachable = checkable->IsReachable(DependencyNotification);

		{
			ObjectLock olock(notification);
			notification->SetNextNotification(Utility::GetTime() + notification->GetInterval());
		}

		{
			Host::Ptr host;
			Service::Ptr service;
			tie(host, service) = GetHostService(checkable);

			ObjectLock olock(checkable);

			if (checkable->GetStateType() == StateTypeSoft)
				continue;

			if ((service && service->GetState() == ServiceOK) || (!service && host->GetState() == HostUp))
				continue;

			if (!reachable || checkable->IsInDowntime() || checkable->IsAcknowledged())
				continue;
		}

		try {
			Log(LogNotice, "NotificationComponent")
			    << "Attempting to send reminder notification '" << notification->GetName() << "'";
			notification->BeginExecuteNotification(NotificationProblem, checkable->GetLastCheckResult(), false, true);
		} catch (const std::exception& ex) {
			Log(LogWarning, "NotificationComponent")
			    << "Exception occured during notification for object '"
			    << GetName() << "': " << DiagnosticInformation(ex);
		}
	}
}
void DummyCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
	const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
	REQUIRE_NOT_NULL(checkable);
	REQUIRE_NOT_NULL(cr);

	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	MacroProcessor::ResolverList resolvers;
	if (service)
		resolvers.emplace_back("service", service);
	resolvers.emplace_back("host", host);
	resolvers.emplace_back("command", commandObj);
	resolvers.emplace_back("icinga", IcingaApplication::GetInstance());

	int dummyState = MacroProcessor::ResolveMacros("$dummy_state$", resolvers, checkable->GetLastCheckResult(),
		nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);

	String dummyText = MacroProcessor::ResolveMacros("$dummy_text$", resolvers, checkable->GetLastCheckResult(),
		nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);

	if (resolvedMacros && !useResolvedMacros)
		return;

	/* Parse output and performance data. */
	std::pair<String, String> co = PluginUtility::ParseCheckOutput(dummyText);

	double now = Utility::GetTime();

	cr->SetOutput(co.first);
	cr->SetPerformanceData(PluginUtility::SplitPerfdata(co.second));
	cr->SetState(PluginUtility::ExitStatusToState(dummyState));
	cr->SetExitStatus(dummyState);
	cr->SetExecutionStart(now);
	cr->SetExecutionEnd(now);

	checkable->ProcessCheckResult(cr);
}
Exemple #4
0
void Checkable::NotifyDowntimeEnd(const Downtime::Ptr& downtime)
{
	/* don't send notifications for flexible downtimes which never triggered */
	if (!downtime->GetFixed() && !downtime->IsTriggered())
		return;

	Checkable::Ptr checkable = downtime->GetCheckable();

	if (!checkable->IsPaused())
		OnNotificationsRequested(checkable, NotificationDowntimeEnd, checkable->GetLastCheckResult(), downtime->GetAuthor(), downtime->GetComment(), nullptr);
}
void PluginCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	MacroProcessor::ResolverList resolvers;
	if (service)
		resolvers.push_back(std::make_pair("service", service));
	resolvers.push_back(std::make_pair("host", host));
	resolvers.push_back(std::make_pair("command", commandObj));
	resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));

	PluginUtility::ExecuteCommand(commandObj, checkable, checkable->GetLastCheckResult(), resolvers, boost::bind(&PluginCheckTask::ProcessFinishedHandler, checkable, cr, _1, _2));
}
Exemple #6
0
Dictionary::Ptr ApiActions::SendCustomNotification(const ConfigObject::Ptr& object,
    const Dictionary::Ptr& params)
{
	Checkable::Ptr checkable = static_pointer_cast<Checkable>(object);

	if (!checkable)
		return ApiActions::CreateResult(404, "Cannot send notification for non-existent object.");

	if (!params->Contains("author"))
		return ApiActions::CreateResult(403, "Parameter 'author' is required.");

	if (!params->Contains("comment"))
		return ApiActions::CreateResult(403, "Parameter 'comment' is required.");

	if (Convert::ToBool(HttpUtility::GetLastParameter(params, "force")))
		checkable->SetForceNextNotification(true);

	Checkable::OnNotificationsRequested(checkable, NotificationCustom, checkable->GetLastCheckResult(),
	    HttpUtility::GetLastParameter(params, "author"), HttpUtility::GetLastParameter(params, "comment"), MessageOrigin::Ptr());

	return ApiActions::CreateResult(200, "Successfully sent custom notification for object '" + checkable->GetName() + "'.");
}
bool Dependency::IsAvailable(DependencyType dt) const
{
	Checkable::Ptr parent = GetParent();

	Host::Ptr parentHost;
	Service::Ptr parentService;
	tie(parentHost, parentService) = GetHostService(parent);

	/* ignore if it's the same checkable object */
	if (parent == GetChild()) {
		Log(LogNotice, "Dependency")
		    << "Dependency '" << GetName() << "' passed: Parent and child " << (parentService ? "service" : "host") << " are identical.";
		return true;
	}

	/* ignore pending  */
	if (!parent->GetLastCheckResult()) {
		Log(LogNotice, "Dependency")
		    << "Dependency '" << GetName() << "' passed: Parent " << (parentService ? "service" : "host") << " '" << parent->GetName() << "' hasn't been checked yet.";
		return true;
	}

	if (GetIgnoreSoftStates()) {
		/* ignore soft states */
		if (parent->GetStateType() == StateTypeSoft) {
			Log(LogNotice, "Dependency")
			    << "Dependency '" << GetName() << "' passed: Parent " << (parentService ? "service" : "host") << " '" << parent->GetName() << "' is in a soft state.";
			return true;
		}
	} else {
		Log(LogNotice, "Dependency")
		    << "Dependency '" << GetName() << "' failed: Parent " << (parentService ? "service" : "host") << " '" << parent->GetName() << "' is in a soft state.";
	}

	int state;

	if (parentService)
		state = ServiceStateToFilter(parentService->GetState());
	else
		state = HostStateToFilter(parentHost->GetState());

	/* check state */
	if (state & GetStateFilter()) {
		Log(LogNotice, "Dependency")
		    << "Dependency '" << GetName() << "' passed: Parent " << (parentService ? "service" : "host") << " '" << parent->GetName() << "' matches state filter.";
		return true;
	}

	/* ignore if not in time period */
	TimePeriod::Ptr tp = GetPeriod();
	if (tp && !tp->IsInside(Utility::GetTime())) {
		Log(LogNotice, "Dependency")
		    << "Dependency '" << GetName() << "' passed: Outside time period.";
		return true;
	}

	if (dt == DependencyCheckExecution && !GetDisableChecks()) {
		Log(LogNotice, "Dependency")
		    << "Dependency '" << GetName() << "' passed: Checks are not disabled.";
		return true;
	} else if (dt == DependencyNotification && !GetDisableNotifications()) {
		Log(LogNotice, "Dependency")
		    << "Dependency '" << GetName() << "' passed: Notifications are not disabled";
		return true;
	}

	Log(LogNotice, "Dependency")
	    << "Dependency '" << GetName() << "' failed. Parent "
	    << (parentService ? "service" : "host") << " '" << parent->GetName() << "' is "
	    << (parentService ? Service::StateToString(parentService->GetState()) : Host::StateToString(parentHost->GetState()));

	return false;
}
int CompatUtility::GetCheckableHasBeenChecked(const Checkable::Ptr& checkable)
{
	return (checkable->GetLastCheckResult() ? 1 : 0);
}
Exemple #9
0
void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
    const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
	Value raw_command = commandObj->GetCommandLine();

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	MacroProcessor::ResolverList resolvers;
	if (service)
		resolvers.push_back(std::make_pair("service", service));
	resolvers.push_back(std::make_pair("host", host));
	resolvers.push_back(std::make_pair("command", commandObj));
	resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));

	String idoType = MacroProcessor::ResolveMacros("$ido_type$", resolvers, checkable->GetLastCheckResult(),
	    NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);

	if (resolvedMacros && !useResolvedMacros)
		return;

	if (idoType.IsEmpty()) {
		cr->SetOutput("Macro 'ido_type' must be set.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	String idoName = MacroProcessor::ResolveMacros("$ido_name$", resolvers, checkable->GetLastCheckResult(),
	    NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);

	if (resolvedMacros && !useResolvedMacros)
		return;

	if (idoName.IsEmpty()) {
		cr->SetOutput("Macro 'ido_name' must be set.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	Type::Ptr type = Type::GetByName(idoType);

	if (!type || !DbConnection::TypeInstance->IsAssignableFrom(type)) {
		cr->SetOutput("IDO type '" + idoType + "' is invalid.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	ConfigType::Ptr dtype = ConfigType::GetByName(idoType);
	VERIFY(dtype);

	DbConnection::Ptr conn = static_pointer_cast<DbConnection>(dtype->GetObject(idoName));

	double qps = conn->GetQueryCount(60) / 60.0;

	if (!conn->GetConnected()) {
		if (conn->GetShouldConnect()) {
			cr->SetOutput("Could not connect to the database server.");
			cr->SetState(ServiceCritical);
		} else {
			cr->SetOutput("Not currently enabled: Another cluster instance is responsible for the IDO database.");
			cr->SetState(ServiceOK);
		}
	} else {
		String schema_version = conn->GetSchemaVersion();

		if (Utility::CompareVersion(IDO_CURRENT_SCHEMA_VERSION, schema_version) < 0) {
			cr->SetOutput("Outdated schema version: " + schema_version + "; Latest version: "  IDO_CURRENT_SCHEMA_VERSION);
			cr->SetState(ServiceWarning);
		} else {
			std::ostringstream msgbuf;
			msgbuf << "Connected to the database server; queries per second: "  << std::fixed << std::setprecision(3) << qps;
			cr->SetOutput(msgbuf.str());
			cr->SetState(ServiceOK);
		}
	}

	Array::Ptr perfdata = new Array();
	perfdata->Add(new PerfdataValue("queries", qps));
	perfdata->Add(new PerfdataValue("queries_1min", conn->GetQueryCount(60)));
	perfdata->Add(new PerfdataValue("queries_5mins", conn->GetQueryCount(5 * 60)));
	perfdata->Add(new PerfdataValue("queries_15mins", conn->GetQueryCount(15 * 60)));
	perfdata->Add(new PerfdataValue("pending_queries", conn->GetPendingQueryCount()));
	cr->SetPerformanceData(perfdata);

	checkable->ProcessCheckResult(cr);
}
Exemple #10
0
void StatusDataWriter::DumpCheckableStatusAttrs(std::ostream& fp, const Checkable::Ptr& checkable)
{
	CheckResult::Ptr cr = checkable->GetLastCheckResult();

	EventCommand::Ptr eventcommand = checkable->GetEventCommand();
	CheckCommand::Ptr checkcommand = checkable->GetCheckCommand();

	fp << "\t" << "check_command=" << CompatUtility::GetCommandName(checkcommand) << "!" << CompatUtility::GetCheckableCommandArgs(checkable) << "\n"
		"\t" "event_handler=" << CompatUtility::GetCommandName(eventcommand) << "\n"
		"\t" "check_interval=" << (checkable->GetCheckInterval() / 60.0) << "\n"
		"\t" "retry_interval=" << (checkable->GetRetryInterval() / 60.0) << "\n"
		"\t" "has_been_checked=" << Convert::ToLong(checkable->HasBeenChecked()) << "\n"
		"\t" "should_be_scheduled=" << checkable->GetEnableActiveChecks() << "\n"
		"\t" "event_handler_enabled=" << Convert::ToLong(checkable->GetEnableEventHandler()) << "\n";

	TimePeriod::Ptr checkPeriod = checkable->GetCheckPeriod();
	if (checkPeriod)
		fp << "\t" "check_period" "\t" << checkPeriod->GetName() << "\n";

	if (cr) {
		fp << "\t" << "check_execution_time=" << Convert::ToString(cr->CalculateExecutionTime()) << "\n"
			"\t" "check_latency=" << Convert::ToString(cr->CalculateLatency()) << "\n";
	}

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	if (service) {
		fp << "\t" "current_state=" << service->GetState() << "\n"
			"\t" "last_hard_state=" << service->GetLastHardState() << "\n"
			"\t" "last_time_ok=" << static_cast<int>(service->GetLastStateOK()) << "\n"
			"\t" "last_time_warn=" << static_cast<int>(service->GetLastStateWarning()) << "\n"
			"\t" "last_time_critical=" << static_cast<int>(service->GetLastStateCritical()) << "\n"
			"\t" "last_time_unknown=" << static_cast<int>(service->GetLastStateUnknown()) << "\n";
	} else {
		int currentState = host->GetState();

		if (currentState != HostUp && !host->IsReachable())
			currentState = 2; /* hardcoded compat state */

		fp << "\t" "current_state=" << currentState << "\n"
			"\t" "last_hard_state=" << host->GetLastHardState() << "\n"
			"\t" "last_time_up=" << static_cast<int>(host->GetLastStateUp()) << "\n"
			"\t" "last_time_down=" << static_cast<int>(host->GetLastStateDown()) << "\n";
	}

	fp << "\t" "state_type=" << checkable->GetStateType() << "\n"
		"\t" "last_check=" << static_cast<long>(host->GetLastCheck()) << "\n";

	if (cr) {
		fp << "\t" "plugin_output=" << CompatUtility::GetCheckResultOutput(cr) << "\n"
			"\t" "long_plugin_output=" << CompatUtility::GetCheckResultLongOutput(cr) << "\n"
			"\t" "performance_data=" << PluginUtility::FormatPerfdata(cr->GetPerformanceData()) << "\n";
	}

	fp << "\t" << "next_check=" << static_cast<long>(checkable->GetNextCheck()) << "\n"
		"\t" "current_attempt=" << checkable->GetCheckAttempt() << "\n"
		"\t" "max_attempts=" << checkable->GetMaxCheckAttempts() << "\n"
		"\t" "last_state_change=" << static_cast<long>(checkable->GetLastStateChange()) << "\n"
		"\t" "last_hard_state_change=" << static_cast<long>(checkable->GetLastHardStateChange()) << "\n"
		"\t" "last_update=" << static_cast<long>(Utility::GetTime()) << "\n"
		"\t" "notifications_enabled=" << Convert::ToLong(checkable->GetEnableNotifications()) << "\n"
		"\t" "active_checks_enabled=" << Convert::ToLong(checkable->GetEnableActiveChecks()) << "\n"
		"\t" "passive_checks_enabled=" << Convert::ToLong(checkable->GetEnablePassiveChecks()) << "\n"
		"\t" "flap_detection_enabled=" << Convert::ToLong(checkable->GetEnableFlapping()) << "\n"
		"\t" "is_flapping=" << Convert::ToLong(checkable->IsFlapping()) << "\n"
		"\t" "percent_state_change=" << checkable->GetFlappingCurrent() << "\n"
		"\t" "problem_has_been_acknowledged=" << (checkable->GetAcknowledgement() != AcknowledgementNone ? 1 : 0) << "\n"
		"\t" "acknowledgement_type=" << checkable->GetAcknowledgement() << "\n"
		"\t" "acknowledgement_end_time=" << checkable->GetAcknowledgementExpiry() << "\n"
		"\t" "scheduled_downtime_depth=" << checkable->GetDowntimeDepth() << "\n"
		"\t" "last_notification=" << CompatUtility::GetCheckableNotificationLastNotification(checkable) << "\n"
		"\t" "next_notification=" << CompatUtility::GetCheckableNotificationNextNotification(checkable) << "\n"
		"\t" "current_notification_number=" << CompatUtility::GetCheckableNotificationNotificationNumber(checkable) << "\n"
		"\t" "is_reachable=" << Convert::ToLong(checkable->IsReachable()) << "\n";
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
	const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
	REQUIRE_NOT_NULL(checkable);
	REQUIRE_NOT_NULL(cr);

	ApiListener::Ptr listener = ApiListener::GetInstance();

	if (!listener) {
		cr->SetOutput("No API listener is configured for this instance.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
	Value raw_command = commandObj->GetCommandLine();

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	MacroProcessor::ResolverList resolvers;
	if (service)
		resolvers.emplace_back("service", service);
	resolvers.emplace_back("host", host);
	resolvers.emplace_back("command", commandObj);
	resolvers.emplace_back("icinga", IcingaApplication::GetInstance());

	String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(),
		nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);

	String missingLagWarning;
	String missingLagCritical;

	double lagWarning = MacroProcessor::ResolveMacros("$cluster_lag_warning$", resolvers, checkable->GetLastCheckResult(),
		&missingLagWarning, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);

	double lagCritical = MacroProcessor::ResolveMacros("$cluster_lag_critical$", resolvers, checkable->GetLastCheckResult(),
		&missingLagCritical, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);

	if (resolvedMacros && !useResolvedMacros)
		return;

	if (zoneName.IsEmpty()) {
		cr->SetOutput("Macro 'cluster_zone' must be set.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	Zone::Ptr zone = Zone::GetByName(zoneName);

	if (!zone) {
		cr->SetOutput("Zone '" + zoneName + "' does not exist.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	bool connected = false;
	double zoneLag = 0;

	double lastMessageSent = 0;
	double lastMessageReceived = 0;
	double messagesSentPerSecond = 0;
	double messagesReceivedPerSecond = 0;
	double bytesSentPerSecond = 0;
	double bytesReceivedPerSecond = 0;

	for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
		if (endpoint->GetConnected())
			connected = true;

		double eplag = ApiListener::CalculateZoneLag(endpoint);

		if (eplag > 0 && eplag > zoneLag)
			zoneLag = eplag;

		if (endpoint->GetLastMessageSent() > lastMessageSent)
			lastMessageSent = endpoint->GetLastMessageSent();

		if (endpoint->GetLastMessageReceived() > lastMessageReceived)
			lastMessageReceived = endpoint->GetLastMessageReceived();

		messagesSentPerSecond += endpoint->GetMessagesSentPerSecond();
		messagesReceivedPerSecond += endpoint->GetMessagesReceivedPerSecond();
		bytesSentPerSecond += endpoint->GetBytesSentPerSecond();
		bytesReceivedPerSecond += endpoint->GetBytesReceivedPerSecond();
	}

	if (connected) {
		cr->SetState(ServiceOK);
		cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));

		/* Check whether the thresholds have been resolved and compare them */
		if (missingLagCritical.IsEmpty() && zoneLag > lagCritical) {
			cr->SetState(ServiceCritical);
			cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
				+ " greater than critical threshold: " + Utility::FormatDuration(lagCritical));
		} else if (missingLagWarning.IsEmpty() && zoneLag > lagWarning) {
			cr->SetState(ServiceWarning);
			cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)
				+ " greater than warning threshold: " + Utility::FormatDuration(lagWarning));
		}
	} else {
		cr->SetState(ServiceCritical);
		cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
	}

	cr->SetPerformanceData(new Array({
		new PerfdataValue("slave_lag", zoneLag, false, "s", lagWarning, lagCritical),
		new PerfdataValue("last_messages_sent", lastMessageSent),
		new PerfdataValue("last_messages_received", lastMessageReceived),
		new PerfdataValue("sum_messages_sent_per_second", messagesSentPerSecond),
		new PerfdataValue("sum_messages_received_per_second", messagesReceivedPerSecond),
		new PerfdataValue("sum_bytes_sent_per_second", bytesSentPerSecond),
		new PerfdataValue("sum_bytes_received_per_second", bytesReceivedPerSecond)
	}));

	checkable->ProcessCheckResult(cr);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
    const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
	ApiListener::Ptr listener = ApiListener::GetInstance();

	if (!listener) {
		cr->SetOutput("No API listener is configured for this instance.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
	Value raw_command = commandObj->GetCommandLine();

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	MacroProcessor::ResolverList resolvers;
	if (service)
		resolvers.push_back(std::make_pair("service", service));
	resolvers.push_back(std::make_pair("host", host));
	resolvers.push_back(std::make_pair("command", commandObj));
	resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));

	String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(),
	    NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);

	if (resolvedMacros && !useResolvedMacros)
		return;

	if (zoneName.IsEmpty()) {
		cr->SetOutput("Macro 'cluster_zone' must be set.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	Zone::Ptr zone = Zone::GetByName(zoneName);

	if (!zone) {
		cr->SetOutput("Zone '" + zoneName + "' does not exist.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	bool connected = false;
	double zoneLag = 0;

	BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
		if (endpoint->GetConnected())
			connected = true;

		double eplag = ApiListener::CalculateZoneLag(endpoint);

		if (eplag > 0 && eplag > zoneLag)
			zoneLag = eplag;
	}

	if (!connected) {
		cr->SetState(ServiceCritical);
		cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag));
	} else {
		cr->SetState(ServiceOK);
		cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag));
	}

	Array::Ptr perfdata = new Array();
	perfdata->Add(new PerfdataValue("slave_lag", zoneLag));
	cr->SetPerformanceData(perfdata);

	checkable->ProcessCheckResult(cr);
}
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr,
    const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
{
	ApiListener::Ptr listener = ApiListener::GetInstance();

	if (!listener) {
		cr->SetOutput("No API listener is configured for this instance.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();
	Value raw_command = commandObj->GetCommandLine();

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	MacroProcessor::ResolverList resolvers;
	if (service)
		resolvers.push_back(std::make_pair("service", service));
	resolvers.push_back(std::make_pair("host", host));
	resolvers.push_back(std::make_pair("command", commandObj));
	resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));

	String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult());

	if (zoneName.IsEmpty()) {
		cr->SetOutput("Macro 'cluster_zone' must be set.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	Zone::Ptr zone = Zone::GetByName(zoneName);

	if (!zone) {
		cr->SetOutput("Zone '" + zoneName + "' does not exist.");
		cr->SetState(ServiceUnknown);
		checkable->ProcessCheckResult(cr);
		return;
	}

	bool connected = false;

	BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
		if (endpoint->IsConnected()) {
			connected = true;
			break;
		}
	}

	if (!connected) {
		cr->SetState(ServiceCritical);
		cr->SetOutput("Zone '" + zoneName + "' is not connected.");
	} else {
		cr->SetState(ServiceOK);
		cr->SetOutput("Zone '" + zoneName + "' is connected.");
	}

	checkable->ProcessCheckResult(cr);
}