Exemple #1
0
double Checkable::CalculateExecutionTime(const CheckResult::Ptr& cr)
{
    if (!cr)
        return 0;

    return cr->GetExecutionEnd() - cr->GetExecutionStart();
}
void ElasticsearchWriter::NotificationSentToAllUsersHandlerInternal(const Notification::Ptr& notification,
	const Checkable::Ptr& checkable, const std::set<User::Ptr>& users, NotificationType type,
	const CheckResult::Ptr& cr, const String& author, const String& text)
{
	AssertOnWorkQueue();

	CONTEXT("Elasticwriter processing notification to all users '" + checkable->GetName() + "'");

	Log(LogDebug, "ElasticsearchWriter")
		<< "Processing notification for '" << checkable->GetName() << "'";

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	String notificationTypeString = Notification::NotificationTypeToString(type);

	Dictionary::Ptr fields = new Dictionary();

	if (service) {
		fields->Set("service", service->GetShortName());
		fields->Set("state", service->GetState());
		fields->Set("last_state", service->GetLastState());
		fields->Set("last_hard_state", service->GetLastHardState());
	} else {
		fields->Set("state", host->GetState());
		fields->Set("last_state", host->GetLastState());
		fields->Set("last_hard_state", host->GetLastHardState());
	}

	fields->Set("host", host->GetName());

	ArrayData userNames;

	for (const User::Ptr& user : users) {
		userNames.push_back(user->GetName());
	}

	fields->Set("users", new Array(std::move(userNames)));
	fields->Set("notification_type", notificationTypeString);
	fields->Set("author", author);
	fields->Set("text", text);

	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();

	if (commandObj)
		fields->Set("check_command", commandObj->GetName());

	double ts = Utility::GetTime();

	if (cr) {
		AddCheckResult(fields, checkable, cr);
		ts = cr->GetExecutionEnd();
	}

	Enqueue(checkable, "notification", fields, ts);
}
Exemple #3
0
void OpenTsdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
	CONTEXT("Processing check result for '" + checkable->GetName() + "'");

	if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata())
		return;

	Service::Ptr service = dynamic_pointer_cast<Service>(checkable);
	Host::Ptr host;

	if (service)
		host = service->GetHost();
	else
		host = static_pointer_cast<Host>(checkable);

	String metric;
	std::map<String, String> tags;

	String escaped_hostName = EscapeTag(host->GetName());
	tags["host"] = escaped_hostName;

	double ts = cr->GetExecutionEnd();

	if (service) {
		String serviceName = service->GetShortName();
		String escaped_serviceName = EscapeMetric(serviceName);
		metric = "icinga.service." + escaped_serviceName;

		SendMetric(metric + ".state", tags, service->GetState(), ts);
	} else {
		metric = "icinga.host";
		SendMetric(metric + ".state", tags, host->GetState(), ts);
	}

	SendMetric(metric + ".state_type", tags, checkable->GetStateType(), ts);
	SendMetric(metric + ".reachable", tags, checkable->IsReachable(), ts);
	SendMetric(metric + ".downtime_depth", tags, checkable->GetDowntimeDepth(), ts);
	SendMetric(metric + ".acknowledgement", tags, checkable->GetAcknowledgement(), ts);

	SendPerfdata(metric, tags, cr, ts);

	metric = "icinga.check";

	if (service) {
		tags["type"] = "service";
		String serviceName = service->GetShortName();
		String escaped_serviceName = EscapeTag(serviceName);
		tags["service"] = escaped_serviceName;
	} else {
		tags["type"] = "host";
	}

	SendMetric(metric + ".current_attempt", tags, checkable->GetCheckAttempt(), ts);
	SendMetric(metric + ".max_check_attempts", tags, checkable->GetMaxCheckAttempts(), ts);
	SendMetric(metric + ".latency", tags, cr->CalculateLatency(), ts);
	SendMetric(metric + ".execution_time", tags, cr->CalculateExecutionTime(), ts);
}
void ElasticsearchWriter::InternalCheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
	AssertOnWorkQueue();

	CONTEXT("Elasticwriter processing check result for '" + checkable->GetName() + "'");

	if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata())
		return;

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	Dictionary::Ptr fields = new Dictionary();

	if (service) {
		fields->Set("service", service->GetShortName());
		fields->Set("state", service->GetState());
		fields->Set("last_state", service->GetLastState());
		fields->Set("last_hard_state", service->GetLastHardState());
	} else {
		fields->Set("state", host->GetState());
		fields->Set("last_state", host->GetLastState());
		fields->Set("last_hard_state", host->GetLastHardState());
	}

	fields->Set("host", host->GetName());
	fields->Set("state_type", checkable->GetStateType());

	fields->Set("current_check_attempt", checkable->GetCheckAttempt());
	fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());

	fields->Set("reachable", checkable->IsReachable());

	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();

	if (commandObj)
		fields->Set("check_command", commandObj->GetName());

	double ts = Utility::GetTime();

	if (cr) {
		AddCheckResult(fields, checkable, cr);
		ts = cr->GetExecutionEnd();
	}

	Enqueue(checkable, "checkresult", fields, ts);
}
void LogstashWriter::NotificationToUserHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable,
    const User::Ptr& user, NotificationType notification_type, CheckResult::Ptr const& cr,
    const String& author, const String& comment_text, const String& command_name)
{
	CONTEXT("Logstash Processing notification to all users '" + checkable->GetName() + "'");

	Log(LogDebug, "LogstashWriter")
	    << "Processing notification for '" << checkable->GetName() << "'";

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	String notification_type_str = Notification::NotificationTypeToString(notification_type);

	String author_comment = "";

	if (notification_type == NotificationCustom || notification_type == NotificationAcknowledgement) {
		author_comment = author + ";" + comment_text;
	}

	double ts = Utility::GetTime();

	Dictionary::Ptr fields = new Dictionary();

	if (service) {
		fields->Set("type", "SERVICE NOTIFICATION");
		fields->Set("service_name", service->GetShortName());
	} else {
		fields->Set("type", "HOST NOTIFICATION");
	}

	if (cr) {
		fields->Set("plugin_output", cr->GetOutput());
		ts = cr->GetExecutionEnd();
	}

	fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));

	fields->Set("host_name", host->GetName());
	fields->Set("command", command_name);
	fields->Set("notification_type", notification_type_str);
	fields->Set("comment", author_comment);

	SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
void ElasticsearchWriter::StateChangeHandlerInternal(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type)
{
	AssertOnWorkQueue();

	CONTEXT("Elasticwriter processing state change '" + checkable->GetName() + "'");

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	Dictionary::Ptr fields = new Dictionary();

	fields->Set("current_check_attempt", checkable->GetCheckAttempt());
	fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
	fields->Set("host", host->GetName());

	if (service) {
		fields->Set("service", service->GetShortName());
		fields->Set("state", service->GetState());
		fields->Set("last_state", service->GetLastState());
		fields->Set("last_hard_state", service->GetLastHardState());
	} else {
		fields->Set("state", host->GetState());
		fields->Set("last_state", host->GetLastState());
		fields->Set("last_hard_state", host->GetLastHardState());
	}

	CheckCommand::Ptr commandObj = checkable->GetCheckCommand();

	if (commandObj)
		fields->Set("check_command", commandObj->GetName());

	double ts = Utility::GetTime();

	if (cr) {
		AddCheckResult(fields, checkable, cr);
		ts = cr->GetExecutionEnd();
	}

	Enqueue(checkable, "statechange", fields, ts);
}
Exemple #7
0
void InfluxdbWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
	CONTEXT("Processing check result for '" + checkable->GetName() + "'");

	if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata())
		return;

	Host::Ptr host;
	Service::Ptr service;
	boost::tie(host, service) = GetHostService(checkable);

	MacroProcessor::ResolverList resolvers;
	if (service)
		resolvers.push_back(std::make_pair("service", service));
	resolvers.push_back(std::make_pair("host", host));
	resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));

	String prefix;

	double ts = cr->GetExecutionEnd();

	// Clone the template and perform an in-place macro expansion of measurement and tag values
	Dictionary::Ptr tmpl_clean = service ? GetServiceTemplate() : GetHostTemplate();
	Dictionary::Ptr tmpl = static_pointer_cast<Dictionary>(tmpl_clean->Clone());
	tmpl->Set("measurement", MacroProcessor::ResolveMacros(tmpl->Get("measurement"), resolvers, cr));

	Dictionary::Ptr tags = tmpl->Get("tags");
	if (tags) {
		ObjectLock olock(tags);
		for (const Dictionary::Pair& pair : tags) {
			// Prevent missing macros from warning; will return an empty value
			// which will be filtered out in SendMetric()
			String missing_macro;
			tags->Set(pair.first,  MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missing_macro));
		}
	}

	SendPerfdata(tmpl, checkable, cr, ts);
}
void LogstashWriter::StateChangeHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type)
{
	CONTEXT("Logstash Processing state change '" + checkable->GetName() + "'");

	Log(LogDebug, "LogstashWriter")
	    << "Processing state change for '" << checkable->GetName() << "'";

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	Dictionary::Ptr fields = new Dictionary();

	fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));
	fields->Set("type", "StateChange");
	fields->Set("current_check_attempt", checkable->GetCheckAttempt());
	fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());
	fields->Set("hostname", host->GetName());

	if (service) {
		fields->Set("service_name", service->GetShortName());
		fields->Set("service_state", Service::StateToString(service->GetState()));
		fields->Set("last_state", service->GetLastState());
		fields->Set("last_hard_state", service->GetLastHardState());
	} else {
		fields->Set("last_state", host->GetLastState());
		fields->Set("last_hard_state", host->GetLastHardState());
        }

	double ts = Utility::GetTime();

	if (cr) {
		fields->Set("plugin_output", cr->GetOutput());
		fields->Set("check_source", cr->GetCheckSource());
		ts = cr->GetExecutionEnd();
	}

	SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
Exemple #9
0
void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrigin::Ptr& origin)
{
    {
        ObjectLock olock(this);
        m_CheckRunning = false;
    }

    double now = Utility::GetTime();

    if (cr->GetScheduleStart() == 0)
        cr->SetScheduleStart(now);

    if (cr->GetScheduleEnd() == 0)
        cr->SetScheduleEnd(now);

    if (cr->GetExecutionStart() == 0)
        cr->SetExecutionStart(now);

    if (cr->GetExecutionEnd() == 0)
        cr->SetExecutionEnd(now);

    if (!origin || origin->IsLocal()) {
        Log(LogDebug, "Checkable")
                << "No origin or local origin for object '" << GetName()
                << "', setting " << IcingaApplication::GetInstance()->GetNodeName()
                << " as check_source.";
        cr->SetCheckSource(IcingaApplication::GetInstance()->GetNodeName());
    }

    Endpoint::Ptr command_endpoint = GetCommandEndpoint();

    /* override check source if command_endpoint was defined */
    if (command_endpoint && !GetExtension("agent_check")) {
        Log(LogDebug, "Checkable")
                << "command_endpoint found for object '" << GetName()
                << "', setting " << command_endpoint->GetName()
                << " as check_source.";
        cr->SetCheckSource(command_endpoint->GetName());
    }

    /* agent checks go through the api */
    if (command_endpoint && GetExtension("agent_check")) {
        ApiListener::Ptr listener = ApiListener::GetInstance();

        if (listener) {
            /* send message back to its origin */
            Dictionary::Ptr message = ClusterEvents::MakeCheckResultMessage(this, cr);
            listener->SyncSendMessage(command_endpoint, message);
        }

        return;

    }

    bool reachable = IsReachable();
    bool notification_reachable = IsReachable(DependencyNotification);

    ASSERT(!OwnsLock());
    ObjectLock olock(this);

    CheckResult::Ptr old_cr = GetLastCheckResult();
    ServiceState old_state = GetStateRaw();
    StateType old_stateType = GetStateType();
    long old_attempt = GetCheckAttempt();
    bool recovery = false;

    if (old_cr && cr->GetExecutionStart() < old_cr->GetExecutionStart())
        return;

    /* The ExecuteCheck function already sets the old state, but we need to do it again
     * in case this was a passive check result. */
    SetLastStateRaw(old_state);
    SetLastStateType(old_stateType);
    SetLastReachable(reachable);

    long attempt = 1;

    std::set<Checkable::Ptr> children = GetChildren();

    if (!old_cr) {
        SetStateType(StateTypeHard);
    } else if (cr->GetState() == ServiceOK) {
        if (old_state == ServiceOK && old_stateType == StateTypeSoft) {
            SetStateType(StateTypeHard); // SOFT OK -> HARD OK
            recovery = true;
        }

        if (old_state != ServiceOK)
            recovery = true; // NOT OK -> SOFT/HARD OK

        ResetNotificationNumbers();
        SetLastStateOK(Utility::GetTime());

        /* update reachability for child objects in OK state */
        if (!children.empty())
            OnReachabilityChanged(this, cr, children, origin);
    } else {
        if (old_attempt >= GetMaxCheckAttempts()) {
            SetStateType(StateTypeHard);
        } else if (old_stateType == StateTypeSoft && old_state != ServiceOK) {
            SetStateType(StateTypeSoft);
            attempt = old_attempt + 1; //NOT-OK -> NOT-OK counter
        } else if (old_state == ServiceOK) {
            SetStateType(StateTypeSoft);
            attempt = 1; //OK -> NOT-OK transition, reset the counter
        } else {
            attempt = old_attempt;
        }

        switch (cr->GetState()) {
        case ServiceOK:
            /* Nothing to do here. */
            break;
        case ServiceWarning:
            SetLastStateWarning(Utility::GetTime());
            break;
        case ServiceCritical:
            SetLastStateCritical(Utility::GetTime());
            break;
        case ServiceUnknown:
            SetLastStateUnknown(Utility::GetTime());
            break;
        }

        /* update reachability for child objects in NOT-OK state */
        if (!children.empty())
            OnReachabilityChanged(this, cr, children, origin);
    }

    if (!reachable)
        SetLastStateUnreachable(Utility::GetTime());

    SetCheckAttempt(attempt);

    ServiceState new_state = cr->GetState();
    SetStateRaw(new_state);

    bool stateChange = (old_state != new_state);
    if (stateChange) {
        SetLastStateChange(now);

        /* remove acknowledgements */
        if (GetAcknowledgement() == AcknowledgementNormal ||
                (GetAcknowledgement() == AcknowledgementSticky && new_state == ServiceOK)) {
            ClearAcknowledgement();
        }

        /* reschedule direct parents */
        BOOST_FOREACH(const Checkable::Ptr& parent, GetParents()) {
            if (parent.get() == this)
                continue;

            ObjectLock olock(parent);
            parent->SetNextCheck(Utility::GetTime());
        }
    }

    bool remove_acknowledgement_comments = false;

    if (GetAcknowledgement() == AcknowledgementNone)
        remove_acknowledgement_comments = true;

    bool hardChange = (GetStateType() == StateTypeHard && old_stateType == StateTypeSoft);

    if (stateChange && old_stateType == StateTypeHard && GetStateType() == StateTypeHard)
        hardChange = true;

    bool is_volatile = GetVolatile();

    if (hardChange || is_volatile) {
        SetLastHardStateRaw(new_state);
        SetLastHardStateChange(now);
    }

    if (new_state != ServiceOK)
        TriggerDowntimes();

    Host::Ptr host;
    Service::Ptr service;
    tie(host, service) = GetHostService(this);

    CheckableType checkable_type = CheckableHost;
    if (service)
        checkable_type = CheckableService;

    /* statistics for external tools */
    Checkable::UpdateStatistics(cr, checkable_type);

    bool in_downtime = IsInDowntime();
    bool send_notification = hardChange && notification_reachable && !in_downtime && !IsAcknowledged();

    if (!old_cr)
        send_notification = false; /* Don't send notifications for the initial state change */

    if (old_state == ServiceOK && old_stateType == StateTypeSoft)
        send_notification = false; /* Don't send notifications for SOFT-OK -> HARD-OK. */

    if (is_volatile && old_state == ServiceOK && new_state == ServiceOK)
        send_notification = false; /* Don't send notifications for volatile OK -> OK changes. */

    bool send_downtime_notification = (GetLastInDowntime() != in_downtime);
    SetLastInDowntime(in_downtime);

    olock.Unlock();

    if (remove_acknowledgement_comments)
        RemoveCommentsByType(CommentAcknowledgement);

    Dictionary::Ptr vars_after = new Dictionary();
    vars_after->Set("state", new_state);
    vars_after->Set("state_type", GetStateType());
    vars_after->Set("attempt", GetCheckAttempt());
    vars_after->Set("reachable", reachable);

    if (old_cr)
        cr->SetVarsBefore(old_cr->GetVarsAfter());

    cr->SetVarsAfter(vars_after);

    olock.Lock();
    SetLastCheckResult(cr);

    bool was_flapping, is_flapping;

    was_flapping = IsFlapping();
    if (GetStateType() == StateTypeHard)
        UpdateFlappingStatus(stateChange);
    is_flapping = IsFlapping();

    olock.Unlock();

//	Log(LogDebug, "Checkable")
//	    << "Flapping: Checkable " << GetName()
//	    << " was: " << (was_flapping)
//	    << " is: " << is_flapping)
//	    << " threshold: " << GetFlappingThreshold()
//	    << "% current: " + GetFlappingCurrent()) << "%.";

    OnNewCheckResult(this, cr, origin);

    /* signal status updates to for example db_ido */
    OnStateChanged(this);

    String old_state_str = (service ? Service::StateToString(old_state) : Host::StateToString(Host::CalculateState(old_state)));
    String new_state_str = (service ? Service::StateToString(new_state) : Host::StateToString(Host::CalculateState(new_state)));

    if (hardChange || is_volatile) {
        OnStateChange(this, cr, StateTypeHard, origin);
        Log(LogNotice, "Checkable")
                << "State Change: Checkable " << GetName() << " hard state change from " << old_state_str << " to " << new_state_str << " detected." << (is_volatile ? " Checkable is volatile." : "");
    } else if (stateChange) {
        OnStateChange(this, cr, StateTypeSoft, origin);
        Log(LogNotice, "Checkable")
                << "State Change: Checkable " << GetName() << " soft state change from " << old_state_str << " to " << new_state_str << " detected.";
    }

    if (GetStateType() == StateTypeSoft || hardChange || recovery || is_volatile)
        ExecuteEventHandler();

    if (send_downtime_notification)
        OnNotificationsRequested(this, in_downtime ? NotificationDowntimeStart : NotificationDowntimeEnd, cr, "", "");

    if (!was_flapping && is_flapping) {
        OnNotificationsRequested(this, NotificationFlappingStart, cr, "", "");

        Log(LogNotice, "Checkable")
                << "Flapping: Checkable " << GetName() << " started flapping (" << GetFlappingThreshold() << "% < " << GetFlappingCurrent() << "%).";

        NotifyFlapping(origin);
    } else if (was_flapping && !is_flapping) {
        OnNotificationsRequested(this, NotificationFlappingEnd, cr, "", "");

        Log(LogNotice, "Checkable")
                << "Flapping: Checkable " << GetName() << " stopped flapping (" << GetFlappingThreshold() << "% >= " << GetFlappingCurrent() << "%).";

        NotifyFlapping(origin);
    } else if (send_notification)
        OnNotificationsRequested(this, recovery ? NotificationRecovery : NotificationProblem, cr, "", "");
}
void LogstashWriter::CheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
	CONTEXT("LOGSTASH Processing check result for '" + checkable->GetName() + "'");

	Log(LogDebug, "LogstashWriter")
	    << "Processing check result for '" << checkable->GetName() << "'";

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	Dictionary::Ptr fields = new Dictionary();

	if (service) {
		fields->Set("service_name", service->GetShortName());
		fields->Set("service_state", Service::StateToString(service->GetState()));
		fields->Set("last_state", service->GetLastState());
		fields->Set("last_hard_state", service->GetLastHardState());
	} else {
		fields->Set("last_state", host->GetLastState());
		fields->Set("last_hard_state", host->GetLastHardState());
	}

	fields->Set("host_name", host->GetName());
	fields->Set("type", "CheckResult");
	fields->Set("state", service ? Service::StateToString(service->GetState()) : Host::StateToString(host->GetState()));

	fields->Set("current_check_attempt", checkable->GetCheckAttempt());
	fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts());

	fields->Set("latency", cr->CalculateLatency());
	fields->Set("execution_time", cr->CalculateExecutionTime());
	fields->Set("reachable",  checkable->IsReachable());

	double ts = Utility::GetTime();

	if (cr) {
		fields->Set("plugin_output", cr->GetOutput());
		fields->Set("check_source", cr->GetCheckSource());
		ts = cr->GetExecutionEnd();
	}

	Array::Ptr perfdata = cr->GetPerformanceData();

	if (perfdata) {
		Dictionary::Ptr perfdataItems = new Dictionary();

		ObjectLock olock(perfdata);
		for (const Value& val : perfdata) {
			PerfdataValue::Ptr pdv;

			if (val.IsObjectType<PerfdataValue>())
				pdv = val;
			else {
				try {
					pdv = PerfdataValue::Parse(val);
				} catch (const std::exception&) {
					Log(LogWarning, "LogstashWriter")
					    << "Ignoring invalid perfdata value: '" << val << "' for object '"
					    << checkable->GetName() << "'.";
					continue;
				}
			}

			Dictionary::Ptr perfdataItem = new Dictionary();
			perfdataItem->Set("value", pdv->GetValue());

			if (pdv->GetMin())
				perfdataItem->Set("min", pdv->GetMin());
			if (pdv->GetMax())
				perfdataItem->Set("max", pdv->GetMax());
			if (pdv->GetWarn())
				perfdataItem->Set("warn", pdv->GetWarn());
			if (pdv->GetCrit())
				perfdataItem->Set("crit", pdv->GetCrit());

			String escaped_key = EscapeMetricLabel(pdv->GetLabel());

			perfdataItems->Set(escaped_key, perfdataItem);
		}

		fields->Set("performance_data", perfdataItems);
	}

	SendLogMessage(ComposeLogstashMessage(fields, GetSource(), ts));
}
void InfluxdbWriter::CheckResultHandlerWQ(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
	AssertOnWorkQueue();

	CONTEXT("Processing check result for '" + checkable->GetName() + "'");

	if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata())
		return;

	Host::Ptr host;
	Service::Ptr service;
	tie(host, service) = GetHostService(checkable);

	MacroProcessor::ResolverList resolvers;
	if (service)
		resolvers.emplace_back("service", service);
	resolvers.emplace_back("host", host);
	resolvers.emplace_back("icinga", IcingaApplication::GetInstance());

	String prefix;

	double ts = cr->GetExecutionEnd();

	// Clone the template and perform an in-place macro expansion of measurement and tag values
	Dictionary::Ptr tmpl_clean = service ? GetServiceTemplate() : GetHostTemplate();
	Dictionary::Ptr tmpl = static_pointer_cast<Dictionary>(tmpl_clean->Clone());
	tmpl->Set("measurement", MacroProcessor::ResolveMacros(tmpl->Get("measurement"), resolvers, cr));

	Dictionary::Ptr tags = tmpl->Get("tags");
	if (tags) {
		ObjectLock olock(tags);
		for (const Dictionary::Pair& pair : tags) {
			String missing_macro;
			Value value = MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missing_macro);

			if (!missing_macro.IsEmpty())
				continue;

			tags->Set(pair.first, value);
		}
	}

	Array::Ptr perfdata = cr->GetPerformanceData();
	if (perfdata) {
		ObjectLock olock(perfdata);
		for (const Value& val : perfdata) {
			PerfdataValue::Ptr pdv;

			if (val.IsObjectType<PerfdataValue>())
				pdv = val;
			else {
				try {
					pdv = PerfdataValue::Parse(val);
				} catch (const std::exception&) {
					Log(LogWarning, "InfluxdbWriter")
						<< "Ignoring invalid perfdata value: " << val;
					continue;
				}
			}

			Dictionary::Ptr fields = new Dictionary();
			fields->Set("value", pdv->GetValue());

			if (GetEnableSendThresholds()) {
				if (pdv->GetCrit())
					fields->Set("crit", pdv->GetCrit());
				if (pdv->GetWarn())
					fields->Set("warn", pdv->GetWarn());
				if (pdv->GetMin())
					fields->Set("min", pdv->GetMin());
				if (pdv->GetMax())
					fields->Set("max", pdv->GetMax());
			}
			if (!pdv->GetUnit().IsEmpty()) {
				fields->Set("unit", pdv->GetUnit());
			}

			SendMetric(tmpl, pdv->GetLabel(), fields, ts);
		}
	}

	if (GetEnableSendMetadata()) {
		Host::Ptr host;
		Service::Ptr service;
		tie(host, service) = GetHostService(checkable);

		Dictionary::Ptr fields = new Dictionary();

		if (service)
			fields->Set("state", new InfluxdbInteger(service->GetState()));
		else
			fields->Set("state", new InfluxdbInteger(host->GetState()));

		fields->Set("current_attempt", new InfluxdbInteger(checkable->GetCheckAttempt()));
		fields->Set("max_check_attempts", new InfluxdbInteger(checkable->GetMaxCheckAttempts()));
		fields->Set("state_type", new InfluxdbInteger(checkable->GetStateType()));
		fields->Set("reachable", checkable->IsReachable());
		fields->Set("downtime_depth", new InfluxdbInteger(checkable->GetDowntimeDepth()));
		fields->Set("acknowledgement", new InfluxdbInteger(checkable->GetAcknowledgement()));
		fields->Set("latency", cr->CalculateLatency());
		fields->Set("execution_time", cr->CalculateExecutionTime());

		SendMetric(tmpl, Empty, fields, ts);
	}
}
void ElasticsearchWriter::AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
{
	String prefix = "check_result.";

	fields->Set(prefix + "output", cr->GetOutput());
	fields->Set(prefix + "check_source", cr->GetCheckSource());
	fields->Set(prefix + "exit_status", cr->GetExitStatus());
	fields->Set(prefix + "command", cr->GetCommand());
	fields->Set(prefix + "state", cr->GetState());
	fields->Set(prefix + "vars_before", cr->GetVarsBefore());
	fields->Set(prefix + "vars_after", cr->GetVarsAfter());

	fields->Set(prefix + "execution_start", FormatTimestamp(cr->GetExecutionStart()));
	fields->Set(prefix + "execution_end", FormatTimestamp(cr->GetExecutionEnd()));
	fields->Set(prefix + "schedule_start", FormatTimestamp(cr->GetScheduleStart()));
	fields->Set(prefix + "schedule_end", FormatTimestamp(cr->GetScheduleEnd()));

	/* Add extra calculated field. */
	fields->Set(prefix + "latency", cr->CalculateLatency());
	fields->Set(prefix + "execution_time", cr->CalculateExecutionTime());

	if (!GetEnableSendPerfdata())
		return;

	Array::Ptr perfdata = cr->GetPerformanceData();

	CheckCommand::Ptr checkCommand = checkable->GetCheckCommand();

	if (perfdata) {
		ObjectLock olock(perfdata);
		for (const Value& val : perfdata) {
			PerfdataValue::Ptr pdv;

			if (val.IsObjectType<PerfdataValue>())
				pdv = val;
			else {
				try {
					pdv = PerfdataValue::Parse(val);
				} catch (const std::exception&) {
					Log(LogWarning, "ElasticsearchWriter")
						<< "Ignoring invalid perfdata for checkable '"
						<< checkable->GetName() << "' and command '"
						<< checkCommand->GetName() << "' with value: " << val;
					continue;
				}
			}

			String escapedKey = pdv->GetLabel();
			boost::replace_all(escapedKey, " ", "_");
			boost::replace_all(escapedKey, ".", "_");
			boost::replace_all(escapedKey, "\\", "_");
			boost::algorithm::replace_all(escapedKey, "::", ".");

			String perfdataPrefix = prefix + "perfdata." + escapedKey;

			fields->Set(perfdataPrefix + ".value", pdv->GetValue());

			if (pdv->GetMin())
				fields->Set(perfdataPrefix + ".min", pdv->GetMin());
			if (pdv->GetMax())
				fields->Set(perfdataPrefix + ".max", pdv->GetMax());
			if (pdv->GetWarn())
				fields->Set(perfdataPrefix + ".warn", pdv->GetWarn());
			if (pdv->GetCrit())
				fields->Set(perfdataPrefix + ".crit", pdv->GetCrit());

			if (!pdv->GetUnit().IsEmpty())
				fields->Set(perfdataPrefix + ".unit", pdv->GetUnit());
		}
	}
}