void ElasticsearchWriter::NotificationSentToAllUsersHandlerInternal(const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set<User::Ptr>& users, NotificationType type, const CheckResult::Ptr& cr, const String& author, const String& text) { AssertOnWorkQueue(); CONTEXT("Elasticwriter processing notification to all users '" + checkable->GetName() + "'"); Log(LogDebug, "ElasticsearchWriter") << "Processing notification for '" << checkable->GetName() << "'"; Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); String notificationTypeString = Notification::NotificationTypeToString(type); Dictionary::Ptr fields = new Dictionary(); if (service) { fields->Set("service", service->GetShortName()); fields->Set("state", service->GetState()); fields->Set("last_state", service->GetLastState()); fields->Set("last_hard_state", service->GetLastHardState()); } else { fields->Set("state", host->GetState()); fields->Set("last_state", host->GetLastState()); fields->Set("last_hard_state", host->GetLastHardState()); } fields->Set("host", host->GetName()); ArrayData userNames; for (const User::Ptr& user : users) { userNames.push_back(user->GetName()); } fields->Set("users", new Array(std::move(userNames))); fields->Set("notification_type", notificationTypeString); fields->Set("author", author); fields->Set("text", text); CheckCommand::Ptr commandObj = checkable->GetCheckCommand(); if (commandObj) fields->Set("check_command", commandObj->GetName()); double ts = Utility::GetTime(); if (cr) { AddCheckResult(fields, checkable, cr); ts = cr->GetExecutionEnd(); } Enqueue(checkable, "notification", fields, ts); }
String CompatUtility::GetCheckableCheckCommand(const Checkable::Ptr& checkable) { String check_command_str; CheckCommand::Ptr checkcommand = checkable->GetCheckCommand(); if (checkcommand) check_command_str = checkcommand->GetName(); return check_command_str; }
void ElasticsearchWriter::InternalCheckResultHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr) { AssertOnWorkQueue(); CONTEXT("Elasticwriter processing check result for '" + checkable->GetName() + "'"); if (!IcingaApplication::GetInstance()->GetEnablePerfdata() || !checkable->GetEnablePerfdata()) return; Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); Dictionary::Ptr fields = new Dictionary(); if (service) { fields->Set("service", service->GetShortName()); fields->Set("state", service->GetState()); fields->Set("last_state", service->GetLastState()); fields->Set("last_hard_state", service->GetLastHardState()); } else { fields->Set("state", host->GetState()); fields->Set("last_state", host->GetLastState()); fields->Set("last_hard_state", host->GetLastHardState()); } fields->Set("host", host->GetName()); fields->Set("state_type", checkable->GetStateType()); fields->Set("current_check_attempt", checkable->GetCheckAttempt()); fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts()); fields->Set("reachable", checkable->IsReachable()); CheckCommand::Ptr commandObj = checkable->GetCheckCommand(); if (commandObj) fields->Set("check_command", commandObj->GetName()); double ts = Utility::GetTime(); if (cr) { AddCheckResult(fields, checkable, cr); ts = cr->GetExecutionEnd(); } Enqueue(checkable, "checkresult", fields, ts); }
void ElasticsearchWriter::StateChangeHandlerInternal(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type) { AssertOnWorkQueue(); CONTEXT("Elasticwriter processing state change '" + checkable->GetName() + "'"); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); Dictionary::Ptr fields = new Dictionary(); fields->Set("current_check_attempt", checkable->GetCheckAttempt()); fields->Set("max_check_attempts", checkable->GetMaxCheckAttempts()); fields->Set("host", host->GetName()); if (service) { fields->Set("service", service->GetShortName()); fields->Set("state", service->GetState()); fields->Set("last_state", service->GetLastState()); fields->Set("last_hard_state", service->GetLastHardState()); } else { fields->Set("state", host->GetState()); fields->Set("last_state", host->GetLastState()); fields->Set("last_hard_state", host->GetLastHardState()); } CheckCommand::Ptr commandObj = checkable->GetCheckCommand(); if (commandObj) fields->Set("check_command", commandObj->GetName()); double ts = Utility::GetTime(); if (cr) { AddCheckResult(fields, checkable, cr); ts = cr->GetExecutionEnd(); } Enqueue(checkable, "statechange", fields, ts); }
/* service */ String CompatUtility::GetCheckableCommandArgs(const Checkable::Ptr& checkable) { CheckCommand::Ptr command = checkable->GetCheckCommand(); Dictionary::Ptr args = new Dictionary(); if (command) { Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); String command_line = GetCommandLine(command); Dictionary::Ptr command_vars = command->GetVars(); if (command_vars) { ObjectLock olock(command_vars); for (const Dictionary::Pair& kv : command_vars) { String macro = "$" + kv.first + "$"; // this is too simple if (command_line.Contains(macro)) args->Set(kv.first, kv.second); } } Dictionary::Ptr host_vars = host->GetVars(); if (host_vars) { ObjectLock olock(host_vars); for (const Dictionary::Pair& kv : host_vars) { String macro = "$" + kv.first + "$"; // this is too simple if (command_line.Contains(macro)) args->Set(kv.first, kv.second); macro = "$host.vars." + kv.first + "$"; if (command_line.Contains(macro)) args->Set(kv.first, kv.second); } } if (service) { Dictionary::Ptr service_vars = service->GetVars(); if (service_vars) { ObjectLock olock(service_vars); for (const Dictionary::Pair& kv : service_vars) { String macro = "$" + kv.first + "$"; // this is too simple if (command_line.Contains(macro)) args->Set(kv.first, kv.second); macro = "$service.vars." + kv.first + "$"; if (command_line.Contains(macro)) args->Set(kv.first, kv.second); } } } String arg_string; ObjectLock olock(args); for (const Dictionary::Pair& kv : args) { arg_string += Convert::ToString(kv.first) + "=" + Convert::ToString(kv.second) + "!"; } return arg_string; } return Empty; }
void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros) { CheckCommand::Ptr commandObj = checkable->GetCheckCommand(); Value raw_command = commandObj->GetCommandLine(); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); MacroProcessor::ResolverList resolvers; if (service) resolvers.push_back(std::make_pair("service", service)); resolvers.push_back(std::make_pair("host", host)); resolvers.push_back(std::make_pair("command", commandObj)); resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance())); String idoType = MacroProcessor::ResolveMacros("$ido_type$", resolvers, checkable->GetLastCheckResult(), NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros); if (resolvedMacros && !useResolvedMacros) return; if (idoType.IsEmpty()) { cr->SetOutput("Macro 'ido_type' must be set."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } String idoName = MacroProcessor::ResolveMacros("$ido_name$", resolvers, checkable->GetLastCheckResult(), NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros); if (resolvedMacros && !useResolvedMacros) return; if (idoName.IsEmpty()) { cr->SetOutput("Macro 'ido_name' must be set."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } Type::Ptr type = Type::GetByName(idoType); if (!type || !DbConnection::TypeInstance->IsAssignableFrom(type)) { cr->SetOutput("IDO type '" + idoType + "' is invalid."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } ConfigType::Ptr dtype = ConfigType::GetByName(idoType); VERIFY(dtype); DbConnection::Ptr conn = static_pointer_cast<DbConnection>(dtype->GetObject(idoName)); double qps = conn->GetQueryCount(60) / 60.0; if (!conn->GetConnected()) { if (conn->GetShouldConnect()) { cr->SetOutput("Could not connect to the database server."); cr->SetState(ServiceCritical); } else { cr->SetOutput("Not currently enabled: Another cluster instance is responsible for the IDO database."); cr->SetState(ServiceOK); } } else { String schema_version = conn->GetSchemaVersion(); if (Utility::CompareVersion(IDO_CURRENT_SCHEMA_VERSION, schema_version) < 0) { cr->SetOutput("Outdated schema version: " + schema_version + "; Latest version: " IDO_CURRENT_SCHEMA_VERSION); cr->SetState(ServiceWarning); } else { std::ostringstream msgbuf; msgbuf << "Connected to the database server; queries per second: " << std::fixed << std::setprecision(3) << qps; cr->SetOutput(msgbuf.str()); cr->SetState(ServiceOK); } } Array::Ptr perfdata = new Array(); perfdata->Add(new PerfdataValue("queries", qps)); perfdata->Add(new PerfdataValue("queries_1min", conn->GetQueryCount(60))); perfdata->Add(new PerfdataValue("queries_5mins", conn->GetQueryCount(5 * 60))); perfdata->Add(new PerfdataValue("queries_15mins", conn->GetQueryCount(15 * 60))); perfdata->Add(new PerfdataValue("pending_queries", conn->GetPendingQueryCount())); cr->SetPerformanceData(perfdata); checkable->ProcessCheckResult(cr); }
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros) { REQUIRE_NOT_NULL(checkable); REQUIRE_NOT_NULL(cr); ApiListener::Ptr listener = ApiListener::GetInstance(); if (!listener) { cr->SetOutput("No API listener is configured for this instance."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } CheckCommand::Ptr commandObj = checkable->GetCheckCommand(); Value raw_command = commandObj->GetCommandLine(); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); MacroProcessor::ResolverList resolvers; if (service) resolvers.emplace_back("service", service); resolvers.emplace_back("host", host); resolvers.emplace_back("command", commandObj); resolvers.emplace_back("icinga", IcingaApplication::GetInstance()); String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(), nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros); String missingLagWarning; String missingLagCritical; double lagWarning = MacroProcessor::ResolveMacros("$cluster_lag_warning$", resolvers, checkable->GetLastCheckResult(), &missingLagWarning, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros); double lagCritical = MacroProcessor::ResolveMacros("$cluster_lag_critical$", resolvers, checkable->GetLastCheckResult(), &missingLagCritical, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros); if (resolvedMacros && !useResolvedMacros) return; if (zoneName.IsEmpty()) { cr->SetOutput("Macro 'cluster_zone' must be set."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } Zone::Ptr zone = Zone::GetByName(zoneName); if (!zone) { cr->SetOutput("Zone '" + zoneName + "' does not exist."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } bool connected = false; double zoneLag = 0; double lastMessageSent = 0; double lastMessageReceived = 0; double messagesSentPerSecond = 0; double messagesReceivedPerSecond = 0; double bytesSentPerSecond = 0; double bytesReceivedPerSecond = 0; for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) { if (endpoint->GetConnected()) connected = true; double eplag = ApiListener::CalculateZoneLag(endpoint); if (eplag > 0 && eplag > zoneLag) zoneLag = eplag; if (endpoint->GetLastMessageSent() > lastMessageSent) lastMessageSent = endpoint->GetLastMessageSent(); if (endpoint->GetLastMessageReceived() > lastMessageReceived) lastMessageReceived = endpoint->GetLastMessageReceived(); messagesSentPerSecond += endpoint->GetMessagesSentPerSecond(); messagesReceivedPerSecond += endpoint->GetMessagesReceivedPerSecond(); bytesSentPerSecond += endpoint->GetBytesSentPerSecond(); bytesReceivedPerSecond += endpoint->GetBytesReceivedPerSecond(); } if (connected) { cr->SetState(ServiceOK); cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)); /* Check whether the thresholds have been resolved and compare them */ if (missingLagCritical.IsEmpty() && zoneLag > lagCritical) { cr->SetState(ServiceCritical); cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag) + " greater than critical threshold: " + Utility::FormatDuration(lagCritical)); } else if (missingLagWarning.IsEmpty() && zoneLag > lagWarning) { cr->SetState(ServiceWarning); cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag) + " greater than warning threshold: " + Utility::FormatDuration(lagWarning)); } } else { cr->SetState(ServiceCritical); cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag)); } cr->SetPerformanceData(new Array({ new PerfdataValue("slave_lag", zoneLag, false, "s", lagWarning, lagCritical), new PerfdataValue("last_messages_sent", lastMessageSent), new PerfdataValue("last_messages_received", lastMessageReceived), new PerfdataValue("sum_messages_sent_per_second", messagesSentPerSecond), new PerfdataValue("sum_messages_received_per_second", messagesReceivedPerSecond), new PerfdataValue("sum_bytes_sent_per_second", bytesSentPerSecond), new PerfdataValue("sum_bytes_received_per_second", bytesReceivedPerSecond) })); checkable->ProcessCheckResult(cr); }
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros) { ApiListener::Ptr listener = ApiListener::GetInstance(); if (!listener) { cr->SetOutput("No API listener is configured for this instance."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } CheckCommand::Ptr commandObj = checkable->GetCheckCommand(); Value raw_command = commandObj->GetCommandLine(); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); MacroProcessor::ResolverList resolvers; if (service) resolvers.push_back(std::make_pair("service", service)); resolvers.push_back(std::make_pair("host", host)); resolvers.push_back(std::make_pair("command", commandObj)); resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance())); String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult(), NULL, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros); if (resolvedMacros && !useResolvedMacros) return; if (zoneName.IsEmpty()) { cr->SetOutput("Macro 'cluster_zone' must be set."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } Zone::Ptr zone = Zone::GetByName(zoneName); if (!zone) { cr->SetOutput("Zone '" + zoneName + "' does not exist."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } bool connected = false; double zoneLag = 0; BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) { if (endpoint->GetConnected()) connected = true; double eplag = ApiListener::CalculateZoneLag(endpoint); if (eplag > 0 && eplag > zoneLag) zoneLag = eplag; } if (!connected) { cr->SetState(ServiceCritical); cr->SetOutput("Zone '" + zoneName + "' is not connected. Log lag: " + Utility::FormatDuration(zoneLag)); } else { cr->SetState(ServiceOK); cr->SetOutput("Zone '" + zoneName + "' is connected. Log lag: " + Utility::FormatDuration(zoneLag)); } Array::Ptr perfdata = new Array(); perfdata->Add(new PerfdataValue("slave_lag", zoneLag)); cr->SetPerformanceData(perfdata); checkable->ProcessCheckResult(cr); }
void ClusterZoneCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros) { ApiListener::Ptr listener = ApiListener::GetInstance(); if (!listener) { cr->SetOutput("No API listener is configured for this instance."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } CheckCommand::Ptr commandObj = checkable->GetCheckCommand(); Value raw_command = commandObj->GetCommandLine(); Host::Ptr host; Service::Ptr service; tie(host, service) = GetHostService(checkable); MacroProcessor::ResolverList resolvers; if (service) resolvers.push_back(std::make_pair("service", service)); resolvers.push_back(std::make_pair("host", host)); resolvers.push_back(std::make_pair("command", commandObj)); resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance())); String zoneName = MacroProcessor::ResolveMacros("$cluster_zone$", resolvers, checkable->GetLastCheckResult()); if (zoneName.IsEmpty()) { cr->SetOutput("Macro 'cluster_zone' must be set."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } Zone::Ptr zone = Zone::GetByName(zoneName); if (!zone) { cr->SetOutput("Zone '" + zoneName + "' does not exist."); cr->SetState(ServiceUnknown); checkable->ProcessCheckResult(cr); return; } bool connected = false; BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) { if (endpoint->IsConnected()) { connected = true; break; } } if (!connected) { cr->SetState(ServiceCritical); cr->SetOutput("Zone '" + zoneName + "' is not connected."); } else { cr->SetState(ServiceOK); cr->SetOutput("Zone '" + zoneName + "' is connected."); } checkable->ProcessCheckResult(cr); }
void ElasticsearchWriter::AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr) { String prefix = "check_result."; fields->Set(prefix + "output", cr->GetOutput()); fields->Set(prefix + "check_source", cr->GetCheckSource()); fields->Set(prefix + "exit_status", cr->GetExitStatus()); fields->Set(prefix + "command", cr->GetCommand()); fields->Set(prefix + "state", cr->GetState()); fields->Set(prefix + "vars_before", cr->GetVarsBefore()); fields->Set(prefix + "vars_after", cr->GetVarsAfter()); fields->Set(prefix + "execution_start", FormatTimestamp(cr->GetExecutionStart())); fields->Set(prefix + "execution_end", FormatTimestamp(cr->GetExecutionEnd())); fields->Set(prefix + "schedule_start", FormatTimestamp(cr->GetScheduleStart())); fields->Set(prefix + "schedule_end", FormatTimestamp(cr->GetScheduleEnd())); /* Add extra calculated field. */ fields->Set(prefix + "latency", cr->CalculateLatency()); fields->Set(prefix + "execution_time", cr->CalculateExecutionTime()); if (!GetEnableSendPerfdata()) return; Array::Ptr perfdata = cr->GetPerformanceData(); CheckCommand::Ptr checkCommand = checkable->GetCheckCommand(); if (perfdata) { ObjectLock olock(perfdata); for (const Value& val : perfdata) { PerfdataValue::Ptr pdv; if (val.IsObjectType<PerfdataValue>()) pdv = val; else { try { pdv = PerfdataValue::Parse(val); } catch (const std::exception&) { Log(LogWarning, "ElasticsearchWriter") << "Ignoring invalid perfdata for checkable '" << checkable->GetName() << "' and command '" << checkCommand->GetName() << "' with value: " << val; continue; } } String escapedKey = pdv->GetLabel(); boost::replace_all(escapedKey, " ", "_"); boost::replace_all(escapedKey, ".", "_"); boost::replace_all(escapedKey, "\\", "_"); boost::algorithm::replace_all(escapedKey, "::", "."); String perfdataPrefix = prefix + "perfdata." + escapedKey; fields->Set(perfdataPrefix + ".value", pdv->GetValue()); if (pdv->GetMin()) fields->Set(perfdataPrefix + ".min", pdv->GetMin()); if (pdv->GetMax()) fields->Set(perfdataPrefix + ".max", pdv->GetMax()); if (pdv->GetWarn()) fields->Set(perfdataPrefix + ".warn", pdv->GetWarn()); if (pdv->GetCrit()) fields->Set(perfdataPrefix + ".crit", pdv->GetCrit()); if (!pdv->GetUnit().IsEmpty()) fields->Set(perfdataPrefix + ".unit", pdv->GetUnit()); } } }