/* Provide a helper function for zone origin name. */
String ApiListener::GetFromZoneName(const Zone::Ptr& fromZone)
{
String fromZoneName;
if (fromZone) {
fromZoneName = fromZone->GetName();
} else {
Zone::Ptr lzone = Zone::GetLocalZone();
if (lzone)
fromZoneName = lzone->GetName();
}
return fromZoneName;
}
void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const
{
Dictionary::Ptr newConfig = new Dictionary();
BOOST_FOREACH(const ZoneFragment& zf, ConfigCompiler::GetZoneDirs(zone->GetName())) {
Dictionary::Ptr newConfigPart = LoadConfigDir(zf.Path);
ObjectLock olock(newConfigPart);
BOOST_FOREACH(const Dictionary::Pair& kv, newConfigPart) {
newConfig->Set("/" + zf.Tag + kv.first, kv.second);
}
}
Dictionary::Ptr ZoneDbObject::GetStatusFields() const
{
Zone::Ptr zone = static_pointer_cast<Zone>(GetObject());
Log(LogDebug, "ZoneDbObject")
<< "update status for zone '" << zone->GetName() << "'";
return new Dictionary({
{ "parent_zone_object_id", zone->GetParent() }
});
}
void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const
{
ConfigDirInformation newConfigInfo;
newConfigInfo.UpdateV1 = new Dictionary();
newConfigInfo.UpdateV2 = new Dictionary();
for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zone->GetName())) {
ConfigDirInformation newConfigPart = LoadConfigDir(zf.Path);
{
ObjectLock olock(newConfigPart.UpdateV1);
for (const Dictionary::Pair& kv : newConfigPart.UpdateV1) {
newConfigInfo.UpdateV1->Set("/" + zf.Tag + kv.first, kv.second);
}
}
{
ObjectLock olock(newConfigPart.UpdateV2);
for (const Dictionary::Pair& kv : newConfigPart.UpdateV2) {
newConfigInfo.UpdateV2->Set("/" + zf.Tag + kv.first, kv.second);
}
}
}
int sumUpdates = newConfigInfo.UpdateV1->GetLength() + newConfigInfo.UpdateV2->GetLength();
if (sumUpdates == 0)
return;
String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName();
Log(LogInformation, "ApiListener")
<< "Copying " << sumUpdates << " zone configuration files for zone '" << zone->GetName() << "' to '" << oldDir << "'.";
Utility::MkDirP(oldDir, 0700);
ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir);
UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, true);
}
void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const
{
String newDir = Application::GetZonesDir() + "/" + zone->GetName();
String oldDir = Application::GetLocalStateDir() + "/lib/icinga2/api/zones/" + zone->GetName();
Log(LogInformation, "ApiListener")
<< "Copying zone configuration files from '" << newDir << "' to '" << oldDir << "'.";
if (!Utility::MkDir(oldDir, 0700)) {
Log(LogCritical, "ApiListener")
<< "mkdir() for path '" << oldDir << "'failed with error code " << errno << ", \"" << Utility::FormatErrorNumber(errno) << "\"";
BOOST_THROW_EXCEPTION(posix_error()
<< boost::errinfo_api_function("mkdir")
<< boost::errinfo_errno(errno)
<< boost::errinfo_file_name(oldDir));
}
Dictionary::Ptr newConfig = LoadConfigDir(newDir);
Dictionary::Ptr oldConfig = LoadConfigDir(oldDir);
UpdateConfigDir(oldConfig, newConfig, oldDir);
}
Value EndpointsTable::ZoneAccessor(const Value& row)
{
Endpoint::Ptr endpoint = static_cast<Endpoint::Ptr>(row);
if (!endpoint)
return Empty;
Zone::Ptr zone = endpoint->GetZone();
if (!zone)
return Empty;
return zone->GetName();
}
BOOST_FOREACH(const Zone::Ptr& zone, ConfigType::GetObjectsByType<Zone>()) {
/* don't connect to global zones */
if (zone->GetGlobal())
continue;
/* only connect to endpoints in a) the same zone b) our parent zone c) immediate child zones */
if (my_zone != zone && my_zone != zone->GetParent() && zone != my_zone->GetParent()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Zone '" << zone->GetName()
<< "' because it's not in the same zone, a parent or a child zone.";
continue;
}
BOOST_FOREACH(const Endpoint::Ptr& endpoint, zone->GetEndpoints()) {
/* don't connect to ourselves */
if (endpoint == GetLocalEndpoint()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName() << "' because that's us.";
continue;
}
/* don't try to connect to endpoints which don't have a host and port */
if (endpoint->GetHost().IsEmpty() || endpoint->GetPort().IsEmpty()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because the host/port attributes are missing.";
continue;
}
/* don't try to connect if there's already a connection attempt */
if (endpoint->GetConnecting()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because we're already trying to connect to it.";
continue;
}
/* don't try to connect if we're already connected */
if (endpoint->GetConnected()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because we're already connected to it.";
continue;
}
boost::thread thread(boost::bind(&ApiListener::AddConnection, this, endpoint));
thread.detach();
}
}
void Checkable::OnAllConfigLoaded(void)
{
ObjectImpl<Checkable>::OnAllConfigLoaded();
Endpoint::Ptr endpoint = GetCommandEndpoint();
if (endpoint) {
Zone::Ptr checkableZone = static_pointer_cast<Zone>(GetZone());
if (!checkableZone)
checkableZone = Zone::GetLocalZone();
Zone::Ptr cmdZone = endpoint->GetZone();
if (checkableZone && cmdZone != checkableZone && cmdZone->GetParent() != checkableZone) {
BOOST_THROW_EXCEPTION(ValidationError(this, boost::assign::list_of("command_endpoint"),
"Command endpoint must be in zone '" + checkableZone->GetName() + "' or in a direct child zone thereof."));
}
}
}
void Checkable::OnAllConfigLoaded()
{
ObjectImpl<Checkable>::OnAllConfigLoaded();
Endpoint::Ptr endpoint = GetCommandEndpoint();
if (endpoint) {
Zone::Ptr checkableZone = static_pointer_cast<Zone>(GetZone());
if (checkableZone) {
Zone::Ptr cmdZone = endpoint->GetZone();
if (cmdZone != checkableZone && cmdZone->GetParent() != checkableZone) {
BOOST_THROW_EXCEPTION(ValidationError(this, { "command_endpoint" },
"Command endpoint must be in zone '" + checkableZone->GetName() + "' or in a direct child zone thereof."));
}
} else {
BOOST_THROW_EXCEPTION(ValidationError(this, { "command_endpoint" },
"Command endpoint must not be set."));
}
}
}
void ApiListener::SendConfigUpdate(const ApiClient::Ptr& aclient)
{
Endpoint::Ptr endpoint = aclient->GetEndpoint();
ASSERT(endpoint);
Zone::Ptr azone = endpoint->GetZone();
Zone::Ptr lzone = Zone::GetLocalZone();
/* don't try to send config updates to our master */
if (!azone->IsChildOf(lzone))
return;
Dictionary::Ptr configUpdate = make_shared<Dictionary>();
String zonesDir = Application::GetLocalStateDir() + "/lib/icinga2/api/zones";
BOOST_FOREACH(const Zone::Ptr& zone, DynamicType::GetObjectsByType<Zone>()) {
String zoneDir = zonesDir + "/" + zone->GetName();
if (!zone->IsChildOf(azone) && !zone->IsGlobal()) {
Log(LogNotice, "ApiListener")
<< "Skipping sync for '" << zone->GetName() << "'. Not a child of zone '" << azone->GetName() << "'.";
continue;
}
if (!Utility::PathExists(zoneDir)) {
Log(LogNotice, "ApiListener")
<< "Ignoring sync for '" << zone->GetName() << "'. Zone directory '" << zoneDir << "' does not exist.";
continue;
}
if (zone->IsGlobal())
Log(LogInformation, "ApiListener")
<< "Syncing global zone '" << zone->GetName() << "'.";
configUpdate->Set(zone->GetName(), LoadConfigDir(zonesDir + "/" + zone->GetName()));
}
bool ApiListener::IsConfigMaster(const Zone::Ptr& zone)
{
String path = Application::GetZonesDir() + "/" + zone->GetName();
return Utility::PathExists(path);
}
Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params)
{
if (!origin->FromClient->GetEndpoint() || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone)))
return Empty;
ApiListener::Ptr listener = ApiListener::GetInstance();
if (!listener) {
Log(LogCritical, "ApiListener", "No instance available.");
return Empty;
}
if (!listener->GetAcceptConfig()) {
Log(LogWarning, "ApiListener")
<< "Ignoring config update. '" << listener->GetName() << "' does not accept config.";
return Empty;
}
Log(LogInformation, "ApiListener")
<< "Applying config update from endpoint '" << origin->FromClient->GetEndpoint()->GetName()
<< "' of zone '" << GetFromZoneName(origin->FromZone) << "'.";
Dictionary::Ptr updateV1 = params->Get("update");
Dictionary::Ptr updateV2 = params->Get("update_v2");
bool configChange = false;
ObjectLock olock(updateV1);
for (const Dictionary::Pair& kv : updateV1) {
Zone::Ptr zone = Zone::GetByName(kv.first);
if (!zone) {
Log(LogWarning, "ApiListener")
<< "Ignoring config update for unknown zone '" << kv.first << "'.";
continue;
}
if (ConfigCompiler::HasZoneConfigAuthority(kv.first)) {
Log(LogWarning, "ApiListener")
<< "Ignoring config update for zone '" << kv.first << "' because we have an authoritative version of the zone's config.";
continue;
}
String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName();
Utility::MkDirP(oldDir, 0700);
ConfigDirInformation newConfigInfo;
newConfigInfo.UpdateV1 = kv.second;
if (updateV2)
newConfigInfo.UpdateV2 = updateV2->Get(kv.first);
Dictionary::Ptr newConfig = kv.second;
ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir);
if (UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, false))
configChange = true;
}
if (configChange) {
Log(LogInformation, "ApiListener", "Restarting after configuration change.");
Application::RequestRestart();
}
return Empty;
}
std::pair<Dictionary::Ptr, Dictionary::Ptr> ApiListener::GetStatus(void)
{
Dictionary::Ptr status = new Dictionary();
Dictionary::Ptr perfdata = new Dictionary();
/* cluster stats */
status->Set("identity", GetIdentity());
double allEndpoints = 0;
Array::Ptr allNotConnectedEndpoints = new Array();
Array::Ptr allConnectedEndpoints = new Array();
Zone::Ptr my_zone = Zone::GetLocalZone();
Dictionary::Ptr connectedZones = new Dictionary();
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
/* only check endpoints in a) the same zone b) our parent zone c) immediate child zones */
if (my_zone != zone && my_zone != zone->GetParent() && zone != my_zone->GetParent()) {
Log(LogDebug, "ApiListener")
<< "Not checking connection to Zone '" << zone->GetName() << "' because it's not in the same zone, a parent or a child zone.";
continue;
}
bool zoneConnected = false;
int countZoneEndpoints = 0;
double zoneLag = 0;
Array::Ptr zoneEndpoints = new Array();
for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
zoneEndpoints->Add(endpoint->GetName());
if (endpoint->GetName() == GetIdentity())
continue;
double eplag = CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
allEndpoints++;
countZoneEndpoints++;
if (!endpoint->GetConnected()) {
allNotConnectedEndpoints->Add(endpoint->GetName());
} else {
allConnectedEndpoints->Add(endpoint->GetName());
zoneConnected = true;
}
}
/* if there's only one endpoint inside the zone, we're not connected - that's us, fake it */
if (zone->GetEndpoints().size() == 1 && countZoneEndpoints == 0)
zoneConnected = true;
Dictionary::Ptr zoneStats = new Dictionary();
zoneStats->Set("connected", zoneConnected);
zoneStats->Set("client_log_lag", zoneLag);
zoneStats->Set("endpoints", zoneEndpoints);
String parentZoneName;
Zone::Ptr parentZone = zone->GetParent();
if (parentZone)
parentZoneName = parentZone->GetName();
zoneStats->Set("parent_zone", parentZoneName);
connectedZones->Set(zone->GetName(), zoneStats);
}
status->Set("num_endpoints", allEndpoints);
status->Set("num_conn_endpoints", allConnectedEndpoints->GetLength());
status->Set("num_not_conn_endpoints", allNotConnectedEndpoints->GetLength());
status->Set("conn_endpoints", allConnectedEndpoints);
status->Set("not_conn_endpoints", allNotConnectedEndpoints);
status->Set("zones", connectedZones);
perfdata->Set("num_endpoints", allEndpoints);
perfdata->Set("num_conn_endpoints", Convert::ToDouble(allConnectedEndpoints->GetLength()));
perfdata->Set("num_not_conn_endpoints", Convert::ToDouble(allNotConnectedEndpoints->GetLength()));
return std::make_pair(status, perfdata);
}
void ApiListener::SyncClient(const JsonRpcConnection::Ptr& aclient, const Endpoint::Ptr& endpoint, bool needSync)
{
Zone::Ptr eZone = endpoint->GetZone();
try {
{
ObjectLock olock(endpoint);
endpoint->SetSyncing(true);
}
Zone::Ptr myZone = Zone::GetLocalZone();
if (myZone->GetParent() == eZone) {
Log(LogInformation, "ApiListener")
<< "Requesting new certificate for this Icinga instance from endpoint '" << endpoint->GetName() << "'.";
JsonRpcConnection::SendCertificateRequest(aclient, nullptr, String());
if (Utility::PathExists(ApiListener::GetCertificateRequestsDir()))
Utility::Glob(ApiListener::GetCertificateRequestsDir() + "/*.json", std::bind(&JsonRpcConnection::SendCertificateRequest, aclient, nullptr, _1), GlobFile);
}
/* Make sure that the config updates are synced
* before the logs are replayed.
*/
Log(LogInformation, "ApiListener")
<< "Sending config updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
/* sync zone file config */
SendConfigUpdate(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending config file updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
/* sync runtime config */
SendRuntimeConfigObjects(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending runtime config updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
if (!needSync) {
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
return;
}
Log(LogInformation, "ApiListener")
<< "Sending replay log for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
ReplayLog(aclient);
if (eZone == Zone::GetLocalZone())
UpdateObjectAuthority();
Log(LogInformation, "ApiListener")
<< "Finished sending replay log for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
} catch (const std::exception& ex) {
{
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
}
Log(LogCritical, "ApiListener")
<< "Error while syncing endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex, false);
Log(LogDebug, "ApiListener")
<< "Error while syncing endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex);
}
Log(LogInformation, "ApiListener")
<< "Finished syncing endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
}
std::pair<Dictionary::Ptr, Dictionary::Ptr> ApiListener::GetStatus()
{
Dictionary::Ptr perfdata = new Dictionary();
/* cluster stats */
double allEndpoints = 0;
Array::Ptr allNotConnectedEndpoints = new Array();
Array::Ptr allConnectedEndpoints = new Array();
Zone::Ptr my_zone = Zone::GetLocalZone();
Dictionary::Ptr connectedZones = new Dictionary();
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
/* only check endpoints in a) the same zone b) our parent zone c) immediate child zones */
if (my_zone != zone && my_zone != zone->GetParent() && zone != my_zone->GetParent()) {
Log(LogDebug, "ApiListener")
<< "Not checking connection to Zone '" << zone->GetName() << "' because it's not in the same zone, a parent or a child zone.";
continue;
}
bool zoneConnected = false;
int countZoneEndpoints = 0;
double zoneLag = 0;
ArrayData zoneEndpoints;
for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
zoneEndpoints.emplace_back(endpoint->GetName());
if (endpoint->GetName() == GetIdentity())
continue;
double eplag = CalculateZoneLag(endpoint);
if (eplag > 0 && eplag > zoneLag)
zoneLag = eplag;
allEndpoints++;
countZoneEndpoints++;
if (!endpoint->GetConnected()) {
allNotConnectedEndpoints->Add(endpoint->GetName());
} else {
allConnectedEndpoints->Add(endpoint->GetName());
zoneConnected = true;
}
}
/* if there's only one endpoint inside the zone, we're not connected - that's us, fake it */
if (zone->GetEndpoints().size() == 1 && countZoneEndpoints == 0)
zoneConnected = true;
String parentZoneName;
Zone::Ptr parentZone = zone->GetParent();
if (parentZone)
parentZoneName = parentZone->GetName();
Dictionary::Ptr zoneStats = new Dictionary({
{ "connected", zoneConnected },
{ "client_log_lag", zoneLag },
{ "endpoints", new Array(std::move(zoneEndpoints)) },
{ "parent_zone", parentZoneName }
});
connectedZones->Set(zone->GetName(), zoneStats);
}
/* connection stats */
size_t jsonRpcClients = GetAnonymousClients().size();
size_t httpClients = GetHttpClients().size();
size_t workQueueItems = JsonRpcConnection::GetWorkQueueLength();
size_t workQueueCount = JsonRpcConnection::GetWorkQueueCount();
size_t syncQueueItems = m_SyncQueue.GetLength();
size_t relayQueueItems = m_RelayQueue.GetLength();
double workQueueItemRate = JsonRpcConnection::GetWorkQueueRate();
double syncQueueItemRate = m_SyncQueue.GetTaskCount(60) / 60.0;
double relayQueueItemRate = m_RelayQueue.GetTaskCount(60) / 60.0;
Dictionary::Ptr status = new Dictionary({
{ "identity", GetIdentity() },
{ "num_endpoints", allEndpoints },
{ "num_conn_endpoints", allConnectedEndpoints->GetLength() },
{ "num_not_conn_endpoints", allNotConnectedEndpoints->GetLength() },
{ "conn_endpoints", allConnectedEndpoints },
{ "not_conn_endpoints", allNotConnectedEndpoints },
{ "zones", connectedZones },
{ "json_rpc", new Dictionary({
{ "clients", jsonRpcClients },
{ "work_queue_items", workQueueItems },
{ "work_queue_count", workQueueCount },
{ "sync_queue_items", syncQueueItems },
{ "relay_queue_items", relayQueueItems },
{ "work_queue_item_rate", workQueueItemRate },
{ "sync_queue_item_rate", syncQueueItemRate },
{ "relay_queue_item_rate", relayQueueItemRate }
}) },
{ "http", new Dictionary({
{ "clients", httpClients }
}) }
});
/* performance data */
perfdata->Set("num_endpoints", allEndpoints);
perfdata->Set("num_conn_endpoints", Convert::ToDouble(allConnectedEndpoints->GetLength()));
perfdata->Set("num_not_conn_endpoints", Convert::ToDouble(allNotConnectedEndpoints->GetLength()));
perfdata->Set("num_json_rpc_clients", jsonRpcClients);
perfdata->Set("num_http_clients", httpClients);
perfdata->Set("num_json_rpc_work_queue_items", workQueueItems);
perfdata->Set("num_json_rpc_work_queue_count", workQueueCount);
perfdata->Set("num_json_rpc_sync_queue_items", syncQueueItems);
perfdata->Set("num_json_rpc_relay_queue_items", relayQueueItems);
perfdata->Set("num_json_rpc_work_queue_item_rate", workQueueItemRate);
perfdata->Set("num_json_rpc_sync_queue_item_rate", syncQueueItemRate);
perfdata->Set("num_json_rpc_relay_queue_item_rate", relayQueueItemRate);
return std::make_pair(status, perfdata);
}