void ApiListener::SyncClient(const JsonRpcConnection::Ptr& aclient, const Endpoint::Ptr& endpoint)
{
try {
{
ObjectLock olock(endpoint);
endpoint->SetSyncing(true);
}
Log(LogInformation, "ApiListener")
<< "Sending updates for endpoint '" << endpoint->GetName() << "'.";
/* sync zone file config */
SendConfigUpdate(aclient);
/* sync runtime config */
SendRuntimeConfigObjects(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending updates for endpoint '" << endpoint->GetName() << "'.";
ReplayLog(aclient);
} catch (const std::exception& ex) {
Log(LogCritical, "ApiListener")
<< "Error while syncing endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex);
}
}
Dictionary::Ptr EndpointDbObject::GetStatusFields(void) const
{
Dictionary::Ptr fields = make_shared<Dictionary>();
Endpoint::Ptr endpoint = static_pointer_cast<Endpoint>(GetObject());
Log(LogDebug, "EndpointDbObject", "update status for endpoint '" + endpoint->GetName() + "'");
fields->Set("identity", endpoint->GetName());
fields->Set("node", IcingaApplication::GetInstance()->GetNodeName());
fields->Set("is_connected", EndpointIsConnected(endpoint));
return fields;
}
void ApiListener::SyncClient(const JsonRpcConnection::Ptr& aclient, const Endpoint::Ptr& endpoint, bool needSync)
{
try {
{
ObjectLock olock(endpoint);
endpoint->SetSyncing(true);
}
/* Make sure that the config updates are synced
* before the logs are replayed.
*/
Log(LogInformation, "ApiListener")
<< "Sending config updates for endpoint '" << endpoint->GetName() << "'.";
/* sync zone file config */
SendConfigUpdate(aclient);
/* sync runtime config */
SendRuntimeConfigObjects(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending config updates for endpoint '" << endpoint->GetName() << "'.";
if (!needSync) {
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
return;
}
Log(LogInformation, "ApiListener")
<< "Sending replay log for endpoint '" << endpoint->GetName() << "'.";
ReplayLog(aclient);
if (endpoint->GetZone() == Zone::GetLocalZone())
UpdateObjectAuthority();
Log(LogInformation, "ApiListener")
<< "Finished sending replay log for endpoint '" << endpoint->GetName() << "'.";
} catch (const std::exception& ex) {
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
Log(LogCritical, "ApiListener")
<< "Error while syncing endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex);
}
}
BOOST_FOREACH(const Endpoint::Ptr& endpoint, ConfigType::GetObjectsByType<Endpoint>()) {
if (!endpoint->GetConnected())
continue;
double ts = endpoint->GetRemoteLogPosition();
if (ts == 0)
continue;
Dictionary::Ptr lparams = new Dictionary();
lparams->Set("log_position", ts);
Dictionary::Ptr lmessage = new Dictionary();
lmessage->Set("jsonrpc", "2.0");
lmessage->Set("method", "log::SetLogPosition");
lmessage->Set("params", lparams);
double maxTs = 0;
BOOST_FOREACH(const JsonRpcConnection::Ptr& client, endpoint->GetClients()) {
if (client->GetTimestamp() > maxTs)
maxTs = client->GetTimestamp();
}
BOOST_FOREACH(const JsonRpcConnection::Ptr& client, endpoint->GetClients()) {
if (client->GetTimestamp() != maxTs)
client->Disconnect();
else
client->SendMessage(lmessage);
}
Log(LogNotice, "ApiListener")
<< "Setting log position for identity '" << endpoint->GetName() << "': "
<< Utility::FormatDateTime("%Y/%m/%d %H:%M:%S", ts);
}
/**
* Creates a new JSON-RPC client and connects to the specified endpoint.
*
* @param endpoint The endpoint.
*/
void ApiListener::AddConnection(const Endpoint::Ptr& endpoint)
{
{
ObjectLock olock(this);
std::shared_ptr<SSL_CTX> sslContext = m_SSLContext;
if (!sslContext) {
Log(LogCritical, "ApiListener", "SSL context is required for AddConnection()");
return;
}
}
String host = endpoint->GetHost();
String port = endpoint->GetPort();
Log(LogInformation, "ApiListener")
<< "Reconnecting to endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'";
TcpSocket::Ptr client = new TcpSocket();
String serverName = endpoint->GetName();
String env = ScriptGlobal::Get("Environment", &Empty);
if (env != "" && env != "production")
serverName += ":" + env;
try {
endpoint->SetConnecting(true);
client->Connect(host, port);
NewClientHandler(client, serverName, RoleClient);
endpoint->SetConnecting(false);
} catch (const std::exception& ex) {
endpoint->SetConnecting(false);
client->Close();
std::ostringstream info;
info << "Cannot connect to host '" << host << "' on port '" << port << "'";
Log(LogCritical, "ApiListener", info.str());
Log(LogDebug, "ApiListener")
<< info.str() << "\n" << DiagnosticInformation(ex);
}
Log(LogInformation, "ApiListener")
<< "Finished reconnecting to endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'";
}
Value EndpointsTable::IdentityAccessor(const Value& row)
{
Endpoint::Ptr endpoint = static_cast<Endpoint::Ptr>(row);
if (!endpoint)
return Empty;
return endpoint->GetName();
}
bool ApiListener::IsMaster(void) const
{
Endpoint::Ptr master = GetMaster();
if (!master)
return false;
return master->GetName() == GetIdentity();
}
Dictionary::Ptr EndpointDbObject::GetConfigFields(void) const
{
Dictionary::Ptr fields = new Dictionary();
Endpoint::Ptr endpoint = static_pointer_cast<Endpoint>(GetObject());
fields->Set("identity", endpoint->GetName());
fields->Set("node", IcingaApplication::GetInstance()->GetNodeName());
return fields;
}
int EndpointDbObject::EndpointIsConnected(const Endpoint::Ptr& endpoint)
{
unsigned int is_connected = endpoint->IsConnected() ? 1 : 0;
/* if identity is equal to node, fake is_connected */
if (endpoint->GetName() == IcingaApplication::GetInstance()->GetNodeName())
is_connected = 1;
return is_connected;
}
/**
* Creates a new JSON-RPC client and connects to the specified endpoint.
*
* @param endpoint The endpoint.
*/
void ApiListener::AddConnection(const Endpoint::Ptr& endpoint)
{
{
ObjectLock olock(this);
boost::shared_ptr<SSL_CTX> sslContext = m_SSLContext;
if (!sslContext) {
Log(LogCritical, "ApiListener", "SSL context is required for AddConnection()");
return;
}
}
String host = endpoint->GetHost();
String port = endpoint->GetPort();
Log(LogInformation, "JsonRpcConnection")
<< "Reconnecting to API endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'";
TcpSocket::Ptr client = new TcpSocket();
try {
endpoint->SetConnecting(true);
client->Connect(host, port);
NewClientHandler(client, endpoint->GetName(), RoleClient);
endpoint->SetConnecting(false);
} catch (const std::exception& ex) {
endpoint->SetConnecting(false);
client->Close();
std::ostringstream info;
info << "Cannot connect to host '" << host << "' on port '" << port << "'";
Log(LogCritical, "ApiListener", info.str());
Log(LogDebug, "ApiListener")
<< info.str() << "\n" << DiagnosticInformation(ex);
}
}
Value EndpointsTable::IsConnectedAccessor(const Value& row)
{
Endpoint::Ptr endpoint = static_cast<Endpoint::Ptr>(row);
if (!endpoint)
return Empty;
unsigned int is_connected = endpoint->IsConnected() ? 1 : 0;
/* if identity is equal to node, fake is_connected */
if (endpoint->GetName() == IcingaApplication::GetInstance()->GetNodeName())
is_connected = 1;
return is_connected;
}
void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient)
{
Endpoint::Ptr endpoint = aclient->GetEndpoint();
ASSERT(endpoint);
Zone::Ptr azone = endpoint->GetZone();
Zone::Ptr lzone = Zone::GetLocalZone();
/* don't try to send config updates to our master */
if (!azone->IsChildOf(lzone))
return;
Dictionary::Ptr configUpdateV1 = new Dictionary();
Dictionary::Ptr configUpdateV2 = new Dictionary();
String zonesDir = Configuration::DataDir + "/api/zones";
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
String zoneDir = zonesDir + "/" + zone->GetName();
if (!zone->IsChildOf(azone) && !zone->IsGlobal())
continue;
if (!Utility::PathExists(zoneDir))
continue;
Log(LogInformation, "ApiListener")
<< "Syncing configuration files for " << (zone->IsGlobal() ? "global " : "")
<< "zone '" << zone->GetName() << "' to endpoint '" << endpoint->GetName() << "'.";
ConfigDirInformation config = LoadConfigDir(zonesDir + "/" + zone->GetName());
configUpdateV1->Set(zone->GetName(), config.UpdateV1);
configUpdateV2->Set(zone->GetName(), config.UpdateV2);
}
Dictionary::Ptr message = new Dictionary({
{ "jsonrpc", "2.0" },
{ "method", "config::Update" },
{ "params", new Dictionary({
{ "update", configUpdateV1 },
{ "update_v2", configUpdateV2 }
}) }
});
aclient->SendMessage(message);
}
void EndpointDbObject::OnConfigUpdate(void)
{
/* update current status on config dump once */
Endpoint::Ptr endpoint = static_pointer_cast<Endpoint>(GetObject());
DbQuery query1;
query1.Table = "endpointstatus";
query1.Type = DbQueryInsert;
Dictionary::Ptr fields1 = make_shared<Dictionary>();
fields1->Set("identity", endpoint->GetName());
fields1->Set("node", IcingaApplication::GetInstance()->GetNodeName());
fields1->Set("is_connected", EndpointIsConnected(endpoint));
fields1->Set("status_update_time", DbValue::FromTimestamp(Utility::GetTime()));
fields1->Set("endpoint_object_id", endpoint);
fields1->Set("instance_id", 0); /* DbConnection class fills in real ID */
query1.Fields = fields1;
OnQuery(query1);
}
void ApiListener::SyncSendMessage(const Endpoint::Ptr& endpoint, const Dictionary::Ptr& message)
{
ObjectLock olock(endpoint);
if (!endpoint->GetSyncing()) {
Log(LogNotice, "ApiListener")
<< "Sending message '" << message->Get("method") << "' to '" << endpoint->GetName() << "'";
double maxTs = 0;
for (const JsonRpcConnection::Ptr& client : endpoint->GetClients()) {
if (client->GetTimestamp() > maxTs)
maxTs = client->GetTimestamp();
}
for (const JsonRpcConnection::Ptr& client : endpoint->GetClients()) {
if (client->GetTimestamp() != maxTs)
continue;
client->SendMessage(message);
}
}
}
void EndpointDbObject::UpdateConnectedStatus(const Endpoint::Ptr& endpoint)
{
bool connected = EndpointIsConnected(endpoint);
Log(LogDebug, "EndpointDbObject")
<< "update is_connected=" << connected << " for endpoint '" << endpoint->GetName() << "'";
DbQuery query1;
query1.Table = "endpointstatus";
query1.Type = DbQueryUpdate;
query1.Category = DbCatState;
Dictionary::Ptr fields1 = new Dictionary();
fields1->Set("is_connected", (connected ? 1 : 0));
fields1->Set("status_update_time", DbValue::FromTimestamp(Utility::GetTime()));
query1.Fields = fields1;
query1.WhereCriteria = new Dictionary();
query1.WhereCriteria->Set("endpoint_object_id", endpoint);
query1.WhereCriteria->Set("instance_id", 0); /* DbConnection class fills in real ID */
OnQuery(query1);
}
void EndpointDbObject::UpdateConnectedStatus(const Endpoint::Ptr& endpoint)
{
bool connected = EndpointIsConnected(endpoint);
Log(LogDebug, "EndpointDbObject", "update is_connected=" + Convert::ToString(connected ? 1 : 0) + " for endpoint '" + endpoint->GetName() + "'");
DbQuery query1;
query1.Table = "endpointstatus";
query1.Type = DbQueryUpdate;
Dictionary::Ptr fields1 = make_shared<Dictionary>();
fields1->Set("is_connected", (connected ? 1 : 0));
fields1->Set("status_update_time", DbValue::FromTimestamp(Utility::GetTime()));
query1.Fields = fields1;
query1.WhereCriteria = make_shared<Dictionary>();
query1.WhereCriteria->Set("endpoint_object_id", endpoint);
query1.WhereCriteria->Set("instance_id", 0); /* DbConnection class fills in real ID */
OnQuery(query1);
}
void ApiListener::SyncClient(const JsonRpcConnection::Ptr& aclient, const Endpoint::Ptr& endpoint, bool needSync)
{
Zone::Ptr eZone = endpoint->GetZone();
try {
{
ObjectLock olock(endpoint);
endpoint->SetSyncing(true);
}
Zone::Ptr myZone = Zone::GetLocalZone();
if (myZone->GetParent() == eZone) {
Log(LogInformation, "ApiListener")
<< "Requesting new certificate for this Icinga instance from endpoint '" << endpoint->GetName() << "'.";
JsonRpcConnection::SendCertificateRequest(aclient, nullptr, String());
if (Utility::PathExists(ApiListener::GetCertificateRequestsDir()))
Utility::Glob(ApiListener::GetCertificateRequestsDir() + "/*.json", std::bind(&JsonRpcConnection::SendCertificateRequest, aclient, nullptr, _1), GlobFile);
}
/* Make sure that the config updates are synced
* before the logs are replayed.
*/
Log(LogInformation, "ApiListener")
<< "Sending config updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
/* sync zone file config */
SendConfigUpdate(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending config file updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
/* sync runtime config */
SendRuntimeConfigObjects(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending runtime config updates for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
if (!needSync) {
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
return;
}
Log(LogInformation, "ApiListener")
<< "Sending replay log for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
ReplayLog(aclient);
if (eZone == Zone::GetLocalZone())
UpdateObjectAuthority();
Log(LogInformation, "ApiListener")
<< "Finished sending replay log for endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
} catch (const std::exception& ex) {
{
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
}
Log(LogCritical, "ApiListener")
<< "Error while syncing endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex, false);
Log(LogDebug, "ApiListener")
<< "Error while syncing endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex);
}
Log(LogInformation, "ApiListener")
<< "Finished syncing endpoint '" << endpoint->GetName() << "' in zone '" << eZone->GetName() << "'.";
}
void ApiListener::ApiReconnectTimerHandler(void)
{
Zone::Ptr my_zone = Zone::GetLocalZone();
for (const Zone::Ptr& zone : ConfigType::GetObjectsByType<Zone>()) {
/* don't connect to global zones */
if (zone->GetGlobal())
continue;
/* only connect to endpoints in a) the same zone b) our parent zone c) immediate child zones */
if (my_zone != zone && my_zone != zone->GetParent() && zone != my_zone->GetParent()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Zone '" << zone->GetName()
<< "' because it's not in the same zone, a parent or a child zone.";
continue;
}
for (const Endpoint::Ptr& endpoint : zone->GetEndpoints()) {
/* don't connect to ourselves */
if (endpoint == GetLocalEndpoint()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName() << "' because that's us.";
continue;
}
/* don't try to connect to endpoints which don't have a host and port */
if (endpoint->GetHost().IsEmpty() || endpoint->GetPort().IsEmpty()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because the host/port attributes are missing.";
continue;
}
/* don't try to connect if there's already a connection attempt */
if (endpoint->GetConnecting()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because we're already trying to connect to it.";
continue;
}
/* don't try to connect if we're already connected */
if (endpoint->GetConnected()) {
Log(LogDebug, "ApiListener")
<< "Not connecting to Endpoint '" << endpoint->GetName()
<< "' because we're already connected to it.";
continue;
}
boost::thread thread(boost::bind(&ApiListener::AddConnection, this, endpoint));
thread.detach();
}
}
Endpoint::Ptr master = GetMaster();
if (master)
Log(LogNotice, "ApiListener")
<< "Current zone master: " << master->GetName();
std::vector<String> names;
for (const Endpoint::Ptr& endpoint : ConfigType::GetObjectsByType<Endpoint>())
if (endpoint->GetConnected())
names.push_back(endpoint->GetName() + " (" + Convert::ToString(endpoint->GetClients().size()) + ")");
Log(LogNotice, "ApiListener")
<< "Connected endpoints: " << Utility::NaturalJoin(names);
}
/**
* Processes a new client connection.
*
* @param client The new client.
*/
void ApiListener::NewClientHandlerInternal(const Socket::Ptr& client, const String& hostname, ConnectionRole role)
{
CONTEXT("Handling new API client connection");
TlsStream::Ptr tlsStream;
{
ObjectLock olock(this);
try {
tlsStream = new TlsStream(client, hostname, role, m_SSLContext);
} catch (const std::exception&) {
Log(LogCritical, "ApiListener", "Cannot create TLS stream from client connection.");
return;
}
}
try {
tlsStream->Handshake();
} catch (const std::exception& ex) {
Log(LogCritical, "ApiListener", "Client TLS handshake failed");
return;
}
boost::shared_ptr<X509> cert = tlsStream->GetPeerCertificate();
String identity;
Endpoint::Ptr endpoint;
bool verify_ok = false;
if (cert) {
try {
identity = GetCertificateCN(cert);
} catch (const std::exception&) {
Log(LogCritical, "ApiListener")
<< "Cannot get certificate common name from cert path: '" << GetCertPath() << "'.";
return;
}
verify_ok = tlsStream->IsVerifyOK();
Log(LogInformation, "ApiListener")
<< "New client connection for identity '" << identity << "'" << (verify_ok ? "" : " (unauthenticated)");
if (verify_ok)
endpoint = Endpoint::GetByName(identity);
} else {
Log(LogInformation, "ApiListener")
<< "New client connection (no client certificate)";
}
bool need_sync = false;
if (endpoint)
need_sync = !endpoint->IsConnected();
ClientType ctype;
if (role == RoleClient) {
Dictionary::Ptr message = new Dictionary();
message->Set("jsonrpc", "2.0");
message->Set("method", "icinga::Hello");
message->Set("params", new Dictionary());
JsonRpc::SendMessage(tlsStream, message);
ctype = ClientJsonRpc;
} else {
tlsStream->WaitForData(5);
if (!tlsStream->IsDataAvailable()) {
Log(LogWarning, "ApiListener", "No data received on new API connection.");
return;
}
char firstByte;
tlsStream->Peek(&firstByte, 1, false);
if (firstByte >= '0' && firstByte <= '9')
ctype = ClientJsonRpc;
else
ctype = ClientHttp;
}
if (ctype == ClientJsonRpc) {
Log(LogNotice, "ApiListener", "New JSON-RPC client");
JsonRpcConnection::Ptr aclient = new JsonRpcConnection(identity, verify_ok, tlsStream, role);
aclient->Start();
if (endpoint) {
endpoint->AddClient(aclient);
if (need_sync) {
{
ObjectLock olock(endpoint);
endpoint->SetSyncing(true);
}
Log(LogInformation, "ApiListener")
<< "Sending updates for endpoint '" << endpoint->GetName() << "'.";
/* sync zone file config */
SendConfigUpdate(aclient);
/* sync runtime config */
SendRuntimeConfigObjects(aclient);
Log(LogInformation, "ApiListener")
<< "Finished sending updates for endpoint '" << endpoint->GetName() << "'.";
ReplayLog(aclient);
}
} else
AddAnonymousClient(aclient);
} else {
Log(LogNotice, "ApiListener", "New HTTP client");
HttpServerConnection::Ptr aclient = new HttpServerConnection(identity, verify_ok, tlsStream);
aclient->Start();
AddHttpClient(aclient);
}
}
void Checkable::ExecuteCheck()
{
CONTEXT("Executing check for object '" + GetName() + "'");
ASSERT(!OwnsLock());
UpdateNextCheck();
bool reachable = IsReachable();
{
ObjectLock olock(this);
/* don't run another check if there is one pending */
if (m_CheckRunning)
return;
m_CheckRunning = true;
SetLastStateRaw(GetStateRaw());
SetLastStateType(GetLastStateType());
SetLastReachable(reachable);
}
/* keep track of scheduling info in case the check type doesn't provide its own information */
double scheduled_start = GetNextCheck();
double before_check = Utility::GetTime();
CheckResult::Ptr cr = new CheckResult();
cr->SetScheduleStart(scheduled_start);
cr->SetExecutionStart(before_check);
Endpoint::Ptr endpoint = GetCommandEndpoint();
bool local = !endpoint || endpoint == Endpoint::GetLocalEndpoint();
if (local) {
GetCheckCommand()->Execute(this, cr, NULL, false);
} else {
Dictionary::Ptr macros = new Dictionary();
GetCheckCommand()->Execute(this, cr, macros, false);
if (endpoint->GetConnected()) {
/* perform check on remote endpoint */
Dictionary::Ptr message = new Dictionary();
message->Set("jsonrpc", "2.0");
message->Set("method", "event::ExecuteCommand");
Host::Ptr host;
Service::Ptr service;
tie(host, service) = GetHostService(this);
Dictionary::Ptr params = new Dictionary();
message->Set("params", params);
params->Set("command_type", "check_command");
params->Set("command", GetCheckCommand()->GetName());
params->Set("host", host->GetName());
if (service)
params->Set("service", service->GetShortName());
params->Set("macros", macros);
ApiListener::Ptr listener = ApiListener::GetInstance();
if (listener)
listener->SyncSendMessage(endpoint, message);
} else if (Application::GetInstance()->GetStartTime() < Utility::GetTime() - 30) {
/* fail to perform check on unconnected endpoint */
cr->SetState(ServiceUnknown);
cr->SetOutput("Remote Icinga instance '" + endpoint->GetName() +
"' " + "is not connected to '" + Endpoint::GetLocalEndpoint()->GetName() + "'");
ProcessCheckResult(cr);
}
{
ObjectLock olock(this);
m_CheckRunning = false;
}
}
}
void ApiListener::ReplayLog(const JsonRpcConnection::Ptr& client)
{
Endpoint::Ptr endpoint = client->GetEndpoint();
if (endpoint->GetLogDuration() == 0) {
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
return;
}
CONTEXT("Replaying log for Endpoint '" + endpoint->GetName() + "'");
int count = -1;
double peer_ts = endpoint->GetLocalLogPosition();
double logpos_ts = peer_ts;
bool last_sync = false;
Endpoint::Ptr target_endpoint = client->GetEndpoint();
ASSERT(target_endpoint);
Zone::Ptr target_zone = target_endpoint->GetZone();
if (!target_zone) {
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
return;
}
for (;;) {
boost::mutex::scoped_lock lock(m_LogLock);
CloseLogFile();
RotateLogFile();
if (count == -1 || count > 50000) {
OpenLogFile();
lock.unlock();
} else {
last_sync = true;
}
count = 0;
std::vector<int> files;
Utility::Glob(GetApiDir() + "log/*", boost::bind(&ApiListener::LogGlobHandler, boost::ref(files), _1), GlobFile);
std::sort(files.begin(), files.end());
for (int ts : files) {
String path = GetApiDir() + "log/" + Convert::ToString(ts);
if (ts < peer_ts)
continue;
Log(LogNotice, "ApiListener")
<< "Replaying log: " << path;
std::fstream *fp = new std::fstream(path.CStr(), std::fstream::in | std::fstream::binary);
StdioStream::Ptr logStream = new StdioStream(fp, true);
String message;
StreamReadContext src;
while (true) {
Dictionary::Ptr pmessage;
try {
StreamReadStatus srs = NetString::ReadStringFromStream(logStream, &message, src);
if (srs == StatusEof)
break;
if (srs != StatusNewItem)
continue;
pmessage = JsonDecode(message);
} catch (const std::exception&) {
Log(LogWarning, "ApiListener")
<< "Unexpected end-of-file for cluster log: " << path;
/* Log files may be incomplete or corrupted. This is perfectly OK. */
break;
}
if (pmessage->Get("timestamp") <= peer_ts)
continue;
Dictionary::Ptr secname = pmessage->Get("secobj");
if (secname) {
ConfigObject::Ptr secobj = ConfigObject::GetObject(secname->Get("type"), secname->Get("name"));
if (!secobj)
continue;
if (!target_zone->CanAccessObject(secobj))
continue;
}
try {
NetString::WriteStringToStream(client->GetStream(), pmessage->Get("message"));
count++;
} catch (const std::exception& ex) {
Log(LogWarning, "ApiListener")
<< "Error while replaying log for endpoint '" << endpoint->GetName() << "': " << DiagnosticInformation(ex);
break;
}
peer_ts = pmessage->Get("timestamp");
if (ts > logpos_ts + 10) {
logpos_ts = ts;
Dictionary::Ptr lparams = new Dictionary();
lparams->Set("log_position", logpos_ts);
Dictionary::Ptr lmessage = new Dictionary();
lmessage->Set("jsonrpc", "2.0");
lmessage->Set("method", "log::SetLogPosition");
lmessage->Set("params", lparams);
JsonRpc::SendMessage(client->GetStream(), lmessage);
}
}
logStream->Close();
}
if (count > 0) {
Log(LogInformation, "ApiListener")
<< "Replayed " << count << " messages.";
}
Log(LogNotice, "ApiListener")
<< "Replayed " << count << " messages.";
if (last_sync) {
{
ObjectLock olock2(endpoint);
endpoint->SetSyncing(false);
}
OpenLogFile();
break;
}
}
}