void LogCollector::onLogUploadResponse(const LogSyncResponse& response)
{
KAA_MUTEX_LOCKING("storageGuard_");
KAA_MUTEX_UNIQUE_DECLARE(storageGuardLock, storageGuard_);
KAA_MUTEX_LOCKED("storageGuard_");
if (!response.deliveryStatuses.is_null()) {
const auto& deliveryStatuses = response.deliveryStatuses.get_array();
for (const auto& status : deliveryStatuses) {
if (!removeDeliveryTimeout(status.requestId)) {
continue;
}
if (status.result == SyncResponseResultType::SUCCESS) {
KAA_LOG_INFO(boost::format("Logs (requestId %ld) successfully delivered") % status.requestId);
storage_->removeRecordBlock(status.requestId);
} else {
storage_->notifyUploadFailed(status.requestId);
KAA_MUTEX_UNLOCKING("storageGuard_");
KAA_UNLOCK(storageGuardLock);
KAA_MUTEX_UNLOCKED("storageGuard_");
if (!status.errorCode.is_null()) {
auto errocCode = status.errorCode.get_LogDeliveryErrorCode();
KAA_LOG_WARN(boost::format("Logs (requestId %ld) failed to deliver (error %d)")
% status.requestId % (int)errocCode);
executorContext_.getCallbackExecutor().add([this, errocCode] ()
{
uploadStrategy_->onFailure(*this, errocCode);
});
} else {
KAA_LOG_WARN("Log delivery failed, but no error code received");
}
KAA_MUTEX_LOCKING("storageGuard_");
KAA_LOCK(storageGuardLock);
KAA_MUTEX_LOCKED("storageGuard_");
}
}
}
KAA_MUTEX_UNLOCKING("storageGuard_");
KAA_UNLOCK(storageGuardLock);
KAA_MUTEX_UNLOCKED("storageGuard_");
processLogUploadDecision(uploadStrategy_->isUploadNeeded(storage_->getStatus()));
}
void ConfigurationPersistenceManager::onConfigurationUpdated(const KaaRootConfiguration& configuration)
{
if (ignoreConfigurationUpdate_) {
ignoreConfigurationUpdate_ = false;
return;
}
KAA_LOG_INFO("Configuration updated.");
AvroByteArrayConverter<KaaRootConfiguration> converter;
SharedDataBuffer buffer = converter.toByteArray(configuration);
KAA_LOG_INFO(boost::format("Going to store configuration using configuration storage %1%") % storage_);
KAA_MUTEX_LOCKING("confPersistenceGuard_");
KAA_MUTEX_UNIQUE_DECLARE(confPersistenceGuardLock, confPersistenceGuard_);
KAA_MUTEX_LOCKED("confPersistenceGuard_");
if (storage_) {
storage_->saveConfiguration(std::vector<std::uint8_t>(buffer.first.get(), buffer.first.get() + buffer.second));
}
KAA_MUTEX_UNLOCKING("confPersistenceGuard_");
KAA_UNLOCK(confPersistenceGuardLock);
KAA_MUTEX_UNLOCKED("confPersistenceGuard_");
configurationHash_ = EndpointObjectHash(buffer);
KAA_LOG_INFO(boost::format("Calculated configuration hash: %1%") % LoggingUtils::ByteArrayToString(configurationHash_.getHashDigest()));
}
void DefaultOperationTcpChannel::onPingTimeout(const boost::system::error_code& err)
{
if (!err) {
sendPingRequest();
} else {
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(channelLock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (err != boost::asio::error::operation_aborted && isConnected_) {
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(channelLock);
KAA_MUTEX_UNLOCKED("channelGuard_")
KAA_LOG_ERROR(boost::format("Channel [%1%] failed to process PING: %2%") % getId() % err.message());
onServerFailed();
return;
} else {
KAA_LOG_DEBUG(boost::format("Channel [%1%] PING timer aborted") % getId());
return;
}
}
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(channelLock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (isConnected_) {
setPingTimer();
}
}
void DefaultOperationTcpChannel::closeConnection()
{
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
bool wasConnected = isConnected_;
KAA_LOG_TRACE(boost::format("Channel [%1%] closing connection: isConnected '%2%'")
% getId()
% boost::io::group(std::boolalpha, wasConnected));
isFirstResponseReceived_ = false;
isConnected_ = false;
isPendingSyncRequest_ = false;
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
if (wasConnected) {
pingTimer_.cancel();
connAckTimer_.cancel();
sendDisconnect();
boost::system::error_code errorCode;
sock_->shutdown(boost::asio::ip::tcp::socket::shutdown_both, errorCode);
sock_->close(errorCode);
responseProcessor.flush();
}
}
void DefaultOperationTcpChannel::doShutdown()
{
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
KAA_LOG_DEBUG(boost::format("Channel [%1%] is shutting down: isShutdown '%2%'")
% getId()
% boost::io::group(std::boolalpha, isShutdown_));
if (!isShutdown_) {
isShutdown_ = true;
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
closeConnection();
KAA_LOG_TRACE(boost::format("Channel [%1%] stopping IO service: isStopped '%2%'")
% getId()
% boost::io::group(std::boolalpha, io_.stopped()));
if (!io_.stopped()) {
io_.stop();
}
stopThreads();
}
}
void DefaultOperationTcpChannel::sync(TransportType type)
{
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (isShutdown_) {
KAA_LOG_WARN(boost::format("Channel [%1%] can't sync: channel is shut down") % getId());
return;
}
if (isPaused_) {
KAA_LOG_WARN(boost::format("Channel [%1%] can't sync: channel is paused") % getId());
return;
}
const auto& suppportedTypes = getSupportedTransportTypes();
auto it = suppportedTypes.find(type);
if (it == suppportedTypes.end() || it->second == ChannelDirection::DOWN) {
KAA_LOG_ERROR(boost::format("Channel [%1%] ignore sync: unsupported transport type %2%")
% getId()
% LoggingUtils::toString(type));
return;
}
if (!currentServer_) {
KAA_LOG_DEBUG(boost::format("Channel [%1%] can't sync: server is null") % getId());
return;
}
if (isFirstResponseReceived_) {
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
std::map<TransportType, ChannelDirection> syncTypes;
syncTypes.insert(std::make_pair(type, it->second));
for (const auto& typeIt : suppportedTypes) {
if (typeIt.first != type) {
syncTypes.insert(std::make_pair(typeIt.first, ChannelDirection::DOWN));
}
}
boost::system::error_code errorCode = sendKaaSync(syncTypes);
if (errorCode) {
KAA_LOG_ERROR(boost::format("Channel [%1%] failed to sync: %2%")
% getId()
% errorCode.message());
onServerFailed();
}
} else {
KAA_LOG_DEBUG(boost::format("Channel [%1%] can't sync: waiting for CONNACK + KAASYNC") % getId());
isPendingSyncRequest_ = true;
}
}
void LogCollector::addLogRecord(const KaaUserLogRecord& record)
{
LogRecordPtr serializedRecord(new LogRecord(record));
KAA_MUTEX_LOCKING("storageGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, storageGuard_);
KAA_MUTEX_LOCKED("storageGuard_");
storage_->addLogRecord(serializedRecord);
KAA_MUTEX_UNLOCKING("storageGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("storageGuard_");
processLogUploadDecision(uploadStrategy_->isUploadNeeded(storage_->getStatus()));
}
void DefaultOperationTcpChannel::onReadEvent(const boost::system::error_code& err)
{
if (!err) {
std::ostringstream responseStream;
responseStream << responseBuffer_.get();
const auto& responseStr = responseStream.str();
try {
if (responseStr.empty()) {
KAA_LOG_ERROR(boost::format("Channel [%1%] no data read from socket") % getId());
} else {
responseProcessor.processResponseBuffer(responseStr.data(), responseStr.size());
}
} catch (const TransportRedirectException& exception) {
KAA_LOG_INFO(boost::format("Channel [%1%] received REDIRECT response") % getId());
return;
} catch (const KaaException& exception) {
KAA_LOG_ERROR(boost::format("Channel [%1%] failed to process data: %2%") % getId() % exception.what());
onServerFailed();
}
} else {
KAA_LOG_WARN(boost::format("Channel [%1%] socket error: %2%") % getId() % err.message());
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(channelLock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (err != boost::asio::error::operation_aborted && isConnected_) {
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(channelLock);
KAA_MUTEX_UNLOCKED("channelGuard_")
onServerFailed();
return;
} else {
KAA_LOG_DEBUG(boost::format("Channel [%1%] socket operations aborted") % getId());
return;
}
}
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(channelLock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (isConnected_) {
readFromSocket();
}
}
void DefaultOperationTcpChannel::pause()
{
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (isShutdown_) {
KAA_LOG_WARN(boost::format("Channel [%1%] can't pause: channel is shut down") % getId());
return;
}
if (!isPaused_) {
isPaused_ = true;
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
closeConnection();
}
}
void DefaultOperationTcpChannel::setServer(ITransportConnectionInfoPtr server)
{
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (isShutdown_) {
KAA_LOG_WARN(boost::format("Channel [%1%] ignore new server: channel is shut down") % getId());
return;
}
if (server->getTransportId() != getTransportProtocolId()) {
KAA_LOG_WARN(boost::format("Channel [%1%] ignore new server: unsupported transport %2%")
% getId()
% LoggingUtils::toString(server->getTransportId()));
return;
}
KAA_LOG_TRACE(boost::format("Channel [%1%] preparing to use new server %2%")
% getId()
% LoggingUtils::toString(*server));
currentServer_ = std::make_shared<IPTransportInfo>(server);
encDec_ = std::make_shared<RsaEncoderDecoder>(clientKeys_.getPublicKey(),
clientKeys_.getPrivateKey(),
currentServer_->getPublicKey(),
context_);
isFailoverInProgress_ = false;
if (!isPaused_) {
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
closeConnection();
KAA_LOG_TRACE(boost::format("Channel [%1%] scheduling open connection")
% getId());
io_.post(std::bind(&DefaultOperationTcpChannel::openConnection, this));
}
}
void EventManager::commit(TransactionIdPtr trxId, IKaaClientContext &context_)
{
auto it = transactions_.find(trxId);
if (it != transactions_.end()) {
KAA_MUTEX_LOCKING("pendingEventsGuard_");
KAA_MUTEX_UNIQUE_DECLARE(eventsLock, pendingEventsGuard_);
KAA_MUTEX_LOCKED("pendingEventsGuard_");
std::list<Event> & events = it->second;
for (Event &e : events) {
pendingEvents_.insert(std::make_pair(currentEventIndex_++, std::move(e)));
}
transactions_.erase(it);
KAA_MUTEX_UNLOCKING("pendingEventsGuard_");
KAA_UNLOCK(eventsLock);
KAA_MUTEX_UNLOCKED("pendingEventsGuard_");
doSync();
}
}
void LogCollector::processTimeout()
{
executorContext_.getCallbackExecutor().add([this] () { uploadStrategy_->onTimeout(*this); });
KAA_LOG_WARN(boost::format("Going to notify log storage of logs delivery timeout..."));
KAA_MUTEX_LOCKING("timeoutsGuard_");
KAA_MUTEX_UNIQUE_DECLARE(timeoutsGuardLock, timeoutsGuard_);
KAA_MUTEX_LOCKED("timeoutsGuard_");
for (const auto& request : timeouts_) {
storage_->notifyUploadFailed(request.first);
}
timeouts_.clear();
KAA_MUTEX_UNLOCKING("timeoutsGuard_");
KAA_UNLOCK(timeoutsGuardLock);
KAA_MUTEX_UNLOCKED("timeoutsGuard_");
processLogUploadDecision(uploadStrategy_->isUploadNeeded(storage_->getStatus()));
}
void LogCollector::addLogRecord(const KaaUserLogRecord& record)
{
/*
* To avoid overhead on copying big-sized log records while capturing in lambdas,
* serialization has been performed before adding task to executor.
*/
LogRecordPtr serializedRecord(new LogRecord(record));
executorContext_.getApiExecutor().add([this, serializedRecord] ()
{
KAA_MUTEX_LOCKING("storageGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, storageGuard_);
KAA_MUTEX_LOCKED("storageGuard_");
storage_->addLogRecord(serializedRecord);
KAA_MUTEX_UNLOCKING("storageGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("storageGuard_");
processLogUploadDecision(uploadStrategy_->isUploadNeeded(storage_->getStatus()));
});
}
void DefaultOperationTcpChannel::syncAll()
{
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (isShutdown_) {
KAA_LOG_WARN(boost::format("Channel [%1%] can't sync: channel is shut down") % getId());
return;
}
if (isPaused_) {
KAA_LOG_WARN(boost::format("Channel [%1%] can't sync: channel is on oause") % getId());
return;
}
if (!currentServer_) {
KAA_LOG_WARN(boost::format("Channel [%1%] can't sync: server is null") % getId());
return;
}
if (isFirstResponseReceived_) {
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
boost::system::error_code errorCode = sendKaaSync(getSupportedTransportTypes());
if (errorCode) {
KAA_LOG_ERROR(boost::format("Channel [%1%]. Failed to sync: %2%") % getId() % errorCode.message());
onServerFailed();
}
} else {
KAA_LOG_DEBUG(boost::format("Can't sync channel [%1%]. Waiting for CONNACK message + KAASYNC message") % getId());
isPendingSyncRequest_ = true;
}
}
void DefaultOperationTcpChannel::onKaaSync(const KaaSyncResponse& message)
{
KAA_LOG_DEBUG(boost::format("Channel [%1%]. KaaSync response received") % getId());
const auto& encodedResponse = message.getPayload();
std::string decodedResponse;
try {
decodedResponse = encDec_->decodeData(encodedResponse.data(), encodedResponse.size());
} catch (const std::exception& e) {
KAA_LOG_ERROR(boost::format("Channel [%1%] unable to decode data: %2%")
% getId()
% e.what());
onServerFailed();
return;
}
auto returnCode = demultiplexer_->processResponse(
std::vector<std::uint8_t>(reinterpret_cast<const std::uint8_t *>(decodedResponse.data()),
reinterpret_cast<const std::uint8_t *>(decodedResponse.data() + decodedResponse.size())));
if (returnCode == DemultiplexerReturnCode::REDIRECT) {
throw TransportRedirectException(boost::format("Channel [%1%] received REDIRECT response") % getId());
} else if (returnCode == DemultiplexerReturnCode::FAILURE) {
onServerFailed();
return;
}
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lock, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
if (!isFirstResponseReceived_) {
KAA_LOG_INFO(boost::format("Channel [%1%] received first response") % getId());
connAckTimer_.cancel();
isFirstResponseReceived_ = true;
}
if (isPendingSyncRequest_) {
isPendingSyncRequest_ = false;
ackTypes_.clear();
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
KAA_LOG_INFO(boost::format("Channel [%1%] has pending request. Starting SYNC...") % getId());
syncAll();
} else if (!ackTypes_.empty()) {
KAA_LOG_INFO(boost::format("Channel [%1%] has %2% pending ACK requests. Starting SYNC...")
% getId()
% ackTypes_.size());
auto ackTypesCopy = ackTypes_;
ackTypes_.clear();
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
if (ackTypesCopy.size() > 1) {
syncAll();
} else {
sync(*ackTypesCopy.begin());
}
}
}
void DefaultOperationTcpChannel::openConnection()
{
if (isConnected_) {
KAA_LOG_WARN(boost::format("Channel [%1%] connection is already opened") % getId());
return;
}
KAA_LOG_TRACE(boost::format("Channel [%1%] opening connection to %2%:%3%")
% getId()
% currentServer_->getHost()
% currentServer_->getPort());
boost::system::error_code errorCode;
boost::asio::ip::tcp::endpoint ep = HttpUtils::resolveEndpoint(currentServer_->getHost(),
currentServer_->getPort(),
errorCode);
if (errorCode) {
KAA_LOG_ERROR(boost::format("Channel [%1%] failed to resolve endpoint: %2%")
% getId()
% errorCode.message());
onServerFailed();
return;
}
responseBuffer_.reset(new boost::asio::streambuf());
sock_.reset(new boost::asio::ip::tcp::socket(io_));
sock_->open(ep.protocol(), errorCode);
if (errorCode) {
KAA_LOG_ERROR(boost::format("Channel [%1%] failed to open socket: %2%")
% getId()
% errorCode.message());
onServerFailed();
return;
}
sock_->connect(ep, errorCode);
if (errorCode) {
KAA_LOG_ERROR(boost::format("Channel [%1%] failed to connect to %2%:%3%: %4%")
% getId()
% ep.address().to_string()
% ep.port()
% errorCode.message());
onServerFailed();
return;
}
channelManager_.onConnected({sock_->local_endpoint().address().to_string(), ep.address().to_string(), getServerType()});
KAA_MUTEX_LOCKING("channelGuard_");
KAA_LOCK(channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
isConnected_ = true;
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(channelGuard_);
KAA_MUTEX_UNLOCKED("channelGuard_");
sendConnect();
setConnAckTimer();
readFromSocket();
setPingTimer();
}
void AbstractHttpChannel::processTypes(const std::map<TransportType, ChannelDirection>& types
#ifdef KAA_THREADSAFE
, KAA_MUTEX_UNIQUE& lock
#endif
)
{
auto postRequest = createRequest(currentServer_, multiplexer_->compileRequest(types));
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lock);
KAA_MUTEX_UNLOCKED("channelGuard_");
try {
// Sending http request
EndpointConnectionInfo connection("", "", getServerType());
auto response = httpClient_.sendRequest(*postRequest, &connection);
channelManager_.onConnected(connection);
KAA_MUTEX_LOCKING("channelGuard_");
KAA_MUTEX_UNIQUE_DECLARE(lockInternal, channelGuard_);
KAA_MUTEX_LOCKED("channelGuard_");
// Retrieving the avro data from the HTTP response
const std::string& processedResponse = retrieveResponse(*response);
KAA_MUTEX_UNLOCKING("channelGuard_");
KAA_UNLOCK(lockInternal);
KAA_MUTEX_UNLOCKED("channelGuard_");
if (!processedResponse.empty()) {
demultiplexer_->processResponse(
std::vector<std::uint8_t>(reinterpret_cast<const std::uint8_t *>(processedResponse.data()),
reinterpret_cast<const std::uint8_t *>(processedResponse.data() + processedResponse.size())));
}
} catch (HttpTransportException& e) {
KAA_LOG_WARN(boost::format("Channel [%1%] failed to connect %2%:%3%: %4%")
% getId()
% currentServer_->getHost()
% currentServer_->getPort()
% e.getHttpStatusCode());
KaaFailoverReason reason;
switch (e.getHttpStatusCode()) {
case HttpStatusCode::UNAUTHORIZED:
reason = KaaFailoverReason::ENDPOINT_NOT_REGISTERED;
break;
case HttpStatusCode::FORBIDDEN:
reason = KaaFailoverReason::CREDENTIALS_REVOKED;
break;
default:
reason = getNotAccessibleFailoverReason(getServerType());
break;
}
onServerFailed(reason);
} catch (TransportException& e) {
KAA_LOG_WARN(boost::format("Channel [%1%] failed to connect %2%:%3%: %4%")
% getId()
% currentServer_->getHost()
% currentServer_->getPort()
% e.getErrorCode().message());
KaaFailoverReason reason = getNotAccessibleFailoverReason(getServerType());
if (connectivityChecker_ && !connectivityChecker_->checkConnectivity()) {
KAA_LOG_WARN(boost::format("Channel [%1%] detected loss of connectivity")
% getId());
reason = KaaFailoverReason::NO_CONNECTIVITY;
}
onServerFailed(reason);
}
}
