//Track the first instance of our App.
//return TRUE on success, else FALSE
BOOL CInstanceChecker::TrackFirstInstanceRunning()
{
//If a previous instance is running, just return prematurely
if (PreviousInstanceRunning())
return FALSE;
//If this is the first instance then copy in our info into the shared memory
//First create the MMF
int nMMFSize = sizeof(CWindowInstance);
g_sinstanceData.hInstanceData = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, nMMFSize, GetMMFFilename());
if (g_sinstanceData.hInstanceData == NULL)
{
TRACE(_T("Failed to create the MMF even though this is the first instance, you might want to consider overriding GetMMFFilename()\n"));
return FALSE;
}
//Open the MMF
CWindowInstance* pInstanceData = static_cast<CWindowInstance*>(MapViewOfFile(g_sinstanceData.hInstanceData, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, nMMFSize));
ASSERT(pInstanceData != NULL); //Opening the MMF should work
//Lock the data prior to updating it
CSingleLock dataLock(&m_instanceDataMutex, TRUE);
pInstanceData->hMainWnd = GetWindowToTrack();
UnmapViewOfFile(pInstanceData);
//Since this will be the last function that will be called
//when this is the first instance we can release the lock
ReleaseLock();
return TRUE;
}
void ReasonerStandardImplementation::receivedReasonerNodeHealth(ReasonerMessageReceived & data, NodeHealth & health){
{ // Disallow other access to aggregated data fields
unique_lock<mutex> dataLock( dataMutex );
if ( role == ReasonerStandardImplementationOptions::Role::SYSTEM ){
(*childNodesHealthMap)[data.reasonerID] = health;
// OUTPUT( "Received status from node reasoner " << data.reasonerID );
}
else if ( role == ReasonerStandardImplementationOptions::Role::PROCESS ){
// aggregate statistics
for (int i=0; i < NODE_STATISTIC_COUNT; i++){
node_statistics[i] += health.statistics[i];
}
// OUTPUT( "CPU: " << health.statistics[CONSUMED_CPU_SECONDS] );
// OUTPUT( "RAPL: " << health.statistics[CONSUMED_ENERGY_JOULE] );
nodeHealth = make_shared<NodeHealth>(health);
// OUTPUT( "Received status from node reasoner " << data.reasonerID );
}
else if ( data.reasonerID == "INJECT" ) // Workaround for testing!
nodeHealth = make_shared<NodeHealth>( health );
else
ERROR( "Received inappropriate message from " << data.reasonerID << "!" );
nPushesReceived++;
}
}
void CInstanceChecker::QuitPreviousInstance(int nExitCode)
{
//Try to open the previous instances MMF
HANDLE hPrevInstance = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, GetMMFFilename());
if (hPrevInstance)
{
// Open up the MMF
int nMMFSize = sizeof(CWindowInstance);
CWindowInstance* pInstanceData = static_cast<CWindowInstance*>(MapViewOfFile(hPrevInstance, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, nMMFSize));
if (pInstanceData != NULL) //Opening the MMF should work
{
// Lock the data prior to reading from it
CSingleLock dataLock(&m_instanceDataMutex, TRUE);
//activate the old window
ASSERT(pInstanceData->hMainWnd); //Something gone wrong with the MMF?
HWND hWnd = pInstanceData->hMainWnd;
//Ask it to exit
HWND hChildWnd = GetLastActivePopup(hWnd);
PostMessage(hChildWnd, WM_QUIT, nExitCode, 0);
}
//Close the file handle now that we
CloseHandle(hPrevInstance);
}
}
void EventStatDialog::OnCbnSelchangeComboTimeslice()
{
CSingleLock dataLock(&m_csEventData);
dataLock.Lock();
#ifdef _DEBUG
//if the number of columns is not 4, you need to manually add data below
ASSERT(sk_numOfColumns == 4);
#endif
switch (m_timeSliceComboBox.GetCurSel())
{
//for now, just hard code, if someone find a better way to deal with them, feel free to change it.
case 0:
m_columnsData[0] = LAST_10S;
m_columnsData[1] = LAST_30S;
m_columnsData[2] = LAST_1MIN;
m_columnsData[3] = LAST_5MINS;
break;
case 1:
m_columnsData[0] = LAST_5MINS;
m_columnsData[1] = LAST_10MINS;
m_columnsData[2] = LAST_15MINS;
m_columnsData[3] = LAST_SINCE_START;
break;
default:
//do nothing
break;
}
UpdateColumnHeader();
dataLock.Unlock();
}
void Close( bool bDeleteDevice )
{
// Make sure thread is attached to JVM/env
JNIEnv *env;
g_JVM->AttachCurrentThread( &env, NULL );
pthread_setspecific( g_ThreadKey, (void*)env );
env->CallVoidMethod( g_HIDDeviceManagerCallbackHandler, g_midHIDDeviceManagerClose, m_nId );
hid_mutex_guard dataLock( &m_dataLock );
m_vecData.clear();
// Clean and release pending feature report reads
hid_mutex_guard cvLock( &m_cvLock );
m_featureReport.clear();
m_bIsWaitingForFeatureReport = false;
m_nFeatureReportError = -ECONNRESET;
pthread_cond_broadcast( &m_cv );
if ( bDeleteDevice )
{
delete m_pDevice;
m_pDevice = nullptr;
}
}
int32
StreamingRingBuffer::Read(void *buffer, size_t length, bool onlyBlockOnNoData)
{
BAutolock readerLock(fReaderLocker);
if (!readerLock.IsLocked())
return B_ERROR;
BAutolock dataLock(fDataLocker);
if (!dataLock.IsLocked())
return B_ERROR;
int32 readSize = 0;
while (length > 0) {
size_t copyLength = min_c(length, fBufferSize - fReadPosition);
copyLength = min_c(copyLength, fReadable);
if (copyLength == 0) {
if (onlyBlockOnNoData && readSize > 0)
return readSize;
fReaderWaiting = true;
dataLock.Unlock();
status_t result;
do {
TRACE("waiting in reader\n");
result = acquire_sem(fReaderNotifier);
TRACE("done waiting in reader with status: 0x%08lx\n", result);
} while (result == B_INTERRUPTED);
if (result != B_OK)
return result;
if (!dataLock.Lock())
return B_ERROR;
continue;
}
// support discarding input
if (buffer != NULL) {
memcpy(buffer, fBuffer + fReadPosition, copyLength);
buffer = (uint8 *)buffer + copyLength;
}
fReadPosition = (fReadPosition + copyLength) % fBufferSize;
fReadable -= copyLength;
readSize += copyLength;
length -= copyLength;
if (fWriterWaiting) {
release_sem_etc(fWriterNotifier, 1, B_DO_NOT_RESCHEDULE);
fWriterWaiting = false;
}
}
return readSize;
}
shared_ptr<SystemHealth> ReasonerStandardImplementation::getSystemHealth(){
{ // Disallow other access to aggregated data fields
unique_lock<mutex> dataLock( dataMutex );
if ( role == ReasonerStandardImplementationOptions::Role::SYSTEM )
return systemHealth;
}
// For other roles, return NULL; this should never happen
return nullptr;
}
void EventStatDialog::CleanUpCounters()
{
CSingleLock dataLock(&m_csEventData);
dataLock.Lock();
for(int i = 0; i < NUMBER_OF_EVENT_TYPES;i++)
{
m_data[i].clear();
}
dataLock.Unlock();
}
shared_ptr<ProcessHealth> ReasonerStandardImplementation::getProcessHealth(){
{ // Disallow other access to aggregated data fields
unique_lock<mutex> dataLock( dataMutex );
if ( role == Role::PROCESS )
return processHealth;
}
// For other roles, return NULL; this should never happen
return nullptr;
}
void DataStorage::storageLoop()
{
isRunning_ = true;
while (isRunning_)
{
//wait until we have data in our queue
{
boost::mutex::scoped_lock dataLock(storageMutex_);
while (dataQueue_.empty())
{
queueEmptyCondition_.wait(dataLock);
}
}
//lock the data again for as long as we're storing it
unsigned int numPackets = 0;
{
boost::mutex::scoped_lock dataLock(storageMutex_);
//create a deferred transaction for the following commands
//Simple explanation: If we do not wrap everything into a transaction here the performance would be VERY bad
sqlite::transaction_guard< > transactionGuard(*connection_);
std::vector<zmq_recorder::Packet>::iterator dataIter = dataQueue_.begin();
for (; dataIter != dataQueue_.end(); ++dataIter)
{
updateData(*dataIter);
}
numPackets = dataQueue_.size();
dataQueue_.clear();
//commit the transaction (or it will rollback)
transactionGuard.commit();
}
std::cout << getCurrentDateStr() << ": Stored " << numPackets << " packet(s)" << std::endl;
//sleep some time, this avoids that we do too many transmissions to the database at once
//it also frees up some time for the CPU
boost::this_thread::sleep(boost::posix_time::milliseconds(1000));
}
}
// Activate the Previous Instance of our Application.
HWND CInstanceChecker::ActivatePreviousInstance()
{
//Try to open the previous instances MMF
HANDLE hPrevInstance = ::OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, GetMMFFilename());
if (hPrevInstance)
{
// Open up the MMF
int nMMFSize = sizeof(CWindowInstance);
CWindowInstance* pInstanceData = (CWindowInstance*) ::MapViewOfFile(hPrevInstance, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, nMMFSize);
if (pInstanceData != NULL) //Opening the MMF should work
{
// Lock the data prior to reading from it
CSingleLock dataLock(&m_instanceDataMutex, TRUE);
//activate the old window
ASSERT(pInstanceData->hMainWnd); //Something gone wrong with the MMF
HWND hWindow = pInstanceData->hMainWnd;
if (hWindow)
{
CWnd wndPrev;
wndPrev.Attach(hWindow);
CWnd* pWndChild = wndPrev.GetLastActivePopup();
// Restore the focus to the previous instance and bring it to the foreground
if (wndPrev.IsIconic())
wndPrev.ShowWindow(SW_RESTORE);
pWndChild->SetForegroundWindow();
//Detach the CWnd we were using
wndPrev.Detach();
}
//Unmap the MMF we were using
::UnmapViewOfFile(pInstanceData);
//Close the file handle now that we
::CloseHandle(hPrevInstance);
//return the Window handle of the previous instance
return hWindow;
}
//Close the file handle now that we
::CloseHandle(hPrevInstance);
//When we have activate the previous instance, we can release the lock
ReleaseLock();
}
return NULL;
}
void EventStatDialog::AddEvent(EventType eventType)
{
if (m_turnOnEventCollection == false)
{
return;
}
time_t currentTime;
time(¤tTime);
CSingleLock dataLock(&m_csEventData);
dataLock.Lock();
m_data[eventType].push_back(currentTime);
dataLock.Unlock();
}
status_t
StreamingRingBuffer::Write(const void *buffer, size_t length)
{
BAutolock writerLock(fWriterLocker);
if (!writerLock.IsLocked())
return B_ERROR;
BAutolock dataLock(fDataLocker);
if (!dataLock.IsLocked())
return B_ERROR;
while (length > 0) {
size_t copyLength = min_c(length, fBufferSize - fWritePosition);
copyLength = min_c(copyLength, fBufferSize - fReadable);
if (copyLength == 0) {
fWriterWaiting = true;
dataLock.Unlock();
status_t result;
do {
TRACE("waiting in writer\n");
result = acquire_sem(fWriterNotifier);
TRACE("done waiting in writer with status: 0x%08lx\n", result);
} while (result == B_INTERRUPTED);
if (result != B_OK)
return result;
if (!dataLock.Lock())
return B_ERROR;
continue;
}
memcpy(fBuffer + fWritePosition, buffer, copyLength);
fWritePosition = (fWritePosition + copyLength) % fBufferSize;
fReadable += copyLength;
buffer = (uint8 *)buffer + copyLength;
length -= copyLength;
if (fReaderWaiting) {
release_sem_etc(fReaderNotifier, 1, B_DO_NOT_RESCHEDULE);
fReaderWaiting = false;
}
}
return B_OK;
}
int ConnectThread::run()
{
MYSQL* sql = m_mysql->lockHandle();
if (!mysql_real_connect(sql, stringOrNull(m_host), stringOrNull(m_user), stringOrNull(m_pass), stringOrNull(m_database), m_port, stringOrNull(m_unixSocket), m_clientFlag))
{
MutexLocker dataLock(m_dataMutex);
m_success = false;
m_error = std::string( mysql_error(sql) );
postEvent( CONNECTION_FINISHED );
m_mysql->unlockHandle();
return 0;
}
else
{
MutexLocker dataLock(m_dataMutex);
m_success = true;
m_error = "";
postEvent( CONNECTION_FINISHED );
m_mysql->unlockHandle();
return 0;
}
}
void ReasonerStandardImplementation::receivedReasonerSystemHealth(ReasonerMessageReceived & data, SystemHealth & health){
{ // Disallow other access to aggregated data fields
unique_lock<mutex> dataLock( dataMutex );
if ( role == ReasonerStandardImplementationOptions::Role::NODE ){
// OUTPUT( "Received status from system reasoner " << data.reasonerID );
systemHealth = make_shared<SystemHealth>(health);
}
else if ( data.reasonerID == "INJECT" ) // Workaround for testing!
systemHealth = make_shared<SystemHealth>( health );
else
ERROR( "Received inappropriate message from " << data.reasonerID << "!" );
}
nPushesReceived++;
}
void ReasonerStandardImplementation::receivedReasonerProcessHealth(ReasonerMessageReceived & data, ProcessHealth & health){
{ // Disallow other access to aggregated data fields
unique_lock<mutex> dataLock( dataMutex );
// Only Node level reasoners should receive process level reports; others ignore them
if ( role == ReasonerStandardImplementationOptions::Role::NODE ){
(*childProcessesHealthMap)[data.reasonerID] = health;
// OUTPUT( "Received status from process reasoner " << data.reasonerID );
}
else if ( data.reasonerID == "INJECT" ) // Workaround for testing!
processHealth = make_shared<ProcessHealth>( health );
else
ERROR( "Received inappropriate message from " << data.reasonerID << "!" );
nPushesReceived++;
}
}
ComponentReport ReasonerStandardImplementation::prepareReport() {
ComponentReport result;
{ // Disallow other access to aggregated data fields
unique_lock<mutex> dataLock( dataMutex );
result.addEntry( "ANOMAL_RUNTIME_MS", ReportEntry( ReportEntry::Type::SIOX_INTERNAL_INFO, VariableDatatype( anomaliesTriggered * update_intervall_ms )));
result.addEntry( "OBSERVED_RUNTIME_MS", ReportEntry( ReportEntry::Type::SIOX_INTERNAL_INFO, VariableDatatype( cyclesTriggered * update_intervall_ms )));
result.addEntry( "STATES_SENT_UPSTREAM", ReportEntry( ReportEntry::Type::SIOX_INTERNAL_INFO, nPushesSent));
result.addEntry( "STATES_RECEIVED", ReportEntry( ReportEntry::Type::SIOX_INTERNAL_INFO, nPushesReceived));
if ( nPushesReceived > 0 ){
const char * text [] = {
"UTILIZATION_CPU",
"UTILIZATION_MEMORY",
"UTILIZATION_IO",
"UTILIZATION_NETWORK_SEND",
"UTILIZATION_NETWORK_RECEIVE",
"CONSUMED_CPU_SECONDS",
"CONSUMED_ENERGY_JOULE",
"CONSUMED_MEMORY_BYTES",
"CONSUMED_NETWORK_BYTES",
"CONSUMED_IO_BYTES",
};
for (int i=0; i < NODE_STATISTIC_COUNT; i++){
result.addEntry( text[i], ReportEntry( ReportEntry::Type::APPLICATION_PERFORMANCE, node_statistics[i] ));
}
}
// for debugging
ostringstream reportText;
reportText << this;
result.addEntry( "TEXT_STATE", ReportEntry( ReportEntry::Type::SIOX_INTERNAL_DEBUG, VariableDatatype( reportText.str() )));
}
return result;
}
void EventStatDialog::UpdateEventsData()
{
CSingleLock dataLock(&m_csEventData);
/*if (dataLock.IsLocked())
{
return;
}*/
dataLock.Lock();
time_t currentTime;
time(¤tTime);
m_hasBadEventRecently = false;
for(int i = 0; i < NUMBER_OF_EVENT_TYPES;i++)
{
int eventCnt[NUMBER_OF_TIME_SLICE_TYPES];
for (int n=0; n < NUMBER_OF_TIME_SLICE_TYPES; n++)
{
eventCnt[n]=0;
}
TimeSliceType currentTimeSlice = LAST_15MINS;
TimeSliceType nextTimeSlice = LAST_10MINS;
deque<time_t>::iterator eventTime;
for( eventTime = m_data[i].begin(); eventTime != m_data[i].end();)
{
if (m_data[i].empty())
{
break;
}
double timeDiff = difftime(currentTime,(*eventTime));
/*
// hard code version
if (timeDiff > GetTime(LAST_15MINS))
{
//delete this item in the queue
m_data[i].pop_front();
eventTime = m_data[i].begin();
continue;
}
else if (timeDiff > GetTime(LAST_10MINS))
{
eventCnt[LAST_15MINS]++;
}
else if (timeDiff > GetTime(LAST_5MINS))
{
eventCnt[LAST_10MINS]++;
}
else if (timeDiff > GetTime(LAST_1MIN))
{
eventCnt[LAST_5MINS]++;
}
else if (timeDiff > GetTime(LAST_30S))
{
eventCnt[LAST_1MIN]++;
}
else if (timeDiff > GetTime(LAST_10S))
{
eventCnt[LAST_30S]++;
}
else
{
eventCnt[LAST_10S]++;
}
*/
if (timeDiff > GetTime(currentTimeSlice))
{
if (currentTimeSlice == LAST_15MINS)
{
//delete this item in the queue
m_data[i].pop_front();
eventTime = m_data[i].begin();
// Increment the RemovedEvents counter to
// keep track of # of removed events
m_numOfRemovedEvents[i]++;
continue;
}
else
{
//do nothing, just skip this
}
}
else if (timeDiff > GetTime(nextTimeSlice))
{
eventCnt[currentTimeSlice]++;
}
else
{
if (nextTimeSlice != LAST_10S)
{
currentTimeSlice = nextTimeSlice;
nextTimeSlice = (TimeSliceType)(nextTimeSlice-1);
continue;
}
else
{
eventCnt[LAST_10S]++;
}
}
eventTime++;
}
//sum up the number of events
/*
//hard code version
eventCnt[LAST_30S]+=eventCnt[LAST_10S];
eventCnt[LAST_1MIN]+=eventCnt[LAST_30S];
eventCnt[LAST_5MINS]+=eventCnt[LAST_1MIN];
eventCnt[LAST_10MINS]+=eventCnt[LAST_5MINS];
eventCnt[LAST_15MINS]+=eventCnt[LAST_10MINS];
*/
for(unsigned int j=0; j < (unsigned int)NUMBER_OF_TIME_SLICE_TYPES - 1; j++)
{
eventCnt[j+1] += eventCnt[j];
}
// Adding # of removed events back to "Since Start" time slice category
eventCnt[NUMBER_OF_TIME_SLICE_TYPES - 1] += m_numOfRemovedEvents[i];
char buff[32];
for(int cn = 0; cn < sk_numOfColumns; cn++)
{
sprintf(buff,"%d",eventCnt[m_columnsData[cn]]);
m_eventTable.SetItemText(i, cn + 1, buff);
}
// check to see if there is bad event
if (IsBadEvent((EventType)i)&&eventCnt[0])
{
m_hasBadEventRecently = true;
}
}
dataLock.Unlock();
}
std::string ConnectThread::error()
{
MutexLocker dataLock(m_dataMutex);
return m_error;
}
bool ConnectThread::wasSuccessful()
{
MutexLocker dataLock(m_dataMutex);
return m_success;
}
void ConnectThread::setClientFlag(unsigned int flag)
{
MutexLocker dataLock(m_dataMutex);
m_clientFlag = flag;
}
void ConnectThread::setUnixSocket(const char* socket)
{
MutexLocker dataLock(m_dataMutex);
m_unixSocket = safeStrToStr(socket);
}
void ConnectThread::setPort(unsigned int port)
{
MutexLocker dataLock(m_dataMutex);
m_port = port;
}
void ConnectThread::setDatabase(const char* db)
{
MutexLocker dataLock(m_dataMutex);
m_database = safeStrToStr(db);
}
void ConnectThread::setUserPassword(const char* user, const char* pass)
{
MutexLocker dataLock(m_dataMutex);
m_user = safeStrToStr(user);
m_pass = safeStrToStr(pass);
}
void ConnectThread::setHost(const char* host)
{
MutexLocker dataLock(m_dataMutex);
m_host = safeStrToStr(host);
}
//Activate the Previous Instance of our Application.
HWND CInstanceChecker::ActivatePreviousInstance(LPCTSTR lpCmdLine, ULONG_PTR dwCopyDataItemData, DWORD dwTimeout)
{
//What will be the return value from this function (assume the worst)
HWND hWindow = NULL;
//Try to open the previous instances MMF
HANDLE hPrevInstance = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, GetMMFFilename());
if (hPrevInstance)
{
//Open up the MMF
int nMMFSize = sizeof(CWindowInstance);
CWindowInstance* pInstanceData = static_cast<CWindowInstance*>(MapViewOfFile(hPrevInstance, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, nMMFSize));
if (pInstanceData) //Opening the MMF should work
{
//Lock the data prior to reading from it
CSingleLock dataLock(&m_instanceDataMutex, TRUE);
//activate the old window
ASSERT(pInstanceData->hMainWnd); //Something gone wrong with the MMF
hWindow = pInstanceData->hMainWnd;
if (::IsWindow(hWindow))
{
CWnd wndPrev;
wndPrev.Attach(hWindow);
CWnd* pWndChild = wndPrev.GetLastActivePopup();
//Restore the focus to the previous instance and bring it to the foreground
if (wndPrev.IsIconic())
wndPrev.ShowWindow(SW_RESTORE);
if (pWndChild)
pWndChild->SetForegroundWindow();
if (lpCmdLine)
{
//Send the current apps command line to the previous instance using WM_COPYDATA
COPYDATASTRUCT cds;
cds.dwData = dwCopyDataItemData;
DWORD dwCmdLength = static_cast<DWORD>(_tcslen(lpCmdLine) + 1);
cds.cbData = dwCmdLength * sizeof(TCHAR);
TCHAR* pszLocalCmdLine = new TCHAR[dwCmdLength]; //We use a local buffer so that we can specify a constant parameter
//to this function
#if (_MSC_VER >= 1400)
_tcscpy_s(pszLocalCmdLine, dwCmdLength, lpCmdLine);
#else
_tcscpy(pszLocalCmdLine, lpCmdLine);
#endif
cds.lpData = pszLocalCmdLine;
CWnd* pMainWindow = AfxGetMainWnd();
HWND hSender = NULL;
if (pMainWindow)
hSender = pMainWindow->GetSafeHwnd();
//Send the message to the previous instance. Use SendMessageTimeout instead of SendMessage to ensure we
//do not hang if the previous instance itself is hung
DWORD_PTR dwResult = 0;
if (SendMessageTimeout(hWindow, WM_COPYDATA, reinterpret_cast<WPARAM>(hSender), reinterpret_cast<LPARAM>(&cds),
SMTO_ABORTIFHUNG, dwTimeout, &dwResult) == 0)
{
//Previous instance is not responding to messages
hWindow = NULL;
}
//Tidy up the heap memory we have used
delete [] pszLocalCmdLine;
}
//Detach the CWnd we were using
wndPrev.Detach();
}
//Unmap the MMF we were using
UnmapViewOfFile(pInstanceData);
}
//Close the file handle now that we
CloseHandle(hPrevInstance);
//When we have activate the previous instance, we can release the lock
ReleaseLock();
}
return hWindow;
}
void ReasonerStandardImplementation::PeriodicRun(){
monitoring_namespace_protect_thread();
bool persistingAnomaly; // Last cycle's anomalyDetected
bool anomalyDetected = false;
while( ! terminated ) {
// OUTPUT( "PeriodicRun started" );
auto timeout = chrono::system_clock::now() + chrono::milliseconds( update_intervall_ms );
cyclesTriggered++;
// Remember last cycle's anomaly state, if persisting anomalies are switched on
persistingAnomaly = anomalyDetected && kpersistentAnomalies;
anomalyDetected = false;
/*
* Query all registered ADPIs for news and aggregate them
*/
{ // Disallow other access to aggregated data fields
unique_lock<mutex> dataLock( dataMutex );
Health * localHealth = nullptr;
switch (role){
case ReasonerStandardImplementationOptions::Role::PROCESS :
localHealth = processHealth.get();
break;
case ReasonerStandardImplementationOptions::Role::NODE :
localHealth = nodeHealth.get();
break;
case ReasonerStandardImplementationOptions::Role::SYSTEM :
localHealth = systemHealth.get();
break;
default:
assert(false && "Invalid process role!");
}
{ // Disallow changes in registered plugins while cycling through ADPIs
unique_lock<mutex> pluginLock( pluginMutex );
// Query recent observations of all connected plugins and integrate them into the local performance issues.
// Loop over all registered ADPIs
for( auto adpi : adpis ) {
// OUTPUT( "ADPI: " << adpi );
// Memory management passes to our responsibility here!
unique_ptr<unordered_map<ComponentID, AnomalyPluginHealthStatistic>> newObservations = adpi->queryRecentObservations();
// OUTPUT( "Reports: " << gatheredStatistics->map.size() << " -> " << newObservations->size() );
// Loop over all reports from current ADPI (by ComponentID)
for (auto report : *newObservations ) {
// OUTPUT( "Report for CID: " << report.first );
// Merge this report into the global set
// TODO: In Funktion auslagern
// Merge occurrences into gatheredStatistics and local Health
for ( int healthState=0; healthState < HEALTH_STATE_COUNT; healthState++ ){
// OUTPUT( "HealthState = " << healthState );
// OUTPUT( "Before: " << gatheredStatistics->map[report.first].occurrences[healthState] );
gatheredStatistics->map[report.first].cid = report.first;
gatheredStatistics->map[report.first].occurrences[healthState] += report.second.occurrences[healthState];
// OUTPUT( "After: " << gatheredStatistics->map[report.first].occurrences[healthState] );
localHealth->occurrences[healthState] += report.second.occurrences[healthState];
// OUTPUT( "Updated: " << gatheredStatistics->map[report.first] );
}
// Merge positive issues into gatheredStatistics and *localHealth
unordered_map<string, HealthIssue> * sourceIssueMap = & (report.second.positiveIssues);
unordered_map<string, HealthIssue> * targetIssueMap = & (gatheredStatistics->map[report.first].positiveIssues);
list<HealthIssue> * targetIssueList = & (localHealth->positiveIssues);
for ( auto issue : *sourceIssueMap )
{
// OUTPUT( "Received issue: " << issue.second );
{ // gatheredStatistics
auto precedent = targetIssueMap->find( issue.first );
if ( precedent == targetIssueMap->end() )
{
(*targetIssueMap)[issue.first] = HealthIssue( issue.first, issue.second.occurrences, issue.second.delta_time_ms );
// OUTPUT( "Created " << (*targetIssueMap)[issue.first] );
}
else
{
precedent->second.add(issue.second);
// OUTPUT( "Added to " << precedent->second );
}
}
{ // *localHealth
HealthIssue * precedent = nullptr;
for( auto candidate : *targetIssueList ){
if ( candidate.name == issue.first )
precedent = & candidate;
}
if( precedent == nullptr )
targetIssueList->push_back( HealthIssue( issue.first, issue.second.occurrences, issue.second.delta_time_ms ) );
else
precedent->add( issue.second );
}
}
// Merge negative issues into gatheredStatistics and *localHealth
sourceIssueMap = & (report.second.negativeIssues);
targetIssueMap = & (gatheredStatistics->map[report.first].negativeIssues);
targetIssueList = & (localHealth->negativeIssues);
for ( auto issue : *sourceIssueMap )
{
// OUTPUT( "Received issue: "<< issue.second );
{ // gatheredStatistics
auto precedent = targetIssueMap->find( issue.first );
if ( precedent == targetIssueMap->end() )
{
(*targetIssueMap)[issue.first] = HealthIssue( issue.first, issue.second.occurrences, issue.second.delta_time_ms );
// OUTPUT( "Created " << (*targetIssueMap)[issue.first] );
}
else
{
precedent->second.add(issue.second);
// OUTPUT( "Added to " << precedent->second );
}
}
{ // *localHealth
HealthIssue * precedent = nullptr;
for( auto candidate : *targetIssueList ){
if ( candidate.name == issue.first ){
precedent = & candidate;
}
}
if( precedent == nullptr )
targetIssueList->push_back( HealthIssue( issue.first, issue.second.occurrences, issue.second.delta_time_ms ) );
else
precedent->add( issue.second );
}
}
}
// OUTPUT( "Reports: " << gatheredStatistics->map.size() << " -> " << newObservations->size() );
}
} // pluginLock
// Remember last step's statistics for comparison
oldObservationCounts = observationCounts;
oldObservationRatios = observationRatios;
oldObservationTotal = observationTotal;
// Reset current counts and total
observationTotal = 0;
for ( int state = 0; state< HEALTH_STATE_COUNT; state++ )
observationCounts[state] = 0;
// Add all gathered statistics by health state
for ( auto report : gatheredStatistics->map )
for ( int state = 0; state < HEALTH_STATE_COUNT; state++ )
observationCounts[state] += report.second.occurrences[state];
// Total up current counts
for ( int state = 0; state< HEALTH_STATE_COUNT; state++ )
observationTotal += observationCounts[state];
// Compute current ratios
if( observationTotal > 0 ){
float scale = 100.0 / observationTotal;
for ( int state = 0; state< HEALTH_STATE_COUNT; state++ ){
observationRatios[state] = observationCounts[state] * scale;
// OUTPUT( observationCounts[state] << "=" << (int) observationRatios[state] << "% " );
}
}
// Check whether anything happened
if ( observationTotal != oldObservationTotal ){
// OUTPUT( "Has news: " << oldObservationTotal << "->" << observationTotal );
// Check whether anything interesting happened
for (int state = 0; state < HEALTH_STATE_COUNT; ++state)
{
// OUTPUT( state << ": " << (int)oldObservationRatios[state] << "->" << (int)observationRatios[state] );
// Did ratio of any non-OK state move by more than 2 points?
if( state != HealthState::OK ){
int diff = observationRatios[state] - oldObservationRatios[state];
if( abs(diff) >= 2 ){
anomalyDetected = true;
// OUTPUT( "Detected an anomaly (moving observation ratio for state " << state << " from " << (int)oldObservationRatios[state] << " to "<< (int)observationRatios[state] << ", or " << observationCounts[state] << " out of " << observationTotal << ")!" );
}
}
// Did number of any ABNORMAL state rise?
if( state == HealthState::ABNORMAL_BAD
|| state == HealthState::ABNORMAL_GOOD
|| state == HealthState::ABNORMAL_OTHER ){
if( observationCounts[state] > oldObservationCounts[state] ){
anomalyDetected = true;
// OUTPUT( "Detected an anomaly (abnormal state " << state << ")!" );
}
}
}
}
// TODO combine health of the next level with my local information:
switch (role){
case ReasonerStandardImplementationOptions::Role::PROCESS :
if ( nodeHealth->overallState != GOOD && nodeHealth->overallState != BAD && nodeHealth->overallState != OK ){
anomalyDetected = true;
}
break;
}
// Run assessment appropriate to reasoner's role;
// then, forward current state to remote reasoners
switch ( role ) {
case ReasonerStandardImplementationOptions::Role::PROCESS:
assessProcessHealth();
if (upstreamReasonerExists){
// OUTPUT( "Pushing process state upstream!" );
comm->pushProcessStateUpstream( processHealth);
nPushesSent++;
}
break;
case ReasonerStandardImplementationOptions::Role::NODE:
assessNodeHealth();
if (upstreamReasonerExists){
// OUTPUT( "Pushing node state upstream!" );
comm->pushNodeStateUpstream( nodeHealth);
nPushesSent++;
}
break;
case ReasonerStandardImplementationOptions::Role::SYSTEM:
assessSystemHealth();
if (upstreamReasonerExists){
// OUTPUT( "Pushing system state upstream!" );
comm->pushSystemStateUpstream( systemHealth);
nPushesSent++;
}
break;
default:
break;
}
} // dataLock
// Determine local performance issues based on recent observations and remote issues.
// now retrieve the nodeStatistics.
if( nodeStatistics != nullptr ) {
nodeStatistics->fetchValues();
for(int i=0; i < NODE_STATISTIC_COUNT ; i++){
nodeHealth->statistics[i] = (*nodeStatistics)[i].toFloat();
// OUTPUT( "CurrentNodeStatistics: " << i << " " << nodeHealth->statistics[i] );
}
}
// Save recentIssues
// TODO
// Update knownIssues based on recentIssues
// TODO
if (anomalyDetected){
anomaliesTriggered++;
}
// Trigger anomaly if any:
{ // Disallow changes in registered plugins while cycling through triggers
unique_lock<mutex> pluginLock( pluginMutex );
if( anomalyDetected || persistingAnomaly ) {
// OUTPUT( "New anomaly: " << anomalyDetected << "; persisting anomaly: " << persistingAnomaly );
// OUTPUT( "Calling triggers!" );
for( auto itr = triggers.begin(); itr != triggers.end() ; itr++ ) {
( *itr )->triggerResponseForAnomaly(false);
}
}
}
// OUTPUT( "PeriodicRun() going to sleep..." );
unique_lock<mutex> lock( recentIssuesMutex );
// TODO: Understand, clarify and comment this!
if( terminated ) {
break;
}
running_condition.wait_until( lock, timeout );
}
// OUTPUT( "PeriodicRun finished" );
}
