void HangDetector() { while (keepRunning) { // increase multiplier during game load to prevent false positives e.g. during pathing const int hangTimeMultiplier = CrashHandler::gameLoading? 3 : 1; const spring_time hangtimeout = spring_msecs(spring_tomsecs(hangTimeout) * hangTimeMultiplier); spring_time curwdt = spring_gettime(); #if defined(USE_GML) && GML_ENABLE_SIM if (gmlMultiThreadSim) { spring_time cursimwdt = simwdt; if (spring_istime(cursimwdt) && curwdt > cursimwdt && (curwdt - cursimwdt) > hangtimeout) { HangHandler(true); simwdt = curwdt; } } #endif spring_time curdrawwdt = drawwdt; if (spring_istime(curdrawwdt) && curwdt > curdrawwdt && (curwdt - curdrawwdt) > hangtimeout) { HangHandler(false); drawwdt = curwdt; } spring_sleep(spring_secs(1)); } }
void BenchmarkSleepFnc(const std::string& name, void (*sleep)(int time), const int runs, const float toMilliSecondsScale) { // waste a few cycles to push the cpu to higher frequency states for (auto spinStopTime = spring_gettime() + spring_secs(2); spring_gettime() < spinStopTime; ) { } spring_time t = spring_gettime(); spring_time tmin, tmax; float tavg = 0; // check lowest possible sleep tick for (int i=0; i<runs; ++i) { sleep(0); spring_time diff = spring_gettime() - t; if ((diff > tmax) || !spring_istime(tmax)) tmax = diff; if ((diff < tmin) || !spring_istime(tmin)) tmin = diff; tavg = float(i * tavg + diff.toNanoSecsi()) / (i + 1); t = spring_gettime(); } // check error in sleeping times spring_time emin, emax; float eavg = 0; if (toMilliSecondsScale != 0) { for (int i=0; i<100; ++i) { const auto sleepTime = (rand() % 50) * 0.1f + 2; // 2..7ms t = spring_gettime(); sleep(sleepTime * toMilliSecondsScale); spring_time diff = (spring_gettime() - t) - spring_msecs(sleepTime); if ((diff > emax) || !emax.isDuration()) emax = diff; if ((diff < emin) || !emin.isDuration()) emin = diff; eavg = float(i * eavg + std::abs(diff.toNanoSecsf())) / (i + 1); } } LOG("[%35s] accuracy:={ err: %+.4fms %+.4fms erravg: %.4fms } min sleep time:={ min: %.6fms avg: %.6fms max: %.6fms }", name.c_str(), emin.toMilliSecsf(), emax.toMilliSecsf(), eavg * 1e-6, tmin.toMilliSecsf(), tavg * 1e-6, tmax.toMilliSecsf()); }
static void HangDetectorLoop() { Threading::SetThreadName("watchdog"); while (!hangDetectorThreadInterrupted) { spring_time curtime = spring_gettime(); bool hangDetected = false; for (unsigned int i = 0; i < WDT_LAST; ++i) { if (!threadSlots[i].active) continue; WatchDogThreadInfo* th_info = registeredThreads[i]; spring_time curwdt = th_info->timer; if (spring_istime(curwdt) && curtime > curwdt && (curtime - curwdt) > hangTimeout) { if (!hangDetected) { LOG_L(L_WARNING, "[Watchdog] Hang detection triggered for Spring %s.", SpringVersion::GetFull().c_str()); #ifdef USE_GML LOG_L(L_WARNING, "MT with %d threads.", GML::ThreadCount()); #endif } LOG_L(L_WARNING, " (in thread: %s)", threadNames[i]); hangDetected = true; th_info->timer = curtime; } } if (hangDetected) { CrashHandler::PrepareStacktrace(); for (unsigned int i = 0; i < WDT_LAST; ++i) { if (!threadSlots[i].active) continue; WatchDogThreadInfo* th_info = registeredThreads[i]; CrashHandler::Stacktrace(th_info->thread, threadNames[i]); LOG_CLEANUP(); } CrashHandler::CleanupStacktrace(); } boost::this_thread::sleep(boost::posix_time::seconds(1)); } }
static void HangDetector() { while (!boost::this_thread::interruption_requested()) { { boost::shared_lock<boost::shared_mutex> lock(mutex); const spring_time curtime = spring_gettime(); for (ThreadsMap::iterator it=registeredThreads.begin(); it != registeredThreads.end(); ++it) { WatchDogThreadInfo& th_info = it->second; if (spring_istime(th_info.timer) && (curtime - th_info.timer) > hangTimeout) { HangHandler(th_info); th_info.timer = curtime; } } } boost::this_thread::sleep(boost::posix_time::seconds(1)); } }