void HangDetector() {
while (keepRunning) {
// increase multiplier during game load to prevent false positives e.g. during pathing
const int hangTimeMultiplier = CrashHandler::gameLoading? 3 : 1;
const spring_time hangtimeout = spring_msecs(spring_tomsecs(hangTimeout) * hangTimeMultiplier);
spring_time curwdt = spring_gettime();
#if defined(USE_GML) && GML_ENABLE_SIM
if (gmlMultiThreadSim) {
spring_time cursimwdt = simwdt;
if (spring_istime(cursimwdt) && curwdt > cursimwdt && (curwdt - cursimwdt) > hangtimeout) {
HangHandler(true);
simwdt = curwdt;
}
}
#endif
spring_time curdrawwdt = drawwdt;
if (spring_istime(curdrawwdt) && curwdt > curdrawwdt && (curwdt - curdrawwdt) > hangtimeout) {
HangHandler(false);
drawwdt = curwdt;
}
spring_sleep(spring_secs(1));
}
}
void BenchmarkSleepFnc(const std::string& name, void (*sleep)(int time), const int runs, const float toMilliSecondsScale)
{
// waste a few cycles to push the cpu to higher frequency states
for (auto spinStopTime = spring_gettime() + spring_secs(2); spring_gettime() < spinStopTime; ) {
}
spring_time t = spring_gettime();
spring_time tmin, tmax;
float tavg = 0;
// check lowest possible sleep tick
for (int i=0; i<runs; ++i) {
sleep(0);
spring_time diff = spring_gettime() - t;
if ((diff > tmax) || !spring_istime(tmax)) tmax = diff;
if ((diff < tmin) || !spring_istime(tmin)) tmin = diff;
tavg = float(i * tavg + diff.toNanoSecsi()) / (i + 1);
t = spring_gettime();
}
// check error in sleeping times
spring_time emin, emax;
float eavg = 0;
if (toMilliSecondsScale != 0) {
for (int i=0; i<100; ++i) {
const auto sleepTime = (rand() % 50) * 0.1f + 2; // 2..7ms
t = spring_gettime();
sleep(sleepTime * toMilliSecondsScale);
spring_time diff = (spring_gettime() - t) - spring_msecs(sleepTime);
if ((diff > emax) || !emax.isDuration()) emax = diff;
if ((diff < emin) || !emin.isDuration()) emin = diff;
eavg = float(i * eavg + std::abs(diff.toNanoSecsf())) / (i + 1);
}
}
LOG("[%35s] accuracy:={ err: %+.4fms %+.4fms erravg: %.4fms } min sleep time:={ min: %.6fms avg: %.6fms max: %.6fms }", name.c_str(), emin.toMilliSecsf(), emax.toMilliSecsf(), eavg * 1e-6, tmin.toMilliSecsf(), tavg * 1e-6, tmax.toMilliSecsf());
}
static void HangDetectorLoop()
{
Threading::SetThreadName("watchdog");
while (!hangDetectorThreadInterrupted) {
spring_time curtime = spring_gettime();
bool hangDetected = false;
for (unsigned int i = 0; i < WDT_LAST; ++i) {
if (!threadSlots[i].active)
continue;
WatchDogThreadInfo* th_info = registeredThreads[i];
spring_time curwdt = th_info->timer;
if (spring_istime(curwdt) && curtime > curwdt && (curtime - curwdt) > hangTimeout) {
if (!hangDetected) {
LOG_L(L_WARNING, "[Watchdog] Hang detection triggered for Spring %s.", SpringVersion::GetFull().c_str());
#ifdef USE_GML
LOG_L(L_WARNING, "MT with %d threads.", GML::ThreadCount());
#endif
}
LOG_L(L_WARNING, " (in thread: %s)", threadNames[i]);
hangDetected = true;
th_info->timer = curtime;
}
}
if (hangDetected) {
CrashHandler::PrepareStacktrace();
for (unsigned int i = 0; i < WDT_LAST; ++i) {
if (!threadSlots[i].active)
continue;
WatchDogThreadInfo* th_info = registeredThreads[i];
CrashHandler::Stacktrace(th_info->thread, threadNames[i]);
LOG_CLEANUP();
}
CrashHandler::CleanupStacktrace();
}
boost::this_thread::sleep(boost::posix_time::seconds(1));
}
}
static void HangDetector()
{
while (!boost::this_thread::interruption_requested()) {
{
boost::shared_lock<boost::shared_mutex> lock(mutex);
const spring_time curtime = spring_gettime();
for (ThreadsMap::iterator it=registeredThreads.begin(); it != registeredThreads.end(); ++it) {
WatchDogThreadInfo& th_info = it->second;
if (spring_istime(th_info.timer) && (curtime - th_info.timer) > hangTimeout) {
HangHandler(th_info);
th_info.timer = curtime;
}
}
}
boost::this_thread::sleep(boost::posix_time::seconds(1));
}
}