void GameSetupDrawer::StartCountdown(unsigned time)
{
if (instance) {
instance->lastTick = spring_gettime(); //FIXME
instance->readyCountdown = spring_msecs(time);
}
}
void HangDetector() {
while (keepRunning) {
// increase multiplier during game load to prevent false positives e.g. during pathing
const int hangTimeMultiplier = CrashHandler::gameLoading? 3 : 1;
const spring_time hangtimeout = spring_msecs(spring_tomsecs(hangTimeout) * hangTimeMultiplier);
spring_time curwdt = spring_gettime();
#if defined(USE_GML) && GML_ENABLE_SIM
if (gmlMultiThreadSim) {
spring_time cursimwdt = simwdt;
if (spring_istime(cursimwdt) && curwdt > cursimwdt && (curwdt - cursimwdt) > hangtimeout) {
HangHandler(true);
simwdt = curwdt;
}
}
#endif
spring_time curdrawwdt = drawwdt;
if (spring_istime(curdrawwdt) && curwdt > curdrawwdt && (curwdt - curdrawwdt) > hangtimeout) {
HangHandler(false);
drawwdt = curwdt;
}
spring_sleep(spring_secs(1));
}
}
__FORCE_ALIGN_STACK__
void CNetProtocol::UpdateLoop()
{
Threading::SetThreadName("heartbeat");
while (keepUpdating) {
Update();
spring_msecs(100).sleep();
}
}
void spring_time::Serialize(creg::ISerializer& s)
{
if (s.IsWriting()) {
int y = spring_tomsecs(*this - spring_gettime());
s.SerializeInt(&y, 4);
} else {
int y;
s.SerializeInt(&y, 4);
*this = *this + spring_msecs(y);
}
}
void GameParticipant::Kill(const std::string& reason, const bool flush)
{
if (link)
{
link->SendData(CBaseNetProtocol::Get().SendQuit(reason));
if (flush) // make sure the Flush() performed by Close() has any effect (forced flushes are undesirable)
spring_sleep(spring_msecs(1000)); // it will cause a slight lag in the game server during kick, but not a big deal
link->Close();
link.reset();
}
linkData[MAX_AIS].link.reset();
#ifdef SYNCCHECK
syncResponse.clear();
#endif
myState = DISCONNECTED;
}
void BenchmarkSleepFnc(const std::string& name, void (*sleep)(int time), const int runs, const float toMilliSecondsScale)
{
// waste a few cycles to push the cpu to higher frequency states
for (auto spinStopTime = spring_gettime() + spring_secs(2); spring_gettime() < spinStopTime; ) {
}
spring_time t = spring_gettime();
spring_time tmin, tmax;
float tavg = 0;
// check lowest possible sleep tick
for (int i=0; i<runs; ++i) {
sleep(0);
spring_time diff = spring_gettime() - t;
if ((diff > tmax) || !spring_istime(tmax)) tmax = diff;
if ((diff < tmin) || !spring_istime(tmin)) tmin = diff;
tavg = float(i * tavg + diff.toNanoSecsi()) / (i + 1);
t = spring_gettime();
}
// check error in sleeping times
spring_time emin, emax;
float eavg = 0;
if (toMilliSecondsScale != 0) {
for (int i=0; i<100; ++i) {
const auto sleepTime = (rand() % 50) * 0.1f + 2; // 2..7ms
t = spring_gettime();
sleep(sleepTime * toMilliSecondsScale);
spring_time diff = (spring_gettime() - t) - spring_msecs(sleepTime);
if ((diff > emax) || !emax.isDuration()) emax = diff;
if ((diff < emin) || !emin.isDuration()) emin = diff;
eavg = float(i * eavg + std::abs(diff.toNanoSecsf())) / (i + 1);
}
}
LOG("[%35s] accuracy:={ err: %+.4fms %+.4fms erravg: %.4fms } min sleep time:={ min: %.6fms avg: %.6fms max: %.6fms }", name.c_str(), emin.toMilliSecsf(), emax.toMilliSecsf(), eavg * 1e-6, tmin.toMilliSecsf(), tavg * 1e-6, tmax.toMilliSecsf());
}
bool CLoadScreen::Draw()
{
//! Limit the Frames Per Second to not lock a singlethreaded CPU from loading the game
if (mtLoading) {
spring_time now = spring_gettime();
unsigned diff_ms = spring_tomsecs(now - last_draw);
static const unsigned wantedFPS = 50;
static const unsigned min_frame_time = 1000 / wantedFPS;
if (diff_ms < min_frame_time) {
spring_time nap = spring_msecs(min_frame_time - diff_ms);
spring_sleep(nap);
}
last_draw = now;
}
//! cause of `curLoadMessage`
boost::recursive_mutex::scoped_lock lck(mutex);
if (LuaIntro != nullptr) {
LuaIntro->Update();
LuaIntro->DrawGenesis();
ClearScreen();
LuaIntro->DrawLoadScreen();
} else {
ClearScreen();
float xDiv = 0.0f;
float yDiv = 0.0f;
const float ratioComp = globalRendering->aspectRatio / aspectRatio;
if (math::fabs(ratioComp - 1.0f) < 0.01f) { //! ~= 1
//! show Load-Screen full screen
//! nothing to do
} else if (ratioComp > 1.0f) {
//! show Load-Screen on part of the screens X-Axis only
xDiv = (1.0f - (1.0f / ratioComp)) * 0.5f;
} else {
//! show Load-Screen on part of the screens Y-Axis only
yDiv = (1.0f - ratioComp) * 0.5f;
}
//! Draw loading screen & print load msg.
if (startupTexture) {
glBindTexture(GL_TEXTURE_2D,startupTexture);
glBegin(GL_QUADS);
glTexCoord2f(0.0f, 1.0f);
glVertex2f(0.0f + xDiv, 0.0f + yDiv);
glTexCoord2f(0.0f, 0.0f);
glVertex2f(0.0f + xDiv, 1.0f - yDiv);
glTexCoord2f(1.0f, 0.0f);
glVertex2f(1.0f - xDiv, 1.0f - yDiv);
glTexCoord2f(1.0f, 1.0f);
glVertex2f(1.0f - xDiv, 0.0f + yDiv);
glEnd();
}
if (showMessages) {
font->Begin();
font->SetTextColor(0.5f,0.5f,0.5f,0.9f);
font->glPrint(0.1f,0.9f, globalRendering->viewSizeY / 35.0f, FONT_NORM,
oldLoadMessages);
font->SetTextColor(0.9f,0.9f,0.9f,0.9f);
float posy = font->GetTextNumLines(oldLoadMessages) * font->GetLineHeight() * globalRendering->viewSizeY / 35.0f;
font->glPrint(0.1f,0.9f - posy * globalRendering->pixelY, globalRendering->viewSizeY / 35.0f, FONT_NORM,
curLoadMessage);
font->End();
}
}
// Always render Spring's license notice
font->Begin();
font->SetOutlineColor(0.0f,0.0f,0.0f,0.65f);
font->SetTextColor(1.0f,1.0f,1.0f,1.0f);
font->glFormat(0.5f,0.06f, globalRendering->viewSizeY / 35.0f, FONT_OUTLINE | FONT_CENTER | FONT_NORM,
"Spring %s", SpringVersion::GetFull().c_str());
font->glFormat(0.5f,0.02f, globalRendering->viewSizeY / 50.0f, FONT_OUTLINE | FONT_CENTER | FONT_NORM,
"This program is distributed under the GNU General Public License, see license.html for more info");
font->End();
if (!mtLoading)
SDL_GL_SwapWindow(globalRendering->window);
return true;
}
void CSoundSource::Play(IAudioChannel* channel, SoundItem* item, float3 pos, float3 velocity, float volume, bool relative)
{
assert(!curStream.Valid());
assert(channel);
if (!item->PlayNow())
return;
Stop();
curVolume = volume;
curPlaying = item;
curChannel = channel;
alSourcei(id, AL_BUFFER, item->buffer->GetId());
alSourcef(id, AL_GAIN, volume * item->GetGain() * channel->volume);
alSourcef(id, AL_PITCH, item->GetPitch() * globalPitch);
velocity *= item->dopplerScale * ELMOS_TO_METERS;
alSource3f(id, AL_VELOCITY, velocity.x, velocity.y, velocity.z);
alSourcei(id, AL_LOOPING, (item->loopTime > 0) ? AL_TRUE : AL_FALSE);
loopStop = spring_gettime() + spring_msecs(item->loopTime);
if (relative || !item->in3D) {
in3D = false;
if (efxEnabled) {
alSource3i(id, AL_AUXILIARY_SEND_FILTER, AL_EFFECTSLOT_NULL, 0, AL_FILTER_NULL);
alSourcei(id, AL_DIRECT_FILTER, AL_FILTER_NULL);
efxEnabled = false;
}
alSourcei(id, AL_SOURCE_RELATIVE, AL_TRUE);
alSourcef(id, AL_ROLLOFF_FACTOR, 0.f);
alSource3f(id, AL_POSITION, 0.0f, 0.0f, -1.0f * ELMOS_TO_METERS);
#ifdef __APPLE__
alSourcef(id, AL_REFERENCE_DISTANCE, referenceDistance * ELMOS_TO_METERS);
#endif
} else {
if (item->buffer->GetChannels() > 1) {
LOG_L(L_WARNING, "Can not play non-mono \"%s\" in 3d.",
item->buffer->GetFilename().c_str());
}
in3D = true;
if (efx->enabled) {
efxEnabled = true;
alSourcef(id, AL_AIR_ABSORPTION_FACTOR, efx->GetAirAbsorptionFactor());
alSource3i(id, AL_AUXILIARY_SEND_FILTER, efx->sfxSlot, 0, AL_FILTER_NULL);
alSourcei(id, AL_DIRECT_FILTER, efx->sfxFilter);
efxUpdates = efx->updates;
}
alSourcei(id, AL_SOURCE_RELATIVE, AL_FALSE);
pos *= ELMOS_TO_METERS;
alSource3f(id, AL_POSITION, pos.x, pos.y, pos.z);
curHeightRolloffModifier = heightRolloffModifier;
alSourcef(id, AL_ROLLOFF_FACTOR, ROLLOFF_FACTOR * item->rolloff * heightRolloffModifier);
#ifdef __APPLE__
alSourcef(id, AL_MAX_DISTANCE, 1000000.0f);
//! Max distance is too small by default on my Mac...
ALfloat gain = channel->volume * item->GetGain() * volume;
if (gain > 1.0f) {
//! OpenAL on Mac cannot handle AL_GAIN > 1 well, so we will adjust settings to get the same output with AL_GAIN = 1.
ALint model = alGetInteger(AL_DISTANCE_MODEL);
ALfloat rolloff = ROLLOFF_FACTOR * item->rolloff * heightRolloffModifier;
ALfloat ref = referenceDistance * ELMOS_TO_METERS;
if ((model == AL_INVERSE_DISTANCE_CLAMPED) || (model == AL_INVERSE_DISTANCE)) {
alSourcef(id, AL_REFERENCE_DISTANCE, ((gain - 1.0f) * ref / rolloff) + ref);
alSourcef(id, AL_ROLLOFF_FACTOR, (gain + rolloff - 1.0f) / gain);
alSourcef(id, AL_GAIN, 1.0f);
}
} else {
alSourcef(id, AL_REFERENCE_DISTANCE, referenceDistance * ELMOS_TO_METERS);
}
#endif
}
alSourcePlay(id);
if (item->buffer->GetId() == 0)
LOG_L(L_WARNING, "CSoundSource::Play: Empty buffer for item %s (file %s)", item->name.c_str(), item->buffer->GetFilename().c_str());
CheckError("CSoundSource::Play");
}
namespace Watchdog
{
const char *threadNames[] = {"main", "sim", "load", "audio"};
static boost::mutex wdmutex;
static unsigned int curorder = 0;
struct WatchDogThreadInfo {
WatchDogThreadInfo()
: threadid(0)
, timer(spring_notime)
, numreg(0)
{}
volatile Threading::NativeThreadHandle thread;
volatile Threading::NativeThreadId threadid;
spring_time timer;
volatile unsigned int numreg;
};
static WatchDogThreadInfo registeredThreadsData[WDT_SIZE];
struct WatchDogThreadSlot {
WatchDogThreadSlot()
: primary(false)
, active(false)
, regorder(0)
{}
volatile bool primary;
volatile bool active;
volatile unsigned int regorder;
};
static WatchDogThreadInfo *registeredThreads[WDT_SIZE] = {NULL};
static WatchDogThreadSlot threadSlots[WDT_SIZE];
static std::map<std::string, unsigned int> threadNameToNum;
static boost::thread* hangDetectorThread = NULL;
static spring_time hangTimeout = spring_msecs(0);
static volatile bool hangDetectorThreadInterrupted = false;
static inline void UpdateActiveThreads(Threading::NativeThreadId num) {
unsigned int active = WDT_LAST;
for (unsigned int i = 0; i < WDT_LAST; ++i) {
WatchDogThreadInfo* th_info = registeredThreads[i];
if (th_info->numreg && Threading::NativeThreadIdsEqual(num, th_info->threadid)) {
threadSlots[i].active = false;
if (threadSlots[i].regorder > threadSlots[active].regorder)
active = i;
}
}
if (active < WDT_LAST)
threadSlots[active].active = true;
}
__FORCE_ALIGN_STACK__
static void HangDetectorLoop()
{
Threading::SetThreadName("watchdog");
while (!hangDetectorThreadInterrupted) {
spring_time curtime = spring_gettime();
bool hangDetected = false;
for (unsigned int i = 0; i < WDT_LAST; ++i) {
if (!threadSlots[i].active)
continue;
WatchDogThreadInfo* th_info = registeredThreads[i];
spring_time curwdt = th_info->timer;
if (spring_istime(curwdt) && (curtime - curwdt) > hangTimeout) {
if (!hangDetected) {
LOG_L(L_WARNING, "[Watchdog] Hang detection triggered for Spring %s.", SpringVersion::GetFull().c_str());
if (GML::Enabled())
LOG_L(L_WARNING, "MT with %d threads.", GML::ThreadCount());
}
LOG_L(L_WARNING, " (in thread: %s)", threadNames[i]);
hangDetected = true;
th_info->timer = curtime;
}
}
if (hangDetected) {
CrashHandler::PrepareStacktrace(LOG_LEVEL_WARNING);
for (unsigned int i = 0; i < WDT_LAST; ++i) {
if (!threadSlots[i].active)
continue;
WatchDogThreadInfo* th_info = registeredThreads[i];
CrashHandler::Stacktrace(th_info->thread, threadNames[i], LOG_LEVEL_WARNING);
}
CrashHandler::CleanupStacktrace(LOG_LEVEL_WARNING);
}
boost::this_thread::sleep(boost::posix_time::seconds(1));
}
}
void RegisterThread(WatchdogThreadnum num, bool primary)
{
boost::mutex::scoped_lock lock(wdmutex);
if (num >= WDT_LAST || registeredThreads[num]->numreg) {
LOG_L(L_ERROR, "[Watchdog::RegisterThread] Invalid thread number");
return;
}
Threading::NativeThreadHandle thread = Threading::GetCurrentThread();
Threading::NativeThreadId threadId = Threading::GetCurrentThreadId();
unsigned int inact = WDT_LAST;
unsigned int i;
for (i = 0; i < WDT_LAST; ++i) {
WatchDogThreadInfo* th_info = ®isteredThreadsData[i];
if (!th_info->numreg)
inact = std::min(i, inact);
else if(Threading::NativeThreadIdsEqual(th_info->threadid, threadId))
break;
}
if (i >= WDT_LAST)
i = inact;
if (i >= WDT_LAST) {
LOG_L(L_ERROR, "[Watchdog::RegisterThread] Internal error");
return;
}
registeredThreads[num] = ®isteredThreadsData[i];
// set threadname
//Threading::SetThreadName(threadNames[num]);
WatchDogThreadInfo* th_info = registeredThreads[num];
th_info->thread = thread;
th_info->threadid = threadId;
th_info->timer = spring_gettime();
++th_info->numreg;
threadSlots[num].primary = primary;
threadSlots[num].regorder = ++curorder;
UpdateActiveThreads(threadId);
}
void DeregisterThread(WatchdogThreadnum num)
{
boost::mutex::scoped_lock lock(wdmutex);
WatchDogThreadInfo* th_info;
if (num >= WDT_LAST || !(th_info = registeredThreads[num])->numreg) {
LOG_L(L_ERROR, "[Watchdog::DeregisterThread] Invalid thread number");
return;
}
threadSlots[num].primary = false;
threadSlots[num].regorder = 0;
UpdateActiveThreads(th_info->threadid);
if(!--(th_info->numreg))
memset(th_info, 0, sizeof(WatchDogThreadInfo));
registeredThreads[num] = ®isteredThreadsData[WDT_LAST];
}
void ClearTimer(bool disable, Threading::NativeThreadId* _threadId)
{
if (hangDetectorThread == NULL)
return; //! Watchdog isn't running
Threading::NativeThreadId threadId;
if (_threadId)
threadId = *_threadId;
else
threadId = Threading::GetCurrentThreadId();
unsigned int num = 0;
for (; num < WDT_LAST; ++num)
if (Threading::NativeThreadIdsEqual(registeredThreads[num]->threadid, threadId))
break;
WatchDogThreadInfo* th_info;
if (num >= WDT_LAST || !(th_info = registeredThreads[num])->numreg) {
LOG_L(L_ERROR, "[Watchdog::ClearTimer] Invalid thread %d", num);
return;
}
th_info->timer = disable ? spring_notime : spring_gettime();
}
void ClearTimer(WatchdogThreadnum num, bool disable)
{
if (hangDetectorThread == NULL)
return; //! Watchdog isn't running
WatchDogThreadInfo* th_info;
if (num >= WDT_LAST || !(th_info = registeredThreads[num])->numreg) {
LOG_L(L_ERROR, "[Watchdog::ClearTimer] Invalid thread number %d", num);
return;
}
th_info->timer = disable ? spring_notime : spring_gettime();
}
void ClearTimer(const std::string &name, bool disable)
{
if (hangDetectorThread == NULL)
return; //! Watchdog isn't running
std::map<std::string, unsigned int>::iterator i = threadNameToNum.find(name);
unsigned int num;
WatchDogThreadInfo* th_info;
if (i == threadNameToNum.end() || (num = i->second) >= WDT_LAST || !(th_info = registeredThreads[num])->numreg) {
LOG_L(L_ERROR, "[Watchdog::ClearTimer] Invalid thread name");
return;
}
th_info->timer = disable ? spring_notime : spring_gettime();
}
void ClearPrimaryTimers(bool disable)
{
if (hangDetectorThread == NULL)
return; //! Watchdog isn't running
for (unsigned int i = 0; i < WDT_LAST; ++i) {
WatchDogThreadInfo* th_info = registeredThreads[i];
if (threadSlots[i].primary)
th_info->timer = disable ? spring_notime : spring_gettime();
}
}
void Install()
{
boost::mutex::scoped_lock lock(wdmutex);
memset(registeredThreadsData, 0, sizeof(registeredThreadsData));
for (unsigned int i = 0; i < WDT_LAST; ++i) {
registeredThreads[i] = ®isteredThreadsData[WDT_LAST];
threadNameToNum[std::string(threadNames[i])] = i;
}
memset(threadSlots, 0, sizeof(threadSlots));
#ifndef _WIN32
// disable if gdb is running
char buf[1024];
SNPRINTF(buf, sizeof(buf), "/proc/%d/cmdline", getppid());
std::ifstream f(buf);
if (f.good()) {
f.read(buf,sizeof(buf));
f.close();
std::string str(buf);
if (str.find("gdb") != std::string::npos) {
LOG("[Watchdog] disabled (gdb detected)");
return;
}
}
#endif
#ifdef USE_VALGRIND
if (RUNNING_ON_VALGRIND) {
LOG("[Watchdog] disabled (Valgrind detected)");
return;
}
#endif
const int hangTimeoutSecs = configHandler->GetInt("HangTimeout");
// HangTimeout = -1 to force disable hang detection
if (hangTimeoutSecs <= 0) {
LOG("[Watchdog] disabled");
return;
}
hangTimeout = spring_secs(hangTimeoutSecs);
// start the watchdog thread
hangDetectorThread = new boost::thread(&HangDetectorLoop);
LOG("[Watchdog] Installed (HangTimeout: %isec)", hangTimeoutSecs);
}
void Uninstall()
{
if (hangDetectorThread == NULL) {
return;
}
boost::mutex::scoped_lock lock(wdmutex);
hangDetectorThreadInterrupted = true;
hangDetectorThread->join();
delete hangDetectorThread;
hangDetectorThread = NULL;
memset(registeredThreadsData, 0, sizeof(registeredThreadsData));
for (unsigned int i = 0; i < WDT_LAST; ++i)
registeredThreads[i] = ®isteredThreadsData[WDT_LAST];
memset(threadSlots, 0, sizeof(threadSlots));
threadNameToNum.clear();
}
}
void sleep_spring(int time) { spring_sleep(spring_msecs(time)); }
namespace Watchdog
{
struct WatchDogThreadInfo {
Threading::NativeThreadHandle thread;
volatile spring_time timer;
std::string name;
};
typedef std::map<Threading::NativeThreadId, WatchDogThreadInfo> ThreadsMap;
typedef std::map<std::string, Threading::NativeThreadId> ThreadNameToIdMap;
static ThreadsMap registeredThreads;
static ThreadNameToIdMap threadNameToId;
static boost::shared_mutex mutex;
static boost::thread* hangdetectorthread = NULL;
static spring_time hangTimeout = spring_msecs(0);
/** Print stack traces for relevant threads. */
static void HangHandler(const WatchDogThreadInfo& threadinfo)
{
logOutput.Print("Hang detection triggered (in thread: %s) for Spring %s.", threadinfo.name.c_str(), SpringVersion::GetFull().c_str());
#ifdef USE_GML
logOutput.Print("MT with %d threads.", gmlThreadCount);
#endif
CrashHandler::Stacktrace(threadinfo.thread);
logOutput.Flush();
}
static void HangDetector()
{
while (!boost::this_thread::interruption_requested()) {
{
boost::shared_lock<boost::shared_mutex> lock(mutex);
const spring_time curtime = spring_gettime();
for (ThreadsMap::iterator it=registeredThreads.begin(); it != registeredThreads.end(); ++it) {
WatchDogThreadInfo& th_info = it->second;
if (spring_istime(th_info.timer) && (curtime - th_info.timer) > hangTimeout) {
HangHandler(th_info);
th_info.timer = curtime;
}
}
}
boost::this_thread::sleep(boost::posix_time::seconds(1));
}
}
void RegisterThread(std::string name)
{
boost::unique_lock<boost::shared_mutex> lock(mutex);
Threading::NativeThreadHandle thread = Threading::_GetCurrentThread();
Threading::NativeThreadId threadId = Threading::_GetCurrentThreadId();
WatchDogThreadInfo& th_info = registeredThreads[threadId];
th_info.thread = thread;
th_info.timer = spring_gettime();
th_info.name = name;
threadNameToId[name] = threadId;
}
/*void DeregisterThread()
{
boost::unique_lock<boost::shared_mutex> lock(mutex);
//FIXME ADD CODE
}*/
void ClearTimer(bool disable, Threading::NativeThreadId* _threadId)
{
boost::shared_lock<boost::shared_mutex> lock(mutex);
if (!hangdetectorthread)
return; //! Watchdog isn't running
Threading::NativeThreadId threadId;
if (_threadId) {
threadId = *_threadId;
} else {
threadId = Threading::_GetCurrentThreadId();
}
ThreadsMap::iterator it = registeredThreads.find(threadId);
if (it == registeredThreads.end()) {
return;
}
assert(it != registeredThreads.end()); //! the thread isn't registered!
WatchDogThreadInfo& th_info = it->second;
th_info.timer = disable ? spring_notime : spring_gettime();
}
void ClearTimer(const std::string& name, bool disable)
{
boost::shared_lock<boost::shared_mutex> lock(mutex);
if (!hangdetectorthread)
return; //! Watchdog isn't running
ThreadNameToIdMap::iterator it = threadNameToId.find(name);
if (it == threadNameToId.end()) {
logOutput.Print("[Watchdog::ClearTimer] No thread found named \"%s\".", name.c_str());
return;
}
ClearTimer(disable, &(it->second));
}
void Install()
{
int hangTimeoutSecs = configHandler->Get("HangTimeout", 0);
//! HangTimeout = -1 to force disable hang detection
if (hangTimeoutSecs < 0)
return;
if (hangTimeoutSecs == 0)
hangTimeoutSecs = 10;
hangTimeout = spring_secs(hangTimeoutSecs);
//! register (this) mainthread
RegisterThread("main");
//! start the watchdog thread
hangdetectorthread = new boost::thread(&HangDetector);
logOutput.Print("[Watchdog] Installed (timeout: %isec)", hangTimeoutSecs);
}
void Uninstall()
{
if (hangdetectorthread) {
hangdetectorthread->interrupt();
hangdetectorthread->join();
delete hangdetectorthread;
hangdetectorthread = NULL;
boost::unique_lock<boost::shared_mutex> lock(mutex);
registeredThreads.clear();
threadNameToId.clear();
}
}
}
namespace CrashHandler {
boost::thread* hangdetectorthread = NULL;
volatile int keepRunning = 1;
// watchdog timers
volatile spring_time simwdt = spring_notime, drawwdt = spring_notime;
spring_time hangTimeout = spring_msecs(0);
volatile bool gameLoading = false;
void SigAbrtHandler(int signal)
{
// cause an exception if on windows
// TODO FIXME do a proper stacktrace dump here
*((int*)(0)) = 0;
}
// Set this to the desired printf style output function.
// Currently we write through the logOutput class to infolog.txt
#define PRINT logOutput.Print
/** Convert exception code to human readable string. */
static const char *ExceptionName(DWORD exceptionCode)
{
switch (exceptionCode) {
case EXCEPTION_ACCESS_VIOLATION: return "Access violation";
case EXCEPTION_DATATYPE_MISALIGNMENT: return "Datatype misalignment";
case EXCEPTION_BREAKPOINT: return "Breakpoint";
case EXCEPTION_SINGLE_STEP: return "Single step";
case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: return "Array bounds exceeded";
case EXCEPTION_FLT_DENORMAL_OPERAND: return "Float denormal operand";
case EXCEPTION_FLT_DIVIDE_BY_ZERO: return "Float divide by zero";
case EXCEPTION_FLT_INEXACT_RESULT: return "Float inexact result";
case EXCEPTION_FLT_INVALID_OPERATION: return "Float invalid operation";
case EXCEPTION_FLT_OVERFLOW: return "Float overflow";
case EXCEPTION_FLT_STACK_CHECK: return "Float stack check";
case EXCEPTION_FLT_UNDERFLOW: return "Float underflow";
case EXCEPTION_INT_DIVIDE_BY_ZERO: return "Integer divide by zero";
case EXCEPTION_INT_OVERFLOW: return "Integer overflow";
case EXCEPTION_PRIV_INSTRUCTION: return "Privileged instruction";
case EXCEPTION_IN_PAGE_ERROR: return "In page error";
case EXCEPTION_ILLEGAL_INSTRUCTION: return "Illegal instruction";
case EXCEPTION_NONCONTINUABLE_EXCEPTION: return "Noncontinuable exception";
case EXCEPTION_STACK_OVERFLOW: return "Stack overflow";
case EXCEPTION_INVALID_DISPOSITION: return "Invalid disposition";
case EXCEPTION_GUARD_PAGE: return "Guard page";
case EXCEPTION_INVALID_HANDLE: return "Invalid handle";
}
return "Unknown exception";
}
static void initImageHlpDll() {
char userSearchPath[8];
STRCPY(userSearchPath, ".");
// Initialize IMAGEHLP.DLL
SymInitialize(GetCurrentProcess(), userSearchPath, TRUE);
}
/** Print out a stacktrace. */
static void Stacktrace(LPEXCEPTION_POINTERS e, HANDLE hThread = INVALID_HANDLE_VALUE) {
PIMAGEHLP_SYMBOL pSym;
STACKFRAME sf;
HANDLE process, thread;
DWORD dwModBase, Disp;
BOOL more = FALSE;
int count = 0;
char modname[MAX_PATH];
pSym = (PIMAGEHLP_SYMBOL)GlobalAlloc(GMEM_FIXED, 16384);
BOOL suspended = FALSE;
CONTEXT c;
if (e) {
c = *e->ContextRecord;
thread = GetCurrentThread();
} else {
SuspendThread(hThread);
suspended = TRUE;
memset(&c, 0, sizeof(CONTEXT));
c.ContextFlags = CONTEXT_FULL;
// FIXME: This does not work if you want to dump the current thread's stack
if (!GetThreadContext(hThread, &c)) {
ResumeThread(hThread);
return;
}
thread = hThread;
}
ZeroMemory(&sf, sizeof(sf));
sf.AddrPC.Offset = c.Eip;
sf.AddrStack.Offset = c.Esp;
sf.AddrFrame.Offset = c.Ebp;
sf.AddrPC.Mode = AddrModeFlat;
sf.AddrStack.Mode = AddrModeFlat;
sf.AddrFrame.Mode = AddrModeFlat;
process = GetCurrentProcess();
// use globalalloc to reduce risk for allocator related deadlock
char* printstrings = (char*)GlobalAlloc(GMEM_FIXED, 0);
bool containsOglDll = false;
while (true) {
more = StackWalk(
IMAGE_FILE_MACHINE_I386, // TODO: fix this for 64 bit windows?
process,
thread,
&sf,
&c,
NULL,
SymFunctionTableAccess,
SymGetModuleBase,
NULL
);
if (!more || sf.AddrFrame.Offset == 0) {
break;
}
dwModBase = SymGetModuleBase(process, sf.AddrPC.Offset);
if (dwModBase) {
GetModuleFileName((HINSTANCE)dwModBase, modname, MAX_PATH);
} else {
strcpy(modname, "Unknown");
}
pSym->SizeOfStruct = sizeof(IMAGEHLP_SYMBOL);
pSym->MaxNameLength = MAX_PATH;
char *printstringsnew = (char *)GlobalAlloc(GMEM_FIXED, (count + 1) * BUFFER_SIZE);
memcpy(printstringsnew, printstrings, count * BUFFER_SIZE);
GlobalFree(printstrings);
printstrings = printstringsnew;
if (SymGetSymFromAddr(process, sf.AddrPC.Offset, &Disp, pSym)) {
// This is the code path taken on VC if debugging syms are found.
SNPRINTF(printstrings + count * BUFFER_SIZE, BUFFER_SIZE, "(%d) %s(%s+%#0lx) [0x%08lX]", count, modname, pSym->Name, Disp, sf.AddrPC.Offset);
} else {
// This is the code path taken on MinGW, and VC if no debugging syms are found.
SNPRINTF(printstrings + count * BUFFER_SIZE, BUFFER_SIZE, "(%d) %s [0x%08lX]", count, modname, sf.AddrPC.Offset);
}
// OpenGL lib names (ATI): "atioglxx.dll" "atioglx2.dll"
containsOglDll = containsOglDll || strstr(modname, "atiogl");
// OpenGL lib names (Nvidia): "nvoglnt.dll" "nvoglv32.dll" "nvoglv64.dll" (last one is a guess)
containsOglDll = containsOglDll || strstr(modname, "nvogl");
++count;
}
if (containsOglDll) {
PRINT("This stack trace indicates a problem with your graphic card driver. "
"Please try upgrading or downgrading it. "
"Specifically recommended is the latest driver, and one that is as old as your graphic card. "
"Make sure to use a driver removal utility, before installing other drivers.");
}
if (suspended) {
ResumeThread(hThread);
}
for (int i = 0; i < count; ++i) {
PRINT("%s", printstrings + i * BUFFER_SIZE);
}
GlobalFree(printstrings);
GlobalFree(pSym);
}
/** Callback for SymEnumerateModules */
#if _MSC_VER >= 1500
static BOOL CALLBACK EnumModules(PCSTR moduleName, ULONG baseOfDll, PVOID userContext)
{
PRINT("0x%08lx\t%s", baseOfDll, moduleName);
return TRUE;
}
#else // _MSC_VER >= 1500
static BOOL CALLBACK EnumModules(LPSTR moduleName, DWORD baseOfDll, PVOID userContext)
{
PRINT("0x%08lx\t%s", baseOfDll, moduleName);
return TRUE;
}
#endif // _MSC_VER >= 1500
/** Called by windows if an exception happens. */
static LONG CALLBACK ExceptionHandler(LPEXCEPTION_POINTERS e)
{
// Prologue.
logOutput.SetSubscribersEnabled(false);
PRINT("Spring %s has crashed.", SpringVersion::GetFull().c_str());
#ifdef USE_GML
PRINT("MT with %d threads.", gmlThreadCount);
#endif
initImageHlpDll();
// Record exception info.
PRINT("Exception: %s (0x%08lx)", ExceptionName(e->ExceptionRecord->ExceptionCode), e->ExceptionRecord->ExceptionCode);
PRINT("Exception Address: 0x%08lx", (unsigned long int) (PVOID) e->ExceptionRecord->ExceptionAddress);
// Record list of loaded DLLs.
PRINT("DLL information:");
SymEnumerateModules(GetCurrentProcess(), EnumModules, NULL);
// Record stacktrace.
PRINT("Stacktrace:");
Stacktrace(e);
// Unintialize IMAGEHLP.DLL
SymCleanup(GetCurrentProcess());
// Cleanup.
SDL_Quit();
logOutput.End(); // Stop writing to log.
// FIXME: update closing of demo to new netcode
// Inform user.
char dir[MAX_PATH], msg[MAX_PATH+200];
GetCurrentDirectory(sizeof(dir) - 1, dir);
SNPRINTF(msg, sizeof(msg),
"Spring has crashed.\n\n"
"A stacktrace has been written to:\n"
"%s\\infolog.txt", dir);
MessageBox(NULL, msg, "Spring: Unhandled exception", 0);
// this seems to silently close the application
return EXCEPTION_EXECUTE_HANDLER;
// this triggers the microsoft "application has crashed" error dialog
//return EXCEPTION_CONTINUE_SEARCH;
// in practice, 100% CPU usage but no continuation of execution
// (tested segmentation fault and division by zero)
//return EXCEPTION_CONTINUE_EXECUTION;
}
/** Print stack traces for relevant threads. */
void HangHandler(bool simhang)
{
PRINT("Hang detection triggered %sfor Spring %s.", simhang ? "(in sim thread) " : "", SpringVersion::GetFull().c_str());
#ifdef USE_GML
PRINT("MT with %d threads.", gmlThreadCount);
#endif
initImageHlpDll();
// Record list of loaded DLLs.
PRINT("DLL information:");
SymEnumerateModules(GetCurrentProcess(), EnumModules, NULL);
if (drawthread != INVALID_HANDLE_VALUE) {
// Record stacktrace.
PRINT("Stacktrace:");
Stacktrace(NULL, drawthread);
}
#if defined(USE_GML) && GML_ENABLE_SIM
if (gmlMultiThreadSim && simthread != INVALID_HANDLE_VALUE) {
PRINT("Stacktrace (sim):");
Stacktrace(NULL, simthread);
}
#endif
// Unintialize IMAGEHLP.DLL
SymCleanup(GetCurrentProcess());
logOutput.Flush();
}
void HangDetector() {
while (keepRunning) {
// increase multiplier during game load to prevent false positives e.g. during pathing
const int hangTimeMultiplier = CrashHandler::gameLoading? 3 : 1;
const spring_time hangtimeout = spring_msecs(spring_tomsecs(hangTimeout) * hangTimeMultiplier);
spring_time curwdt = spring_gettime();
#if defined(USE_GML) && GML_ENABLE_SIM
if (gmlMultiThreadSim) {
spring_time cursimwdt = simwdt;
if (spring_istime(cursimwdt) && curwdt > cursimwdt && (curwdt - cursimwdt) > hangtimeout) {
HangHandler(true);
simwdt = curwdt;
}
}
#endif
spring_time curdrawwdt = drawwdt;
if (spring_istime(curdrawwdt) && curwdt > curdrawwdt && (curwdt - curdrawwdt) > hangtimeout) {
HangHandler(false);
drawwdt = curwdt;
}
spring_sleep(spring_secs(1));
}
}
void InstallHangHandler() {
DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(),
&drawthread, 0, TRUE, DUPLICATE_SAME_ACCESS);
int hangTimeoutMS = configHandler->Get("HangTimeout", 0);
CrashHandler::gameLoading = false;
// HangTimeout = -1 to force disable hang detection
if (hangTimeoutMS >= 0) {
if (hangTimeoutMS == 0) {
hangTimeoutMS = 10;
}
hangTimeout = spring_secs(hangTimeoutMS);
hangdetectorthread = new boost::thread(&HangDetector);
}
InitializeSEH();
}
void ClearDrawWDT(bool disable) { drawwdt = disable ? spring_notime : spring_gettime(); }
void ClearSimWDT(bool disable) { simwdt = disable ? spring_notime : spring_gettime(); }
void GameLoading(bool loading) { CrashHandler::gameLoading = loading; }
void UninstallHangHandler()
{
if (hangdetectorthread) {
keepRunning = 0;
hangdetectorthread->join();
delete hangdetectorthread;
}
if (drawthread != INVALID_HANDLE_VALUE) {
CloseHandle(drawthread);
}
if (simthread != INVALID_HANDLE_VALUE) {
CloseHandle(simthread);
}
}
/** Install crash handler. */
void Install()
{
SetUnhandledExceptionFilter(ExceptionHandler);
signal(SIGABRT, SigAbrtHandler);
}
/** Uninstall crash handler. */
void Remove()
{
SetUnhandledExceptionFilter(NULL);
signal(SIGABRT, SIG_DFL);
}
}; // namespace CrashHandler
