void HangDetector() {
while (keepRunning) {
// increase multiplier during game load to prevent false positives e.g. during pathing
const int hangTimeMultiplier = CrashHandler::gameLoading? 3 : 1;
const spring_time hangtimeout = spring_msecs(spring_tomsecs(hangTimeout) * hangTimeMultiplier);
spring_time curwdt = spring_gettime();
#if defined(USE_GML) && GML_ENABLE_SIM
if (gmlMultiThreadSim) {
spring_time cursimwdt = simwdt;
if (spring_istime(cursimwdt) && curwdt > cursimwdt && (curwdt - cursimwdt) > hangtimeout) {
HangHandler(true);
simwdt = curwdt;
}
}
#endif
spring_time curdrawwdt = drawwdt;
if (spring_istime(curdrawwdt) && curwdt > curdrawwdt && (curwdt - curdrawwdt) > hangtimeout) {
HangHandler(false);
drawwdt = curwdt;
}
spring_sleep(spring_secs(1));
}
}
void InstallHangHandler() {
DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(),
&drawthread, 0, TRUE, DUPLICATE_SAME_ACCESS);
int hangTimeoutMS = configHandler->Get("HangTimeout", 0);
CrashHandler::gameLoading = false;
// HangTimeout = -1 to force disable hang detection
if (hangTimeoutMS >= 0) {
if (hangTimeoutMS == 0) {
hangTimeoutMS = 10;
}
hangTimeout = spring_secs(hangTimeoutMS);
hangdetectorthread = new boost::thread(&HangDetector);
}
InitializeSEH();
}
void Install()
{
boost::mutex::scoped_lock lock(wdmutex);
memset(registeredThreadsData, 0, sizeof(registeredThreadsData));
for (unsigned int i = 0; i < WDT_LAST; ++i) {
registeredThreads[i] = ®isteredThreadsData[WDT_LAST];
threadNameToNum[std::string(threadNames[i])] = i;
}
memset(threadSlots, 0, sizeof(threadSlots));
#ifndef _WIN32
// disable if gdb is running
char buf[1024];
SNPRINTF(buf, sizeof(buf), "/proc/%d/cmdline", getppid());
std::ifstream f(buf);
if (f.good()) {
f.read(buf,sizeof(buf));
f.close();
std::string str(buf);
if (str.find("gdb") != std::string::npos) {
LOG("[Watchdog] disabled (gdb detected)");
return;
}
}
#endif
#ifdef USE_VALGRIND
if (RUNNING_ON_VALGRIND) {
LOG("[Watchdog] disabled (Valgrind detected)");
return;
}
#endif
const int hangTimeoutSecs = configHandler->GetInt("HangTimeout");
// HangTimeout = -1 to force disable hang detection
if (hangTimeoutSecs <= 0) {
LOG("[Watchdog] disabled");
return;
}
hangTimeout = spring_secs(hangTimeoutSecs);
// start the watchdog thread
hangDetectorThread = new boost::thread(&HangDetectorLoop);
LOG("[Watchdog] Installed (HangTimeout: %isec)", hangTimeoutSecs);
}
static void DrawThreadBarcode()
{
const float maxHist_f = 4.0f;
const spring_time curTime = spring_now();
const spring_time maxHist = spring_secs(maxHist_f);
auto& coreProf = profiler.profileCore;
const auto numThreads = coreProf.size();
const float drawArea[4] = {0.01f, 0.30f, (start_x / 2), 0.35f};
// background
CVertexArray* va = GetVertexArray();
va->Initialize();
va->AddVertex0(drawArea[0] - 10 * globalRendering->pixelX, drawArea[1] - 10 * globalRendering->pixelY, 0.0f);
va->AddVertex0(drawArea[0] - 10 * globalRendering->pixelX, drawArea[3] + 10 * globalRendering->pixelY, 0.0f);
va->AddVertex0(drawArea[2] + 10 * globalRendering->pixelX, drawArea[3] + 10 * globalRendering->pixelY, 0.0f);
va->AddVertex0(drawArea[2] + 10 * globalRendering->pixelX, drawArea[1] - 10 * globalRendering->pixelY, 0.0f);
glColor4f(0.0f,0.0f,0.0f, 0.5f);
va->DrawArray0(GL_QUADS);
// title
font->glFormat(drawArea[0], drawArea[3], 0.7f, FONT_TOP | DBG_FONT_FLAGS, "ThreadPool (%.0fsec)", maxHist_f);
// bars
glColor4f(1.0f,0.0f,0.0f, 0.6f);
int i = 0;
for (auto& frames: coreProf) {
float drawArea2[4] = {drawArea[0], 0.f, drawArea[2], 0.f};
drawArea2[1] = drawArea[1] + ((drawArea[3] - drawArea[1]) / numThreads) * i++;
drawArea2[3] = drawArea[1] + ((drawArea[3] - drawArea[1]) / numThreads) * i - 4 * globalRendering->pixelY;
DrawTimeSlice(frames, curTime, maxHist, drawArea2);
}
// feeder
//const float y1 = 0.0f;
//const float y2 = 0.1f * numThreads;
//CVertexArray* va = GetVertexArray();
va->Initialize();
const float r = (curTime % maxHist).toSecsf() / maxHist_f;
const float xf = drawArea[0] + r * (drawArea[2] - drawArea[0]);
va->AddVertex0(xf, drawArea[1], 0.0f);
va->AddVertex0(xf, drawArea[3], 0.0f);
va->AddVertex0(xf + 5 * globalRendering->pixelX, drawArea[3], 0.0f);
va->AddVertex0(xf + 5 * globalRendering->pixelX, drawArea[1], 0.0f);
glColor3f(1.0f,0.0f,0.0f);
va->DrawArray0(GL_QUADS);
}
void Install()
{
int hangTimeoutSecs = configHandler->Get("HangTimeout", 0);
//! HangTimeout = -1 to force disable hang detection
if (hangTimeoutSecs < 0)
return;
if (hangTimeoutSecs == 0)
hangTimeoutSecs = 10;
hangTimeout = spring_secs(hangTimeoutSecs);
//! register (this) mainthread
RegisterThread("main");
//! start the watchdog thread
hangdetectorthread = new boost::thread(&HangDetector);
logOutput.Print("[Watchdog] Installed (timeout: %isec)", hangTimeoutSecs);
}
void BenchmarkSleepFnc(const std::string& name, void (*sleep)(int time), const int runs, const float toMilliSecondsScale)
{
// waste a few cycles to push the cpu to higher frequency states
for (auto spinStopTime = spring_gettime() + spring_secs(2); spring_gettime() < spinStopTime; ) {
}
spring_time t = spring_gettime();
spring_time tmin, tmax;
float tavg = 0;
// check lowest possible sleep tick
for (int i=0; i<runs; ++i) {
sleep(0);
spring_time diff = spring_gettime() - t;
if ((diff > tmax) || !spring_istime(tmax)) tmax = diff;
if ((diff < tmin) || !spring_istime(tmin)) tmin = diff;
tavg = float(i * tavg + diff.toNanoSecsi()) / (i + 1);
t = spring_gettime();
}
// check error in sleeping times
spring_time emin, emax;
float eavg = 0;
if (toMilliSecondsScale != 0) {
for (int i=0; i<100; ++i) {
const auto sleepTime = (rand() % 50) * 0.1f + 2; // 2..7ms
t = spring_gettime();
sleep(sleepTime * toMilliSecondsScale);
spring_time diff = (spring_gettime() - t) - spring_msecs(sleepTime);
if ((diff > emax) || !emax.isDuration()) emax = diff;
if ((diff < emin) || !emin.isDuration()) emin = diff;
eavg = float(i * eavg + std::abs(diff.toNanoSecsf())) / (i + 1);
}
}
LOG("[%35s] accuracy:={ err: %+.4fms %+.4fms erravg: %.4fms } min sleep time:={ min: %.6fms avg: %.6fms max: %.6fms }", name.c_str(), emin.toMilliSecsf(), emax.toMilliSecsf(), eavg * 1e-6, tmin.toMilliSecsf(), tavg * 1e-6, tmax.toMilliSecsf());
}
int main(int argc, char* argv[])
{
Threading::SetMainThread();
try {
spring_clock::PushTickRate();
// initialize start time (can safely be done before SDL_Init
// since we are not using SDL_GetTicks as our clock anymore)
spring_time::setstarttime(spring_time::gettime(true));
CLogOutput::LogSystemInfo();
std::string scriptName;
std::string scriptText;
std::string binaryName = argv[0];
gflags::SetUsageMessage("Usage: " + binaryName + " [options] path_to_script.txt");
gflags::SetVersionString(SpringVersion::GetFull());
gflags::ParseCommandLineFlags(&argc, &argv, true);
ParseCmdLine(argc, argv, scriptName);
GlobalConfig::Instantiate();
FileSystemInitializer::InitializeLogOutput();
FileSystemInitializer::Initialize();
// Initialize crash reporting
CrashHandler::Install();
LOG("report any errors to Mantis or the forums.");
LOG("loading script from file: %s", scriptName.c_str());
// server will take ownership of these
std::shared_ptr<ClientSetup> dsClientSetup(new ClientSetup());
std::shared_ptr<GameData> dsGameData(new GameData());
std::shared_ptr<CGameSetup> dsGameSetup(new CGameSetup());
CFileHandler fh(scriptName);
if (!fh.FileExists())
throw content_error("script does not exist in given location: " + scriptName);
if (!fh.LoadStringData(scriptText))
throw content_error("script cannot be read: " + scriptName);
dsClientSetup->LoadFromStartScript(scriptText);
if (!dsGameSetup->Init(scriptText)) {
// read the script provided by cmdline
LOG_L(L_ERROR, "failed to load script %s", scriptName.c_str());
return 1;
}
// Create the server, it will run in a separate thread
CGlobalUnsyncedRNG rng;
const unsigned sleepTime = FLAGS_sleeptime;
const unsigned randSeed = time(nullptr) % ((spring_gettime().toNanoSecsi() + 1) * 9007);
rng.Seed(randSeed);
dsGameData->SetRandomSeed(rng.NextInt());
// Use script provided hashes if they exist
if (dsGameSetup->mapHash != 0) {
dsGameData->SetMapChecksum(dsGameSetup->mapHash);
dsGameSetup->LoadStartPositions(false); // reduced mode
} else {
dsGameData->SetMapChecksum(archiveScanner->GetArchiveCompleteChecksum(dsGameSetup->mapName));
CFileHandler f("maps/" + dsGameSetup->mapName);
if (!f.FileExists())
vfsHandler->AddArchiveWithDeps(dsGameSetup->mapName, false);
dsGameSetup->LoadStartPositions(); // full mode
}
if (dsGameSetup->modHash != 0) {
dsGameData->SetModChecksum(dsGameSetup->modHash);
} else {
const std::string& modArchive = archiveScanner->ArchiveFromName(dsGameSetup->modName);
const unsigned int modCheckSum = archiveScanner->GetArchiveCompleteChecksum(modArchive);
dsGameData->SetModChecksum(modCheckSum);
}
LOG("starting server...");
{
dsGameData->SetSetupText(dsGameSetup->setupText);
CGameServer server(dsClientSetup, dsGameData, dsGameSetup);
while (!server.HasGameID()) {
// wait until gameID has been generated or
// a timeout occurs (if no clients connect)
if (server.HasFinished())
break;
spring_sleep(spring_secs(sleepTime));
}
while (!server.HasFinished()) {
static bool printData = (server.GetDemoRecorder() != nullptr);
if (printData) {
printData = false;
const std::unique_ptr<CDemoRecorder>& demoRec = server.GetDemoRecorder();
const std::uint8_t* gameID = (demoRec->GetFileHeader()).gameID;
LOG("recording demo: %s", (demoRec->GetName()).c_str());
LOG("using mod: %s", (dsGameSetup->modName).c_str());
LOG("using map: %s", (dsGameSetup->mapName).c_str());
LOG("GameID: %02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", gameID[0], gameID[1], gameID[2], gameID[3], gameID[4], gameID[5], gameID[6], gameID[7], gameID[8], gameID[9], gameID[10], gameID[11], gameID[12], gameID[13], gameID[14], gameID[15]);
}
spring_secs(sleepTime).sleep(true);
}
}
LOG("exiting");
FileSystemInitializer::Cleanup();
GlobalConfig::Deallocate();
DataDirLocater::FreeInstance();
spring_clock::PopTickRate();
LOG("exited");
}
CATCH_SPRING_ERRORS
return 0;
}
static void DrawFrameBarcode()
{
const float maxHist_f = 0.5f;
const spring_time curTime = spring_now();
const spring_time maxHist = spring_secs(maxHist_f);
float drawArea[4] = {0.01f, 0.2f, start_x - 0.05f, 0.25f};
// background
CVertexArray* va = GetVertexArray();
va->Initialize();
va->AddVertex0(drawArea[0] - 10 * globalRendering->pixelX, drawArea[1] - 10 * globalRendering->pixelY, 0.0f);
va->AddVertex0(drawArea[0] - 10 * globalRendering->pixelX, drawArea[3] + 20 * globalRendering->pixelY, 0.0f);
va->AddVertex0(drawArea[2] + 10 * globalRendering->pixelX, drawArea[3] + 20 * globalRendering->pixelY, 0.0f);
va->AddVertex0(drawArea[2] + 10 * globalRendering->pixelX, drawArea[1] - 10 * globalRendering->pixelY, 0.0f);
glColor4f(0.0f,0.0f,0.0f, 0.5f);
va->DrawArray0(GL_QUADS);
// title and legend
font->glFormat(drawArea[0], drawArea[3] + 10 * globalRendering->pixelY, 0.7f, FONT_TOP | DBG_FONT_FLAGS,
"Frame Grapher (%.2fsec)"
"\xff\xff\x80\xff GC"
"\xff\xff\xff\x01 Unsynced"
"\xff\x01\x01\xff Swap"
"\xff\x01\xff\x01 Video"
"\xff\xff\x01\x01 Sim"
, maxHist_f);
// gc frames
glColor4f(1.0f,0.5f,1.0f, 0.55f);
DrawTimeSlice(lgcFrames, curTime, maxHist, drawArea);
// updateunsynced frames
glColor4f(1.0f,1.0f,0.0f, 0.9f);
DrawTimeSlice(uusFrames, curTime, maxHist, drawArea);
// video swap frames
glColor4f(0.0f,0.0f,1.0f, 0.55f);
DrawTimeSlice(swpFrames, curTime, maxHist, drawArea);
// video frames
glColor4f(0.0f,1.0f,0.0f, 0.55f);
DrawTimeSlice(vidFrames, curTime, maxHist, drawArea);
// sim frames
glColor4f(1.0f,0.0f,0.0f, 0.55f);
DrawTimeSlice(simFrames, curTime, maxHist, drawArea);
// draw `feeder` indicating current time pos
//CVertexArray* va = GetVertexArray();
va->Initialize();
// draw feeder
const float r = (curTime % maxHist).toSecsf() / maxHist_f;
const float xf = drawArea[0] + r * (drawArea[2] - drawArea[0]);
va->AddVertex0(xf, drawArea[1], 0.0f);
va->AddVertex0(xf, drawArea[3], 0.0f);
va->AddVertex0(xf + 10 * globalRendering->pixelX, drawArea[3], 0.0f);
va->AddVertex0(xf + 10 * globalRendering->pixelX, drawArea[1], 0.0f);
// draw scale (horizontal bar that indicates 30FPS timing length)
const float xs1 = drawArea[2] - 1.f/(30.f*maxHist_f) * (drawArea[2] - drawArea[0]);
const float xs2 = drawArea[2] + 0.0f * (drawArea[2] - drawArea[0]);
va->AddVertex0(xs1, drawArea[3] + 2 * globalRendering->pixelY, 0.0f);
va->AddVertex0(xs1, drawArea[3] + 10 * globalRendering->pixelY, 0.0f);
va->AddVertex0(xs2, drawArea[3] + 10 * globalRendering->pixelY, 0.0f);
va->AddVertex0(xs2, drawArea[3] + 2 * globalRendering->pixelY, 0.0f);
glColor4f(1.0f,0.0f,0.0f, 1.0f);
va->DrawArray0(GL_QUADS);
}
