void
LateWriteObserver::Observe(IOInterposeObserver::Observation& aOb)
{
#ifdef OBSERVE_LATE_WRITES
// Crash if that is the shutdown check mode
if (gShutdownChecks == SCM_CRASH) {
MOZ_CRASH();
}
// If we have shutdown mode SCM_NOTHING or we can't record then abort
if (gShutdownChecks == SCM_NOTHING || !Telemetry::CanRecordExtended()) {
return;
}
// Write the stack and loaded libraries to a file. We can get here
// concurrently from many writes, so we use multiple temporary files.
std::vector<uintptr_t> rawStack;
NS_StackWalk(RecordStackWalker, /* skipFrames */ 0, /* maxFrames */ 0,
reinterpret_cast<void*>(&rawStack), 0, nullptr);
Telemetry::ProcessedStack stack = Telemetry::GetStackAndModules(rawStack);
nsPrintfCString nameAux("%s%s%s", mProfileDirectory,
NS_SLASH, "Telemetry.LateWriteTmpXXXXXX");
char* name;
nameAux.GetMutableData(&name);
// We want the sha1 of the entire file, so please don't write to fd
// directly; use sha1Stream.
FILE* stream;
#ifdef XP_WIN
HANDLE hFile;
do {
// mkstemp isn't supported so keep trying until we get a file
int result = _mktemp_s(name, strlen(name) + 1);
hFile = CreateFileA(name, GENERIC_WRITE, 0, nullptr, CREATE_NEW,
FILE_ATTRIBUTE_NORMAL, nullptr);
} while (GetLastError() == ERROR_FILE_EXISTS);
if (hFile == INVALID_HANDLE_VALUE) {
NS_RUNTIMEABORT("Um, how did we get here?");
}
// http://support.microsoft.com/kb/139640
int fd = _open_osfhandle((intptr_t)hFile, _O_APPEND);
if (fd == -1) {
NS_RUNTIMEABORT("Um, how did we get here?");
}
stream = _fdopen(fd, "w");
#else
int fd = mkstemp(name);
stream = fdopen(fd, "w");
#endif
SHA1Stream sha1Stream(stream);
size_t numModules = stack.GetNumModules();
sha1Stream.Printf("%u\n", (unsigned)numModules);
for (size_t i = 0; i < numModules; ++i) {
Telemetry::ProcessedStack::Module module = stack.GetModule(i);
sha1Stream.Printf("%s %s\n", module.mBreakpadId.c_str(),
module.mName.c_str());
}
size_t numFrames = stack.GetStackSize();
sha1Stream.Printf("%u\n", (unsigned)numFrames);
for (size_t i = 0; i < numFrames; ++i) {
const Telemetry::ProcessedStack::Frame& frame = stack.GetFrame(i);
// NOTE: We write the offsets, while the atos tool expects a value with
// the virtual address added. For example, running otool -l on the the firefox
// binary shows
// cmd LC_SEGMENT_64
// cmdsize 632
// segname __TEXT
// vmaddr 0x0000000100000000
// so to print the line matching the offset 123 one has to run
// atos -o firefox 0x100000123.
sha1Stream.Printf("%d %x\n", frame.mModIndex, (unsigned)frame.mOffset);
}
SHA1Sum::Hash sha1;
sha1Stream.Finish(sha1);
// Note: These files should be deleted by telemetry once it reads them. If
// there were no telemetry runs by the time we shut down, we just add files
// to the existing ones instead of replacing them. Given that each of these
// files is a bug to be fixed, that is probably the right thing to do.
// We append the sha1 of the contents to the file name. This provides a simple
// client side deduplication.
nsPrintfCString finalName("%s%s", mProfileDirectory,
"/Telemetry.LateWriteFinal-");
for (int i = 0; i < 20; ++i) {
finalName.AppendPrintf("%02x", sha1[i]);
}
PR_Delete(finalName.get());
PR_Rename(name, finalName.get());
#endif
}
void
ThreadMain(void*)
{
PR_SetCurrentThreadName("Hang Monitor");
MonitorAutoLock lock(*gMonitor);
// In order to avoid issues with the hang monitor incorrectly triggering
// during a general system stop such as sleeping, the monitor thread must
// run twice to trigger hang protection.
PRIntervalTime lastTimestamp = 0;
int waitCount = 0;
#ifdef REPORT_CHROME_HANGS
Telemetry::ProcessedStack stack;
int32_t systemUptime = -1;
int32_t firefoxUptime = -1;
auto annotations = MakeUnique<ChromeHangAnnotations>();
#endif
while (true) {
if (gShutdown) {
return; // Exit the thread
}
// avoid rereading the volatile value in this loop
PRIntervalTime timestamp = gTimestamp;
PRIntervalTime now = PR_IntervalNow();
if (timestamp != PR_INTERVAL_NO_WAIT &&
now < timestamp) {
// 32-bit overflow, reset for another waiting period
timestamp = 1; // lowest legal PRInterval value
}
if (timestamp != PR_INTERVAL_NO_WAIT &&
timestamp == lastTimestamp &&
gTimeout > 0) {
++waitCount;
#ifdef REPORT_CHROME_HANGS
// Capture the chrome-hang stack + Firefox & system uptimes after
// the minimum hang duration has been reached (not when the hang ends)
if (waitCount == 2) {
GetChromeHangReport(stack, systemUptime, firefoxUptime);
ChromeHangAnnotatorCallout(*annotations);
}
#else
// This is the crash-on-hang feature.
// See bug 867313 for the quirk in the waitCount comparison
if (waitCount >= 2) {
int32_t delay =
int32_t(PR_IntervalToSeconds(now - timestamp));
if (delay >= gTimeout) {
MonitorAutoUnlock unlock(*gMonitor);
Crash();
}
}
#endif
} else {
#ifdef REPORT_CHROME_HANGS
if (waitCount >= 2) {
uint32_t hangDuration = PR_IntervalToSeconds(now - lastTimestamp);
Telemetry::RecordChromeHang(hangDuration, stack, systemUptime,
firefoxUptime, Move(annotations));
stack.Clear();
annotations = MakeUnique<ChromeHangAnnotations>();
}
#endif
lastTimestamp = timestamp;
waitCount = 0;
}
PRIntervalTime timeout;
if (gTimeout <= 0) {
timeout = PR_INTERVAL_NO_TIMEOUT;
} else {
timeout = PR_MillisecondsToInterval(gTimeout * 500);
}
lock.Wait(timeout);
}
}
void
ThreadMain(void*)
{
PR_SetCurrentThreadName("Hang Monitor");
MonitorAutoLock lock(*gMonitor);
// In order to avoid issues with the hang monitor incorrectly triggering
// during a general system stop such as sleeping, the monitor thread must
// run twice to trigger hang protection.
PRIntervalTime lastTimestamp = 0;
int waitCount = 0;
#ifdef REPORT_CHROME_HANGS
Telemetry::ProcessedStack stack;
#endif
while (true) {
if (gShutdown) {
return; // Exit the thread
}
// avoid rereading the volatile value in this loop
PRIntervalTime timestamp = gTimestamp;
PRIntervalTime now = PR_IntervalNow();
if (timestamp != PR_INTERVAL_NO_WAIT &&
now < timestamp) {
// 32-bit overflow, reset for another waiting period
timestamp = 1; // lowest legal PRInterval value
}
if (timestamp != PR_INTERVAL_NO_WAIT &&
timestamp == lastTimestamp &&
gTimeout > 0) {
++waitCount;
if (waitCount == 2) {
#ifdef REPORT_CHROME_HANGS
GetChromeHangReport(stack);
#else
int32_t delay =
int32_t(PR_IntervalToSeconds(now - timestamp));
if (delay > gTimeout) {
MonitorAutoUnlock unlock(*gMonitor);
Crash();
}
#endif
}
}
else {
#ifdef REPORT_CHROME_HANGS
if (waitCount >= 2) {
uint32_t hangDuration = PR_IntervalToSeconds(now - lastTimestamp);
Telemetry::RecordChromeHang(hangDuration, stack);
stack.Clear();
}
#endif
lastTimestamp = timestamp;
waitCount = 0;
}
PRIntervalTime timeout;
if (gTimeout <= 0) {
timeout = PR_INTERVAL_NO_TIMEOUT;
}
else {
timeout = PR_MillisecondsToInterval(gTimeout * 500);
}
lock.Wait(timeout);
}
}
