static char * appendSignalName(char *buf, int signo) { switch (signo) { case SIGABRT: buf = appendText(buf, "SIGABRT"); break; case SIGSEGV: buf = appendText(buf, "SIGSEGV"); break; case SIGBUS: buf = appendText(buf, "SIGBUS"); break; case SIGFPE: buf = appendText(buf, "SIGFPE"); break; case SIGILL: buf = appendText(buf, "SIGILL"); break; default: return appendULL(buf, (unsigned long long) signo); } buf = appendText(buf, "("); buf = appendULL(buf, (unsigned long long) signo); buf = appendText(buf, ")"); return buf; }
static int runInSubprocessWithTimeLimit(AbortHandlerState &state, Callback callback, void *userData, int timeLimit) { char *end; pid_t child; int p[2], e; if (pipe(p) == -1) { e = errno; end = state.messageBuf; end = appendText(end, "Could not create subprocess: pipe() failed with errno="); end = appendULL(end, e); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); return -1; } child = asyncFork(); if (child == 0) { close(p[0]); callback(state, userData); _exit(0); return -1; } else if (child == -1) { e = errno; close(p[0]); close(p[1]); end = state.messageBuf; end = appendText(end, "Could not create subprocess: fork() failed with errno="); end = appendULL(end, e); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); return -1; } else { int status; close(p[1]); // We give the child process a time limit. If it doesn't succeed in // exiting within the time limit, we assume that it has frozen // and we kill it. struct pollfd fd; fd.fd = p[0]; fd.events = POLLIN | POLLHUP | POLLERR; if (poll(&fd, 1, timeLimit) <= 0) { kill(child, SIGKILL); safePrintErr("Could not run child process: it did not exit in time\n"); } close(p[0]); if (waitpid(child, &status, 0) == child) { return status; } else { return -1; } } }
static void dumpFileDescriptorInfoWithLsof(AbortHandlerState &state, void *userData) { char *end; end = state.messageBuf; end = appendULL(end, state.pid); *end = '\0'; closeAllFileDescriptors(2, true); execlp("lsof", "lsof", "-p", state.messageBuf, "-nP", (const char * const) 0); end = state.messageBuf; end = appendText(end, "ERROR: cannot execute command 'lsof': errno="); end = appendULL(end, errno); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); _exit(1); }
static void dumpWithCrashWatch(AbortHandlerState &state) { char *messageBuf = state.messageBuf; const char *pidStr = messageBuf; char *end = messageBuf; end = appendULL(end, (unsigned long long) state.pid); *end = '\0'; pid_t child = asyncFork(); if (child == 0) { closeAllFileDescriptors(2, true); execlp("crash-watch", "crash-watch", "--dump", pidStr, (char * const) 0); if (errno == ENOENT) { safePrintErr("Crash-watch is not installed. Please install it with 'gem install crash-watch' " "or download it from https://github.com/FooBarWidget/crash-watch.\n"); } else { int e = errno; end = messageBuf; end = appendText(end, "crash-watch is installed, but it could not be executed! "); end = appendText(end, "(execlp() returned errno="); end = appendULL(end, e); end = appendText(end, ") Please check your file permissions or something.\n"); write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); } _exit(1); } else if (child == -1) { int e = errno; end = messageBuf; end = appendText(end, "Could not execute crash-watch: fork() failed with errno="); end = appendULL(end, e); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); } else { waitpid(child, NULL, 0); } }
static bool createCrashLogFile(char *filename, time_t t) { char *end = filename; end = appendText(end, "/var/tmp/passenger-crash-log."); end = appendULL(end, (unsigned long long) t); *end = '\0'; int fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd == -1) { end = filename; end = appendText(end, "/tmp/passenger-crash-log."); end = appendULL(end, (unsigned long long) t); *end = '\0'; fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0600); } if (fd == -1) { *filename = '\0'; return false; } else { close(fd); return true; } }
static void dumpFileDescriptorInfo(AbortHandlerState &state) { char *messageBuf = state.messageBuf; char *end; struct stat buf; int status; end = messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] Open files and file descriptors:\n"); write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); status = runInSubprocessWithTimeLimit(state, dumpFileDescriptorInfoWithLsof, NULL, 4000); if (status != 0) { safePrintErr("Falling back to another mechanism for dumping file descriptors.\n"); end = messageBuf; end = appendText(end, "/proc/"); end = appendULL(end, state.pid); end = appendText(end, "/fd"); *end = '\0'; if (stat(messageBuf, &buf) == 0) { dumpFileDescriptorInfoWithLs(state, end + 1); } else { end = messageBuf; end = appendText(end, "/dev/fd"); *end = '\0'; if (stat(messageBuf, &buf) == 0) { dumpFileDescriptorInfoWithLs(state, end + 1); } else { end = messageBuf; end = appendText(end, "ERROR: No other file descriptor dumping mechanism on current platform detected.\n"); write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); } } } }
static void abortHandler(int signo, siginfo_t *info, void *ctx) { pid_t pid = getpid(); char messageBuf[1024]; #ifdef LIBC_HAS_BACKTRACE_FUNC void *backtraceStore[512]; backtraceStore[0] = '\0'; // Don't let gdb print uninitialized contents. #endif char *end = messageBuf; end = appendText(end, "[ pid="); end = appendULL(end, (unsigned long long) pid); end = appendText(end, ", timestamp="); end = appendULL(end, (unsigned long long) time(NULL)); end = appendText(end, " ] Process aborted! signo="); end = appendSignalName(end, signo); end = appendText(end, ", reason="); end = appendSignalReason(end, info); // It is important that writing the message and the backtrace are two // seperate operations because it's not entirely clear whether the // latter is async signal safe and thus can crash. #ifdef LIBC_HAS_BACKTRACE_FUNC end = appendText(end, ", backtrace available.\n"); #else end = appendText(end, "\n"); #endif write(STDERR_FILENO, messageBuf, end - messageBuf); #ifdef LIBC_HAS_BACKTRACE_FUNC /* For some reason, it would appear that fatal signal * handlers have a deadline on some systems: the process will * be killed if the signal handler doesn't finish in time. * This killing appears to be triggered at some system calls, * including but not limited to nanosleep(). * backtrace() might be slow and running crash-watch is * definitely slow, so we do our work in a child process * in order not to be affected by the deadline. But preferably * we don't fork because forking will cause us to lose * thread information. */ #ifdef __linux__ bool hasDeadline = false; #else // Mac OS X has a deadline. Not sure about other systems. bool hasDeadline = true; #endif if (!hasDeadline || fork() == 0) { int frames = backtrace(backtraceStore, sizeof(backtraceStore) / sizeof(void *)); end = messageBuf; end = appendText(end, "--------------------------------------\n"); end = appendText(end, "[ pid="); end = appendULL(end, (unsigned long long) pid); end = appendText(end, " ] Backtrace with "); end = appendULL(end, (unsigned long long) frames); end = appendText(end, " frames:\n"); write(STDERR_FILENO, messageBuf, end - messageBuf); backtrace_symbols_fd(backtraceStore, frames, STDERR_FILENO); end = messageBuf; end = appendText(end, "--------------------------------------\n"); end = appendText(end, "[ pid="); end = appendULL(end, (unsigned long long) pid); end = appendText(end, " ] Dumping a more detailed backtrace with crash-watch " "('gem install crash-watch' if you don't have it)...\n"); write(STDERR_FILENO, messageBuf, end - messageBuf); end = messageBuf; end = appendText(end, "crash-watch --dump "); end = appendULL(end, (unsigned long long) getpid()); *end = '\0'; system(messageBuf); _exit(1); } #endif // Run default signal handler. kill(getpid(), signo); }
// Must be async signal safe. static char * appendSignalReason(char *buf, siginfo_t *info) { bool handled = true; switch (info->si_code) { SI_CODE_HANDLER(SI_USER); #ifdef SI_KERNEL SI_CODE_HANDLER(SI_KERNEL); #endif SI_CODE_HANDLER(SI_QUEUE); SI_CODE_HANDLER(SI_TIMER); #ifdef SI_ASYNCIO SI_CODE_HANDLER(SI_ASYNCIO); #endif #ifdef SI_MESGQ SI_CODE_HANDLER(SI_MESGQ); #endif #ifdef SI_SIGIO SI_CODE_HANDLER(SI_SIGIO); #endif #ifdef SI_TKILL SI_CODE_HANDLER(SI_TKILL); #endif default: switch (info->si_signo) { case SIGSEGV: switch (info->si_code) { SI_CODE_HANDLER(SEGV_MAPERR); SI_CODE_HANDLER(SEGV_ACCERR); default: handled = false; break; } break; case SIGBUS: switch (info->si_code) { SI_CODE_HANDLER(BUS_ADRALN); SI_CODE_HANDLER(BUS_ADRERR); SI_CODE_HANDLER(BUS_OBJERR); default: handled = false; break; } break; }; if (!handled) { buf = appendText(buf, "#"); buf = appendULL(buf, (unsigned long long) info->si_code); } break; } if (info->si_code <= 0) { buf = appendText(buf, ", signal sent by PID "); buf = appendULL(buf, (unsigned long long) info->si_pid); buf = appendText(buf, " with UID "); buf = appendULL(buf, (unsigned long long) info->si_uid); } return buf; }
static void abortHandler(int signo, siginfo_t *info, void *ctx) { AbortHandlerState state; state.pid = getpid(); state.signo = signo; state.info = info; pid_t child; time_t t = time(NULL); char crashLogFile[256]; abortHandlerCalled++; if (abortHandlerCalled > 1) { // The abort handler itself crashed! char *end = state.messageBuf; end = appendText(end, "[ origpid="); end = appendULL(end, (unsigned long long) state.pid); end = appendText(end, ", pid="); end = appendULL(end, (unsigned long long) getpid()); end = appendText(end, ", timestamp="); end = appendULL(end, (unsigned long long) t); if (abortHandlerCalled == 2) { // This is the first time it crashed. end = appendText(end, " ] Abort handler crashed! signo="); end = appendSignalName(end, state.signo); end = appendText(end, ", reason="); end = appendSignalReason(end, state.info); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); // Run default signal handler. raise(signo); } else { // This is the second time it crashed, meaning it failed to // invoke the default signal handler to abort the process! end = appendText(end, " ] Abort handler crashed again! Force exiting this time. signo="); end = appendSignalName(end, state.signo); end = appendText(end, ", reason="); end = appendSignalReason(end, state.info); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); _exit(1); } return; } if (emergencyPipe1[0] != -1) { close(emergencyPipe1[0]); } if (emergencyPipe1[1] != -1) { close(emergencyPipe1[1]); } if (emergencyPipe2[0] != -1) { close(emergencyPipe2[0]); } if (emergencyPipe2[1] != -1) { close(emergencyPipe2[1]); } emergencyPipe1[0] = emergencyPipe1[1] = -1; emergencyPipe2[0] = emergencyPipe2[1] = -1; /* We want to dump the entire crash log to both stderr and a log file. * We use 'tee' for this. */ if (createCrashLogFile(crashLogFile, t)) { forkAndRedirectToTee(crashLogFile); } char *end = state.messagePrefix; end = appendText(end, "[ pid="); end = appendULL(end, (unsigned long long) state.pid); *end = '\0'; end = state.messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, ", timestamp="); end = appendULL(end, (unsigned long long) t); end = appendText(end, " ] Process aborted! signo="); end = appendSignalName(end, state.signo); end = appendText(end, ", reason="); end = appendSignalReason(end, state.info); end = appendText(end, ", randomSeed="); end = appendULL(end, (unsigned long long) randomSeed); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); end = state.messageBuf; if (*crashLogFile != '\0') { end = appendText(end, state.messagePrefix); end = appendText(end, " ] Crash log dumped to "); end = appendText(end, crashLogFile); end = appendText(end, "\n"); } else { end = appendText(end, state.messagePrefix); end = appendText(end, " ] Could not create crash log file, so dumping to stderr only.\n"); } write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); if (beepOnAbort) { end = state.messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] PASSENGER_BEEP_ON_ABORT on, executing beep...\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); child = asyncFork(); if (child == 0) { closeAllFileDescriptors(2, true); #ifdef __APPLE__ execlp("osascript", "osascript", "-e", "beep 2", (const char * const) 0); safePrintErr("Cannot execute 'osascript' command\n"); #else execlp("beep", "beep", (const char * const) 0); safePrintErr("Cannot execute 'beep' command\n"); #endif _exit(1); } else if (child == -1) { int e = errno; end = state.messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] Could fork a child process for invoking a beep: fork() failed with errno="); end = appendULL(end, e); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); } } if (stopOnAbort) { end = state.messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] PASSENGER_STOP_ON_ABORT on, so process stopped. Send SIGCONT when you want to continue.\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); raise(SIGSTOP); } // It isn't safe to call any waiting functions in this signal handler, // not even read() and waitpid() even though they're async signal safe. // So we fork a child process and let it dump as much diagnostics as possible // instead of doing it in this process. child = asyncFork(); if (child == 0) { // Sleep for a short while to allow the parent process to raise SIGSTOP. // usleep() and nanosleep() aren't async signal safe so we use select() // instead. struct timeval tv; tv.tv_sec = 0; tv.tv_usec = 100000; select(0, NULL, NULL, NULL, &tv); resetSignalHandlersAndMask(); child = asyncFork(); if (child == 0) { // OS X: for some reason the SIGPIPE handler may be reset to default after forking. // Later in this program we're going to pipe backtrace_symbols_fd() into the backtrace // sanitizer, which may fail, and we don't want the diagnostics process to crash // with SIGPIPE as a result, so we ignore SIGPIPE again. ignoreSigpipe(); dumpDiagnostics(state); // The child process may or may or may not resume the original process. // We do it ourselves just to be sure. kill(state.pid, SIGCONT); _exit(0); } else if (child == -1) { int e = errno; end = state.messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, "] Could fork a child process for dumping diagnostics: fork() failed with errno="); end = appendULL(end, e); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); _exit(1); } else { // Exit immediately so that child process is adopted by init process. _exit(0); } } else if (child == -1) { int e = errno; end = state.messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] Could fork a child process for dumping diagnostics: fork() failed with errno="); end = appendULL(end, e); end = appendText(end, "\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); } else { raise(SIGSTOP); // Will continue after the child process has done its job. } // Run default signal handler. raise(signo); }
// This function is performed in a child process. static void dumpDiagnostics(AbortHandlerState &state) { char *messageBuf = state.messageBuf; char *end; pid_t pid; int status; end = messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] Date, uname and ulimits:\n"); write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); // Dump human-readable time string and string. pid = asyncFork(); if (pid == 0) { closeAllFileDescriptors(2, true); execlp("date", "date", (const char * const) 0); _exit(1); } else if (pid == -1) { safePrintErr("ERROR: Could not fork a process to dump the time!\n"); } else if (waitpid(pid, &status, 0) != pid || status != 0) { safePrintErr("ERROR: Could not run 'date'!\n"); } // Dump system uname. pid = asyncFork(); if (pid == 0) { closeAllFileDescriptors(2, true); execlp("uname", "uname", "-mprsv", (const char * const) 0); _exit(1); } else if (pid == -1) { safePrintErr("ERROR: Could not fork a process to dump the uname!\n"); } else if (waitpid(pid, &status, 0) != pid || status != 0) { safePrintErr("ERROR: Could not run 'uname -mprsv'!\n"); } // Dump ulimit. pid = asyncFork(); if (pid == 0) { closeAllFileDescriptors(2, true); execlp("ulimit", "ulimit", "-a", (const char * const) 0); // On Linux 'ulimit' is a shell builtin, not a command. execlp("/bin/sh", "/bin/sh", "-c", "ulimit -a", (const char * const) 0); _exit(1); } else if (pid == -1) { safePrintErr("ERROR: Could not fork a process to dump the ulimit!\n"); } else if (waitpid(pid, &status, 0) != pid || status != 0) { safePrintErr("ERROR: Could not run 'ulimit -a'!\n"); } end = messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] Phusion Passenger version: " PASSENGER_VERSION "\n"); write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); if (lastAssertionFailure.filename != NULL) { end = messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] Last assertion failure: ("); end = appendText(end, lastAssertionFailure.expression); end = appendText(end, "), "); if (lastAssertionFailure.function != NULL) { end = appendText(end, "function "); end = appendText(end, lastAssertionFailure.function); end = appendText(end, ", "); } end = appendText(end, "file "); end = appendText(end, lastAssertionFailure.filename); end = appendText(end, ", line "); end = appendULL(end, lastAssertionFailure.line); end = appendText(end, ".\n"); write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); } // It is important that writing the message and the backtrace are two // seperate operations because it's not entirely clear whether the // latter is async signal safe and thus can crash. end = messageBuf; end = appendText(end, state.messagePrefix); #ifdef LIBC_HAS_BACKTRACE_FUNC end = appendText(end, " ] libc backtrace available!\n"); #else end = appendText(end, " ] libc backtrace not available.\n"); #endif write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); #ifdef LIBC_HAS_BACKTRACE_FUNC runInSubprocessWithTimeLimit(state, dumpBacktrace, NULL, 4000); #endif safePrintErr("--------------------------------------\n"); if (customDiagnosticsDumper != NULL) { end = messageBuf; end = appendText(end, state.messagePrefix); end = appendText(end, " ] Dumping additional diagnostical information...\n"); write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); safePrintErr("--------------------------------------\n"); runInSubprocessWithTimeLimit(state, runCustomDiagnosticsDumper, NULL, 2000); safePrintErr("--------------------------------------\n"); } dumpFileDescriptorInfo(state); safePrintErr("--------------------------------------\n"); if (shouldDumpWithCrashWatch) { end = messageBuf; end = appendText(end, state.messagePrefix); #ifdef LIBC_HAS_BACKTRACE_FUNC end = appendText(end, " ] Dumping a more detailed backtrace with crash-watch...\n"); #else end = appendText(end, " ] Dumping a backtrace with crash-watch...\n"); #endif write_nowarn(STDERR_FILENO, messageBuf, end - messageBuf); dumpWithCrashWatch(state); } else { write_nowarn(STDERR_FILENO, "\n", 1); } }
static void dumpBacktrace(AbortHandlerState &state, void *userData) { void *backtraceStore[512]; int frames = backtrace(backtraceStore, sizeof(backtraceStore) / sizeof(void *)); char *end = state.messageBuf; end = appendText(end, "--------------------------------------\n"); end = appendText(end, "[ pid="); end = appendULL(end, (unsigned long long) state.pid); end = appendText(end, " ] Backtrace with "); end = appendULL(end, (unsigned long long) frames); end = appendText(end, " frames:\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); if (backtraceSanitizerCommand != NULL) { int p[2]; if (pipe(p) == -1) { int e = errno; end = state.messageBuf; end = appendText(end, "Could not dump diagnostics through backtrace sanitizer: pipe() failed with errno="); end = appendULL(end, e); end = appendText(end, "\n"); end = appendText(end, "Falling back to writing to stderr directly...\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); backtrace_symbols_fd(backtraceStore, frames, STDERR_FILENO); return; } pid_t pid = asyncFork(); if (pid == 0) { const char *pidStr = end = state.messageBuf; end = appendULL(end, (unsigned long long) state.pid); *end = '\0'; end++; close(p[1]); dup2(p[0], STDIN_FILENO); closeAllFileDescriptors(2, true); char *command = end; end = appendText(end, "exec "); end = appendText(end, backtraceSanitizerCommand); if (backtraceSanitizerPassProgramInfo) { end = appendText(end, " \""); end = appendText(end, argv0); end = appendText(end, "\" "); end = appendText(end, pidStr); } *end = '\0'; end++; execlp("/bin/sh", "/bin/sh", "-c", command, (const char * const) 0); end = state.messageBuf; end = appendText(end, "ERROR: cannot execute '"); end = appendText(end, backtraceSanitizerCommand); end = appendText(end, "' for sanitizing the backtrace, trying 'cat'...\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); execlp("cat", "cat", (const char * const) 0); execlp("/bin/cat", "cat", (const char * const) 0); execlp("/usr/bin/cat", "cat", (const char * const) 0); safePrintErr("ERROR: cannot execute 'cat'\n"); _exit(1); } else if (pid == -1) { close(p[0]); close(p[1]); int e = errno; end = state.messageBuf; end = appendText(end, "Could not dump diagnostics through backtrace sanitizer: fork() failed with errno="); end = appendULL(end, e); end = appendText(end, "\n"); end = appendText(end, "Falling back to writing to stderr directly...\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); backtrace_symbols_fd(backtraceStore, frames, STDERR_FILENO); } else { int status = -1; close(p[0]); backtrace_symbols_fd(backtraceStore, frames, p[1]); close(p[1]); if (waitpid(pid, &status, 0) == -1 || status != 0) { end = state.messageBuf; end = appendText(end, "ERROR: cannot execute '"); end = appendText(end, backtraceSanitizerCommand); end = appendText(end, "' for sanitizing the backtrace, writing to stderr directly...\n"); write_nowarn(STDERR_FILENO, state.messageBuf, end - state.messageBuf); backtrace_symbols_fd(backtraceStore, frames, STDERR_FILENO); } } } else { backtrace_symbols_fd(backtraceStore, frames, STDERR_FILENO); } }