void processAndLogNewSpawnException(SpawnException &e, const Options &options,
ResourceLocator &resourceLocator, RandomGenerator &randomGenerator)
{
ErrorRenderer renderer(resourceLocator);
string errorId = randomGenerator.generateHexString(4);
string appMessage = e.getErrorPage();
char filename[PATH_MAX];
stringstream stream;
e.set("ERROR_ID", errorId);
if (appMessage.empty()) {
appMessage = "none";
}
try {
int fd = -1;
FdGuard guard(fd, true);
string errorPage;
errorPage = renderer.renderWithDetails(appMessage, options, &e);
#if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)
snprintf(filename, PATH_MAX, "%s/passenger-error-XXXXXX.html",
getSystemTempDir());
fd = mkstemps(filename, sizeof(".html") - 1);
#else
snprintf(filename, PATH_MAX, "%s/passenger-error.XXXXXX",
getSystemTempDir());
fd = mkstemp(filename);
#endif
if (fd == -1) {
int e = errno;
throw SystemException("Cannot generate a temporary filename",
e);
}
writeExact(fd, errorPage);
} catch (const SystemException &e2) {
filename[0] = '\0';
P_ERROR("Cannot render an error page: " << e2.what() << "\n" <<
e2.backtrace());
}
stream << "Could not spawn process for application " << options.appRoot <<
": " << e.what() << "\n" <<
" Error ID: " << errorId << "\n";
if (filename[0] != '\0') {
stream << " Error details saved to: " << filename << "\n";
}
stream << " Message from application: " << appMessage << "\n";
P_ERROR(stream.str());
}
void processAndLogNewSpawnException(SpawnException &e, const Options &options,
const SpawningKit::ConfigPtr &config)
{
TRACE_POINT();
UnionStation::TransactionPtr transaction;
ErrorRenderer renderer(*config->resourceLocator);
string appMessage = e.getErrorPage();
string errorId;
char filename[PATH_MAX];
stringstream stream;
if (options.analytics && config->unionStationCore != NULL) {
try {
UPDATE_TRACE_POINT();
transaction = config->unionStationCore->newTransaction(
options.getAppGroupName(),
"exceptions",
options.unionStationKey);
errorId = transaction->getTxnId();
} catch (const tracable_exception &e2) {
transaction.reset();
P_WARN("Cannot log to Union Station: " << e2.what() <<
"\n Backtrace:\n" << e2.backtrace());
}
}
UPDATE_TRACE_POINT();
if (appMessage.empty()) {
appMessage = "none";
}
if (errorId.empty()) {
errorId = config->randomGenerator->generateHexString(4);
}
e.set("error_id", errorId);
try {
int fd = -1;
string errorPage;
UPDATE_TRACE_POINT();
errorPage = renderer.renderWithDetails(appMessage, options, &e);
#if (defined(__linux__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 11))) || defined(__APPLE__) || defined(__FreeBSD__)
snprintf(filename, PATH_MAX, "%s/passenger-error-XXXXXX.html",
getSystemTempDir());
fd = mkstemps(filename, sizeof(".html") - 1);
#else
snprintf(filename, PATH_MAX, "%s/passenger-error.XXXXXX",
getSystemTempDir());
fd = mkstemp(filename);
#endif
FdGuard guard(fd, NULL, 0, true);
if (fd == -1) {
int e = errno;
throw SystemException("Cannot generate a temporary filename",
e);
}
UPDATE_TRACE_POINT();
writeExact(fd, errorPage);
} catch (const SystemException &e2) {
filename[0] = '\0';
P_ERROR("Cannot render an error page: " << e2.what() << "\n" <<
e2.backtrace());
}
if (transaction != NULL) {
try {
UPDATE_TRACE_POINT();
transaction->message("Context: spawning");
transaction->message("Message: " +
jsonString(e.what()));
transaction->message("App message: " +
jsonString(appMessage));
const char *kind;
switch (e.getErrorKind()) {
case SpawnException::PRELOADER_STARTUP_ERROR:
kind = "PRELOADER_STARTUP_ERROR";
break;
case SpawnException::PRELOADER_STARTUP_PROTOCOL_ERROR:
kind = "PRELOADER_STARTUP_PROTOCOL_ERROR";
break;
case SpawnException::PRELOADER_STARTUP_TIMEOUT:
kind = "PRELOADER_STARTUP_TIMEOUT";
break;
case SpawnException::PRELOADER_STARTUP_EXPLAINABLE_ERROR:
kind = "PRELOADER_STARTUP_EXPLAINABLE_ERROR";
break;
case SpawnException::APP_STARTUP_ERROR:
kind = "APP_STARTUP_ERROR";
break;
case SpawnException::APP_STARTUP_PROTOCOL_ERROR:
kind = "APP_STARTUP_PROTOCOL_ERROR";
break;
case SpawnException::APP_STARTUP_TIMEOUT:
kind = "APP_STARTUP_TIMEOUT";
break;
case SpawnException::APP_STARTUP_EXPLAINABLE_ERROR:
kind = "APP_STARTUP_EXPLAINABLE_ERROR";
break;
default:
kind = "UNDEFINED_ERROR";
break;
}
transaction->message(string("Kind: ") + kind);
Json::Value details;
const map<string, string> &annotations = e.getAnnotations();
map<string, string>::const_iterator it, end = annotations.end();
for (it = annotations.begin(); it != end; it++) {
details[it->first] = it->second;
}
// This information is not very useful. Union Station
// already collects system metrics.
details.removeMember("system_metrics");
// Don't include environment variables because they may
// contain sensitive information.
details.removeMember("envvars");
transaction->message("Details: " + stringifyJson(details));
} catch (const tracable_exception &e2) {
P_WARN("Cannot log to Union Station: " << e2.what() <<
"\n Backtrace:\n" << e2.backtrace());
}
}
UPDATE_TRACE_POINT();
stream << "Could not spawn process for application " << options.appRoot <<
": " << e.what() << "\n" <<
" Error ID: " << errorId << "\n";
if (filename[0] != '\0') {
stream << " Error details saved to: " << filename << "\n";
}
stream << " Message from application: " << appMessage << "\n";
P_ERROR(stream.str());
if (config->agentsOptions != NULL) {
HookScriptOptions hOptions;
hOptions.name = "spawn_failed";
hOptions.spec = config->agentsOptions->get("hook_spawn_failed", false);
hOptions.agentsOptions = config->agentsOptions;
hOptions.environment.push_back(make_pair("PASSENGER_APP_ROOT", options.appRoot));
hOptions.environment.push_back(make_pair("PASSENGER_APP_GROUP_NAME", options.getAppGroupName()));
hOptions.environment.push_back(make_pair("PASSENGER_ERROR_MESSAGE", e.what()));
hOptions.environment.push_back(make_pair("PASSENGER_ERROR_ID", errorId));
hOptions.environment.push_back(make_pair("PASSENGER_APP_ERROR_MESSAGE", appMessage));
oxt::thread(boost::bind(runHookScripts, hOptions),
"Hook: spawn_failed", 256 * 1024);
}
}
void
writeExact(int fd, const StaticString &data, unsigned long long *timeout) {
const char * restrict data_ptr = data.data();
writeExact(fd, data_ptr, data.size(), timeout);
}
//For each event in FIFO, build a message line for repeatereventlistener and send it
//This procedure accesses eventFifo, but there is no need for synchronization (with main repeater process)
//because we got our own copy of memory when kernel started us
static int doEventWork(void)
{
connectionEvent *connEv;
sessionEvent *sessEv;
startEndEvent *seEv;
int eventNum;
repeaterEvent ev;
int msgLen;
#ifdef EVENTS_USE_LISTENER
int connection;
char eventMessageToListener[MAX_EVENT_MSG_LEN];
char eventListenerIp[MAX_IP_LEN];
if (strcmp(eventHandlerType,"eventlistener") == 0) {
connection = openConnectionToEventListener(eventListenerHost, eventListenerPort, eventListenerIp, MAX_IP_LEN);
if (-1 != connection) {
while(!isFifoEmpty()) {
ev = eventFifo[fifoTailInd];
itemsInFifo--;
fifoTailInd = advanceFifoIndex(fifoTailInd);
eventNum = ev.eventNum;
switch (eventNum) {
case VIEWER_CONNECT:
case VIEWER_DISCONNECT:
case SERVER_CONNECT:
case SERVER_DISCONNECT:
connEv = (connectionEvent *) ev.extraInfo;
msgLen = snprintf(eventMessageToListener, MAX_EVENT_MSG_LEN,
"EvMsgVer:%d,EvNum:%d,Time:%ld,Pid:%d,TblInd:%d,Code:%ld,Mode:%d,Ip:%d.%d.%d.%d\n",
REP_EVENT_VERSION, eventNum, ev.timeStamp, ev.repeaterProcessId,
connEv -> tableIndex, connEv -> code, connEv -> connMode,
connEv->peerIp.a,connEv->peerIp.b,connEv->peerIp.c,connEv->peerIp.d);
break;
case VIEWER_SERVER_SESSION_START:
case VIEWER_SERVER_SESSION_END:
sessEv = (sessionEvent *) ev.extraInfo;
msgLen = snprintf(eventMessageToListener, MAX_EVENT_MSG_LEN,
"EvMsgVer:%d,EvNum:%d,Time:%ld,Pid:%d,SvrTblInd:%d,VwrTblInd:%d,"
"Code:%ld,Mode:%d,SvrIp:%d.%d.%d.%d,VwrIp:%d.%d.%d.%d\n",
REP_EVENT_VERSION, eventNum, ev.timeStamp, ev.repeaterProcessId,
sessEv -> serverTableIndex, sessEv -> viewerTableIndex, sessEv -> code, sessEv -> connMode,
sessEv->serverIp.a, sessEv->serverIp.b, sessEv->serverIp.c, sessEv->serverIp.d,
sessEv->viewerIp.a, sessEv->viewerIp.b, sessEv->viewerIp.c, sessEv->viewerIp.d);
break;
case REPEATER_STARTUP:
case REPEATER_SHUTDOWN:
case REPEATER_HEARTBEAT:
seEv = (startEndEvent *) ev.extraInfo;
msgLen = snprintf(eventMessageToListener, MAX_EVENT_MSG_LEN,
"EvMsgVer:%d,EvNum:%d,Time:%ld,Pid:%d,MaxSessions:%d\n",
REP_EVENT_VERSION, eventNum, ev.timeStamp, ev.repeaterProcessId, seEv -> maxSessions);
break;
default:
msgLen = 0;
strlcpy(eventMessageToListener, "\n", MAX_EVENT_MSG_LEN);
break;
}
if (msgLen > 0) {
writeExact(connection, eventMessageToListener,
strlen(eventMessageToListener), TIMEOUT_5SECS);
debug(LEVEL_3, "%s", eventMessageToListener);
}
}
close(connection);
}
}
#endif //EVENTS_USE_LISTENER
#ifdef EVENTS_USE_MYSQL
char eventMessageSQL[MAX_EVENT_SQL_LEN];
debug(LEVEL_3,"Event handler type: %s, length: %i, strcmp: %i\n",eventHandlerType,strlen(eventHandlerType),strcmp(eventHandlerType,"mysql"));
if (strcmp(eventHandlerType,"mysql") == 0) {
int status = 0;
MYSQL mysql;
mysql_init(&mysql);
if (!mysql_real_connect(&mysql,mysqlHost,mysqlUser,mysqlPass,mysqlDb,0,NULL,0))
{
debug(LEVEL_1, "Failed to connect to database (%s:%i, %s): Error: %s\n",mysqlHost,mysqlPort,mysqlDb,mysql_error(&mysql));
}
else
{
debug(LEVEL_3,"Connected to MySQL Database\n");
while(!isFifoEmpty()) {
ev = eventFifo[fifoTailInd];
itemsInFifo--;
fifoTailInd = advanceFifoIndex(fifoTailInd);
eventNum = ev.eventNum;
char cuid_str[36];
char puid_str[36];
char cuid_str2[36];
memset(cuid_str, 0, 36);
memset(puid_str, 0, 36);
memset(cuid_str2, 0, 36);
switch (eventNum) {
case VIEWER_CONNECT:
status = 1;
connEv = (connectionEvent *) ev.extraInfo;
uuid_unparse(connEv->uid,cuid_str);
uuid_unparse(process_uid,puid_str);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"INSERT INTO viewers (uid,repeater_uid,lasttime,status,table_index,code,mode,ip) VALUES ('%s','%s',%ld,%d,%d,%ld,%d,'%d.%d.%d.%d');\n",
cuid_str,puid_str, ev.timeStamp, status,connEv->tableIndex,connEv->code,connEv->connMode,
connEv->peerIp.a,connEv->peerIp.b,connEv->peerIp.c,connEv->peerIp.d);
break;
case VIEWER_DISCONNECT:
status = 0;
connEv = (connectionEvent *) ev.extraInfo;
uuid_unparse(connEv->uid,cuid_str);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"UPDATE viewers SET lasttime = %ld,status = %d WHERE uid = '%s';\n",
ev.timeStamp, status,cuid_str);
break;
case SERVER_CONNECT:
status = 1;
connEv = (connectionEvent *) ev.extraInfo;
uuid_unparse(connEv->uid,cuid_str);
uuid_unparse(process_uid,puid_str);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"INSERT INTO servers (uid,repeater_uid,lasttime,status,table_index,code,mode,ip) VALUES ('%s','%s',%ld,%d,%d,%ld,%d,'%d.%d.%d.%d');\n",
cuid_str,puid_str,ev.timeStamp, status,connEv->tableIndex,connEv->code,connEv->connMode,
connEv->peerIp.a,connEv->peerIp.b,connEv->peerIp.c,connEv->peerIp.d);
break;
case SERVER_DISCONNECT:
status = 0;
connEv = (connectionEvent *) ev.extraInfo;
uuid_unparse(connEv->uid,cuid_str);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"UPDATE servers SET lasttime = %ld,status = %d WHERE uid = '%s';\n",
ev.timeStamp, status,cuid_str);
break;
case VIEWER_SERVER_SESSION_START:
status = 1;
sessEv = (sessionEvent *) ev.extraInfo;
uuid_unparse(sessEv->server_uid,cuid_str);
uuid_unparse(sessEv->viewer_uid,cuid_str2);
uuid_unparse(process_uid,puid_str);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"INSERT INTO sessions (repeater_uid,status,lasttime,server_index,server_ip,server_uid,viewer_index,viewer_ip,viewer_uid,code,mode) "
"VALUES ('%s',%d,%ld,%d,'%d.%d.%d.%d','%s',%d,'%d.%d.%d.%d','%s',%ld,%d);\n",
puid_str,status,ev.timeStamp,
sessEv -> serverTableIndex,sessEv->serverIp.a, sessEv->serverIp.b, sessEv->serverIp.c, sessEv->serverIp.d,cuid_str,
sessEv -> viewerTableIndex,sessEv->viewerIp.a, sessEv->viewerIp.b, sessEv->viewerIp.c, sessEv->viewerIp.d,cuid_str2,
sessEv -> code, sessEv -> connMode);
break;
case VIEWER_SERVER_SESSION_END:
status = 0;
sessEv = (sessionEvent *) ev.extraInfo;
uuid_unparse(process_uid,puid_str);
uuid_unparse(sessEv->server_uid,cuid_str);
uuid_unparse(sessEv->viewer_uid,cuid_str2);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"UPDATE sessions SET status=%d,lasttime=%ld WHERE repeater_uid='%s' AND server_uid='%s' AND viewer_uid='%s';\n",
status,ev.timeStamp,puid_str,cuid_str,cuid_str2);
break;
case REPEATER_STARTUP:
status = 1;
seEv = (startEndEvent *) ev.extraInfo;
uuid_unparse(process_uid,puid_str);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"INSERT INTO repeaters (uid,process_id,lasttime,status,maxsessions,ip,server_port,viewer_port) VALUES ('%s',%d,%ld,%d,%d,'%s',%d,%d);\n",
puid_str,ev.repeaterProcessId,ev.timeStamp, status, seEv -> maxSessions, ownIpAddress,serverPort,viewerPort);
break;
case REPEATER_SHUTDOWN:
status = 0;
seEv = (startEndEvent *) ev.extraInfo;
uuid_unparse(process_uid,puid_str);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"UPDATE repeaters SET lasttime=%ld,status=%d WHERE uid='%s';\n",
ev.timeStamp, status, puid_str);
break;
case REPEATER_HEARTBEAT:
uuid_unparse(process_uid,puid_str);
msgLen = snprintf(eventMessageSQL, MAX_EVENT_SQL_LEN,
"UPDATE repeaters SET lasttime=%ld WHERE uid='%s';\n",
ev.timeStamp, puid_str);
break;
default:
msgLen = 0;
strlcpy(eventMessageSQL, "\n", MAX_EVENT_SQL_LEN);
break;
}
if (msgLen > 0) {
mysql_query(&mysql,eventMessageSQL);
if(mysql_errno(&mysql))
{
debug(LEVEL_1,"MYSQL ERROR: %s\n",mysql_error(&mysql));
}
debug(LEVEL_1, "%s", eventMessageSQL);
}
}
mysql_close(&mysql);
}
}
#endif //EVENTS_USE_MYSQL
return 0;
}
