// The 'self' parameter is for keeping the current Group object alive while this thread is running.
void
Group::finalizeRestart(GroupPtr self, Options options, RestartMethod method,
SpawnerFactoryPtr spawnerFactory, unsigned int restartsInitiated,
vector<Callback> postLockActions)
{
TRACE_POINT();
Pool::runAllActions(postLockActions);
postLockActions.clear();
this_thread::disable_interruption di;
this_thread::disable_syscall_interruption dsi;
// Create a new spawner.
SpawnerPtr newSpawner = spawnerFactory->create(options);
SpawnerPtr oldSpawner;
UPDATE_TRACE_POINT();
PoolPtr pool = getPool();
Pool::DebugSupportPtr debug = pool->debugSupport;
if (debug != NULL && debug->restarting) {
this_thread::restore_interruption ri(di);
this_thread::restore_syscall_interruption rsi(dsi);
this_thread::interruption_point();
debug->debugger->send("About to end restarting");
debug->messages->recv("Finish restarting");
}
ScopedLock l(pool->syncher);
if (!isAlive()) {
P_DEBUG("Group " << name << " is shutting down, so aborting restart");
return;
}
if (restartsInitiated != this->restartsInitiated) {
// Before this restart could be finalized, another restart command was given.
// The spawner we just created might be out of date now so we abort.
P_DEBUG("Restart of group " << name << " aborted because a new restart was initiated concurrently");
if (debug != NULL && debug->restarting) {
debug->debugger->send("Restarting aborted");
}
return;
}
// Run some sanity checks.
pool->fullVerifyInvariants();
assert(m_restarting);
UPDATE_TRACE_POINT();
// Atomically swap the new spawner with the old one.
resetOptions(options);
oldSpawner = spawner;
spawner = newSpawner;
m_restarting = false;
if (shouldSpawn()) {
spawn();
} else if (isWaitingForCapacity()) {
P_INFO("Group " << name << " is waiting for capacity to become available. "
"Trying to shutdown another idle process to free capacity...");
if (pool->forceFreeCapacity(this, postLockActions) != NULL) {
spawn();
} else {
P_INFO("There are no processes right now that are eligible "
"for shutdown. Will try again later.");
}
}
verifyInvariants();
l.unlock();
oldSpawner.reset();
Pool::runAllActions(postLockActions);
P_DEBUG("Restart of group " << name << " done");
if (debug != NULL && debug->restarting) {
debug->debugger->send("Restarting done");
}
}
static gid_t gidFor(const string &groupName) {
return getgrnam(groupName.c_str())->gr_gid;
}
static string primaryGroupFor(const string &userName) {
gid_t gid = getpwnam(userName.c_str())->pw_gid;
return getgrgid(gid)->gr_name;
}
TEST_METHOD(1) {
// Basic spawning test.
Options options = createOptions();
options.appRoot = "stub/rack";
options.startCommand = "ruby\t" "start.rb";
options.startupFile = "start.rb";
SpawnerPtr spawner = createSpawner(options);
process = spawner->spawn(options);
process->requiresShutdown = false;
ensure_equals(process->sockets->size(), 1u);
Connection conn = process->sockets->front().checkoutConnection();
ScopeGuard guard(boost::bind(checkin, process, &conn));
writeExact(conn.fd, "ping\n");
ensure_equals(readAll(conn.fd), "pong\n");
}
TEST_METHOD(2) {
// It enforces the given start timeout.
Options options = createOptions();
options.appRoot = "stub";
options.startCommand = "sleep\t" "60";
void
Group::spawnThreadRealMain(const SpawnerPtr &spawner, const Options &options, unsigned int restartsInitiated) {
TRACE_POINT();
this_thread::disable_interruption di;
this_thread::disable_syscall_interruption dsi;
PoolPtr pool = getPool();
Pool::DebugSupportPtr debug = pool->debugSupport;
bool done = false;
while (!done) {
bool shouldFail = false;
if (debug != NULL && debug->spawning) {
UPDATE_TRACE_POINT();
this_thread::restore_interruption ri(di);
this_thread::restore_syscall_interruption rsi(dsi);
this_thread::interruption_point();
string iteration;
{
LockGuard g(debug->syncher);
debug->spawnLoopIteration++;
iteration = toString(debug->spawnLoopIteration);
}
P_DEBUG("Begin spawn loop iteration " << iteration);
debug->debugger->send("Begin spawn loop iteration " +
iteration);
vector<string> cases;
cases.push_back("Proceed with spawn loop iteration " + iteration);
cases.push_back("Fail spawn loop iteration " + iteration);
MessagePtr message = debug->messages->recvAny(cases);
shouldFail = message->name == "Fail spawn loop iteration " + iteration;
}
ProcessPtr process;
ExceptionPtr exception;
try {
UPDATE_TRACE_POINT();
this_thread::restore_interruption ri(di);
this_thread::restore_syscall_interruption rsi(dsi);
if (shouldFail) {
throw SpawnException("Simulated failure");
} else {
process = spawner->spawn(options);
process->setGroup(shared_from_this());
}
} catch (const thread_interrupted &) {
break;
} catch (const tracable_exception &e) {
exception = copyException(e);
// Let other (unexpected) exceptions crash the program so
// gdb can generate a backtrace.
}
UPDATE_TRACE_POINT();
ScopeGuard guard(boost::bind(Process::forceTriggerShutdownAndCleanup, process));
boost::unique_lock<boost::mutex> lock(pool->syncher);
if (!isAlive()) {
if (process != NULL) {
P_DEBUG("Group is being shut down so dropping process " <<
process->inspect() << " which we just spawned and exiting spawn loop");
} else {
P_DEBUG("The group is being shut down. A process failed "
"to be spawned anyway, so ignoring this error and exiting "
"spawn loop");
}
// We stop immediately because any previously assumed invariants
// may have been violated.
break;
} else if (restartsInitiated != this->restartsInitiated) {
if (process != NULL) {
P_DEBUG("A restart was issued for the group, so dropping process " <<
process->inspect() << " which we just spawned and exiting spawn loop");
} else {
P_DEBUG("A restart was issued for the group. A process failed "
"to be spawned anyway, so ignoring this error and exiting "
"spawn loop");
}
// We stop immediately because any previously assumed invariants
// may have been violated.
break;
}
verifyInvariants();
assert(m_spawning);
assert(processesBeingSpawned > 0);
processesBeingSpawned--;
assert(processesBeingSpawned == 0);
UPDATE_TRACE_POINT();
vector<Callback> actions;
if (process != NULL) {
AttachResult result = attach(process, actions);
if (result == AR_OK) {
guard.clear();
if (getWaitlist.empty()) {
pool->assignSessionsToGetWaiters(actions);
} else {
assignSessionsToGetWaiters(actions);
}
P_DEBUG("New process count = " << enabledCount <<
", remaining get waiters = " << getWaitlist.size());
} else {
done = true;
P_DEBUG("Unable to attach spawned process " << process->inspect());
if (result == AR_ANOTHER_GROUP_IS_WAITING_FOR_CAPACITY) {
pool->possiblySpawnMoreProcessesForExistingGroups();
}
}
} else {
// TODO: sure this is the best thing? if there are
// processes currently alive we should just use them.
P_ERROR("Could not spawn process for group " << name <<
": " << exception->what() << "\n" <<
exception->backtrace());
if (enabledCount == 0) {
enableAllDisablingProcesses(actions);
}
Pool::assignExceptionToGetWaiters(getWaitlist, exception, actions);
pool->assignSessionsToGetWaiters(actions);
done = true;
}
done = done
|| (processLowerLimitsSatisfied() && getWaitlist.empty())
|| processUpperLimitsReached()
|| pool->atFullCapacity(false);
m_spawning = !done;
if (done) {
P_DEBUG("Spawn loop done");
} else {
processesBeingSpawned++;
P_DEBUG("Continue spawning");
}
UPDATE_TRACE_POINT();
pool->fullVerifyInvariants();
lock.unlock();
UPDATE_TRACE_POINT();
runAllActions(actions);
UPDATE_TRACE_POINT();
}
if (debug != NULL && debug->spawning) {
debug->debugger->send("Spawn loop done");
}
}
// Included in DirectSpawnerTest.cpp and SmartSpawnerTest.cpp.
typedef boost::shared_ptr<Spawner> SpawnerPtr;
TEST_METHOD(1) {
set_test_name("Basic spawning test");
SpawningKit::AppPoolOptions options = createOptions();
options.appRoot = "stub/rack";
options.startCommand = "ruby start.rb";
options.startupFile = "start.rb";
SpawnerPtr spawner = createSpawner(options);
result = spawner->spawn(options);
ensure_equals(result.sockets.size(), 1u);
FileDescriptor fd(connectToServer(result.sockets[0].address,
__FILE__, __LINE__), NULL, 0);
writeExact(fd, "ping\n");
ensure_equals(readAll(fd, 1024).first, "pong\n");
}
TEST_METHOD(2) {
set_test_name("It enforces the given start timeout");
SpawningKit::AppPoolOptions options = createOptions();
options.appRoot = "stub";
options.startCommand = "sleep 60";
options.startupFile = ".";
options.startTimeout = 100;
if (defaultLogLevel == (LoggingKit::Level) DEFAULT_LOG_LEVEL) {
// If the user did not customize the test's log level,
// then we'll want to tone down the noise.
