static int do_procs(SnapshotPtr &snap, char *args[])
{
list<ProcessPtr> procs;
list<ProcessPtr>::iterator it;
procs = snap->procs();
Sizes::scale_kbytes();
cout << "PID";
for (int i = 0; i < Sizes::NUM_SIZES; ++i) {
cout << "\t" << Sizes::size_name(i);
}
cout << "\t" << "CMD";
cout << "\n";
for (it = procs.begin(); it != procs.end(); ++it) {
ProcessPtr proc = *it;
SizesPtr sizes = proc->sizes();
cout << proc->pid();
for (int i = 0; i < Sizes::NUM_SIZES; ++i) {
cout << "\t" << sizes->sval(i);
}
cout << "\t" << proc->cmdline();
cout << "\n";
}
return 0;
}
DisableResult
Pool::disableProcess(const StaticString &gupid) {
ScopedLock l(syncher);
ProcessPtr process = findProcessByGupid(gupid, false);
if (process != NULL) {
Group *group = process->getGroup();
// Must be a boost::shared_ptr to be interruption-safe.
boost::shared_ptr<DisableWaitTicket> ticket = boost::make_shared<DisableWaitTicket>();
DisableResult result = group->disable(process,
boost::bind(syncDisableProcessCallback, _1, _2, ticket));
group->verifyInvariants();
group->verifyExpensiveInvariants();
if (result == DR_DEFERRED) {
l.unlock();
ScopedLock l2(ticket->syncher);
while (!ticket->done) {
ticket->cond.wait(l2);
}
return ticket->result;
} else {
return result;
}
} else {
return DR_NOOP;
}
}
ProcessPtr
Pool::findOldestIdleProcess(const Group *exclude) const {
ProcessPtr oldestIdleProcess;
GroupMap::ConstIterator g_it(groups);
while (*g_it != NULL) {
const GroupPtr &group = g_it.getValue();
if (group.get() == exclude) {
g_it.next();
continue;
}
const ProcessList &processes = group->enabledProcesses;
ProcessList::const_iterator p_it, p_end = processes.end();
for (p_it = processes.begin(); p_it != p_end; p_it++) {
const ProcessPtr process = *p_it;
if (process->busyness() == 0
&& (oldestIdleProcess == NULL
|| process->lastUsed < oldestIdleProcess->lastUsed)
) {
oldestIdleProcess = process;
}
}
g_it.next();
}
return oldestIdleProcess;
}
ProcessPtr DatabaseSubsystem::getProcess(int processID)
{
RWLock::ScopedLock lock(_dbLock);
ProcessPtr result;
Session session = getSession();
session << "SELECT * FROM process WHERE process_id = ?",
use(processID), into(result), now;
if (!result.isNull())
getProcessParams(session, result);
return result;
}
SizesPtr File::sizes()
{
if (_procs.empty()) {
warn << "File::sizes - no processes for file " << name() << "\n";
SizesPtr null_sizes;
return null_sizes;
}
// std::set doesn't have .front
ProcessPtr proc = *(_procs.begin());
// This goes over all procs (because of the _maps), the proc is
// we're only using the proc to get to the pagepool.
return Map::sum_sizes(proc->page_pool(), _maps);
}
SizesPtr File::sizes(const RangePtr &elf_range)
{
SizesPtr null_sizes; // failure return
stringstream pref;
pref << "File::sizes " << name() << ": ";
if (_procs.empty()) {
warn << pref.str() << "no processes for file\n";
return null_sizes;
}
SizesPtr totals(new Sizes);
SizesPtr sizes;
set<ProcessPtr>::iterator proc_it;
list<MapPtr>::iterator map_it;
list<MapPtr> maps_for_proc;
RangePtr subrange, map_elf_range;
ProcessPtr firstproc = *(_procs.begin()); // set has no ->front()
PagePoolPtr page_pool = firstproc->page_pool();
// We need to loop through the procs, because the mapping from
// ELF virtual address to actual address can be different in each
for (proc_it = _procs.begin(); proc_it != _procs.end(); ++proc_it) {
maps_for_proc = Map::intersect_lists((*proc_it)->maps(),
_maps);
if (maps_for_proc.empty()) {
warn << pref.str() << "no maps for process "
<< (*proc_it)->pid() << "\n";
return null_sizes;
}
for (map_it = maps_for_proc.begin();
map_it != maps_for_proc.end();
++map_it) {
map_elf_range = (*map_it)->elf_range();
if (!map_elf_range) continue;
subrange = map_elf_range->intersect(elf_range);
if (!subrange) continue;
RangePtr mem_range = (*map_it)->elf_to_mem_range(subrange);
sizes = (*map_it)->sizes_for_mem_range(page_pool,
mem_range);
if (sizes) {
totals->add(sizes);
}
}
}
return totals;
}
bool
Group::shouldInitiateOobw(const ProcessPtr &process) const {
return process->oobwStatus == Process::OOBW_REQUESTED
&& process->enabled != Process::DETACHED
&& process->isAlive()
&& oobwAllowed();
}
bool ExeExcuter::run(
const OJString & exeFile,
const OJString & inputFile,
const OJString & outputFile,
OJInt32_t limitTime,
OJInt32_t limitMemory
)
{
ProcessPtr wp = ProcessFactory::create(ProcessType::WithUser, inputFile, outputFile);
wp->create(exeFile, limitTime, limitMemory);
result_ = wp->getExitCodeEx();
runTime_ = wp->getRunTime();
runMemory_ = wp->getRunMemory();
return isAccept();
}
ProcessPtr DatabaseSubsystem::getProcess(ClassificationObjectPtr clo)
{
RWLock::ScopedLock lock(_dbLock);
ProcessPtr result;
Session session = getSession();
session << "SELECT * FROM process INNER JOIN data_descriptor ON"
" data_descriptor.process_id = process.process_id WHERE"
" data_descriptor.descr_id IN "
" (SELECT descr_id FROM classification_object_data WHERE"
" object_id = ?)"
"LIMIT 1",
use(clo->objectID), into(result), now;
if (!result.isNull())
getProcessParams(session, result);
return result;
}
/**
* Calls Group::detach() so be sure to fix up the invariants afterwards.
* See the comments for Group::detach() and the code for detachProcessUnlocked().
*/
ProcessPtr
Pool::forceFreeCapacity(const Group *exclude,
boost::container::vector<Callback> &postLockActions)
{
ProcessPtr process = findOldestIdleProcess(exclude);
if (process != NULL) {
P_DEBUG("Forcefully detaching process " << process->inspect() <<
" in order to free capacity in the pool");
Group *group = process->getGroup();
assert(group != NULL);
assert(group->getWaitlist.empty());
group->detach(process, postLockActions);
}
return process;
}
// ProcessManager::tick
// - run through the list of processes and update them
//
void ProcessManager::tick(uint64_t deltaMilliseconds)
{
ProcessPtr next;
ProcessList::iterator i = m_processList.begin();
for (; i != m_processList.end(); i++)
{
ProcessPtr p = ProcessPtr(*i);
if ( p->isDead() )
{
// Check for a child process and add if exists
next = p->getNext();
if ( next )
{
p->setNext(ProcessPtr((Process *)NULL));
attach( next );
}
detach( p );
}
else if ( p->isActive() && !p->isPaused() )
{
p->update(deltaMilliseconds);
}
}
}
void
Group::requestOOBW(const ProcessPtr &process) {
// Standard resource management boilerplate stuff...
PoolPtr pool = getPool();
boost::unique_lock<boost::mutex> lock(pool->syncher);
if (isAlive() && process->isAlive() && process->oobwStatus == Process::OOBW_NOT_ACTIVE) {
process->oobwStatus = Process::OOBW_REQUESTED;
}
}
void
Group::initiateOobw(const ProcessPtr &process) {
assert(process->oobwStatus == Process::OOBW_REQUESTED);
process->oobwStatus = Process::OOBW_IN_PROGRESS;
if (process->enabled == Process::ENABLED
|| process->enabled == Process::DISABLING)
{
// We want the process to be disabled. However, disabling a process is potentially
// asynchronous, so we pass a callback which will re-aquire the lock and call this
// method again.
P_DEBUG("Disabling process " << process->inspect() << " in preparation for OOBW");
DisableResult result = disable(process,
boost::bind(&Group::lockAndMaybeInitiateOobw, this,
_1, _2, shared_from_this()));
switch (result) {
case DR_SUCCESS:
// Continue code flow.
break;
case DR_DEFERRED:
// lockAndMaybeInitiateOobw() will eventually be called.
return;
case DR_ERROR:
case DR_NOOP:
P_DEBUG("Out-of-band work for process " << process->inspect() << " aborted "
"because the process could not be disabled");
process->oobwStatus = Process::OOBW_NOT_ACTIVE;
return;
default:
P_BUG("Unexpected disable() result " << result);
}
}
assert(process->enabled == Process::DISABLED);
assert(process->sessions == 0);
P_DEBUG("Initiating OOBW request for process " << process->inspect());
interruptableThreads.create_thread(
boost::bind(&Group::spawnThreadOOBWRequest, this, shared_from_this(), process),
"OOBW request thread for process " + process->inspect(),
POOL_HELPER_THREAD_STACK_SIZE);
}
void JavaExcuter::run(
const OJString & exeFile,
const OJString & inputFile,
const OJString & outputFile,
OJInt32_t limitTime,
OJInt32_t limitMemory
)
{
OJString exePath = FileTool::GetFilePath(exeFile);
OJString exeFileName = FileTool::GetFileName(exeFile);//get only name
OJString cmdBuffer;
FormatString(cmdBuffer, OJStr("java -cp %s %s"), exePath.c_str(), exeFileName.c_str());
ProcessPtr wp = ProcessFactory::create(ProcessType::Excuter, inputFile, outputFile);
wp->create(cmdBuffer, limitTime*30, limitMemory*10);
result_ = wp->getResult();
runTime_ = wp->getRunTime();
runMemory_ = wp->getRunMemory();
}
void
Group::onSessionInitiateFailure(const ProcessPtr &process, Session *session) {
vector<Callback> actions;
TRACE_POINT();
// Standard resource management boilerplate stuff...
PoolPtr pool = getPool();
boost::unique_lock<boost::mutex> lock(pool->syncher);
assert(process->isAlive());
assert(isAlive() || getLifeStatus() == SHUTTING_DOWN);
UPDATE_TRACE_POINT();
P_DEBUG("Could not initiate a session with process " <<
process->inspect() << ", detaching from pool if possible");
if (!pool->detachProcessUnlocked(process, actions)) {
P_DEBUG("Process was already detached");
}
pool->fullVerifyInvariants();
lock.unlock();
runAllActions(actions);
}
// The 'self' parameter is for keeping the current Group object alive
void
Group::lockAndMaybeInitiateOobw(const ProcessPtr &process, DisableResult result, GroupPtr self) {
TRACE_POINT();
// Standard resource management boilerplate stuff...
PoolPtr pool = getPool();
boost::unique_lock<boost::mutex> lock(pool->syncher);
if (OXT_UNLIKELY(!process->isAlive() || !isAlive())) {
return;
}
assert(process->oobwStatus == Process::OOBW_IN_PROGRESS);
if (result == DR_SUCCESS) {
if (process->enabled == Process::DISABLED) {
P_DEBUG("Process " << process->inspect() << " disabled; proceeding " <<
"with out-of-band work");
process->oobwStatus = Process::OOBW_REQUESTED;
if (shouldInitiateOobw(process)) {
initiateOobw(process);
} else {
// We do not re-enable the process because it's likely that the
// administrator has explicitly changed the state.
P_DEBUG("Out-of-band work for process " << process->inspect() << " aborted "
"because the process no longer requests out-of-band work");
process->oobwStatus = Process::OOBW_NOT_ACTIVE;
}
} else {
// We do not re-enable the process because it's likely that the
// administrator has explicitly changed the state.
P_DEBUG("Out-of-band work for process " << process->inspect() << " aborted "
"because the process was reenabled after disabling");
process->oobwStatus = Process::OOBW_NOT_ACTIVE;
}
} else {
P_DEBUG("Out-of-band work for process " << process->inspect() << " aborted "
"because the process could not be disabled");
process->oobwStatus = Process::OOBW_NOT_ACTIVE;
}
}
bool
Pool::detachProcessUnlocked(const ProcessPtr &process,
boost::container::vector<Callback> &postLockActions)
{
if (OXT_LIKELY(process->isAlive())) {
verifyInvariants();
Group *group = process->getGroup();
group->detach(process, postLockActions);
// 'process' may now be a stale pointer so don't use it anymore.
assignSessionsToGetWaiters(postLockActions);
possiblySpawnMoreProcessesForExistingGroups();
group->verifyInvariants();
verifyInvariants();
verifyExpensiveInvariants();
return true;
} else {
return false;
}
}
void FTTask::setProcessParameters(ProcessPtr process) const
{
process->setWindowFunction(_windowFunction);
process->setOverlap(_overlap);
process->setWindowSize(_windowSize);
process->parameters["transformCount"] =
Poco::NumberFormatter::format(_transforms.size());
int trIndex = 1;
for (vector<MatrixTransform*>::const_iterator it = _transforms.begin();
it != _transforms.end(); ++it, ++trIndex)
{
string paramName = "transform" + Poco::NumberFormatter::format(trIndex);
process->parameters[paramName] = (*it)->name();
MatrixTransform::TransformParameters tp = (*it)->getParameters();
for (MatrixTransform::TransformParameters::const_iterator
pit = tp.begin(); pit != tp.end(); ++pit)
{
string tparamName = paramName + pit->first;
process->parameters[tparamName] = pit->second;
}
}
}
bool
Pool::detachProcess(const string &gupid, const AuthenticationOptions &options) {
ScopedLock l(syncher);
ProcessPtr process = findProcessByGupid(gupid, false);
if (process != NULL) {
const Group *group = process->getGroup();
if (group->authorizeByUid(options.uid)
|| group->authorizeByApiKey(options.apiKey))
{
boost::container::vector<Callback> actions;
bool result = detachProcessUnlocked(process, actions);
fullVerifyInvariants();
l.unlock();
runAllActions(actions);
return result;
} else {
throw SecurityException("Operation unauthorized");
}
} else {
return false;
}
}
// The 'self' parameter is for keeping the current Group object alive while this thread is running.
void
Group::spawnThreadOOBWRequest(GroupPtr self, ProcessPtr process) {
TRACE_POINT();
this_thread::disable_interruption di;
this_thread::disable_syscall_interruption dsi;
Socket *socket;
Connection connection;
PoolPtr pool = getPool();
Pool::DebugSupportPtr debug = pool->debugSupport;
UPDATE_TRACE_POINT();
P_DEBUG("Performing OOBW request for process " << process->inspect());
if (debug != NULL && debug->oobw) {
debug->debugger->send("OOBW request about to start");
debug->messages->recv("Proceed with OOBW request");
}
UPDATE_TRACE_POINT();
{
// Standard resource management boilerplate stuff...
boost::unique_lock<boost::mutex> lock(pool->syncher);
if (OXT_UNLIKELY(!process->isAlive()
|| process->enabled == Process::DETACHED
|| !isAlive()))
{
return;
}
if (process->enabled != Process::DISABLED) {
UPDATE_TRACE_POINT();
P_INFO("Out-of-Band Work canceled: process " << process->inspect() <<
" was concurrently re-enabled.");
if (debug != NULL && debug->oobw) {
debug->debugger->send("OOBW request canceled");
}
return;
}
assert(process->oobwStatus == Process::OOBW_IN_PROGRESS);
assert(process->sessions == 0);
socket = process->sessionSockets.top();
assert(socket != NULL);
}
UPDATE_TRACE_POINT();
unsigned long long timeout = 1000 * 1000 * 60; // 1 min
try {
this_thread::restore_interruption ri(di);
this_thread::restore_syscall_interruption rsi(dsi);
// Grab a connection. The connection is marked as fail in order to
// ensure it is closed / recycled after this request (otherwise we'd
// need to completely read the response).
connection = socket->checkoutConnection();
connection.fail = true;
ScopeGuard guard(boost::bind(&Socket::checkinConnection, socket, connection));
// This is copied from RequestHandler when it is sending data using the
// "session" protocol.
char sizeField[sizeof(uint32_t)];
SmallVector<StaticString, 10> data;
data.push_back(StaticString(sizeField, sizeof(uint32_t)));
data.push_back(makeStaticStringWithNull("REQUEST_METHOD"));
data.push_back(makeStaticStringWithNull("OOBW"));
data.push_back(makeStaticStringWithNull("PASSENGER_CONNECT_PASSWORD"));
data.push_back(makeStaticStringWithNull(process->connectPassword));
uint32_t dataSize = 0;
for (unsigned int i = 1; i < data.size(); i++) {
dataSize += (uint32_t) data[i].size();
}
Uint32Message::generate(sizeField, dataSize);
gatheredWrite(connection.fd, &data[0], data.size(), &timeout);
// We do not care what the actual response is ... just wait for it.
UPDATE_TRACE_POINT();
waitUntilReadable(connection.fd, &timeout);
} catch (const SystemException &e) {
P_ERROR("*** ERROR: " << e.what() << "\n" << e.backtrace());
} catch (const TimeoutException &e) {
P_ERROR("*** ERROR: " << e.what() << "\n" << e.backtrace());
}
UPDATE_TRACE_POINT();
vector<Callback> actions;
{
// Standard resource management boilerplate stuff...
PoolPtr pool = getPool();
boost::unique_lock<boost::mutex> lock(pool->syncher);
if (OXT_UNLIKELY(!process->isAlive() || !isAlive())) {
return;
}
process->oobwStatus = Process::OOBW_NOT_ACTIVE;
if (process->enabled == Process::DISABLED) {
enable(process, actions);
assignSessionsToGetWaiters(actions);
}
pool->fullVerifyInvariants();
initiateNextOobwRequest();
}
UPDATE_TRACE_POINT();
runAllActions(actions);
actions.clear();
UPDATE_TRACE_POINT();
P_DEBUG("Finished OOBW request for process " << process->inspect());
if (debug != NULL && debug->oobw) {
debug->debugger->send("OOBW request finished");
}
}
void
Group::spawnThreadRealMain(const SpawnerPtr &spawner, const Options &options, unsigned int restartsInitiated) {
TRACE_POINT();
this_thread::disable_interruption di;
this_thread::disable_syscall_interruption dsi;
PoolPtr pool = getPool();
Pool::DebugSupportPtr debug = pool->debugSupport;
bool done = false;
while (!done) {
bool shouldFail = false;
if (debug != NULL && debug->spawning) {
UPDATE_TRACE_POINT();
this_thread::restore_interruption ri(di);
this_thread::restore_syscall_interruption rsi(dsi);
this_thread::interruption_point();
string iteration;
{
LockGuard g(debug->syncher);
debug->spawnLoopIteration++;
iteration = toString(debug->spawnLoopIteration);
}
P_DEBUG("Begin spawn loop iteration " << iteration);
debug->debugger->send("Begin spawn loop iteration " +
iteration);
vector<string> cases;
cases.push_back("Proceed with spawn loop iteration " + iteration);
cases.push_back("Fail spawn loop iteration " + iteration);
MessagePtr message = debug->messages->recvAny(cases);
shouldFail = message->name == "Fail spawn loop iteration " + iteration;
}
ProcessPtr process;
ExceptionPtr exception;
try {
UPDATE_TRACE_POINT();
this_thread::restore_interruption ri(di);
this_thread::restore_syscall_interruption rsi(dsi);
if (shouldFail) {
throw SpawnException("Simulated failure");
} else {
process = spawner->spawn(options);
process->setGroup(shared_from_this());
}
} catch (const thread_interrupted &) {
break;
} catch (const tracable_exception &e) {
exception = copyException(e);
// Let other (unexpected) exceptions crash the program so
// gdb can generate a backtrace.
}
UPDATE_TRACE_POINT();
ScopeGuard guard(boost::bind(Process::forceTriggerShutdownAndCleanup, process));
boost::unique_lock<boost::mutex> lock(pool->syncher);
if (!isAlive()) {
if (process != NULL) {
P_DEBUG("Group is being shut down so dropping process " <<
process->inspect() << " which we just spawned and exiting spawn loop");
} else {
P_DEBUG("The group is being shut down. A process failed "
"to be spawned anyway, so ignoring this error and exiting "
"spawn loop");
}
// We stop immediately because any previously assumed invariants
// may have been violated.
break;
} else if (restartsInitiated != this->restartsInitiated) {
if (process != NULL) {
P_DEBUG("A restart was issued for the group, so dropping process " <<
process->inspect() << " which we just spawned and exiting spawn loop");
} else {
P_DEBUG("A restart was issued for the group. A process failed "
"to be spawned anyway, so ignoring this error and exiting "
"spawn loop");
}
// We stop immediately because any previously assumed invariants
// may have been violated.
break;
}
verifyInvariants();
assert(m_spawning);
assert(processesBeingSpawned > 0);
processesBeingSpawned--;
assert(processesBeingSpawned == 0);
UPDATE_TRACE_POINT();
vector<Callback> actions;
if (process != NULL) {
AttachResult result = attach(process, actions);
if (result == AR_OK) {
guard.clear();
if (getWaitlist.empty()) {
pool->assignSessionsToGetWaiters(actions);
} else {
assignSessionsToGetWaiters(actions);
}
P_DEBUG("New process count = " << enabledCount <<
", remaining get waiters = " << getWaitlist.size());
} else {
done = true;
P_DEBUG("Unable to attach spawned process " << process->inspect());
if (result == AR_ANOTHER_GROUP_IS_WAITING_FOR_CAPACITY) {
pool->possiblySpawnMoreProcessesForExistingGroups();
}
}
} else {
// TODO: sure this is the best thing? if there are
// processes currently alive we should just use them.
P_ERROR("Could not spawn process for group " << name <<
": " << exception->what() << "\n" <<
exception->backtrace());
if (enabledCount == 0) {
enableAllDisablingProcesses(actions);
}
Pool::assignExceptionToGetWaiters(getWaitlist, exception, actions);
pool->assignSessionsToGetWaiters(actions);
done = true;
}
done = done
|| (processLowerLimitsSatisfied() && getWaitlist.empty())
|| processUpperLimitsReached()
|| pool->atFullCapacity(false);
m_spawning = !done;
if (done) {
P_DEBUG("Spawn loop done");
} else {
processesBeingSpawned++;
P_DEBUG("Continue spawning");
}
UPDATE_TRACE_POINT();
pool->fullVerifyInvariants();
lock.unlock();
UPDATE_TRACE_POINT();
runAllActions(actions);
UPDATE_TRACE_POINT();
}
if (debug != NULL && debug->spawning) {
debug->debugger->send("Spawn loop done");
}
}
// 'lockNow == false' may only be used during unit tests. Normally we
// should never call the callback while holding the lock.
void
Pool::asyncGet(const Options &options, const GetCallback &callback, bool lockNow, UnionStation::StopwatchLog **stopwatchLog) {
DynamicScopedLock lock(syncher, lockNow);
assert(lifeStatus == ALIVE || lifeStatus == PREPARED_FOR_SHUTDOWN);
verifyInvariants();
P_TRACE(2, "asyncGet(appGroupName=" << options.getAppGroupName() << ")");
boost::container::vector<Callback> actions;
Group *existingGroup = findMatchingGroup(options);
if (stopwatchLog != NULL) {
// Log some essentials stats about what this request is facing in its upcoming journey through the queue:
// 1) position in the queue upon entry, and 2) whether spawning activity is occurring (which takes cycles
// but also indicates the server has headroom to handle the load).
Json::Value data;
if (!existingGroup) {
data["message"] = "spawning.."; // the first of this group, so keep it simple (also: we don't know maxQ yet)
} else {
char queueMaxStr[10];
int queueMax = existingGroup->options.maxRequestQueueSize;
if (queueMax > 0) {
snprintf(queueMaxStr, 10, "%d", queueMax);
}
char message[50];
snprintf(message, 100, "queue: %zu / %s, spawning: %s", existingGroup->getWaitlist.size(),
(queueMax == 0 ? "inf" : queueMaxStr),
(existingGroup->processesBeingSpawned == 0 ? "no" : "yes"));
data["message"] = message;
}
Json::Value json;
json["data"] = data;
json["data_type"] = "generic";
json["name"] = "Await available process";
*stopwatchLog = new UnionStation::StopwatchLog(options.transaction, "Pool::asyncGet", stringifyJson(json).c_str());
}
if (OXT_LIKELY(existingGroup != NULL)) {
/* Best case: the app group is already in the pool. Let's use it. */
P_TRACE(2, "Found existing Group");
existingGroup->verifyInvariants();
SessionPtr session = existingGroup->get(options, callback, actions);
existingGroup->verifyInvariants();
verifyInvariants();
P_TRACE(2, "asyncGet() finished");
if (lockNow) {
lock.unlock();
}
if (session != NULL) {
callback(session, ExceptionPtr());
}
} else if (!atFullCapacityUnlocked()) {
/* The app super group isn't in the pool and we have enough free
* resources to make a new one.
*/
P_DEBUG("Spawning new Group");
GroupPtr group = createGroupAndAsyncGetFromIt(options,
callback, actions);
group->verifyInvariants();
verifyInvariants();
P_DEBUG("asyncGet() finished");
} else {
/* Uh oh, the app super group isn't in the pool but we don't
* have the resources to make a new one. The sysadmin should
* configure the system to let something like this happen
* as least as possible, but let's try to handle it as well
* as we can.
*/
ProcessPtr freedProcess = forceFreeCapacity(NULL, actions);
if (freedProcess == NULL) {
/* No process is eligible for killing. This could happen if, for example,
* all (super)groups are currently initializing/restarting/spawning/etc.
* We have no choice but to satisfy this get() action later when resources
* become available.
*/
P_DEBUG("Could not free a process; putting request to top-level getWaitlist");
getWaitlist.push_back(GetWaiter(
options.copyAndPersist().detachFromUnionStationTransaction(),
callback));
} else {
/* Now that a process has been trashed we can create
* the missing Group.
*/
P_DEBUG("Creating new Group");
GroupPtr group = createGroup(options);
SessionPtr session = group->get(options, callback,
actions);
/* The Group is now spawning a process so the callback
* should now have been put on the wait list,
* unless something has changed and we forgot to update
* some code here or if options.noop...
*/
if (session != NULL) {
assert(options.noop);
actions.push_back(boost::bind(GetCallback::call,
callback, session, ExceptionPtr()));
}
freedProcess->getGroup()->verifyInvariants();
group->verifyInvariants();
}
assert(atFullCapacityUnlocked());
verifyInvariants();
verifyExpensiveInvariants();
P_TRACE(2, "asyncGet() finished");
}
if (!actions.empty()) {
if (lockNow) {
if (lock.owns_lock()) {
lock.unlock();
}
runAllActions(actions);
} else {
// This state is not allowed. If we reach
// here then it probably indicates a bug in
// the test suite.
abort();
}
}
}
// CProcessManager::Detach
// - Detach a process from the process list, but don't delete it
//
void ProcessManager::detach(ProcessPtr process)
{
m_processList.remove(process);
process->setAttached(false);
}
// 'lockNow == false' may only be used during unit tests. Normally we
// should never call the callback while holding the lock.
void
Pool::asyncGet(const Options &options, const GetCallback &callback, bool lockNow) {
DynamicScopedLock lock(syncher, lockNow);
assert(lifeStatus == ALIVE || lifeStatus == PREPARED_FOR_SHUTDOWN);
verifyInvariants();
P_TRACE(2, "asyncGet(appGroupName=" << options.getAppGroupName() << ")");
boost::container::vector<Callback> actions;
Group *existingGroup = findMatchingGroup(options);
if (OXT_LIKELY(existingGroup != NULL)) {
/* Best case: the app group is already in the pool. Let's use it. */
P_TRACE(2, "Found existing Group");
existingGroup->verifyInvariants();
SessionPtr session = existingGroup->get(options, callback, actions);
existingGroup->verifyInvariants();
verifyInvariants();
P_TRACE(2, "asyncGet() finished");
if (lockNow) {
lock.unlock();
}
if (session != NULL) {
callback(session, ExceptionPtr());
}
} else if (!atFullCapacityUnlocked()) {
/* The app super group isn't in the pool and we have enough free
* resources to make a new one.
*/
P_DEBUG("Spawning new Group");
GroupPtr group = createGroupAndAsyncGetFromIt(options,
callback, actions);
group->verifyInvariants();
verifyInvariants();
P_DEBUG("asyncGet() finished");
} else {
/* Uh oh, the app super group isn't in the pool but we don't
* have the resources to make a new one. The sysadmin should
* configure the system to let something like this happen
* as least as possible, but let's try to handle it as well
* as we can.
*/
ProcessPtr freedProcess = forceFreeCapacity(NULL, actions);
if (freedProcess == NULL) {
/* No process is eligible for killing. This could happen if, for example,
* all (super)groups are currently initializing/restarting/spawning/etc.
* We have no choice but to satisfy this get() action later when resources
* become available.
*/
P_DEBUG("Could not free a process; putting request to top-level getWaitlist");
getWaitlist.push_back(GetWaiter(
options.copyAndPersist(),
callback));
} else {
/* Now that a process has been trashed we can create
* the missing Group.
*/
P_DEBUG("Creating new Group");
GroupPtr group = createGroup(options);
SessionPtr session = group->get(options, callback,
actions);
/* The Group is now spawning a process so the callback
* should now have been put on the wait list,
* unless something has changed and we forgot to update
* some code here or if options.noop...
*/
if (session != NULL) {
assert(options.noop);
actions.push_back(boost::bind(GetCallback::call,
callback, session, ExceptionPtr()));
}
freedProcess->getGroup()->verifyInvariants();
group->verifyInvariants();
}
assert(atFullCapacityUnlocked());
verifyInvariants();
verifyExpensiveInvariants();
P_TRACE(2, "asyncGet() finished");
}
if (!actions.empty()) {
if (lockNow) {
if (lock.owns_lock()) {
lock.unlock();
}
runAllActions(actions);
} else {
// This state is not allowed. If we reach
// here then it probably indicates a bug in
// the test suite.
abort();
}
}
}
bool ExmapTest::run()
{
SysInfoPtr sysinfo(new LinuxSysInfo);
Snapshot snap(sysinfo);
is(snap.num_procs(), 0, "zero procs before load");
ok(snap.load(), "can load snapshot");
ok(snap.num_procs() > 0, "some procs after load");
list<ProcessPtr> allprocs = snap.procs();
ok(!allprocs.empty(), "can get a list of procs");
list<ProcessPtr>::iterator proc_it;
list<ProcessPtr> procs;
bool failed_to_get_sizes = false;
SizesPtr sizes;
for (proc_it = allprocs.begin(); proc_it != allprocs.end(); ++proc_it) {
string cmdline = (*proc_it)->cmdline();
if (cmdline== SA_EXE) {
procs.push_back(*proc_it);
}
sizes = (*proc_it)->sizes();
if (!sizes) {
failed_to_get_sizes = true;
}
}
ok(!failed_to_get_sizes, "can get sizes for every proc");
is((int) procs.size(), NUM_INSTANCES, "can find all our sharedarray procs");
ProcessPtr proc = procs.front();
sizes = proc->sizes();
ok(sizes->val(Sizes::VM) > NUM_ARRAYS * ARRAY_SIZE, "image is big enough");
double ps_size = get_pid_size_from_ps(proc->pid());
is_approx_rel(sizes->val(Sizes::VM),
ps_size,
0.001,
"exmap info matches ps output");
ok(sizes->val(Sizes::RESIDENT) > 0, "nonzero resident size");
ok(sizes->val(Sizes::EFFECTIVE_RESIDENT) > 0, "nonzero eff resident size");
ok(sizes->val(Sizes::EFFECTIVE_RESIDENT) < sizes->val(Sizes::RESIDENT),
"effective is smaller than eff resident");
ok(sizes->val(Sizes::MAPPED) > 0, "nonzero mapped size");
ok(sizes->val(Sizes::EFFECTIVE_MAPPED) > 0, "nonzero eff mapped size");
ok(sizes->val(Sizes::EFFECTIVE_MAPPED) < sizes->val(Sizes::MAPPED),
"effective is smaller than eff mapped");
list<FilePtr> files;
list<FilePtr>::iterator file_it;
list<FilePtr> all_files = snap.files();
ok(all_files.size() > 0, "can find some files");
Regexp re;
re.compile(SA_LIB + "$");
failed_to_get_sizes = false;
for (file_it = all_files.begin(); file_it != all_files.end(); ++file_it) {
string name = (*file_it)->name();
if (re.matches(name)) {
files.push_back(*file_it);
}
sizes = (*file_it)->sizes();
if (!sizes) {
failed_to_get_sizes = true;
}
}
ok(!failed_to_get_sizes, "can get sizes for every file");
is((int) files.size(), 1, "shared lib only listed once");
FilePtr shlib_file = files.front();
ok(shlib_file->is_elf(), "shared lib recognised as elf file");
list<ProcessPtr> procs_per_file = shlib_file->procs();
is((int) procs_per_file.size(), NUM_INSTANCES,
"right number of procs mapping the file");
for (proc_it = procs_per_file.begin();
proc_it != procs_per_file.end();
++proc_it) {
ok(proc->cmdline() == SA_EXE, "each proc has correct cmdline");
}
for (proc_it = procs.begin();
proc_it != procs.end();
++proc_it) {
sizes = proc->sizes(shlib_file);
long arrays_size = NUM_ARRAYS * ARRAY_SIZE;
float delta = std::abs(arrays_size - (long) sizes->val(Sizes::VM));
delta /= arrays_size;
ok(delta < 0.01, "Shared lib has correct size in each proc");
}
Elf::SectionPtr text = shlib_file->elf()->section(".text");
ok(text, "can find text section");
ok(text->size() > 0, "text section has nonzero size");
sizes = procs.front()->sizes(shlib_file, text->mem_range());
/// This appears to be compiler and system dependent...
/*
is_approx_rel(sizes->val(Sizes::RESIDENT),
2.0 * text->size(),
0.001,
"lib text is resident and mapped twice (!)");
*/
Elf::SectionPtr bss = shlib_file->elf()->section(".bss");
ok(bss, "can find bss section");
ok(bss->size() > 0, "bss section has nonzero size");
SizesPtr bss_sizes = procs.front()->sizes(shlib_file, bss->mem_range());
ok(bss_sizes, "can get sizes for bss section");
Elf::SectionPtr data = shlib_file->elf()->section(".data");
ok(data, "can find data section");
ok(data->size() > 0, "data section has nonzero size");
SizesPtr data_sizes = procs.front()->sizes(shlib_file, data->mem_range());
ok(data_sizes, "can get sizes for data section");
is(data->size(), bss->size(), "data and bss sections have same size");
is_approx(data_sizes->val(Sizes::MAPPED),
bss_sizes->val(Sizes::MAPPED),
Elf::page_size(),
"data and bss mapped within page of each other");
is_approx(data_sizes->val(Sizes::RESIDENT),
bss_sizes->val(Sizes::RESIDENT),
Elf::page_size(),
"data and bss resident within page of each other");
// Now get the all-proc sizes from the file
sizes = shlib_file->sizes(bss->mem_range());
// The totals should be a multiple of the individual
is(sizes->val(Sizes::RESIDENT),
bss_sizes->val(Sizes::RESIDENT) * NUM_INSTANCES,
"Total resident for all procs the same as multiplying up one proc");
is(sizes->val(Sizes::MAPPED),
bss_sizes->val(Sizes::MAPPED) * NUM_INSTANCES,
"Total mapped for all procs the same as multiplying up one proc");
double bss_resident_arrays_size = 0;
double data_resident_arrays_size = 0;
double bss_writable_arrays_size = 0;
double data_writable_arrays_size = 0;
for (int i = 0; i < NUM_ARRAYS; ++i) {
if (ARRAY_INFO[i].initialised) {
data_resident_arrays_size
+= ARRAY_SIZE * ARRAY_INFO[i].resident_percent / 100;
data_writable_arrays_size
+= ARRAY_SIZE * ARRAY_INFO[i].writable_percent / 100;
}
else {
bss_resident_arrays_size
+= ARRAY_SIZE * ARRAY_INFO[i].resident_percent / 100;
bss_writable_arrays_size
+= ARRAY_SIZE * ARRAY_INFO[i].writable_percent / 100;
}
}
for (proc_it = procs.begin();
proc_it != procs.end();
++proc_it) {
sizes = proc->sizes(shlib_file, data->mem_range());
is_approx(sizes->val(Sizes::RESIDENT),
data_resident_arrays_size,
Elf::page_size(),
"resident size for data in proc correct with a page");
is_approx(sizes->val(Sizes::WRITABLE),
data_writable_arrays_size,
Elf::page_size(),
"writable size for data in proc correct with a page");
sizes = proc->sizes(shlib_file, bss->mem_range());
is_approx_rel(sizes->val(Sizes::RESIDENT),
bss_resident_arrays_size,
0.001,
"correct resident size for bss in proc");
is_approx_rel(sizes->val(Sizes::WRITABLE),
bss_writable_arrays_size,
0.001,
"correct writable size for bss in proc");
}
for (int i = 0; i < NUM_ARRAYS; ++i) {
const struct array_info *info = ARRAY_INFO + i;
string symname = info->name;
Elf::SymbolPtr sym = shlib_file->elf()->symbol(symname);
ok(sym, "can find symbol " + symname);
is(sym->size(), ARRAY_SIZE, symname + " has correct size");
sizes = proc->sizes(shlib_file, sym->range());
is_approx_rel((int) sizes->val(Sizes::RESIDENT),
ARRAY_SIZE * info->resident_percent / 100,
0.001,
symname + " has correct resident size");
is_approx_rel(sizes->val(Sizes::WRITABLE),
ARRAY_SIZE * info->writable_percent / 100.0,
0.001,
symname + " has correct writable size");
// Uninitialised space which is only read appears to be shared
// amongst every proc in the system (a 'zero page'?). This is
// good from a low-memusage point of view, but it means that
// it is shared among nearly all running procs to varying
// degrees. So we can't really account for it.
if (symname == "uninit_readme" || symname == "uninit_readhalf") {
continue;
}
int expected_eff_resident = ARRAY_SIZE * info->resident_percent / 100;
if (info->shared) {
expected_eff_resident /= NUM_INSTANCES;
}
// approximate match since the percentage (fixed pt)
// arithmetic could put us off by a factor of 1/100 (and does...:-)
is_approx((int) sizes->val(Sizes::EFFECTIVE_RESIDENT),
expected_eff_resident,
1 + (expected_eff_resident / 100),
symname + " has correct effective size");
}
// Non-elf maps
procs.clear();
for (proc_it = allprocs.begin(); proc_it != allprocs.end(); ++proc_it) {
string cmdline = (*proc_it)->cmdline();
if (cmdline == MI_EXE) {
procs.push_back(*proc_it);
}
}
is((int) procs.size(), NUM_INSTANCES, "can find all our mapit procs");
proc = procs.front();
files.clear();
re.compile("/" + MI_DAT + "$");
for (file_it = all_files.begin(); file_it != all_files.end(); ++file_it) {
string name = (*file_it)->name();
if (re.matches(name)) {
files.push_back(*file_it);
}
}
is((int) files.size(), 1, MI_DAT + " file only listed once");
FilePtr mapped_file = files.front();
ok(!mapped_file->is_elf(), MI_DAT + " isn't an elf file");
off_t fsize = 0;
double mi_data_size = 0;
ok(file_size(MI_DAT, fsize), "can get file size");
mi_data_size = fsize;
ok(mi_data_size > 0, "file has nonzero size");
for (proc_it = procs.begin();
proc_it != procs.end();
++proc_it) {
sizes = (*proc_it)->sizes(mapped_file);
is_approx_rel(sizes->val(Sizes::VM),
mi_data_size,
0.001,
"correct data file size");
is_approx_rel(sizes->val(Sizes::RESIDENT),
mi_data_size,
0.001,
"whole data file is resident");
is_approx_rel(sizes->val(Sizes::EFFECTIVE_RESIDENT),
mi_data_size / NUM_INSTANCES,
0.001,
"data file is shared between instances correctly");
}
return true;
}
void
Group::onSessionClose(const ProcessPtr &process, Session *session) {
TRACE_POINT();
// Standard resource management boilerplate stuff...
PoolPtr pool = getPool();
boost::unique_lock<boost::mutex> lock(pool->syncher);
assert(process->isAlive());
assert(isAlive() || getLifeStatus() == SHUTTING_DOWN);
P_TRACE(2, "Session closed for process " << process->inspect());
verifyInvariants();
UPDATE_TRACE_POINT();
/* Update statistics. */
process->sessionClosed(session);
assert(process->getLifeStatus() == Process::ALIVE);
assert(process->enabled == Process::ENABLED
|| process->enabled == Process::DISABLING
|| process->enabled == Process::DETACHED);
if (process->enabled == Process::ENABLED) {
pqueue.decrease(process->pqHandle, process->busyness());
}
/* This group now has a process that's guaranteed to be not
* totally busy.
*/
assert(!process->isTotallyBusy());
bool detachingBecauseOfMaxRequests = false;
bool detachingBecauseCapacityNeeded = false;
bool shouldDetach =
( detachingBecauseOfMaxRequests = (
options.maxRequests > 0
&& process->processed >= options.maxRequests
)) || (
detachingBecauseCapacityNeeded = (
process->sessions == 0
&& getWaitlist.empty()
&& (
!pool->getWaitlist.empty()
|| anotherGroupIsWaitingForCapacity()
)
)
);
bool shouldDisable =
process->enabled == Process::DISABLING
&& process->sessions == 0
&& enabledCount > 0;
if (shouldDetach || shouldDisable) {
vector<Callback> actions;
if (shouldDetach) {
if (detachingBecauseCapacityNeeded) {
/* Someone might be trying to get() a session for a different
* group that couldn't be spawned because of lack of pool capacity.
* If this group isn't under sufficiently load (as apparent by the
* checked conditions) then now's a good time to detach
* this process or group in order to free capacity.
*/
P_DEBUG("Process " << process->inspect() << " is no longer totally "
"busy; detaching it in order to make room in the pool");
} else {
/* This process has processed its maximum number of requests,
* so we detach it.
*/
P_DEBUG("Process " << process->inspect() <<
" has reached its maximum number of requests (" <<
options.maxRequests << "); detaching it");
}
pool->detachProcessUnlocked(process, actions);
} else {
removeProcessFromList(process, disablingProcesses);
addProcessToList(process, disabledProcesses);
removeFromDisableWaitlist(process, DR_SUCCESS, actions);
maybeInitiateOobw(process);
}
pool->fullVerifyInvariants();
lock.unlock();
runAllActions(actions);
} else {
// This could change process->enabled.
maybeInitiateOobw(process);
if (!getWaitlist.empty() && process->enabled == Process::ENABLED) {
/* If there are clients on this group waiting for a process to
* become available then call them now.
*/
UPDATE_TRACE_POINT();
// Already calls verifyInvariants().
assignSessionsToGetWaitersQuickly(lock);
}
}
}
void
Group::setupAttachOrDetachHook(const ProcessPtr process, HookScriptOptions &options) const {
options.environment.push_back(make_pair("PASSENGER_PROCESS_PID", toString(process->getPid())));
options.environment.push_back(make_pair("PASSENGER_APP_ROOT", this->options.appRoot));
}
void
Session::requestOOBW() {
ProcessPtr process = getProcess();
assert(process->isAlive());
process->getGroup()->requestOOBW(process);
}
void
Group::detachedProcessesCheckerMain(GroupPtr self) {
TRACE_POINT();
PoolPtr pool = getPool();
Pool::DebugSupportPtr debug = pool->debugSupport;
if (debug != NULL && debug->detachedProcessesChecker) {
debug->debugger->send("About to start detached processes checker");
debug->messages->recv("Proceed with starting detached processes checker");
}
boost::unique_lock<boost::mutex> lock(pool->syncher);
while (true) {
assert(detachedProcessesCheckerActive);
if (getLifeStatus() == SHUT_DOWN || this_thread::interruption_requested()) {
UPDATE_TRACE_POINT();
P_DEBUG("Stopping detached processes checker");
detachedProcessesCheckerActive = false;
break;
}
UPDATE_TRACE_POINT();
if (!detachedProcesses.empty()) {
P_TRACE(2, "Checking whether any of the " << detachedProcesses.size() <<
" detached processes have exited...");
ProcessList::iterator it = detachedProcesses.begin();
ProcessList::iterator end = detachedProcesses.end();
while (it != end) {
const ProcessPtr process = *it;
switch (process->getLifeStatus()) {
case Process::ALIVE:
if (process->canTriggerShutdown()) {
P_DEBUG("Detached process " << process->inspect() <<
" has 0 active sessions now. Triggering shutdown.");
process->triggerShutdown();
assert(process->getLifeStatus() == Process::SHUTDOWN_TRIGGERED);
}
it++;
break;
case Process::SHUTDOWN_TRIGGERED:
if (process->canCleanup()) {
P_DEBUG("Detached process " << process->inspect() << " has shut down. Cleaning up associated resources.");
process->cleanup();
assert(process->getLifeStatus() == Process::DEAD);
it++;
removeProcessFromList(process, detachedProcesses);
} else if (process->shutdownTimeoutExpired()) {
P_WARN("Detached process " << process->inspect() <<
" didn't shut down within " PROCESS_SHUTDOWN_TIMEOUT_DISPLAY
". Forcefully killing it with SIGKILL.");
kill(process->pid, SIGKILL);
it++;
} else {
it++;
}
break;
default:
P_BUG("Unknown 'lifeStatus' state " << (int) process->getLifeStatus());
}
}
}
UPDATE_TRACE_POINT();
if (detachedProcesses.empty()) {
UPDATE_TRACE_POINT();
P_DEBUG("Stopping detached processes checker");
detachedProcessesCheckerActive = false;
vector<Callback> actions;
if (shutdownCanFinish()) {
UPDATE_TRACE_POINT();
finishShutdown(actions);
}
verifyInvariants();
verifyExpensiveInvariants();
lock.unlock();
UPDATE_TRACE_POINT();
runAllActions(actions);
break;
} else {
UPDATE_TRACE_POINT();
verifyInvariants();
verifyExpensiveInvariants();
}
// Not all processes can be shut down yet. Sleep for a while unless
// someone wakes us up.
UPDATE_TRACE_POINT();
detachedProcessesCheckerCond.timed_wait(lock,
posix_time::milliseconds(100));
}
}
void WheelAnimalProcessManager::runProcess(WheelAnimalProcess::PROCESS process,Orz::Event * evt)
{
_referenced = evt->getData<ReferenceCount *>()->reference();
ProcessPtr pro = _processes.at(process);
_updateFun = boost::bind(&WheelAnimalProcess::update, pro.get(), _1);
}
