void Group::spawnThreadRealMain(const SpawnerPtr &spawner, const Options &options, unsigned int restartsInitiated) { TRACE_POINT(); this_thread::disable_interruption di; this_thread::disable_syscall_interruption dsi; PoolPtr pool = getPool(); Pool::DebugSupportPtr debug = pool->debugSupport; bool done = false; while (!done) { bool shouldFail = false; if (debug != NULL && debug->spawning) { UPDATE_TRACE_POINT(); this_thread::restore_interruption ri(di); this_thread::restore_syscall_interruption rsi(dsi); this_thread::interruption_point(); string iteration; { LockGuard g(debug->syncher); debug->spawnLoopIteration++; iteration = toString(debug->spawnLoopIteration); } P_DEBUG("Begin spawn loop iteration " << iteration); debug->debugger->send("Begin spawn loop iteration " + iteration); vector<string> cases; cases.push_back("Proceed with spawn loop iteration " + iteration); cases.push_back("Fail spawn loop iteration " + iteration); MessagePtr message = debug->messages->recvAny(cases); shouldFail = message->name == "Fail spawn loop iteration " + iteration; } ProcessPtr process; ExceptionPtr exception; try { UPDATE_TRACE_POINT(); this_thread::restore_interruption ri(di); this_thread::restore_syscall_interruption rsi(dsi); if (shouldFail) { throw SpawnException("Simulated failure"); } else { process = spawner->spawn(options); process->setGroup(shared_from_this()); } } catch (const thread_interrupted &) { break; } catch (const tracable_exception &e) { exception = copyException(e); // Let other (unexpected) exceptions crash the program so // gdb can generate a backtrace. } UPDATE_TRACE_POINT(); ScopeGuard guard(boost::bind(Process::forceTriggerShutdownAndCleanup, process)); boost::unique_lock<boost::mutex> lock(pool->syncher); if (!isAlive()) { if (process != NULL) { P_DEBUG("Group is being shut down so dropping process " << process->inspect() << " which we just spawned and exiting spawn loop"); } else { P_DEBUG("The group is being shut down. A process failed " "to be spawned anyway, so ignoring this error and exiting " "spawn loop"); } // We stop immediately because any previously assumed invariants // may have been violated. break; } else if (restartsInitiated != this->restartsInitiated) { if (process != NULL) { P_DEBUG("A restart was issued for the group, so dropping process " << process->inspect() << " which we just spawned and exiting spawn loop"); } else { P_DEBUG("A restart was issued for the group. A process failed " "to be spawned anyway, so ignoring this error and exiting " "spawn loop"); } // We stop immediately because any previously assumed invariants // may have been violated. break; } verifyInvariants(); assert(m_spawning); assert(processesBeingSpawned > 0); processesBeingSpawned--; assert(processesBeingSpawned == 0); UPDATE_TRACE_POINT(); vector<Callback> actions; if (process != NULL) { AttachResult result = attach(process, actions); if (result == AR_OK) { guard.clear(); if (getWaitlist.empty()) { pool->assignSessionsToGetWaiters(actions); } else { assignSessionsToGetWaiters(actions); } P_DEBUG("New process count = " << enabledCount << ", remaining get waiters = " << getWaitlist.size()); } else { done = true; P_DEBUG("Unable to attach spawned process " << process->inspect()); if (result == AR_ANOTHER_GROUP_IS_WAITING_FOR_CAPACITY) { pool->possiblySpawnMoreProcessesForExistingGroups(); } } } else { // TODO: sure this is the best thing? if there are // processes currently alive we should just use them. P_ERROR("Could not spawn process for group " << name << ": " << exception->what() << "\n" << exception->backtrace()); if (enabledCount == 0) { enableAllDisablingProcesses(actions); } Pool::assignExceptionToGetWaiters(getWaitlist, exception, actions); pool->assignSessionsToGetWaiters(actions); done = true; } done = done || (processLowerLimitsSatisfied() && getWaitlist.empty()) || processUpperLimitsReached() || pool->atFullCapacity(false); m_spawning = !done; if (done) { P_DEBUG("Spawn loop done"); } else { processesBeingSpawned++; P_DEBUG("Continue spawning"); } UPDATE_TRACE_POINT(); pool->fullVerifyInvariants(); lock.unlock(); UPDATE_TRACE_POINT(); runAllActions(actions); UPDATE_TRACE_POINT(); } if (debug != NULL && debug->spawning) { debug->debugger->send("Spawn loop done"); } }