void slave_notifyProcessed(Slave* slave, guint numberEventsProcessed, guint numberNodesWithEvents) {
MAGIC_ASSERT(slave);
_slave_lock(slave);
slave->numEventsCurrentInterval += numberEventsProcessed;
slave->numNodesWithEventsCurrentInterval += numberNodesWithEvents;
_slave_unlock(slave);
countdownlatch_countDownAwait(slave->processingLatch);
countdownlatch_countDownAwait(slave->barrierLatch);
}
void engine_notifyProcessed(Engine* engine, guint numberEventsProcessed, guint numberNodesWithEvents) {
MAGIC_ASSERT(engine);
_engine_lock(engine);
engine->numEventsCurrentInterval += numberEventsProcessed;
engine->numNodesWithEventsCurrentInterval += numberNodesWithEvents;
_engine_unlock(engine);
countdownlatch_countDownAwait(engine->processingLatch);
countdownlatch_countDownAwait(engine->barrierLatch);
}
Event* scheduler_pop(Scheduler* scheduler) {
MAGIC_ASSERT(scheduler);
/* this function should block until a non-null event is available for the worker to run.
* return NULL only to signal the worker thread to quit */
while(scheduler->isRunning) {
/* pop from a queue based on the policy */
Event* nextEvent = scheduler->policy->pop(scheduler->policy, scheduler->currentRound.endTime);
if(nextEvent != NULL) {
/* we have an event, let the worker run it */
return nextEvent;
} else if(scheduler->policyType == SP_SERIAL_GLOBAL) {
/* the running thread has no more events to execute this round, but we only have a
* single, global, serial queue, so returning NULL without blocking is OK. */
return NULL;
} else {
/* the running thread has no more events to execute this round and we need to block it
* so that we can wait for all threads to finish events from this round. We want to
* track idle times, so let's start by making sure we have timer elements in place. */
GTimer* executeEventsBarrierWaitTime = g_hash_table_lookup(scheduler->threadToWaitTimerMap, GUINT_TO_POINTER(pthread_self()));
/* wait for all other worker threads to finish their events too, and track wait time */
if(executeEventsBarrierWaitTime) {
g_timer_continue(executeEventsBarrierWaitTime);
}
countdownlatch_countDownAwait(scheduler->executeEventsBarrier);
if(executeEventsBarrierWaitTime) {
g_timer_stop(executeEventsBarrierWaitTime);
}
/* now all threads reached the current round end barrier time.
* asynchronously collect some stats that the main thread will use. */
if(scheduler->policy->getNextTime) {
SimulationTime nextTime = scheduler->policy->getNextTime(scheduler->policy);
g_mutex_lock(&(scheduler->globalLock));
scheduler->currentRound.minNextEventTime = MIN(scheduler->currentRound.minNextEventTime, nextTime);
g_mutex_unlock(&(scheduler->globalLock));
}
/* clear all log messages from the last round */
logger_flushRecords(logger_getDefault(), pthread_self());
/* wait for other threads to finish their collect step */
countdownlatch_countDownAwait(scheduler->collectInfoBarrier);
/* now wait for main thread to process a barrier update for the next round */
countdownlatch_countDownAwait(scheduler->prepareRoundBarrier);
}
}
/* scheduler is done, return NULL to stop worker */
return NULL;
}
SimulationTime scheduler_awaitNextRound(Scheduler* scheduler) {
/* this function is called by the slave main thread */
if(scheduler->policyType != SP_SERIAL_GLOBAL) {
/* other workers will also wait at this barrier when they are finished with their events */
countdownlatch_countDownAwait(scheduler->executeEventsBarrier);
countdownlatch_reset(scheduler->executeEventsBarrier);
/* then they collect stats and wait at this barrier */
countdownlatch_countDownAwait(scheduler->collectInfoBarrier);
countdownlatch_reset(scheduler->collectInfoBarrier);
}
SimulationTime minNextEventTime = SIMTIME_MAX;
g_mutex_lock(&scheduler->globalLock);
minNextEventTime = scheduler->currentRound.minNextEventTime;
g_mutex_unlock(&scheduler->globalLock);
return minNextEventTime;
}
void scheduler_awaitStart(Scheduler* scheduler) {
/* set up the thread timer map */
g_mutex_lock(&scheduler->globalLock);
if(!g_hash_table_lookup(scheduler->threadToWaitTimerMap, GUINT_TO_POINTER(pthread_self()))) {
GTimer* waitTimer = g_timer_new();
g_timer_stop(waitTimer);
g_hash_table_insert(scheduler->threadToWaitTimerMap, GUINT_TO_POINTER(pthread_self()), waitTimer);
}
g_mutex_unlock(&scheduler->globalLock);
/* wait until all threads are waiting to start */
countdownlatch_countDownAwait(scheduler->startBarrier);
/* each thread will boot their own hosts */
_scheduler_startHosts(scheduler);
/* everyone is waiting for the next round to be ready */
countdownlatch_countDownAwait(scheduler->prepareRoundBarrier);
}
void scheduler_awaitFinish(Scheduler* scheduler) {
/* each thread will run cleanup on their own hosts */
g_mutex_lock(&scheduler->globalLock);
scheduler->isRunning = FALSE;
g_mutex_unlock(&scheduler->globalLock);
_scheduler_stopHosts(scheduler);
/* wait until all threads are waiting to finish */
countdownlatch_countDownAwait(scheduler->finishBarrier);
}
void scheduler_finish(Scheduler* scheduler) {
/* make sure when the workers wake up they know we are done */
g_mutex_lock(&scheduler->globalLock);
scheduler->isRunning = FALSE;
g_mutex_unlock(&scheduler->globalLock);
if(scheduler->policyType != SP_SERIAL_GLOBAL) {
/* wake up threads from their waiting for the next round.
* because isRunning is now false, they will all exit and wait at finishBarrier */
countdownlatch_countDownAwait(scheduler->prepareRoundBarrier);
/* wait for them to be ready to finish */
countdownlatch_countDownAwait(scheduler->finishBarrier);
}
g_mutex_lock(&scheduler->globalLock);
if(g_hash_table_size(scheduler->hostIDToHostMap) > 0) {
g_hash_table_remove_all(scheduler->hostIDToHostMap);
}
g_mutex_unlock(&scheduler->globalLock);
}
void scheduler_start(Scheduler* scheduler) {
_scheduler_assignHosts(scheduler);
g_mutex_lock(&scheduler->globalLock);
scheduler->isRunning = TRUE;
g_mutex_unlock(&scheduler->globalLock);
if(scheduler->policyType != SP_SERIAL_GLOBAL) {
/* this will cause a worker to execute the locked initialization in awaitStart */
countdownlatch_countDownAwait(scheduler->startBarrier);
}
}
void scheduler_continueNextRound(Scheduler* scheduler, SimulationTime windowStart, SimulationTime windowEnd) {
g_mutex_lock(&scheduler->globalLock);
scheduler->currentRound.endTime = windowEnd;
scheduler->currentRound.minNextEventTime = SIMTIME_MAX;
g_mutex_unlock(&scheduler->globalLock);
if(scheduler->policyType != SP_SERIAL_GLOBAL) {
/* workers are waiting for preparation of the next round
* this will cause them to start running events */
countdownlatch_countDownAwait(scheduler->prepareRoundBarrier);
/* workers are running events now, and will wait at executeEventsBarrier
* when blocked because there are no more events available in the current round */
countdownlatch_reset(scheduler->prepareRoundBarrier);
}
}
static gint _engine_distributeEvents(Engine* engine) {
MAGIC_ASSERT(engine);
GList* nodeList = internetwork_getAllNodes(engine->internet);
/* assign nodes to the worker threads so they get processed */
GSList* listArray[engine->config->nWorkerThreads];
memset(listArray, 0, engine->config->nWorkerThreads * sizeof(GSList*));
gint counter = 0;
GList* item = g_list_first(nodeList);
while(item) {
Node* node = item->data;
gint i = counter % engine->config->nWorkerThreads;
listArray[i] = g_slist_append(listArray[i], node);
counter++;
item = g_list_next(item);
}
/* we will track when workers finish processing their nodes */
engine->processingLatch = countdownlatch_new(engine->config->nWorkerThreads + 1);
/* after the workers finish processing, wait for barrier update */
engine->barrierLatch = countdownlatch_new(engine->config->nWorkerThreads + 1);
/* start up the workers */
GSList* workerThreads = NULL;
for(gint i = 0; i < engine->config->nWorkerThreads; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i+1));
GThread* t = g_thread_new(name->str, (GThreadFunc)worker_run, (gpointer)listArray[i]);
workerThreads = g_slist_append(workerThreads, t);
g_string_free(name, TRUE);
}
/* process all events in the priority queue */
while(engine->executeWindowStart < engine->endTime)
{
/* wait for the workers to finish processing nodes before we touch them */
countdownlatch_countDownAwait(engine->processingLatch);
/* we are in control now, the workers are waiting at barrierLatch */
message("execution window [%lu--%lu] ran %u events from %u active nodes",
engine->executeWindowStart, engine->executeWindowEnd,
engine->numEventsCurrentInterval,
engine->numNodesWithEventsCurrentInterval);
/* check if we should take 1 step ahead or fast-forward our execute window.
* since looping through all the nodes to find the minimum event is
* potentially expensive, we use a heuristic of only trying to jump ahead
* if the last interval had only a few events in it. */
if(engine->numEventsCurrentInterval < 10) {
/* we had no events in that interval, lets try to fast forward */
SimulationTime minNextEventTime = SIMTIME_INVALID;
item = g_list_first(nodeList);
while(item) {
Node* node = item->data;
EventQueue* eventq = node_getEvents(node);
Event* nextEvent = eventqueue_peek(eventq);
if(nextEvent && (nextEvent->time < minNextEventTime)) {
minNextEventTime = nextEvent->time;
}
item = g_list_next(item);
}
/* fast forward to the next event */
engine->executeWindowStart = minNextEventTime;
} else {
/* we still have events, lets just step one interval */
engine->executeWindowStart = engine->executeWindowEnd;
}
/* make sure we dont run over the end */
engine->executeWindowEnd = engine->executeWindowStart + engine->minTimeJump;
if(engine->executeWindowEnd > engine->endTime) {
engine->executeWindowEnd = engine->endTime;
}
/* reset for next round */
countdownlatch_reset(engine->processingLatch);
engine->numEventsCurrentInterval = 0;
engine->numNodesWithEventsCurrentInterval = 0;
/* if we are done, make sure the workers know about it */
if(engine->executeWindowStart >= engine->endTime) {
engine->killed = TRUE;
}
/* release the workers for the next round, or to exit */
countdownlatch_countDownAwait(engine->barrierLatch);
countdownlatch_reset(engine->barrierLatch);
}
/* wait for the threads to finish their cleanup */
GSList* threadItem = workerThreads;
while(threadItem) {
GThread* t = threadItem->data;
g_thread_join(t);
g_thread_unref(t);
threadItem = g_slist_next(threadItem);
}
g_slist_free(workerThreads);
for(gint i = 0; i < engine->config->nWorkerThreads; i++) {
g_slist_free(listArray[i]);
}
countdownlatch_free(engine->processingLatch);
countdownlatch_free(engine->barrierLatch);
/* frees the list struct we own, but not the nodes it holds (those were
* taken care of by the workers) */
g_list_free(nodeList);
return 0;
}
void slave_runParallel(Slave* slave) {
MAGIC_ASSERT(slave);
GList* nodeList = _slave_getAllHosts(slave);
/* assign nodes to the worker threads so they get processed */
WorkLoad workArray[slave->nWorkers];
memset(workArray, 0, slave->nWorkers * sizeof(WorkLoad));
gint counter = 0;
GList* item = g_list_first(nodeList);
while(item) {
Host* node = item->data;
gint i = counter % slave->nWorkers;
workArray[i].hosts = g_list_append(workArray[i].hosts, node);
counter++;
item = g_list_next(item);
}
/* we will track when workers finish processing their nodes */
slave->processingLatch = countdownlatch_new(slave->nWorkers + 1);
/* after the workers finish processing, wait for barrier update */
slave->barrierLatch = countdownlatch_new(slave->nWorkers + 1);
/* start up the workers */
GSList* workerThreads = NULL;
for(gint i = 0; i < slave->nWorkers; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i+1));
workArray[i].slave = slave;
workArray[i].master = slave->master;
GThread* t = g_thread_new(name->str, (GThreadFunc)worker_runParallel, &(workArray[i]));
workerThreads = g_slist_append(workerThreads, t);
g_string_free(name, TRUE);
}
message("started %i worker threads", slave->nWorkers);
/* process all events in the priority queue */
while(master_getExecuteWindowStart(slave->master) < master_getEndTime(slave->master))
{
/* wait for the workers to finish processing nodes before we touch them */
countdownlatch_countDownAwait(slave->processingLatch);
/* we are in control now, the workers are waiting at barrierLatch */
info("execution window [%"G_GUINT64_FORMAT"--%"G_GUINT64_FORMAT"] ran %u events from %u active nodes",
master_getExecuteWindowStart(slave->master), master_getExecuteWindowEnd(slave->master),
slave->numEventsCurrentInterval, slave->numNodesWithEventsCurrentInterval);
/* check if we should take 1 step ahead or fast-forward our execute window.
* since looping through all the nodes to find the minimum event is
* potentially expensive, we use a heuristic of only trying to jump ahead
* if the last interval had only a few events in it. */
SimulationTime minNextEventTime = SIMTIME_INVALID;
if(slave->numEventsCurrentInterval < 10) {
/* we had no events in that interval, lets try to fast forward */
item = g_list_first(nodeList);
/* fast forward to the next event, the new window start will be the next event time */
while(item) {
Host* node = item->data;
EventQueue* eventq = host_getEvents(node);
Event* nextEvent = eventqueue_peek(eventq);
SimulationTime nextEventTime = shadowevent_getTime(nextEvent);
if(nextEvent && (nextEventTime < minNextEventTime)) {
minNextEventTime = nextEventTime;
}
item = g_list_next(item);
}
if(minNextEventTime == SIMTIME_INVALID) {
minNextEventTime = master_getExecuteWindowEnd(slave->master);
}
} else {
/* we still have events, lets just step one interval,
* consider the next event as the previous window end */
minNextEventTime = master_getExecuteWindowEnd(slave->master);
}
/* notify master that we finished this round, and what our next event is */
master_slaveFinishedCurrentWindow(slave->master, minNextEventTime);
/* reset for next round */
countdownlatch_reset(slave->processingLatch);
slave->numEventsCurrentInterval = 0;
slave->numNodesWithEventsCurrentInterval = 0;
/* release the workers for the next round, or to exit */
countdownlatch_countDownAwait(slave->barrierLatch);
countdownlatch_reset(slave->barrierLatch);
}
message("waiting for %i worker threads to finish", slave->nWorkers);
/* wait for the threads to finish their cleanup */
GSList* threadItem = workerThreads;
while(threadItem) {
GThread* t = threadItem->data;
/* the join will consume the reference, so unref is not needed */
g_thread_join(t);
threadItem = g_slist_next(threadItem);
}
g_slist_free(workerThreads);
message("%i worker threads finished", slave->nWorkers);
for(gint i = 0; i < slave->nWorkers; i++) {
WorkLoad w = workArray[i];
g_list_free(w.hosts);
}
countdownlatch_free(slave->processingLatch);
countdownlatch_free(slave->barrierLatch);
/* frees the list struct we own, but not the nodes it holds (those were
* taken care of by the workers) */
g_list_free(nodeList);
}
