static void _scheduler_free(Scheduler* scheduler) {
MAGIC_ASSERT(scheduler);
guint nWorkers = g_queue_get_length(scheduler->threadItems);
while(!g_queue_is_empty(scheduler->threadItems)) {
SchedulerThreadItem* item = g_queue_pop_head(scheduler->threadItems);
countdownlatch_free(item->notifyDoneRunning);
g_free(item);
}
g_queue_free(scheduler->threadItems);
scheduler->policy->free(scheduler->policy);
random_free(scheduler->random);
countdownlatch_free(scheduler->executeEventsBarrier);
countdownlatch_free(scheduler->collectInfoBarrier);
countdownlatch_free(scheduler->prepareRoundBarrier);
countdownlatch_free(scheduler->startBarrier);
countdownlatch_free(scheduler->finishBarrier);
if(scheduler->threadToWaitTimerMap) {
g_hash_table_destroy(scheduler->threadToWaitTimerMap);
}
g_mutex_clear(&(scheduler->globalLock));
message("%i worker threads finished", nWorkers);
MAGIC_CLEAR(scheduler);
g_free(scheduler);
}
static gint _engine_distributeEvents(Engine* engine) {
MAGIC_ASSERT(engine);
GList* nodeList = internetwork_getAllNodes(engine->internet);
/* assign nodes to the worker threads so they get processed */
GSList* listArray[engine->config->nWorkerThreads];
memset(listArray, 0, engine->config->nWorkerThreads * sizeof(GSList*));
gint counter = 0;
GList* item = g_list_first(nodeList);
while(item) {
Node* node = item->data;
gint i = counter % engine->config->nWorkerThreads;
listArray[i] = g_slist_append(listArray[i], node);
counter++;
item = g_list_next(item);
}
/* we will track when workers finish processing their nodes */
engine->processingLatch = countdownlatch_new(engine->config->nWorkerThreads + 1);
/* after the workers finish processing, wait for barrier update */
engine->barrierLatch = countdownlatch_new(engine->config->nWorkerThreads + 1);
/* start up the workers */
GSList* workerThreads = NULL;
for(gint i = 0; i < engine->config->nWorkerThreads; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i+1));
GThread* t = g_thread_new(name->str, (GThreadFunc)worker_run, (gpointer)listArray[i]);
workerThreads = g_slist_append(workerThreads, t);
g_string_free(name, TRUE);
}
/* process all events in the priority queue */
while(engine->executeWindowStart < engine->endTime)
{
/* wait for the workers to finish processing nodes before we touch them */
countdownlatch_countDownAwait(engine->processingLatch);
/* we are in control now, the workers are waiting at barrierLatch */
message("execution window [%lu--%lu] ran %u events from %u active nodes",
engine->executeWindowStart, engine->executeWindowEnd,
engine->numEventsCurrentInterval,
engine->numNodesWithEventsCurrentInterval);
/* check if we should take 1 step ahead or fast-forward our execute window.
* since looping through all the nodes to find the minimum event is
* potentially expensive, we use a heuristic of only trying to jump ahead
* if the last interval had only a few events in it. */
if(engine->numEventsCurrentInterval < 10) {
/* we had no events in that interval, lets try to fast forward */
SimulationTime minNextEventTime = SIMTIME_INVALID;
item = g_list_first(nodeList);
while(item) {
Node* node = item->data;
EventQueue* eventq = node_getEvents(node);
Event* nextEvent = eventqueue_peek(eventq);
if(nextEvent && (nextEvent->time < minNextEventTime)) {
minNextEventTime = nextEvent->time;
}
item = g_list_next(item);
}
/* fast forward to the next event */
engine->executeWindowStart = minNextEventTime;
} else {
/* we still have events, lets just step one interval */
engine->executeWindowStart = engine->executeWindowEnd;
}
/* make sure we dont run over the end */
engine->executeWindowEnd = engine->executeWindowStart + engine->minTimeJump;
if(engine->executeWindowEnd > engine->endTime) {
engine->executeWindowEnd = engine->endTime;
}
/* reset for next round */
countdownlatch_reset(engine->processingLatch);
engine->numEventsCurrentInterval = 0;
engine->numNodesWithEventsCurrentInterval = 0;
/* if we are done, make sure the workers know about it */
if(engine->executeWindowStart >= engine->endTime) {
engine->killed = TRUE;
}
/* release the workers for the next round, or to exit */
countdownlatch_countDownAwait(engine->barrierLatch);
countdownlatch_reset(engine->barrierLatch);
}
/* wait for the threads to finish their cleanup */
GSList* threadItem = workerThreads;
while(threadItem) {
GThread* t = threadItem->data;
g_thread_join(t);
g_thread_unref(t);
threadItem = g_slist_next(threadItem);
}
g_slist_free(workerThreads);
for(gint i = 0; i < engine->config->nWorkerThreads; i++) {
g_slist_free(listArray[i]);
}
countdownlatch_free(engine->processingLatch);
countdownlatch_free(engine->barrierLatch);
/* frees the list struct we own, but not the nodes it holds (those were
* taken care of by the workers) */
g_list_free(nodeList);
return 0;
}
void slave_runParallel(Slave* slave) {
MAGIC_ASSERT(slave);
GList* nodeList = _slave_getAllHosts(slave);
/* assign nodes to the worker threads so they get processed */
WorkLoad workArray[slave->nWorkers];
memset(workArray, 0, slave->nWorkers * sizeof(WorkLoad));
gint counter = 0;
GList* item = g_list_first(nodeList);
while(item) {
Host* node = item->data;
gint i = counter % slave->nWorkers;
workArray[i].hosts = g_list_append(workArray[i].hosts, node);
counter++;
item = g_list_next(item);
}
/* we will track when workers finish processing their nodes */
slave->processingLatch = countdownlatch_new(slave->nWorkers + 1);
/* after the workers finish processing, wait for barrier update */
slave->barrierLatch = countdownlatch_new(slave->nWorkers + 1);
/* start up the workers */
GSList* workerThreads = NULL;
for(gint i = 0; i < slave->nWorkers; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i+1));
workArray[i].slave = slave;
workArray[i].master = slave->master;
GThread* t = g_thread_new(name->str, (GThreadFunc)worker_runParallel, &(workArray[i]));
workerThreads = g_slist_append(workerThreads, t);
g_string_free(name, TRUE);
}
message("started %i worker threads", slave->nWorkers);
/* process all events in the priority queue */
while(master_getExecuteWindowStart(slave->master) < master_getEndTime(slave->master))
{
/* wait for the workers to finish processing nodes before we touch them */
countdownlatch_countDownAwait(slave->processingLatch);
/* we are in control now, the workers are waiting at barrierLatch */
info("execution window [%"G_GUINT64_FORMAT"--%"G_GUINT64_FORMAT"] ran %u events from %u active nodes",
master_getExecuteWindowStart(slave->master), master_getExecuteWindowEnd(slave->master),
slave->numEventsCurrentInterval, slave->numNodesWithEventsCurrentInterval);
/* check if we should take 1 step ahead or fast-forward our execute window.
* since looping through all the nodes to find the minimum event is
* potentially expensive, we use a heuristic of only trying to jump ahead
* if the last interval had only a few events in it. */
SimulationTime minNextEventTime = SIMTIME_INVALID;
if(slave->numEventsCurrentInterval < 10) {
/* we had no events in that interval, lets try to fast forward */
item = g_list_first(nodeList);
/* fast forward to the next event, the new window start will be the next event time */
while(item) {
Host* node = item->data;
EventQueue* eventq = host_getEvents(node);
Event* nextEvent = eventqueue_peek(eventq);
SimulationTime nextEventTime = shadowevent_getTime(nextEvent);
if(nextEvent && (nextEventTime < minNextEventTime)) {
minNextEventTime = nextEventTime;
}
item = g_list_next(item);
}
if(minNextEventTime == SIMTIME_INVALID) {
minNextEventTime = master_getExecuteWindowEnd(slave->master);
}
} else {
/* we still have events, lets just step one interval,
* consider the next event as the previous window end */
minNextEventTime = master_getExecuteWindowEnd(slave->master);
}
/* notify master that we finished this round, and what our next event is */
master_slaveFinishedCurrentWindow(slave->master, minNextEventTime);
/* reset for next round */
countdownlatch_reset(slave->processingLatch);
slave->numEventsCurrentInterval = 0;
slave->numNodesWithEventsCurrentInterval = 0;
/* release the workers for the next round, or to exit */
countdownlatch_countDownAwait(slave->barrierLatch);
countdownlatch_reset(slave->barrierLatch);
}
message("waiting for %i worker threads to finish", slave->nWorkers);
/* wait for the threads to finish their cleanup */
GSList* threadItem = workerThreads;
while(threadItem) {
GThread* t = threadItem->data;
/* the join will consume the reference, so unref is not needed */
g_thread_join(t);
threadItem = g_slist_next(threadItem);
}
g_slist_free(workerThreads);
message("%i worker threads finished", slave->nWorkers);
for(gint i = 0; i < slave->nWorkers; i++) {
WorkLoad w = workArray[i];
g_list_free(w.hosts);
}
countdownlatch_free(slave->processingLatch);
countdownlatch_free(slave->barrierLatch);
/* frees the list struct we own, but not the nodes it holds (those were
* taken care of by the workers) */
g_list_free(nodeList);
}
