void slave_runParallel(Slave* slave) {
MAGIC_ASSERT(slave);
GList* nodeList = _slave_getAllHosts(slave);
/* assign nodes to the worker threads so they get processed */
WorkLoad workArray[slave->nWorkers];
memset(workArray, 0, slave->nWorkers * sizeof(WorkLoad));
gint counter = 0;
GList* item = g_list_first(nodeList);
while(item) {
Host* node = item->data;
gint i = counter % slave->nWorkers;
workArray[i].hosts = g_list_append(workArray[i].hosts, node);
counter++;
item = g_list_next(item);
}
/* we will track when workers finish processing their nodes */
slave->processingLatch = countdownlatch_new(slave->nWorkers + 1);
/* after the workers finish processing, wait for barrier update */
slave->barrierLatch = countdownlatch_new(slave->nWorkers + 1);
/* start up the workers */
GSList* workerThreads = NULL;
for(gint i = 0; i < slave->nWorkers; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i+1));
workArray[i].slave = slave;
workArray[i].master = slave->master;
GThread* t = g_thread_new(name->str, (GThreadFunc)worker_runParallel, &(workArray[i]));
workerThreads = g_slist_append(workerThreads, t);
g_string_free(name, TRUE);
}
message("started %i worker threads", slave->nWorkers);
/* process all events in the priority queue */
while(master_getExecuteWindowStart(slave->master) < master_getEndTime(slave->master))
{
/* wait for the workers to finish processing nodes before we touch them */
countdownlatch_countDownAwait(slave->processingLatch);
/* we are in control now, the workers are waiting at barrierLatch */
info("execution window [%"G_GUINT64_FORMAT"--%"G_GUINT64_FORMAT"] ran %u events from %u active nodes",
master_getExecuteWindowStart(slave->master), master_getExecuteWindowEnd(slave->master),
slave->numEventsCurrentInterval, slave->numNodesWithEventsCurrentInterval);
/* check if we should take 1 step ahead or fast-forward our execute window.
* since looping through all the nodes to find the minimum event is
* potentially expensive, we use a heuristic of only trying to jump ahead
* if the last interval had only a few events in it. */
SimulationTime minNextEventTime = SIMTIME_INVALID;
if(slave->numEventsCurrentInterval < 10) {
/* we had no events in that interval, lets try to fast forward */
item = g_list_first(nodeList);
/* fast forward to the next event, the new window start will be the next event time */
while(item) {
Host* node = item->data;
EventQueue* eventq = host_getEvents(node);
Event* nextEvent = eventqueue_peek(eventq);
SimulationTime nextEventTime = shadowevent_getTime(nextEvent);
if(nextEvent && (nextEventTime < minNextEventTime)) {
minNextEventTime = nextEventTime;
}
item = g_list_next(item);
}
if(minNextEventTime == SIMTIME_INVALID) {
minNextEventTime = master_getExecuteWindowEnd(slave->master);
}
} else {
/* we still have events, lets just step one interval,
* consider the next event as the previous window end */
minNextEventTime = master_getExecuteWindowEnd(slave->master);
}
/* notify master that we finished this round, and what our next event is */
master_slaveFinishedCurrentWindow(slave->master, minNextEventTime);
/* reset for next round */
countdownlatch_reset(slave->processingLatch);
slave->numEventsCurrentInterval = 0;
slave->numNodesWithEventsCurrentInterval = 0;
/* release the workers for the next round, or to exit */
countdownlatch_countDownAwait(slave->barrierLatch);
countdownlatch_reset(slave->barrierLatch);
}
message("waiting for %i worker threads to finish", slave->nWorkers);
/* wait for the threads to finish their cleanup */
GSList* threadItem = workerThreads;
while(threadItem) {
GThread* t = threadItem->data;
/* the join will consume the reference, so unref is not needed */
g_thread_join(t);
threadItem = g_slist_next(threadItem);
}
g_slist_free(workerThreads);
message("%i worker threads finished", slave->nWorkers);
for(gint i = 0; i < slave->nWorkers; i++) {
WorkLoad w = workArray[i];
g_list_free(w.hosts);
}
countdownlatch_free(slave->processingLatch);
countdownlatch_free(slave->barrierLatch);
/* frees the list struct we own, but not the nodes it holds (those were
* taken care of by the workers) */
g_list_free(nodeList);
}
static gint _engine_distributeEvents(Engine* engine) {
MAGIC_ASSERT(engine);
GList* nodeList = internetwork_getAllNodes(engine->internet);
/* assign nodes to the worker threads so they get processed */
GSList* listArray[engine->config->nWorkerThreads];
memset(listArray, 0, engine->config->nWorkerThreads * sizeof(GSList*));
gint counter = 0;
GList* item = g_list_first(nodeList);
while(item) {
Node* node = item->data;
gint i = counter % engine->config->nWorkerThreads;
listArray[i] = g_slist_append(listArray[i], node);
counter++;
item = g_list_next(item);
}
/* we will track when workers finish processing their nodes */
engine->processingLatch = countdownlatch_new(engine->config->nWorkerThreads + 1);
/* after the workers finish processing, wait for barrier update */
engine->barrierLatch = countdownlatch_new(engine->config->nWorkerThreads + 1);
/* start up the workers */
GSList* workerThreads = NULL;
for(gint i = 0; i < engine->config->nWorkerThreads; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i+1));
GThread* t = g_thread_new(name->str, (GThreadFunc)worker_run, (gpointer)listArray[i]);
workerThreads = g_slist_append(workerThreads, t);
g_string_free(name, TRUE);
}
/* process all events in the priority queue */
while(engine->executeWindowStart < engine->endTime)
{
/* wait for the workers to finish processing nodes before we touch them */
countdownlatch_countDownAwait(engine->processingLatch);
/* we are in control now, the workers are waiting at barrierLatch */
message("execution window [%lu--%lu] ran %u events from %u active nodes",
engine->executeWindowStart, engine->executeWindowEnd,
engine->numEventsCurrentInterval,
engine->numNodesWithEventsCurrentInterval);
/* check if we should take 1 step ahead or fast-forward our execute window.
* since looping through all the nodes to find the minimum event is
* potentially expensive, we use a heuristic of only trying to jump ahead
* if the last interval had only a few events in it. */
if(engine->numEventsCurrentInterval < 10) {
/* we had no events in that interval, lets try to fast forward */
SimulationTime minNextEventTime = SIMTIME_INVALID;
item = g_list_first(nodeList);
while(item) {
Node* node = item->data;
EventQueue* eventq = node_getEvents(node);
Event* nextEvent = eventqueue_peek(eventq);
if(nextEvent && (nextEvent->time < minNextEventTime)) {
minNextEventTime = nextEvent->time;
}
item = g_list_next(item);
}
/* fast forward to the next event */
engine->executeWindowStart = minNextEventTime;
} else {
/* we still have events, lets just step one interval */
engine->executeWindowStart = engine->executeWindowEnd;
}
/* make sure we dont run over the end */
engine->executeWindowEnd = engine->executeWindowStart + engine->minTimeJump;
if(engine->executeWindowEnd > engine->endTime) {
engine->executeWindowEnd = engine->endTime;
}
/* reset for next round */
countdownlatch_reset(engine->processingLatch);
engine->numEventsCurrentInterval = 0;
engine->numNodesWithEventsCurrentInterval = 0;
/* if we are done, make sure the workers know about it */
if(engine->executeWindowStart >= engine->endTime) {
engine->killed = TRUE;
}
/* release the workers for the next round, or to exit */
countdownlatch_countDownAwait(engine->barrierLatch);
countdownlatch_reset(engine->barrierLatch);
}
/* wait for the threads to finish their cleanup */
GSList* threadItem = workerThreads;
while(threadItem) {
GThread* t = threadItem->data;
g_thread_join(t);
g_thread_unref(t);
threadItem = g_slist_next(threadItem);
}
g_slist_free(workerThreads);
for(gint i = 0; i < engine->config->nWorkerThreads; i++) {
g_slist_free(listArray[i]);
}
countdownlatch_free(engine->processingLatch);
countdownlatch_free(engine->barrierLatch);
/* frees the list struct we own, but not the nodes it holds (those were
* taken care of by the workers) */
g_list_free(nodeList);
return 0;
}
Scheduler* scheduler_new(SchedulerPolicyType policyType, guint nWorkers, gpointer threadUserData,
guint schedulerSeed, SimulationTime endTime) {
Scheduler* scheduler = g_new0(Scheduler, 1);
MAGIC_INIT(scheduler);
/* global lock */
g_mutex_init(&(scheduler->globalLock));
scheduler->startBarrier = countdownlatch_new(nWorkers+1);
scheduler->finishBarrier = countdownlatch_new(nWorkers+1);
scheduler->executeEventsBarrier = countdownlatch_new(nWorkers+1);
scheduler->collectInfoBarrier = countdownlatch_new(nWorkers+1);
scheduler->prepareRoundBarrier = countdownlatch_new(nWorkers+1);
scheduler->endTime = endTime;
scheduler->currentRound.endTime = scheduler->endTime;// default to one single round
scheduler->currentRound.minNextEventTime = SIMTIME_MAX;
scheduler->threadToWaitTimerMap = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, (GDestroyNotify)g_timer_destroy);
scheduler->hostIDToHostMap = g_hash_table_new(g_direct_hash, g_direct_equal);
scheduler->random = random_new(schedulerSeed);
/* ensure we have sane default modes for the number of workers we are using */
if(nWorkers == 0) {
scheduler->policyType = SP_SERIAL_GLOBAL;
} else if(nWorkers > 0 && policyType == SP_SERIAL_GLOBAL) {
policyType = SP_PARALLEL_HOST_STEAL;
} else {
scheduler->policyType = policyType;
}
/* create the configured policy to handle queues */
switch(scheduler->policyType) {
case SP_PARALLEL_HOST_SINGLE: {
scheduler->policy = schedulerpolicyhostsingle_new();
break;
}
case SP_PARALLEL_HOST_STEAL: {
scheduler->policy = schedulerpolicyhostsingle_new();
break;
}
case SP_PARALLEL_THREAD_SINGLE: {
scheduler->policy = schedulerpolicythreadsingle_new();
break;
}
case SP_PARALLEL_THREAD_PERTHREAD: {
scheduler->policy = schedulerpolicythreadperthread_new();
break;
}
case SP_PARALLEL_THREAD_PERHOST: {
scheduler->policy = schedulerpolicythreadperhost_new();
break;
}
case SP_SERIAL_GLOBAL:
default: {
scheduler->policy = schedulerpolicyglobalsingle_new();
break;
}
}
utility_assert(scheduler->policy);
/* make sure our ref count is set before starting the threads */
scheduler->referenceCount = 1;
scheduler->threadItems = g_queue_new();
/* start up threads and create worker storage, each thread will call worker_new,
* and wait at startBarrier until we are ready to launch */
for(gint i = 0; i < nWorkers; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i));
SchedulerThreadItem* item = g_new0(SchedulerThreadItem, 1);
item->notifyDoneRunning = countdownlatch_new(1);
WorkerRunData* runData = g_new0(WorkerRunData, 1);
runData->userData = threadUserData;
runData->scheduler = scheduler;
runData->threadID = i;
runData->notifyDoneRunning = item->notifyDoneRunning;
gint returnVal = pthread_create(&(item->thread), NULL, (void*(*)(void*))worker_run, runData);
if(returnVal != 0) {
critical("unable to create worker thread");
return NULL;
}
returnVal = pthread_setname_np(item->thread, name->str);
if(returnVal != 0) {
warning("unable to set name of worker thread to '%s'", name->str);
}
utility_assert(item->thread);
g_queue_push_tail(scheduler->threadItems, item);
logger_register(logger_getDefault(), item->thread);
g_string_free(name, TRUE);
}
message("main scheduler thread will operate with %u worker threads", nWorkers);
return scheduler;
}