gpointer worker_runSerial(WorkLoad* workload) {
utility_assert(workload);
Worker* worker = _worker_getPrivate();
Event* nextEvent = eventqueue_peek(worker->serialEventQueue);
if(nextEvent) {
worker->clock_now = SIMTIME_INVALID;
worker->clock_last = 0;
/* process all events in the priority queue */
while(nextEvent &&
(shadowevent_getTime(nextEvent) < slave_getExecuteWindowEnd(worker->slave)) &&
(shadowevent_getTime(nextEvent) < slave_getEndTime(worker->slave)))
{
/* get next event */
nextEvent = eventqueue_pop(worker->serialEventQueue);
worker->cached_event = nextEvent;
worker->cached_node = shadowevent_getNode(nextEvent);
/* ensure priority */
worker->clock_now = shadowevent_getTime(worker->cached_event);
// engine->clock = worker->clock_now;
utility_assert(worker->clock_now >= worker->clock_last);
gboolean complete = shadowevent_run(worker->cached_event);
if(complete) {
shadowevent_free(worker->cached_event);
}
worker->cached_event = NULL;
worker->cached_node = NULL;
worker->clock_last = worker->clock_now;
worker->clock_now = SIMTIME_INVALID;
nextEvent = eventqueue_peek(worker->serialEventQueue);
}
}
slave_setKilled(worker->slave, TRUE);
/* in single thread mode, we must free the nodes */
GList* hosts = workload->hosts;
while(hosts) {
worker->cached_node = hosts->data;
host_freeAllApplications(worker->cached_node);
worker->cached_node = NULL;
hosts = hosts->next;
}
g_list_foreach(workload->hosts, (GFunc) host_free, NULL);
return NULL;
}
static guint _worker_processNode(Worker* worker, Node* node, SimulationTime barrier) {
/* update cache, reset clocks */
worker->cached_node = node;
worker->clock_last = SIMTIME_INVALID;
worker->clock_now = SIMTIME_INVALID;
worker->clock_barrier = barrier;
/* lock the node */
node_lock(worker->cached_node);
EventQueue* eventq = node_getEvents(worker->cached_node);
Event* nextEvent = eventqueue_peek(eventq);
/* process all events in the nodes local queue */
guint nEventsProcessed = 0;
while(nextEvent && (nextEvent->time < worker->clock_barrier))
{
worker->cached_event = eventqueue_pop(eventq);
MAGIC_ASSERT(worker->cached_event);
/* make sure we don't jump backward in time */
worker->clock_now = worker->cached_event->time;
if(worker->clock_last != SIMTIME_INVALID) {
g_assert(worker->clock_now >= worker->clock_last);
}
/* do the local task */
gboolean complete = shadowevent_run(worker->cached_event);
/* update times */
worker->clock_last = worker->clock_now;
worker->clock_now = SIMTIME_INVALID;
/* finished event can now be destroyed */
if(complete) {
shadowevent_free(worker->cached_event);
nEventsProcessed++;
}
/* get the next event, or NULL will tell us to break */
nextEvent = eventqueue_peek(eventq);
}
/* unlock, clear cache */
node_unlock(worker->cached_node);
worker->cached_node = NULL;
worker->cached_event = NULL;
return nEventsProcessed;
}
static gint _engine_distributeEvents(Engine* engine) {
MAGIC_ASSERT(engine);
GList* nodeList = internetwork_getAllNodes(engine->internet);
/* assign nodes to the worker threads so they get processed */
GSList* listArray[engine->config->nWorkerThreads];
memset(listArray, 0, engine->config->nWorkerThreads * sizeof(GSList*));
gint counter = 0;
GList* item = g_list_first(nodeList);
while(item) {
Node* node = item->data;
gint i = counter % engine->config->nWorkerThreads;
listArray[i] = g_slist_append(listArray[i], node);
counter++;
item = g_list_next(item);
}
/* we will track when workers finish processing their nodes */
engine->processingLatch = countdownlatch_new(engine->config->nWorkerThreads + 1);
/* after the workers finish processing, wait for barrier update */
engine->barrierLatch = countdownlatch_new(engine->config->nWorkerThreads + 1);
/* start up the workers */
GSList* workerThreads = NULL;
for(gint i = 0; i < engine->config->nWorkerThreads; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i+1));
GThread* t = g_thread_new(name->str, (GThreadFunc)worker_run, (gpointer)listArray[i]);
workerThreads = g_slist_append(workerThreads, t);
g_string_free(name, TRUE);
}
/* process all events in the priority queue */
while(engine->executeWindowStart < engine->endTime)
{
/* wait for the workers to finish processing nodes before we touch them */
countdownlatch_countDownAwait(engine->processingLatch);
/* we are in control now, the workers are waiting at barrierLatch */
message("execution window [%lu--%lu] ran %u events from %u active nodes",
engine->executeWindowStart, engine->executeWindowEnd,
engine->numEventsCurrentInterval,
engine->numNodesWithEventsCurrentInterval);
/* check if we should take 1 step ahead or fast-forward our execute window.
* since looping through all the nodes to find the minimum event is
* potentially expensive, we use a heuristic of only trying to jump ahead
* if the last interval had only a few events in it. */
if(engine->numEventsCurrentInterval < 10) {
/* we had no events in that interval, lets try to fast forward */
SimulationTime minNextEventTime = SIMTIME_INVALID;
item = g_list_first(nodeList);
while(item) {
Node* node = item->data;
EventQueue* eventq = node_getEvents(node);
Event* nextEvent = eventqueue_peek(eventq);
if(nextEvent && (nextEvent->time < minNextEventTime)) {
minNextEventTime = nextEvent->time;
}
item = g_list_next(item);
}
/* fast forward to the next event */
engine->executeWindowStart = minNextEventTime;
} else {
/* we still have events, lets just step one interval */
engine->executeWindowStart = engine->executeWindowEnd;
}
/* make sure we dont run over the end */
engine->executeWindowEnd = engine->executeWindowStart + engine->minTimeJump;
if(engine->executeWindowEnd > engine->endTime) {
engine->executeWindowEnd = engine->endTime;
}
/* reset for next round */
countdownlatch_reset(engine->processingLatch);
engine->numEventsCurrentInterval = 0;
engine->numNodesWithEventsCurrentInterval = 0;
/* if we are done, make sure the workers know about it */
if(engine->executeWindowStart >= engine->endTime) {
engine->killed = TRUE;
}
/* release the workers for the next round, or to exit */
countdownlatch_countDownAwait(engine->barrierLatch);
countdownlatch_reset(engine->barrierLatch);
}
/* wait for the threads to finish their cleanup */
GSList* threadItem = workerThreads;
while(threadItem) {
GThread* t = threadItem->data;
g_thread_join(t);
g_thread_unref(t);
threadItem = g_slist_next(threadItem);
}
g_slist_free(workerThreads);
for(gint i = 0; i < engine->config->nWorkerThreads; i++) {
g_slist_free(listArray[i]);
}
countdownlatch_free(engine->processingLatch);
countdownlatch_free(engine->barrierLatch);
/* frees the list struct we own, but not the nodes it holds (those were
* taken care of by the workers) */
g_list_free(nodeList);
return 0;
}
void slave_runParallel(Slave* slave) {
MAGIC_ASSERT(slave);
GList* nodeList = _slave_getAllHosts(slave);
/* assign nodes to the worker threads so they get processed */
WorkLoad workArray[slave->nWorkers];
memset(workArray, 0, slave->nWorkers * sizeof(WorkLoad));
gint counter = 0;
GList* item = g_list_first(nodeList);
while(item) {
Host* node = item->data;
gint i = counter % slave->nWorkers;
workArray[i].hosts = g_list_append(workArray[i].hosts, node);
counter++;
item = g_list_next(item);
}
/* we will track when workers finish processing their nodes */
slave->processingLatch = countdownlatch_new(slave->nWorkers + 1);
/* after the workers finish processing, wait for barrier update */
slave->barrierLatch = countdownlatch_new(slave->nWorkers + 1);
/* start up the workers */
GSList* workerThreads = NULL;
for(gint i = 0; i < slave->nWorkers; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i+1));
workArray[i].slave = slave;
workArray[i].master = slave->master;
GThread* t = g_thread_new(name->str, (GThreadFunc)worker_runParallel, &(workArray[i]));
workerThreads = g_slist_append(workerThreads, t);
g_string_free(name, TRUE);
}
message("started %i worker threads", slave->nWorkers);
/* process all events in the priority queue */
while(master_getExecuteWindowStart(slave->master) < master_getEndTime(slave->master))
{
/* wait for the workers to finish processing nodes before we touch them */
countdownlatch_countDownAwait(slave->processingLatch);
/* we are in control now, the workers are waiting at barrierLatch */
info("execution window [%"G_GUINT64_FORMAT"--%"G_GUINT64_FORMAT"] ran %u events from %u active nodes",
master_getExecuteWindowStart(slave->master), master_getExecuteWindowEnd(slave->master),
slave->numEventsCurrentInterval, slave->numNodesWithEventsCurrentInterval);
/* check if we should take 1 step ahead or fast-forward our execute window.
* since looping through all the nodes to find the minimum event is
* potentially expensive, we use a heuristic of only trying to jump ahead
* if the last interval had only a few events in it. */
SimulationTime minNextEventTime = SIMTIME_INVALID;
if(slave->numEventsCurrentInterval < 10) {
/* we had no events in that interval, lets try to fast forward */
item = g_list_first(nodeList);
/* fast forward to the next event, the new window start will be the next event time */
while(item) {
Host* node = item->data;
EventQueue* eventq = host_getEvents(node);
Event* nextEvent = eventqueue_peek(eventq);
SimulationTime nextEventTime = shadowevent_getTime(nextEvent);
if(nextEvent && (nextEventTime < minNextEventTime)) {
minNextEventTime = nextEventTime;
}
item = g_list_next(item);
}
if(minNextEventTime == SIMTIME_INVALID) {
minNextEventTime = master_getExecuteWindowEnd(slave->master);
}
} else {
/* we still have events, lets just step one interval,
* consider the next event as the previous window end */
minNextEventTime = master_getExecuteWindowEnd(slave->master);
}
/* notify master that we finished this round, and what our next event is */
master_slaveFinishedCurrentWindow(slave->master, minNextEventTime);
/* reset for next round */
countdownlatch_reset(slave->processingLatch);
slave->numEventsCurrentInterval = 0;
slave->numNodesWithEventsCurrentInterval = 0;
/* release the workers for the next round, or to exit */
countdownlatch_countDownAwait(slave->barrierLatch);
countdownlatch_reset(slave->barrierLatch);
}
message("waiting for %i worker threads to finish", slave->nWorkers);
/* wait for the threads to finish their cleanup */
GSList* threadItem = workerThreads;
while(threadItem) {
GThread* t = threadItem->data;
/* the join will consume the reference, so unref is not needed */
g_thread_join(t);
threadItem = g_slist_next(threadItem);
}
g_slist_free(workerThreads);
message("%i worker threads finished", slave->nWorkers);
for(gint i = 0; i < slave->nWorkers; i++) {
WorkLoad w = workArray[i];
g_list_free(w.hosts);
}
countdownlatch_free(slave->processingLatch);
countdownlatch_free(slave->barrierLatch);
/* frees the list struct we own, but not the nodes it holds (those were
* taken care of by the workers) */
g_list_free(nodeList);
}