gint master_run(Master* master) {
MAGIC_ASSERT(master);
message("loading and initializing simulation data");
/* start loading and initializing simulation data */
_master_loadConfiguration(master);
gboolean isSuccess = _master_loadTopology(master);
if(!isSuccess) {
return 1;
}
_master_initializeTimeWindows(master);
/* the master will be responsible for distributing the actions to the slaves so that
* they all have a consistent view of the simulation, topology, etc.
* For now we only have one slave so send it everything. */
guint slaveSeed = random_nextUInt(master->random);
master->slave = slave_new(master, master->options, master->endTime, slaveSeed);
message("registering plugins and hosts");
/* register the components needed by each slave.
* this must be done after slaves are available so we can send them messages */
_master_registerPlugins(master);
_master_registerHosts(master);
message("running simulation");
/* dont buffer log messages in debug mode */
if(options_getLogLevel(master->options) != LOGLEVEL_DEBUG) {
message("log message buffering is enabled for efficiency");
logger_setEnableBuffering(logger_getDefault(), TRUE);
}
/* start running each slave */
slave_run(master->slave);
/* only need to disable buffering if it was enabled, otherwise
* don't log the message as it may confuse the user. */
if(options_getLogLevel(master->options) != LOGLEVEL_DEBUG) {
message("log message buffering is disabled during cleanup");
logger_setEnableBuffering(logger_getDefault(), FALSE);
}
message("simulation finished, cleaning up now");
return slave_free(master->slave);
}
Event* scheduler_pop(Scheduler* scheduler) {
MAGIC_ASSERT(scheduler);
/* this function should block until a non-null event is available for the worker to run.
* return NULL only to signal the worker thread to quit */
while(scheduler->isRunning) {
/* pop from a queue based on the policy */
Event* nextEvent = scheduler->policy->pop(scheduler->policy, scheduler->currentRound.endTime);
if(nextEvent != NULL) {
/* we have an event, let the worker run it */
return nextEvent;
} else if(scheduler->policyType == SP_SERIAL_GLOBAL) {
/* the running thread has no more events to execute this round, but we only have a
* single, global, serial queue, so returning NULL without blocking is OK. */
return NULL;
} else {
/* the running thread has no more events to execute this round and we need to block it
* so that we can wait for all threads to finish events from this round. We want to
* track idle times, so let's start by making sure we have timer elements in place. */
GTimer* executeEventsBarrierWaitTime = g_hash_table_lookup(scheduler->threadToWaitTimerMap, GUINT_TO_POINTER(pthread_self()));
/* wait for all other worker threads to finish their events too, and track wait time */
if(executeEventsBarrierWaitTime) {
g_timer_continue(executeEventsBarrierWaitTime);
}
countdownlatch_countDownAwait(scheduler->executeEventsBarrier);
if(executeEventsBarrierWaitTime) {
g_timer_stop(executeEventsBarrierWaitTime);
}
/* now all threads reached the current round end barrier time.
* asynchronously collect some stats that the main thread will use. */
if(scheduler->policy->getNextTime) {
SimulationTime nextTime = scheduler->policy->getNextTime(scheduler->policy);
g_mutex_lock(&(scheduler->globalLock));
scheduler->currentRound.minNextEventTime = MIN(scheduler->currentRound.minNextEventTime, nextTime);
g_mutex_unlock(&(scheduler->globalLock));
}
/* clear all log messages from the last round */
logger_flushRecords(logger_getDefault(), pthread_self());
/* wait for other threads to finish their collect step */
countdownlatch_countDownAwait(scheduler->collectInfoBarrier);
/* now wait for main thread to process a barrier update for the next round */
countdownlatch_countDownAwait(scheduler->prepareRoundBarrier);
}
}
/* scheduler is done, return NULL to stop worker */
return NULL;
}
gint main_runShadow(gint argc, gchar* argv[]) {
/* check the compiled GLib version */
if (!GLIB_CHECK_VERSION(2, 32, 0)) {
g_printerr("** GLib version 2.32.0 or above is required but Shadow was compiled against version %u.%u.%u\n",
(guint)GLIB_MAJOR_VERSION, (guint)GLIB_MINOR_VERSION, (guint)GLIB_MICRO_VERSION);
return EXIT_FAILURE;
}
if(GLIB_MAJOR_VERSION == 2 && GLIB_MINOR_VERSION == 40) {
g_printerr("** You compiled against GLib version %u.%u.%u, which has bugs known to break Shadow. Please update to a newer version of GLib.\n",
(guint)GLIB_MAJOR_VERSION, (guint)GLIB_MINOR_VERSION, (guint)GLIB_MICRO_VERSION);
return EXIT_FAILURE;
}
/* check the that run-time GLib matches the compiled version */
const gchar* mismatch = glib_check_version(GLIB_MAJOR_VERSION, GLIB_MINOR_VERSION, GLIB_MICRO_VERSION);
if(mismatch) {
g_printerr("** The version of the run-time GLib library (%u.%u.%u) is not compatible with the version against which Shadow was compiled (%u.%u.%u). GLib message: '%s'\n",
glib_major_version, glib_minor_version, glib_micro_version,
(guint)GLIB_MAJOR_VERSION, (guint)GLIB_MINOR_VERSION, (guint)GLIB_MICRO_VERSION,
mismatch);
return EXIT_FAILURE;
}
/* parse the options from the command line */
gchar* cmds = g_strjoinv(" ", argv);
gchar** cmdv = g_strsplit(cmds, " ", 0);
g_free(cmds);
Options* options = options_new(argc, cmdv);
g_strfreev(cmdv);
if(!options) {
return EXIT_FAILURE;
}
/* if they just want the shadow version, print it and exit */
if(options_doRunPrintVersion(options)) {
g_printerr("%s running GLib v%u.%u.%u and IGraph v%s\n%s\n",
SHADOW_VERSION_STRING,
(guint)GLIB_MAJOR_VERSION, (guint)GLIB_MINOR_VERSION, (guint)GLIB_MICRO_VERSION,
#if defined(IGRAPH_VERSION)
IGRAPH_VERSION,
#else
"(n/a)",
#endif
SHADOW_INFO_STRING);
options_free(options);
return EXIT_SUCCESS;
}
/* start up the logging subsystem to handle all future messages */
Logger* shadowLogger = logger_new(options_getLogLevel(options));
logger_setDefault(shadowLogger);
/* disable buffering during startup so that we see every message immediately in the terminal */
logger_setEnableBuffering(shadowLogger, FALSE);
gint returnCode = _main_helper(options);
options_free(options);
Logger* logger = logger_getDefault();
if(logger) {
logger_setDefault(NULL);
logger_unref(logger);
}
g_printerr("** Stopping Shadow, returning code %i (%s)\n", returnCode, (returnCode == 0) ? "success" : "error");
return returnCode;
}
static gint _main_helper(Options* options) {
/* start off with some status messages */
#if defined(IGRAPH_VERSION)
gint igraphMajor = -1, igraphMinor = -1, igraphPatch = -1;
igraph_version(NULL, &igraphMajor, &igraphMinor, &igraphPatch);
gchar* startupStr = g_strdup_printf("Starting %s with GLib v%u.%u.%u and IGraph v%i.%i.%i",
SHADOW_VERSION_STRING,
(guint)GLIB_MAJOR_VERSION, (guint)GLIB_MINOR_VERSION, (guint)GLIB_MICRO_VERSION,
igraphMajor, igraphMinor, igraphPatch);
#else
gchar* startupStr = g_strdup_printf("Starting %s with GLib v%u.%u.%u (IGraph version not available)",
SHADOW_VERSION_STRING,
(guint)GLIB_MAJOR_VERSION, (guint)GLIB_MINOR_VERSION, (guint)GLIB_MICRO_VERSION);
#endif
message("%s", startupStr);
/* avoid logging the message to stderr twice (only log if this is not a relaunch) */
if(g_getenv("SHADOW_SPAWNED") == NULL) {
g_printerr("** %s\n", startupStr);
}
g_free(startupStr);
message(SHADOW_INFO_STRING);
message("logging current startup arguments and environment");
gchar** envlist = g_get_environ();
gchar** arglist = g_strsplit(options_getArgumentString(options), " ", 0);
_main_logEnvironment(arglist, envlist);
g_strfreev(arglist);
g_strfreev(envlist);
/* check if we still need to setup our required environment and relaunch */
if(g_getenv("SHADOW_SPAWNED") == NULL) {
message("shadow will automatically adjust environment and relaunch");
message("loading shadow configuration file");
/* we need to relaunch.
* first lets load the config file to help us setup the environment */
const GString* fileName = options_getInputXMLFilename(options);
if(!fileName) {
return EXIT_FAILURE;
}
GString* file = utility_getFileContents(fileName->str);
if(!file) {
critical("unable to read config file contents");
return EXIT_FAILURE;
}
Configuration* config = configuration_new(options, file);
g_string_free(file, TRUE);
file = NULL;
if(!config) {
critical("there was a problem parsing the Shadow config file, and we can't run without it");
return EXIT_FAILURE;
}
message("shadow configuration file loaded, parsed, and passed validation");
/* now start to set up the environment */
gchar** envlist = g_get_environ();
GString* commandBuffer = g_string_new(options_getArgumentString(options));
if(!envlist || !commandBuffer) {
critical("there was a problem loading existing environment");
configuration_free(config);
return EXIT_FAILURE;
}
message("setting up LD_PRELOAD environment");
/* compute the proper LD_PRELOAD value, extract the shadow preload file and
* set it as a command line argument if needed */
gchar* preloadArgValue = NULL;
{
/* before we restart, we should:
* -set the shadow preload lib as a command line arg
* -make sure it does not exist in LD_PRELOAD, but otherwise leave LD_PRELOAD in place
* we need to search for our preload lib. our order of preference follows.
*/
/* 1. existing "--preload=" option value */
if(_main_isValidPathToPreloadLib(options_getPreloadString(options))) {
preloadArgValue = g_strdup(options_getPreloadString(options));
}
/* 2. the 'preload' attribute value of the 'shadow' element in shadow.config.xml */
/* we only need to search if we haven't already found a valid path */
if(!preloadArgValue) {
ConfigurationShadowElement* element = configuration_getShadowElement(config);
if(element && element->preloadPath.isSet) {
gchar* path = element->preloadPath.string->str;
if(_main_isValidPathToPreloadLib(path)) {
preloadArgValue = g_strdup(path);
}
}
}
/* 3. the LD_PRELOAD value */
GString* preloadEnvValueBuffer = NULL;
/* we always search the env variable and remove existing Shadow preload libs */
if(g_environ_getenv(envlist, "LD_PRELOAD") != NULL) {
gchar** tokens = g_strsplit(g_environ_getenv(envlist, "LD_PRELOAD"), ":", 0);
for(gint i = 0; tokens[i] != NULL; i++) {
/* each token in the env variable should be an absolute path */
if(_main_isValidPathToPreloadLib(tokens[i])) {
/* found a valid path, only save it if we don't have one yet (from options or config) */
if(!preloadArgValue) {
preloadArgValue = g_strdup(tokens[i]);
}
} else {
/* maintain non-shadow entries */
if(preloadEnvValueBuffer) {
g_string_append_printf(preloadEnvValueBuffer, ":%s", tokens[i]);
} else {
preloadEnvValueBuffer = g_string_new(tokens[i]);
}
}
}
g_strfreev(tokens);
}
/* 4. the 'environment' attribute of the 'shadow' configuration element in shadow.config.xml */
/* always go through the 'environment' attribute of the 'shadow' element to pull in any keys defined there */
{
ConfigurationShadowElement* element = configuration_getShadowElement(config);
if(element && element->environment.isSet) {
/* entries are split by ';' */
gchar** envTokens = g_strsplit(element->environment.string->str, ";", 0);
for(gint i = 0; envTokens[i] != NULL; i++) {
/* each env entry is key=value, get 2 tokens max */
gchar** items = g_strsplit(envTokens[i], "=", 2);
gchar* key = items[0];
gchar* value = items[1];
if(key != NULL && value != NULL) {
/* check if the key is LD_PRELOAD */
if(!g_ascii_strncasecmp(key, "LD_PRELOAD", 10)) {
gchar** preloadTokens = g_strsplit(value, ":", 0);
for(gint j = 0; preloadTokens[j] != NULL; j++) {
/* each token in the env variable should be an absolute path */
if(_main_isValidPathToPreloadLib(preloadTokens[j])) {
/* found a valid path, only save it if we don't have one yet (from options or config) */
if(!preloadArgValue) {
preloadArgValue = g_strdup(preloadTokens[j]);
}
} else {
/* maintain non-shadow entries */
if(preloadEnvValueBuffer) {
g_string_append_printf(preloadEnvValueBuffer, ":%s", preloadTokens[j]);
} else {
preloadEnvValueBuffer = g_string_new(preloadTokens[j]);
}
}
}
g_strfreev(preloadTokens);
} else {
/* set the key=value pair, but don't overwrite any existing settings */
envlist = g_environ_setenv(envlist, key, value, 0);
}
}
g_strfreev(items);
}
g_strfreev(envTokens);
}
}
/* save away the non-shadow preload libs */
if(preloadEnvValueBuffer) {
envlist = g_environ_setenv(envlist, "LD_PRELOAD", preloadEnvValueBuffer->str, 1);
g_string_free(preloadEnvValueBuffer, TRUE);
preloadEnvValueBuffer = NULL;
} else {
envlist = g_environ_unsetenv(envlist, "LD_PRELOAD");
}
/* 5. as a last hope, try looking in RPATH since shadow is built with one */
/* we only need to search if we haven't already found a valid path */
if(!preloadArgValue) {
gchar* rpathStr = _main_getRPath();
if(rpathStr != NULL) {
gchar** tokens = g_strsplit(rpathStr, ":", 0);
for(gint i = 0; tokens[i] != NULL; i++) {
GString* candidateBuffer = g_string_new(NULL);
/* rpath specifies directories, so look inside */
g_string_printf(candidateBuffer, "%s/%s", tokens[i], INTERPOSELIBSTR);
gchar* candidate = g_string_free(candidateBuffer, FALSE);
if(_main_isValidPathToPreloadLib(candidate)) {
preloadArgValue = candidate;
break;
} else {
g_free(candidate);
}
}
g_strfreev(tokens);
}
g_free(rpathStr);
}
/* if we still didn't find our preload lib, that is a user error */
if(!preloadArgValue) {
critical("can't find path to %s, did you specify an absolute path to an existing readable file?", INTERPOSELIBSTR);
configuration_free(config);
return EXIT_FAILURE;
}
/* now that we found the correct path to the preload lib, first remove any possibly
* incomplete path that might exist in the command line args, and then replace it
* with the path that we found and verified is correct. */
{
/* first remove all preload options */
gchar** tokens = g_strsplit(commandBuffer->str, " ", 0);
g_string_free(commandBuffer, TRUE);
commandBuffer = NULL;
for(gint i = 0; tokens[i] != NULL; i++) {
/* use -1 to search the entire string */
if(!g_ascii_strncasecmp(tokens[i], "--preload=", 10)) {
/* skip this key=value string */
} else if(!g_ascii_strncasecmp(tokens[i], "-p", 2)) {
/* skip this key, and also the next arg which is the value */
i++;
} else {
if(commandBuffer) {
g_string_append_printf(commandBuffer, " %s", tokens[i]);
} else {
commandBuffer = g_string_new(tokens[i]);
}
}
}
g_strfreev(tokens);
/* now add back in the preload option */
g_string_append_printf(commandBuffer, " --preload=%s", preloadArgValue);
}
}
message("setting up LD_STATIC_TLS_EXTRA environment");
/* compute the proper TLS size we need for dlmopen()ing all of the plugins,
* but only do this if the user didn't manually specify a size */
if(g_environ_getenv(envlist, "LD_STATIC_TLS_EXTRA") == NULL) {
gchar* staticTLSValue = _main_getStaticTLSValue(options, config, preloadArgValue);
envlist = g_environ_setenv(envlist, "LD_STATIC_TLS_EXTRA", staticTLSValue, 0);
message("we need %s total bytes of static TLS storage", staticTLSValue);
g_free(staticTLSValue);
}
/* cleanup unused string */
if(preloadArgValue) {
g_free(preloadArgValue);
preloadArgValue = NULL;
}
/* are we running valgrind */
if(options_doRunValgrind(options)) {
message("setting up environment for valgrind");
/* make glib friendlier to valgrind */
envlist = g_environ_setenv(envlist, "G_DEBUG", "gc-friendly", 0);
envlist = g_environ_setenv(envlist, "G_SLICE", "always-malloc", 0);
/* add the valgrind command and some default options */
g_string_prepend(commandBuffer,
"valgrind --leak-check=full --show-reachable=yes --track-origins=yes --trace-children=yes --log-file=shadow-valgrind-%p.log --error-limit=no ");
} else {
/* The following can be used to add internal GLib memory validation that
* will abort the program if it finds an error. This is only useful outside
* of the valgrind context, as otherwise valgrind will complain about
* the implementation of the GLib validator.
* e.g. $ G_SLICE=debug-blocks shadow --file
*
* envlist = g_environ_setenv(envlist, "G_SLICE", "debug-blocks", 0);
*/
}
/* keep track that we are relaunching shadow */
envlist = g_environ_setenv(envlist, "SHADOW_SPAWNED", "TRUE", 1);
gchar* command = g_string_free(commandBuffer, FALSE);
gchar** arglist = g_strsplit(command, " ", 0);
g_free(command);
configuration_free(config);
message("environment was updated; shadow is relaunching now with new environment");
Logger* logger = logger_getDefault();
if(logger) {
logger_setDefault(NULL);
logger_unref(logger);
}
/* execvpe only returns if there is an error, otherwise the current process
* image is replaced with a new process */
gint returnValue = execvpe(arglist[0], arglist, envlist);
/* cleanup */
if(envlist) {
g_strfreev(envlist);
}
if(arglist) {
g_strfreev(arglist);
}
critical("** Error %i while re-launching shadow process: %s", returnValue, g_strerror(returnValue));
return EXIT_FAILURE;
}
/* we dont need to relaunch, so we can run the simulation */
/* make sure we have initialized static tls */
if(!_main_verifyStaticTLS()) {
critical("** Shadow Setup Check Failed: LD_STATIC_TLS_EXTRA does not contain a nonzero value");
return EXIT_FAILURE;
}
/* make sure we have the shadow preload lib */
if(!_main_isValidPathToPreloadLib(options_getPreloadString(options))) {
critical("** Shadow Setup Check Failed: cannot find absolute path to "INTERPOSELIBSTR"");
return EXIT_FAILURE;
}
/* now load the preload library into shadow's namespace */
if(!_main_loadShadowPreload(options)) {
critical("** Shadow Setup Check Failed: unable to load preload library");
return EXIT_FAILURE;
}
/* tell the preload lib we are ready for action */
extern int interposer_setShadowIsLoaded(int);
int interposerResult = interposer_setShadowIsLoaded(1);
if(interposerResult != 0) {
/* it was not intercepted, meaning our preload library is not set up properly */
critical("** Shadow Setup Check Failed: preload library is not correctly interposing functions");
return EXIT_FAILURE;
}
message("startup checks passed, we are ready to start simulation");
/* pause for debugger attachment if the option is set */
if(options_doRunDebug(options)) {
gint pid = (gint)getpid();
message("Pausing with SIGTSTP to enable debugger attachment (pid %i)", pid);
g_printerr("** Pausing with SIGTSTP to enable debugger attachment (pid %i)\n", pid);
raise(SIGTSTP);
message("Resuming now");
}
/* allocate and initialize our main simulation driver */
gint returnCode = 0;
shadowMaster = master_new(options);
message("log message buffering is enabled for efficiency");
logger_setEnableBuffering(logger_getDefault(), TRUE);
if(shadowMaster) {
/* run the simulation */
returnCode = master_run(shadowMaster);
/* cleanup */
master_free(shadowMaster);
shadowMaster = NULL;
}
message("%s simulation was shut down cleanly, returning code %i", SHADOW_VERSION_STRING, returnCode);
return returnCode;
}
Scheduler* scheduler_new(SchedulerPolicyType policyType, guint nWorkers, gpointer threadUserData,
guint schedulerSeed, SimulationTime endTime) {
Scheduler* scheduler = g_new0(Scheduler, 1);
MAGIC_INIT(scheduler);
/* global lock */
g_mutex_init(&(scheduler->globalLock));
scheduler->startBarrier = countdownlatch_new(nWorkers+1);
scheduler->finishBarrier = countdownlatch_new(nWorkers+1);
scheduler->executeEventsBarrier = countdownlatch_new(nWorkers+1);
scheduler->collectInfoBarrier = countdownlatch_new(nWorkers+1);
scheduler->prepareRoundBarrier = countdownlatch_new(nWorkers+1);
scheduler->endTime = endTime;
scheduler->currentRound.endTime = scheduler->endTime;// default to one single round
scheduler->currentRound.minNextEventTime = SIMTIME_MAX;
scheduler->threadToWaitTimerMap = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, (GDestroyNotify)g_timer_destroy);
scheduler->hostIDToHostMap = g_hash_table_new(g_direct_hash, g_direct_equal);
scheduler->random = random_new(schedulerSeed);
/* ensure we have sane default modes for the number of workers we are using */
if(nWorkers == 0) {
scheduler->policyType = SP_SERIAL_GLOBAL;
} else if(nWorkers > 0 && policyType == SP_SERIAL_GLOBAL) {
policyType = SP_PARALLEL_HOST_STEAL;
} else {
scheduler->policyType = policyType;
}
/* create the configured policy to handle queues */
switch(scheduler->policyType) {
case SP_PARALLEL_HOST_SINGLE: {
scheduler->policy = schedulerpolicyhostsingle_new();
break;
}
case SP_PARALLEL_HOST_STEAL: {
scheduler->policy = schedulerpolicyhostsingle_new();
break;
}
case SP_PARALLEL_THREAD_SINGLE: {
scheduler->policy = schedulerpolicythreadsingle_new();
break;
}
case SP_PARALLEL_THREAD_PERTHREAD: {
scheduler->policy = schedulerpolicythreadperthread_new();
break;
}
case SP_PARALLEL_THREAD_PERHOST: {
scheduler->policy = schedulerpolicythreadperhost_new();
break;
}
case SP_SERIAL_GLOBAL:
default: {
scheduler->policy = schedulerpolicyglobalsingle_new();
break;
}
}
utility_assert(scheduler->policy);
/* make sure our ref count is set before starting the threads */
scheduler->referenceCount = 1;
scheduler->threadItems = g_queue_new();
/* start up threads and create worker storage, each thread will call worker_new,
* and wait at startBarrier until we are ready to launch */
for(gint i = 0; i < nWorkers; i++) {
GString* name = g_string_new(NULL);
g_string_printf(name, "worker-%i", (i));
SchedulerThreadItem* item = g_new0(SchedulerThreadItem, 1);
item->notifyDoneRunning = countdownlatch_new(1);
WorkerRunData* runData = g_new0(WorkerRunData, 1);
runData->userData = threadUserData;
runData->scheduler = scheduler;
runData->threadID = i;
runData->notifyDoneRunning = item->notifyDoneRunning;
gint returnVal = pthread_create(&(item->thread), NULL, (void*(*)(void*))worker_run, runData);
if(returnVal != 0) {
critical("unable to create worker thread");
return NULL;
}
returnVal = pthread_setname_np(item->thread, name->str);
if(returnVal != 0) {
warning("unable to set name of worker thread to '%s'", name->str);
}
utility_assert(item->thread);
g_queue_push_tail(scheduler->threadItems, item);
logger_register(logger_getDefault(), item->thread);
g_string_free(name, TRUE);
}
message("main scheduler thread will operate with %u worker threads", nWorkers);
return scheduler;
}
