/**
@details
-# Wait for all job dependencies to complete. Requirement [@ref r_exec_thread_6]
-# Call the job. Requirement [@ref r_exec_periodic_0]
-# If the job is a system job, check to see if the next job call time is the lowest next time by
calling Trick::ScheduledJobQueue::test_next_job_call_time(Trick::JobData *, long long)
-# Set the job complete flag
*/
static int call_next_job(Trick::JobData * curr_job, Trick::ScheduledJobQueue & job_queue, bool rt_nap, long long curr_time_tics) {
Trick::JobData * depend_job ;
unsigned int ii ;
int ret = 0 ;
//cout << "time = " << curr_time_tics << " " << curr_job->name << " job next = "
// << curr_job->next_tics << " id = " << curr_job->id << endl ;
/* Wait for all jobs that the current job depends on to complete. */
for ( ii = 0 ; ii < curr_job->depends.size() ; ii++ ) {
depend_job = curr_job->depends[ii] ;
while (! depend_job->complete) {
if (rt_nap == true) {
RELEASE();
}
}
}
/* Call the current scheduled job. */
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , "scheduled job did not return 0") ;
}
/* System jobs next call time are not set until after they run. We test their next job call time here. */
if ( curr_job->system_job_class ) {
job_queue.test_next_job_call_time(curr_job , curr_time_tics) ;
}
curr_job->complete = true ;
return 0 ;
}
int Trick::Executive::get_freeze_job(std::string sim_object_name) {
freeze_job = get_job(sim_object_name + ".sched_freeze_to_exec_command") ;
if ( freeze_job == NULL ) {
exec_terminate_with_return(-1 , __FILE__ , __LINE__ , "Executive could not find freeze job" ) ;
} else {
freeze_job->next_tics = TRICK_MAX_LONG_LONG ;
}
return 0 ;
}
/**
@details
-# If the timer is enabled
-# If the frame time is valid
-# Set the system itimer to expire one time at the desired frame period.
-# Else termiate the simulation with the error message stating that the
itimer frame period is invalid.
*/
int Trick::ITimer::start(double in_frame_time) {
struct itimerval ivalue;
int ret ;
unsigned long long frame_sec ;
unsigned int frame_usec ;
if ( enabled ) {
// clear out any built up signals.
while ( (ret = sem_trywait(semaphore)) == 0 ) ;
frame_sec = ( unsigned long long )in_frame_time ;
frame_usec = (unsigned int)((in_frame_time - frame_sec) * 1000000) ;
// check minimum time > 10ms
if ( frame_sec == 0 && frame_usec < 10000 ) {
active = false ;
return(0) ;
} else if ( frame_usec < 2000 && frame_sec > 0 ) {
// adjust times with usec < 2000 , we'll be subtracting 2ms from the frame_usec
frame_sec -= 1 ;
frame_usec += 1000000 ;
}
frame_usec -= 2000 ;
if ( in_frame_time > 0 ) {
/* Set timer interval in micro-seconds */
ivalue.it_interval.tv_sec = 0;
ivalue.it_interval.tv_usec = 0;
ivalue.it_value.tv_sec = frame_sec;
ivalue.it_value.tv_usec = (long) frame_usec;
setitimer(ITIMER_REAL, &ivalue, NULL);
active = true ;
} else {
char error_message[256];
sprintf(error_message, "itimer frame_time is not set\n" ) ;
exec_terminate_with_return(-1, __FILE__, __LINE__ , error_message);
}
}
return (0) ;
}
/**
@details
-# If real-time is active:
-# If the freeze_terminate flag is set, terminate the simulation
-# Set the sleep timer frame period to software frame period
-# Start real-time setting the real-time clock to the current simulation time.
*/
int Trick::RealtimeSync::unfreeze(long long sim_time_tics, double software_frame_sec) {
if ( active ) {
/* If the overrun freeze_shutdown condition was met terminate the simulation */
if (freeze_shutdown) {
exec_terminate_with_return(-1 , __FILE__ , __LINE__ , "Freeze-Shutdown condition reached.");
}
/* Adjust the real-time clock reference by the amount of time we were frozen */
rt_clock->adjust_ref_time(freeze_time_tics - sim_time_tics) ;
/* Set top of frame time for 1st frame (used in frame logging). */
last_clock_time = rt_clock->clock_time() ;
/* Start the sleep timer with the software frame expiration */
sleep_timer->start(software_frame_sec / rt_clock->get_rt_clock_ratio()) ;
}
return(0) ;
}
int Trick::SlaveInfo::read_slave_status() {
MS_SIM_COMMAND slave_command ;
MS_SIM_COMMAND exec_command ;
/** @par Detailed Design: */
/** @li If the slave is an active synchronization partner (activated == true) */
if (activated == true) {
/** @li read the current slave exec_command */
slave_command = connection->read_command() ;
//printf("DEBUG master read %d command from slave\n", slave_command);fflush(stdout);
exec_command = (MS_SIM_COMMAND)exec_get_exec_command() ;
// fixup: is it possible we won't get slave's Exit command over socket when it terminates?, set it here if that happens
if (dynamic_cast<MSSocket*>(connection)) {
if ((slave_command == MS_ErrorCmd) && (reconnect_wait_limit > 0.0) && (reconnect_count == 0)) {
slave_command = MS_ExitCmd;
}
}
/** @li If the master is not currently exiting, change modes if the slave is freezing/exiting or has an error */
if ( exec_command != MS_ExitCmd ) {
switch ( slave_command ) {
case (MS_ErrorCmd):
/** @li if the user has set a reconnect_wait_limit, continue on if we are still under that limit, otherwise
if the current slave mode cannot be read, exit the master if sync_error_terminate == true,
otherwise set the activated flag to false */
if ( (reconnect_count * exec_get_freeze_frame()) < reconnect_wait_limit) {
reconnect_count++;
} else if (sync_error_terminate == true) {
message_publish(MSG_ERROR, "Master lost sync with slave, so master is terminating.\n") ;
exec_terminate_with_return(-1, __FILE__, __LINE__ , "Master lost sync with slave.") ;
}
else {
message_publish(MSG_ERROR, "Master lost sync with slave, so slave is being deactivated.\n") ;
activated = false ;
return(0) ;
}
break ;
case (MS_ExitCmd):
/** @li if the current slave mode is exiting, exit the master if sync_error_terminate == true.
otherwise wait for slave to reconnect. when wait limit is 0, set the activated flag to false */
if (sync_error_terminate == true){
message_publish(MSG_WARNING, "sync_error_terminate is true: Slave is exiting, so master is terminating.\n") ;
exec_terminate_with_return(-1, __FILE__, __LINE__ , "Slave is exiting, so is the master.") ;
}
else {
message_publish(MSG_WARNING, "Slave is exiting.\n") ;
// if reconnect_wait_limit is set, master waits for slave to reconnect (e.g. dmtcp restarting)
if (reconnect_wait_limit > 0.0) {
message_publish(MSG_WARNING, "Master will wait %f seconds for slave to reconnect.\n", reconnect_wait_limit) ;
// make reads (shared mem connection) return quickly so we don't overrun waiting for reconnect
// TODO: for socket connection we will overrun in the accept call (see restart_dmtcp_slave)
connection->set_sync_wait_limit(exec_get_freeze_frame());
if (chkpnt_binary) {
restart_dmtcp_slave(); // restart the slave dmtcp executable
}
}
else {
message_publish(MSG_WARNING, "reconnect_wait_limit: 0.0 - Master will stop communicating with slave.\n") ;
activated = false ;
}
return(0) ;
}
break ;
case (MS_ChkpntLoadBinCmd):
// slave has received our load command and is now sending us his dmtcp port and checkpoint file name
dmtcp_port = connection->read_port() ;
connection->read_name(chkpnt_name, sizeof(chkpnt_name)); // dir/filename
message_publish(MSG_WARNING , "Master received DMTCP Port and Checkpoint Filename from slave.\n");
connection->write_command((MS_SIM_COMMAND)exec_get_exec_command()) ; // send this as an ack so slove can shut down
break ;
case (MS_FreezeCmd):
/** @li if the current slave is freezing, freeze the master too */
message_publish(MSG_INFO, "Slave is freezing.\n") ;
exec_set_exec_command(FreezeCmd) ;
reconnect_count = 0;
break ;
case (MS_ReconnectCmd):
// set the sync wait limit back to its default
connection->set_sync_wait_limit(sync_wait_limit);
message_publish(MSG_INFO, "Master has reconnected to slave.\n") ;
reconnect_count = 0;
break ;
default:
break ;
}
}
}
return(0) ;
}
/**
@details
-# Block all signals to the child.
-# Set the thread cancel type to asynchronous to allow master to this child at any time.
-# Lock the go mutex so the master has to wait until this child is ready before staring execution.
-# Set thread priority and CPU affinity
-# The child enters an infinite loop
-# Blocks on mutex or frame trigger until master signals to start processing
-# Switch if the child is a synchronous thread
-# For each scheduled jobs whose next call time is equal to the current simulation time [@ref ScheduledJobQueue]
-# Call call_next_job(Trick::JobData * curr_job, Trick::ScheduledJobQueue & job_queue, bool rt_nap, long long curr_time_tics)
-# Switch if the child is a asynchronous must finish thread
-# Do while the job queue time is less than the time of the next AMF sync time.
-# For each scheduled jobs whose next call time is equal to the current queue time
-# Call call_next_job(Trick::JobData * curr_job, Trick::ScheduledJobQueue & job_queue, bool rt_nap, long long curr_time_tics)
-# Switch if the child is a asynchronous thread
-# For each scheduled jobs
-# Call call_next_job(Trick::JobData * curr_job, Trick::ScheduledJobQueue & job_queue, bool rt_nap, long long curr_time_tics)
-# Set the child complete flag
*/
void * Trick::Threads::thread_body() {
/* Lock the go mutex so the master has to wait until this child is ready before staring execution. */
pthread_mutex_lock(&go_mutex);
/* signal the master that the child is ready and running */
child_complete = true;
running = true ;
try {
do {
/* Block child on go mutex or frame trigger until master signals. */
if (rt_semaphores == true) {
pthread_cond_wait(&go_cv, &go_mutex);
} else {
/* Else the child process frame is being started when a shared memory flag... */
while (frame_trigger == false) {} ;
frame_trigger = false ;
}
if ( enabled ) {
switch ( process_type ) {
case PROCESS_TYPE_SCHEDULED:
/* Loop through all jobs currently scheduled to run at this simulation time step. */
job_queue.reset_curr_index() ;
job_queue.set_next_job_call_time(TRICK_MAX_LONG_LONG) ;
while ( (curr_job = job_queue.find_next_job( curr_time_tics )) != NULL ) {
call_next_job(curr_job, job_queue, rt_nap, curr_time_tics) ;
}
break ;
case PROCESS_TYPE_AMF_CHILD:
/* call the AMF top of frame jobs */
top_of_frame_queue.reset_curr_index() ;
while ( (curr_job = top_of_frame_queue.get_next_job()) != NULL ) {
int ret ;
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , " top_of_frame job did not return 0") ;
}
}
/* Loop through all jobs currently up to the point of the next AMF frame sync */
do {
job_queue.reset_curr_index() ;
job_queue.set_next_job_call_time(amf_next_tics) ;
while ( (curr_job = job_queue.find_next_job( curr_time_tics )) != NULL ) {
call_next_job(curr_job, job_queue, rt_nap, curr_time_tics) ;
}
curr_time_tics = job_queue.get_next_job_call_time() ;
} while ( curr_time_tics < amf_next_tics ) ;
/* call the AMF end of frame jobs */
end_of_frame_queue.reset_curr_index() ;
while ( (curr_job = end_of_frame_queue.get_next_job()) != NULL ) {
int ret ;
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , " end_of_frame job did not return 0") ;
}
}
break ;
case PROCESS_TYPE_ASYNC_CHILD:
/* Loop through all jobs once */
if ( amf_cycle_tics == 0 ) {
// Old behavior, run all jobs once and return.
job_queue.reset_curr_index() ;
job_queue.set_next_job_call_time(TRICK_MAX_LONG_LONG) ;
while ( (curr_job = job_queue.get_next_job()) != NULL ) {
call_next_job(curr_job, job_queue, rt_nap, curr_time_tics) ;
}
} else {
// catch up job next times to current frame.
job_queue.reset_curr_index() ;
while ( (curr_job = job_queue.get_next_job()) != NULL ) {
long long start_frame = amf_next_tics - amf_cycle_tics ;
while ( curr_job->next_tics < start_frame ) {
curr_job->next_tics += curr_job->cycle_tics ;
}
}
// New behavior, run a mini scheduler.
/* call the AMF top of frame jobs */
top_of_frame_queue.reset_curr_index() ;
while ( (curr_job = top_of_frame_queue.get_next_job()) != NULL ) {
int ret ;
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , " top_of_frame job did not return 0") ;
}
}
/* Loop through all jobs currently up to the point of the next AMF frame sync */
do {
job_queue.reset_curr_index() ;
job_queue.set_next_job_call_time(amf_next_tics) ;
while ( (curr_job = job_queue.find_next_job( curr_time_tics )) != NULL ) {
call_next_job(curr_job, job_queue, rt_nap, curr_time_tics) ;
}
curr_time_tics = job_queue.get_next_job_call_time() ;
} while ( curr_time_tics < amf_next_tics ) ;
/* call the AMF end of frame jobs */
end_of_frame_queue.reset_curr_index() ;
while ( (curr_job = end_of_frame_queue.get_next_job()) != NULL ) {
int ret ;
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , " end_of_frame job did not return 0") ;
}
}
}
break ;
default:
break ;
}
}
/* After all jobs have completed, set the child_complete flag to true. */
child_complete = true;
} while (1);
} catch (Trick::ExecutiveException & ex ) {
fprintf(stderr, "\nCHILD THREAD %d TERMINATED with exec_terminate\n ROUTINE: %s\n DIAGNOSTIC: %s\n"
" THREAD STOP TIME: %f\n" ,
thread_id, ex.file.c_str(), ex.message.c_str(), exec_get_sim_time()) ;
exit(ex.ret_code) ;
} catch (const std::exception &ex) {
std::string except_file ;
if ( curr_job != NULL ) {
except_file = curr_job->name ;
} else {
except_file = "somewhere in Executive::run" ;
}
fprintf(stderr, "\nCHILD THREAD %d TERMINATED with exec_terminate\n ROUTINE: %s\n DIAGNOSTIC: %s\n"
" THREAD STOP TIME: %f\n" ,
thread_id, except_file.c_str() , ex.what(), exec_get_sim_time()) ;
exit(-1) ;
#ifdef __linux
#ifdef __GNUC__
#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 2
// for post gcc 4.1.2
} catch (abi::__forced_unwind&) {
//pthread_exit and pthread_cancel will cause an abi::__forced_unwind to be thrown. Rethrow it.
throw;
#endif
#endif
#endif
} catch (...) {
/*
In gcc 4.1.2 I cannot find the catch type for the pthread_cancel forced_unwind exception so I changed
the catch here to just rethrow all unknown exceptions. For the other architectures
we signal the main thread for an orderly shutdown.
*/
#ifdef __linux
#ifdef __GNUC__
#if __GNUC__ == 4 && __GNUC_MINOR__ == 1
throw;
#else
std::string except_file ;
std::string except_message ;
if ( curr_job != NULL ) {
except_file = curr_job->name ;
} else {
except_file = "somewhere in Executive::run" ;
}
except_message = "unknown error" ;
fprintf(stderr, "\nExecutive::loop terminated with unknown exception\n ROUTINE: %s\n DIAGNOSTIC: %s\n"
" STOP TIME: %f\n" , except_file.c_str() , except_message.c_str() , exec_get_sim_time()) ;
exit(-1) ;
#endif
#endif
#endif
}
pthread_exit(NULL) ;
return 0 ;
}
int throw_exception() {
return exec_terminate_with_return(-1, "throw_exception", 1 , "exec_terminate called") ;
}
/**
@details
-# If real-time is not active:
-# If real-time synchronization has been enabled:
-# Set the active flag to true.
-# Start real-time setting the real-time clock to the current simulation time.
-# exit end of frame routine
-# If real-time is active:
-# If real-time synchronization has been disabled:
-# Set the active flag to false.
-# Get the current real-time.
-# Calculate the real-time taken for the last frame of execution.
-# if the frame has overrun
-# Increment the number of consecutive overruns
-# If the maximum number of consecutive overrun frames has
been reached or the maximum single overrun time has been exceeded
-# If the freeze/terminate action was set
-# set the freeze_terminate flag
-# freeze the simulation
-# Else terminate the simulation
-# Stop the sleep timer.
-# Else the frame has underrun
-# Reset the number of consecutive overruns to 0.
-# Pause for the sleep timer to expire
-# Spin for the real-time clock to match the simulation time
-# Reset the sleep timer for the next frame
-# Save the current real-time as the start of the frame reference
*/
int Trick::RealtimeSync::rt_monitor(long long sim_time_tics) {
long long curr_clock_time ;
char buf[512];
/* determine if the state of real-time has changed this frame */
if ( ! active ) {
if ( enable_flag ) {
active = true ;
enable_flag = false ;
start_realtime(exec_get_software_frame() , sim_time_tics) ;
}
if ( disable_flag ) {
disable_flag = false ;
}
return(0) ;
}
if ( enable_flag ) {
enable_flag = false ;
}
if ( disable_flag ) {
active = false ;
disable_flag = false ;
}
/* calculate the current underrun/overrun */
curr_clock_time = rt_clock->clock_time() ;
frame_overrun_time = 0 ;
frame_sched_time = curr_clock_time - last_clock_time ;
frame_overrun_time = curr_clock_time - sim_time_tics ;
/* If the wall clock time is greater than the sim time an overrun occurred. */
if (curr_clock_time > sim_time_tics) {
/* Update the overrun counter and current overrun time */
frame_overrun_cnt++;
total_overrun++;
/* If the number overruns surpass the maximum allowed freeze or shutdown. */
if (frame_overrun_cnt >= rt_max_overrun_cnt || frame_overrun_time >= rt_max_overrun_time_tics) {
/* If the rt_overrun_freeze flag is set, enter freeze mode else terminate the simulation. */
if (rt_overrun_freeze == true) {
freeze_shutdown = true ;
message_publish(MSG_ERROR, "\nMaximum overrun condition exceeded:\n"
"consecutive overruns/allowed overruns: %d/%d\n"
"total overrun time/allowed time: %f/%g\n"
"Entering Freeze-Shutdown Mode\n" ,
frame_overrun_cnt, rt_max_overrun_cnt,
(double)(frame_overrun_time/tics_per_sec), rt_max_overrun_time);
exec_freeze() ;
} else {
sprintf(buf, "\nMaximum overrun condition exceeded:\n"
"consecutive overruns/allowed overruns: %d/%d\n"
"total overrun time/allowed time: %f/%g\n",
frame_overrun_cnt, rt_max_overrun_cnt,
(double)(frame_overrun_time/tics_per_sec), rt_max_overrun_time);
exec_terminate_with_return(-1 , __FILE__ , __LINE__ , buf);
}
}
/* stop the sleep timer in an overrun condition */
sleep_timer->stop() ;
/* Call clock_spin to allow interrupt driven clocks to service their interrupts */
curr_clock_time = rt_clock->clock_spin(sim_time_tics) ;
} else {
/* Else an underrun condition occurred. */
/* Reset consecutive overrun counter frame_overrun_cnt */
frame_overrun_cnt = 0;
/* pause for the timer to signal the end of frame */
sleep_timer->pause() ;
/* Spin to make sure that we are at the top of the frame */
curr_clock_time = rt_clock->clock_spin(sim_time_tics) ;
/* If the timer requires to be reset at the end of each frame, reset it here. */
sleep_timer->reset(exec_get_software_frame() / rt_clock->get_rt_clock_ratio()) ;
}
/* Set the next frame overrun/underrun reference time to the current time */
last_clock_time = curr_clock_time ;
return(0) ;
}
/**
@details
-# Wait for all synchronous threads to finish initializing before entering infinite loop
-# Enter an infinite loop. A call to exec_terminate(const char *) or
exec_terminate_with_return(int, const char *, int , const char * ) will exit the loop.
-# For all asynchronous must finish threads that are to finish at this time, wait for their completion
-# If the exec_command equals freeze mode, call Trick::Executive::freeze_loop(). Transition from
freeze back to run will continue execution at this point.
Requirement [@ref r_exec_mode_1]
-# Set the job complete flags for all jobs to false.
-# Check to see if all child threads are alive. If a thread has exited, call
Trick::Executive::exec_terminate_with_return(int, char *, int, char *)
Requirement [@ref r_exec_thread_7]
-# Signal threads to start the next time step of processing.
-# For each scheduled jobs whose next call time is equal to the current simulation time [@ref ScheduledJobQueue]
-# Wait for all job dependencies to complete. Requirement [@ref r_exec_thread_6]
-# Call the job. Requirement [@ref r_exec_periodic_0]
-# If the job is a system job, check to see if the next job call time is the lowest next time by
calling Trick::ScheduledJobQueue::test_next_job_call_time(Trick::JobData *, long long)
-# If the exec_command equals ExitCmd
-# Call Trick::Executive::exec_terminate_with_return(int, char *, int, char *)
-# If the elapsed time has reached the termination time
-# Call Trick::Executive::exec_terminate_with_return(int, char *, int, char *)
-# If the elapsed time equals the next software frame time
-# Call the end_of_frame jobs. Requirement [@ref r_exec_periodic_2]
-# Set the end of frame execution time to the current time + software_frame
*/
int Trick::Executive::loop_multi_thread() {
JobData * depend_job ;
unsigned int ii ;
std::stringstream error_message ;
Trick::ScheduledJobQueue * main_sched_queue ;
int ret = 0 ;
/* Wait for all threads to finish initializing and set the child_complete flag. */
for (ii = 1; ii < threads.size() ; ii++) {
Threads * curr_thread = threads[ii] ;
while (curr_thread->child_complete == false ) {
if (rt_nap == true) {
RELEASE();
}
}
}
/* The main scheduler queue is the queue in thread 0 */
main_sched_queue = &(threads[0]->job_queue) ;
while (1) {
/* Give aynchronous jobs to the top of the next time start to finish executing previous frame */
for (ii = 0; ii < threads.size() ; ii++) {
Trick::Threads * curr_thread = threads[ii] ;
if ( curr_thread->process_type == PROCESS_TYPE_ASYNC_CHILD ) {
if ( curr_thread->child_complete == true ) {
if (curr_thread->amf_cycle_tics != 0 ) {
// catch up async next_tic time to a time greater than the time last pass
while ( curr_thread->amf_next_tics < time_tics ) {
curr_thread->amf_next_tics += curr_thread->amf_cycle_tics ;
}
}
}
}
}
/* Call freeze_loop() if commanded by freeze() or a <CTRL-C> signal was caught. */
if (exec_command == FreezeCmd) {
exec_command = NoCmd;
freeze_loop();
}
/* Call all top of frame jobs if the simulation time equals to the time software frame boundary. */
if (time_tics == next_frame_check_tics - software_frame_tics ) {
top_of_frame_queue.reset_curr_index() ;
while ( (curr_job = top_of_frame_queue.get_next_job()) != NULL ) {
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , "top_of_frame job did not return 0") ;
}
}
frame_count++ ;
}
/* Call the input_processor_run queue jobs. */
input_processor_run_queue.reset_curr_index() ;
while ( (curr_job = input_processor_run_queue.find_next_job( time_tics )) != NULL ) {
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , "input_processor_run job did not return 0") ;
}
// input processor jobs set their own next job call time. The test_next_job_call_time
// will adjust the next call time for this queue
input_processor_run_queue.test_next_job_call_time(curr_job , time_tics) ;
/* System jobs next call time are not set until after they run.
Test their next job call time after they have been called */
if ( curr_job->system_job_class ) {
main_sched_queue->test_next_job_call_time(curr_job , time_tics) ;
}
}
/* Go through all of the job queues and mark all jobs that are to run this time step to not complete. */
for (ii = 0; ii < threads.size() ; ii++) {
Trick::Threads * curr_thread = threads[ii] ;
/* For all threads that are waiting to start the next cycle (child_complete == true)
reset job completion flags */
if ( isThreadReadyToRun(curr_thread, time_tics) ) {
/* For all jobs in all threads that will run for this time_tic, */
/* Set job complete flags to false. */
/* The job complete flags are used for job depends_on checks. */
curr_thread->job_queue.reset_curr_index();
while ( (curr_job = curr_thread->job_queue.find_job(time_tics)) != NULL ) {
curr_job->complete = false;
}
}
}
/* After all jobs on all threads that are going to run are set not complete, start the threads */
for (ii = 1; ii < threads.size() ; ii++) {
Trick::Threads * curr_thread = threads[ii] ;
/* For all threads that are waiting to start the next cycle (child_complete == true)
signal them to start through either the thread mutex or frame trigger. */
if ( isThreadReadyToRun(curr_thread, time_tics) ) {
curr_thread->curr_time_tics = time_tics ;
curr_thread->child_complete = false ;
curr_thread->amf_next_tics += curr_thread->amf_cycle_tics ;
if (curr_thread->rt_semaphores == true ) {
pthread_mutex_lock(&(curr_thread->go_mutex));
pthread_cond_signal(&(curr_thread->go_cv));
pthread_mutex_unlock(&(curr_thread->go_mutex));
} else {
curr_thread->frame_trigger = true ;
}
}
}
/* Get next job scheduled to run at the current simulation time step. */
main_sched_queue->reset_curr_index() ;
while ( (curr_job = main_sched_queue->find_next_job( time_tics )) != NULL ) {
/* Wait for all jobs that the current job depends on to complete. */
for ( ii = 0 ; ii < curr_job->depends.size() ; ii++ ) {
depend_job = curr_job->depends[ii] ;
while (! depend_job->complete) {
if (rt_nap == true) {
RELEASE();
}
}
}
/* Call the current job scheduled to run at the current simulation time step. */
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , "scheduled job did not return 0") ;
}
/* System jobs next call time are not set until after they run.
Test their next job call time after they have been called */
if ( curr_job->system_job_class ) {
main_sched_queue->test_next_job_call_time(curr_job , time_tics) ;
}
curr_job->complete = true ;
}
/* Call Executive::exec_terminate_with_return(int , const char * , int , const char *)
if exec_command equals ExitCmd. */
if (exec_command == ExitCmd) {
exec_terminate_with_return( 0 , __FILE__ , __LINE__ , "Sim control Shutdown" ) ;
}
/* We are already in run... clear any redundant exec_commands to go to run */
if ( exec_command == RunCmd ) {
exec_command = NoCmd ;
}
/* Call Executive::exec_terminate_with_return( int , const char * , int , const char *)
if simulation elapsed time has reached the termination time. */
if (terminate_time <= time_last_pass_tics ) {
time_tics = time_last_pass_tics ;
exec_terminate_with_return( 0 , __FILE__ , __LINE__ , "Reached termination time" ) ;
}
/* Call all end of frame jobs if the simulation time equals to the time software frame boundary. */
if (time_tics == next_frame_check_tics ) {
end_of_frame_queue.reset_curr_index() ;
while ( (curr_job = end_of_frame_queue.get_next_job()) != NULL ) {
ret = curr_job->call() ;
if ( ret != 0 ) {
exec_terminate_with_return(ret , curr_job->name.c_str() , 0 , "end_of_frame job did not return 0") ;
}
}
next_frame_check_tics += software_frame_tics ;
}
}
return(0) ;
}
/**
@details
-# verify that all jobs in the checkpoint are present in the current simulation
-# Clear all current job information in scheduler
-# Clear all current sim_object information in scheduler
-# Add the sim_objects in checkpoint to the scheduler
-# Call post_checkpoint to delete memory for checkpoint
*/
int Trick::Executive::restart() {
unsigned int ii ;
std::multimap<std::string, Trick::JobData *>::iterator it ;
std::pair<std::multimap<std::string, Trick::JobData *>::iterator, std::multimap<std::string, Trick::JobData *>::iterator> ret ;
std::vector <Trick::SimObject *>::iterator sit ;
std::vector <Trick::JobData *>::iterator jit ;
std::multimap<std::string, Trick::JobData *> all_jobs_from_checkpointed_objects ;
/*
Save the current job index position of the input processor queue. This queue position is special
because of the possibility of restarting from a checkpoint read in from the input file. When
we restart from a checkpoint we reset the queues. If we reset the input_processor queue we'll
call the input_processor again... and we can end up in recursive hell. Save the current position
and replace the queue position after the queues have been rebuilt. This ensures that we don't
call the input file twice, or more!
*/
unsigned int current_ip_index = input_processor_queue.get_curr_index() ;
/* clear all current job information in scheduler. These will be repopulated when we call add_job_to_queue */
clear_scheduled_queues() ;
all_jobs.clear() ;
all_jobs_vector.clear() ;
all_tagged_jobs.clear() ;
/* clear all sim_object information in scheduler */
num_sim_objects = 0 ;
sim_objects.clear() ;
/* restore the list of sim_objects from the checkpoint. */
restore_stl(sim_objects , std::string("trick_sys") , std::string("sim_objects")) ;
/* Create a temporary all_jobs map to use to restore job data from all_jobs_for_checkpoint */
for ( sit = sim_objects.begin() ; sit != sim_objects.end() ; sit++ ) {
for ( jit = (*sit)->jobs.begin() ; jit != (*sit)->jobs.end() ; jit++ ) {
/* Dynamically created sim_objects, like data_recording objects, require that
some of the job class data be repopulated before we start
copying job data back into the sim. */
(*jit)->sim_object_id = (*sit)->id ;
(*jit)->parent_object = (*sit) ;
(*jit)->frame_id = (*jit)->sim_object_id + ((double)(*jit)->id / 100.0);
std::string so_name_and_period = (*sit)->name + "." ;
if ( strncmp( (*jit)->name.c_str() , so_name_and_period.c_str(), so_name_and_period.length())) {
(*jit)->name = (*sit)->name + "." + (*jit)->name ;
}
all_jobs_from_checkpointed_objects.insert(std::pair<std::string, JobData *>((*jit)->name,(*jit))) ;
}
}
/* verify that all jobs in the checkpoint are present in the current simulation */
for ( ii = 0 ; ii < num_all_jobs ; ii++ ) {
ret = all_jobs_from_checkpointed_objects.equal_range(all_jobs_for_checkpoint[ii].name) ;
if ( ret.first == all_jobs_from_checkpointed_objects.end() ) {
message_publish(MSG_ERROR, "Could not find job %s\n", all_jobs_for_checkpoint[ii].name.c_str()) ;
exec_terminate_with_return( -1 , __FILE__ , __LINE__ , "Job in checkpoint file does not exist in current sim." ) ;
} else {
for ( it = ret.first ; it != ret.second ; it++ ) {
// The JobData::id and JobData::sim_object_id together uniquely identify a job.
if (( it->second->id == all_jobs_for_checkpoint[ii].id ) &&
( it->second->sim_object_id == all_jobs_for_checkpoint[ii].sim_object_id )) {
// copy the job information from the checkpoint back to the job.
it->second->copy_from_checkpoint(&all_jobs_for_checkpoint[ii]) ;
}
}
}
}
/* restore the executive sim_objects vector from the checkpoint and add back all of
the jobs to the schedulers */
for ( sit = sim_objects.begin() ; sit != sim_objects.end() ; sit++ ) {
add_jobs_to_queue(*sit, true) ;
for ( ii = 0 ; ii < other_schedulers.size() ; ii++ ) {
other_schedulers[ii]->add_sim_object(*sit) ;
}
}
num_sim_objects = sim_objects.size() ;
// The queues have been rebuilt, restore the current position of the input processor queue.
input_processor_queue.set_curr_index(current_ip_index) ;
// set the restart_called flag to true. This flag is checked during Executive::init. If
// we have restarted from a checkpoint, we will be skipping initialization jobs.
restart_called = true ;
/* Delete temporary memory used to restore jobs */
TMM_delete_var_a(all_jobs_for_checkpoint) ;
all_jobs_for_checkpoint = NULL ;
// restore the vector of freeze times
restore_stl(freeze_times , std::string("trick_sys") , std::string("freeze_times")) ;
/* Set the main thread current time to the simulation time tics value, used with Executive::get_sim_time() */
threads[0]->curr_time_tics = time_tics ;
// if we restarted from scratch without calling init jobs, need to create child threads
create_threads() ;
// make sure asynchronous must finish jobs restart at correct time
for (ii = 1; ii < threads.size() ; ii++) {
Threads * curr_thread = threads[ii] ;
if ( (curr_thread->process_type == PROCESS_TYPE_AMF_CHILD) ||
(curr_thread->process_type == PROCESS_TYPE_ASYNC_CHILD) ) {
if ( curr_thread->amf_cycle_tics > 0 and time_tics % curr_thread->amf_cycle_tics ) {
curr_thread->amf_next_tics = time_tics + curr_thread->amf_cycle_tics - (time_tics % curr_thread->amf_cycle_tics) ;
} else {
curr_thread->amf_next_tics = time_tics ;
}
}
}
// If we are in freeze, reset the freeze_scheduled queue, we'll restart freeze time at 0
if ( mode == Freeze ) {
init_freeze_scheduled() ;
}
return(0) ;
}
