/**
* \brief This function prints the messages associated with each error flag.
* \param all_flags The array that will hold all of the error flags from the run
*/
void printFlags(int **all_flags){
int noError = 0;
int i,j;
// Print System errors, if needed.
for(j = 0; j < SYS_ERR_SIZE; j++){
//printf("Error flag[0][%d] = %d\n", j, all_flags[0][j]);
if(all_flags[0][j] > 0){
EmitLog(MyRank, SCHEDULER_THREAD, build_msg(0,j), all_flags[0][j], PRINT_ALWAYS);
} else {
noError++;
}
}
// Print Plan errors
for(i = 1; i < ERR_FLAG_SIZE; i++){
for(j = 0; j < plan_list[i - 1]->esize + GEN_SIZE; j++){
//printf("Error flag[%d][%d] = %d\n", i, j, all_flags[i][j]);
if(all_flags[i][j] > 0){
EmitLog(MyRank, SCHEDULER_THREAD, build_msg(i,j), all_flags[i][j], PRINT_ALWAYS);
} else {
noError++;
}
}
}
/*
if (noError == ERR_FLAG_SIZE) {
EmitLog(MyRank, SCHEDULER_THREAD, "No flagged errors have occured in this time period.", -1, PRINT_ALWAYS);
}
*/
} /* printFlags */
/**
* \brief Stores (and optionally displays) performance data for the plan.
* \param [in] plan The Plan structure containing the plan data.
* \returns An integer error code.
* \sa parseFFT2Plan
* \sa makeFFT2Plan
* \sa initFFT2Plan
* \sa execFFT2Plan
* \sa killFFT2Plan
*/
int perfFFT2Plan(void *plan){
int ret = ~ERR_CLEAN;
uint64_t points;
uint64_t opcounts[NUM_TIMERS];
Plan *p;
FFTdata *d;
p = (Plan *)plan;
d = (FFTdata *)p->vptr;
if(p->exec_count > 0){
points = (uint64_t)d->M * (uint64_t)d->M; // Number of data points in the calculations.
opcounts[TIMER0] = (10 * points * FFTlog2((uint64_t)d->M)) * p->exec_count; // Method used by authors of the fftw3 library - see http://www.fftw.org/speed
opcounts[TIMER1] = (5 * points * FFTlog2(points)) * p->exec_count;
opcounts[TIMER2] = 0;
perf_table_update(&p->timers, opcounts, p->name);
#ifdef HAVE_PAPI
PAPI_table_update(p->name, p->PAPI_Results, p->PAPI_Times, p->PAPI_Num_Events);
#endif //HAVE_PAPI
double flops_forward = ((double)opcounts[TIMER0] / perftimer_gettime(&p->timers, TIMER0)) / 1e6;
EmitLogfs(MyRank, 9999, "FFT2D plan performance:", flops_forward, "MFLOPS", PRINT_SOME);
EmitLog (MyRank, 9999, "FFT2D execution count :", p->exec_count, PRINT_SOME);
ret = ERR_CLEAN;
}
return ret;
} /* perfFFT2Plan */
void QBaseThread::ThreadFinished( )
{
QString strLog = "Finished";
EmitLog( strLog );
deleteLater( );
}
/**
* \brief Calculate and store performance data for the plan.
* \param plan The Plan structure that holds the plan's data.
* \returns Success or failure in calculating the performance.
*/
int perfDOPENACCGEMMPlan(void *plan){
int ret = ~ERR_CLEAN;
uint64_t opcounts[NUM_TIMERS];
Plan *p;
DOPENACCGEMM_DATA *d;
p = (Plan *)plan;
d = (DOPENACCGEMM_DATA *)p->vptr;
if(p->exec_count > 0){ // Ensures the plan has been executed at least once...
// Assign appropriate plan-specific operation counts to the opcount[] array, such that the
// indices correspond with the timers used in the exec function.
const unsigned long M = d->M;
const unsigned long M2 = M * M;
opcounts[TIMER0] = p->exec_count * 2 * M2 * (M + 1); //* YOUR_OPERATIONS_PER_EXECUTION; // Where operations can be a function of the input size.
perf_table_update(&p->timers, opcounts, p->name); // Updates the global table with the performance data.
#ifdef HAVE_PAPI
PAPI_table_update(p->name, p->PAPI_Results, p->PAPI_Times, p->PAPI_Num_Events);
#endif //HAVE_PAPI
double flops = ((double)opcounts[TIMER0] / perftimer_gettime(&p->timers, TIMER0)) / 1e6; // Example for computing MFLOPS
EmitLogfs(MyRank, 9999, "DOPENACCGEMM plan performance:", flops, "MFLOPS", PRINT_SOME); // Displays calculated performance when the '-v2' command line option is passed.
EmitLog (MyRank, 9999, "DOPENACCGEMM execution count :", p->exec_count, PRINT_SOME);
ret = ERR_CLEAN;
}
return ret;
} /* perfDOPENACCGEMMPlan */
/* Helper function to test PAPI return codes */
inline void TEST_PAPI(int res, int test, int rank, int tnum, int debug){
char message[512];
if(res != test){
snprintf(message, 512, "PAPI error %d: %s\n", res, PAPI_strerror(res));
EmitLog(rank, tnum, message, -1, debug);
}
}
/** \brief Stop the workers */
void StopWorkerThreads(){
int i;
/* tell them all to finish */
for(i = 0; i < num_workers; i++){
EmitLog(MyRank, SCHEDULER_THREAD, "Stopping Worker Thread",WorkerHandle[i].Num, PRINT_OFTEN);
pthread_rwlock_wrlock(&(WorkerHandle[i].Lock));
WorkerHandle[i].Plan = NULL;
pthread_rwlock_unlock(&(WorkerHandle[i].Lock));
#ifndef ASYNC_WORKERS
pthread_join(WorkerHandle[i].ID, NULL);
#endif
}
return;
}
void QBaseThread::ThreadEventInfo( bool bPostEvent, QEvent *pEvent )
{
QString strEvent = "";
ParkSolution::ThreadEvent eEvent = ( ParkSolution::ThreadEvent ) pEvent->type( );
switch ( eEvent ) {
case ParkSolution::ThreadExit :
strEvent = "ThreadExit";
break;
}
QString strLog = QString( " %1 %2 Event" ).arg( bPostEvent ? "Post" : "Process", strEvent );
EmitLog( strLog );
}
/**
* \brief Sets up the performance data gathering system.
* Calls functions to calibrate the system timers and to initialize the global
* table for storing performance data.
*/
void performance_init(){
if(MyRank == ROOT){
EmitLog(MyRank, SCHEDULER_THREAD, "Calibrating performance timers.", -1, PRINT_ALWAYS);
}
ORB_calibrate();
#ifdef HAVE_PAPI
/* Initialize PAPI and PAPI threads */
//TEST_PAPI(PAPI_library_init, PAPI_VER_CURRENT, MyRank, SCHEDULER_THREAD, PRINT_ALWAYS, PAPI_VER_CURRENT);
TEST_PAPI(PAPI_library_init(PAPI_VER_CURRENT), PAPI_VER_CURRENT, MyRank, SCHEDULER_THREAD, PRINT_ALWAYS);
//TEST_PAPI(PAPI_thread_init, PAPI_OK, MyRank, SCHEDULER_THREAD, PRINT_ALWAYS, pthread_self);
TEST_PAPI(PAPI_thread_init(pthread_self), PAPI_OK, MyRank, SCHEDULER_THREAD, PRINT_ALWAYS);
#endif //HAVE_PAPI
/* Initialize performance tables */
perf_table_init();
} /* performance_init */
/** \brief Start up the worker threads that run the plans that compose the workforce of SystemBurn */
void StartWorkerThreads(){
int i, one = 1;
data p[1];
p->i = &one;
/* construct worker thread's control structure start them with sleep */
for(i = 0; i < num_workers; i++){
pthread_rwlock_init(&(WorkerHandle[i].Lock),0);
pthread_attr_init(&(WorkerHandle[i].Attr));
WorkerHandle[i].Num = i;
WorkerHandle[i].Plan = (plan_list[SLEEP]->make)(p);
pthread_create(&(WorkerHandle[i].ID), &(WorkerHandle[i].Attr), WorkerThread, &(WorkerHandle[i]));
EmitLog(MyRank, SCHEDULER_THREAD, "Starting Worker Thread",WorkerHandle[i].Num, PRINT_OFTEN);
#ifdef ASYNC_WORKERS
pthread_detach(WorkerHandle[i].ID);
#endif
}
return;
} /* StartWorkerThreads */
void QSmsThread::OnRecvMsg( int nMsgCount )
{
QVariant var;
QVariant nYear = 0, nMonth = 0, nDay = 0, nHour = 0, nMinute = 0, nSecond = 0;
QString strTel, strCnt;
strTel.reserve( 32 );
strCnt.reserve( 200 );
QString strLogPattern = "Received message :%1 %2";
QString strLog;
while ( 0 < nMsgCount ) {
var = pAxCtrl->dynamicCall( "RecvMsg( QString&, QString&, int&, int&, int&, int&, int&, int& )",
strTel, strCnt, nYear, nMonth, nDay, nHour, nMinute, nSecond );
strLog = strLogPattern.arg( strTel, strCnt );
EmitLog( strLog );
qDebug( ) << strLog << endl;
nMsgCount--;
}
}
/**
* \brief Calculates (and optionally displays) performance data for the plan.
* \param [in] plan The Plan structure that contains all the plan data.
* \returns An integer error code.
* \sa parseCUDAMEMPlan
* \sa makeCUDAMEMPlan
* \sa initCUDAMEMPlan
* \sa execCUDAMEMPlan
* \sa killCUDAMEMPlan
*/
int perfCUDAMEMPlan (void *plan) {
int ret = ~ERR_CLEAN;
uint64_t opcounts[NUM_TIMERS];
Plan *p;
CUDAMEMdata *d;
p = (Plan *)plan;
d = (CUDAMEMdata *)p->vptr;
if (p->exec_count > 0) {
uint64_t M2 = (uint64_t)d->M * (uint64_t)d->M;
opcounts[TIMER0] = (M2 * (2 * (uint64_t)d->M + 1)) * (uint64_t)d->nLoopCount * p->exec_count; // Count # of floating point operations
opcounts[TIMER1] = 0;
opcounts[TIMER2] = 0;
perf_table_update(&p->timers, opcounts, p->name);
double flops = ((double)opcounts[TIMER0]/perftimer_gettime(&p->timers, TIMER0))/1e6;
EmitLogfs(MyRank, 9999, "CUDAMEM plan performance:", flops, "MFLOPS", PRINT_SOME);
EmitLog (MyRank, 9999, "CUDAMEM execution count :", p->exec_count, PRINT_SOME);
ret = ERR_CLEAN;
}
return ret;
}
/**
* \brief Calculates (and optionally displays) performance data for the plan.
* \param [in] plan The Plan structure that contains all the plan data.
* \returns An integer error code.
* \sa parseCBAPlan
* \sa makeCBAPlan
* \sa initCBAPlan
* \sa execCBAPlan
* \sa killCBAPlan
*/
int perfCBAPlan(void *plan){
int ret = ~ERR_CLEAN;
uint64_t opcounts[NUM_TIMERS];
Plan *p;
CBA_data *d;
p = (Plan *)plan;
d = (CBA_data *)p->vptr;
if(p->exec_count > 0){
opcounts[TIMER0] = ((uint64_t)d->ncols * (uint64_t)d->niter * 8ULL) * p->exec_count; // count # of bytes processed
opcounts[TIMER1] = 0;
opcounts[TIMER2] = 0;
perf_table_update(&p->timers, opcounts, p->name);
#ifdef HAVE_PAPI
PAPI_table_update(p->name, p->PAPI_Results, p->PAPI_Times, p->PAPI_Num_Events);
#endif //HAVE_PAPI
double ips = ((double)opcounts[TIMER0] / perftimer_gettime(&p->timers, TIMER0)) / 1e9;
EmitLogfs(MyRank, 9999, "CBA plan performance:", ips, "GB/s", PRINT_SOME);
EmitLog (MyRank, 9999, "CBA execution count :", p->exec_count, PRINT_SOME);
ret = ERR_CLEAN;
}
return ret;
} /* perfCBAPlan */
void QSmsThread::OnConnectResult( int nStatus )
{
QString strLog = QString( "Connect SGM Modem %1" ).arg( ( 0 == nStatus ) ? "Success" : "Failure" );
EmitLog( strLog );
qDebug( ) << strLog << endl;
}
void QSmsThread::HandleException( int code, const QString &source, const QString &desc, const QString &help )
{
QString strLog = QString( "nCode:%1 Source:%2 Desc:%3 Help:%4" ).arg( QString::number( code ), source, desc, help );
EmitLog( strLog );
qDebug( ) << strLog << endl;
}
/** \brief The main function for the worker threads, which determines what they run and what inputs they receive.
\param p Pointer to the ThreadHandle struct that holds the information for the worker thread
*/
void *WorkerThread(void *p){
long cpucoreid, numcpucores;
int init_flag, run_flag, perf_flag;
int one = 1;
data sleep_pass[1];
sleep_pass->i = &one;
ThreadHandle *MyHandle = (ThreadHandle *)p;
Plan *WorkerPlan, *BossPlan;
#ifdef LINUX_PLACEMENT
int affin_flag;
cpucoreid = sched_getcpu();
//syscall(__NR_getcpu(&cpucoreid, NULL, NULL));
numcpucores = sysconf(_SC_NPROCESSORS_ONLN);
EmitLog(MyRank, MyHandle->Num, "Starting on processor core", cpucoreid, PRINT_SOME);
#else
cpucoreid = -1;
numcpucores = -1;
EmitLog(MyRank, MyHandle->Num, "Starting...", -1, PRINT_SOME);
#endif
WorkerPlan = NULL;
for(;; ){
pthread_rwlock_rdlock( &(MyHandle->Lock) );
BossPlan = MyHandle->Plan;
pthread_rwlock_unlock( &(MyHandle->Lock) );
if(BossPlan == NULL){
if(DO_PERF){
EmitLog(MyRank, MyHandle->Num, "Printing performance data.", -1, PRINT_SOME);
perf_flag = perfPlan(WorkerPlan);
if(perf_flag != ERR_CLEAN){
EmitLog(MyRank, MyHandle->Num, "Performance recording error flag triggered, error number:", perf_flag, PRINT_SOME);
}
} //DO_PERF
EmitLog(MyRank, MyHandle->Num, "Thread exiting", -1, PRINT_SOME);
WorkerPlan = killPlan(WorkerPlan); /* clean up old plan */
pthread_exit((void *)0);
} else {
if(BossPlan != WorkerPlan){ /* if the plan was updated */
if((WorkerPlan != NULL) && (WorkerPlan->name != SLEEP) ){
if(DO_PERF){
EmitLog(MyRank, MyHandle->Num, "Printing performance data.", -1, PRINT_SOME);
perf_flag = perfPlan(WorkerPlan);
if(perf_flag != ERR_CLEAN){
EmitLog(MyRank, MyHandle->Num, "Performance recording error flag triggered, error number:", perf_flag, PRINT_SOME);
}
} //DO_PERF
}
#ifdef LINUX_PLACEMENT
numcpucores = sysconf(_SC_NPROCESSORS_ONLN);
cpucoreid = sched_getcpu();
//syscall(__NR_getcpu(&cpucoreid, NULL, NULL));
cpu_set_t cpuset;
affin_flag = pthread_getaffinity_np(MyHandle->ID, sizeof(cpu_set_t), &cpuset);
if(affin_flag != 0){
add_error(MyHandle,SYSTEM,2);
}
EmitLog(MyRank, MyHandle->Num, "New plan detected. Switching plans on core", cpucoreid, PRINT_SOME);
if(PRINT_RARELY <= verbose_flag){
int i;
EmitLog(MyRank, MyHandle->Num, "CPU set for this thread:", -1, PRINT_SOME);
printf("\t\t\t\t");
for(i = 0; i < numcpucores; i++){
if(CPU_ISSET(i, &cpuset)){
printf(" %d", i);
}
}
printf("\n");
}
#else /* ifdef LINUX_PLACEMENT */
EmitLog(MyRank, MyHandle->Num, "New plan detected. Switching plans.", -1, PRINT_SOME);
#endif /* ifdef LINUX_PLACEMENT */
WorkerPlan = killPlan(WorkerPlan); /* clean up old plan */
WorkerPlan = BossPlan; /* switch plans */
init_flag = InitPlan(WorkerPlan); /* initialize new plan */
if(init_flag != ERR_CLEAN){
add_error(MyHandle, WorkerPlan->name,init_flag);
EmitLog(MyRank, MyHandle->Num, "Initialization error flag triggered, error number:", init_flag, PRINT_ALWAYS);
MyHandle->Plan = (plan_list[SLEEP]->make)(sleep_pass);
continue;
}
}
run_flag = runPlan(WorkerPlan);
if(run_flag != ERR_CLEAN){
add_error(MyHandle, WorkerPlan->name,run_flag);
//MyHandle->Plan = (plan_list[SLEEP]->make)(sleep_pass);
EmitLog(MyRank, MyHandle->Num, "Runtime error flag triggered, error number:", run_flag, PRINT_ALWAYS);
}
}
}
return((void *)0); /* not reached */
} /* WorkerThread */
void QBaseThread::ThreadStarted( )
{
QString strLog = "Started";
EmitLog( strLog );
}
void QSmsThread::OnSendMsg( int nResult, int nMsgIndex )
{
QString strLog = QString( "Send Message %1 DestNumber:%2" ).arg( ( 0 == nResult ) ? "Success" : "Failure", QString::number( nMsgIndex ) );
EmitLog( strLog );
qDebug( ) << strLog << endl;
}
/**
* \brief Main..
*/
int main(int argc, char **argv, char **envp) {
int i, last;
Load load_data;
char **load_names = NULL;
int load_filesize = 0;
char *load_buffer = NULL;
int config_filesize = 0;
char *config_buffer = NULL;
char *config_file = NULL;
char *log_file = NULL;
int num_loads = 0;
struct timeval StartTime, CurrentTime;
int **errorFlags;
int err = ERR_CLEAN;
int pflag = 0;
Plan *CommPlan;
int iflag;
unsigned int nap;
comm_setup(&argc, &argv);
MyRank = comm_getrank();
last = 0;
num_loads = initialize(argc, argv, &log_file, &config_file, &load_names);
/* Initialize ROOT's global variables using input files. */
if(MyRank == ROOT) {
config_filesize = initConfigOptions(config_file, &config_buffer); /* parse configuration file. */
if((config_filesize <= 0) || (config_buffer == NULL) ) {
EmitLog(MyRank, SCHEDULER_THREAD, "Aborting run - A config file could not be opened/read.", -1, PRINT_ALWAYS);
config_filesize = 0;
if(config_buffer != NULL) {
free(config_buffer);
}
}
comm_broadcast_int(&config_filesize);
} else {
comm_broadcast_int(&config_filesize);
config_buffer = getFileBuffer(config_filesize);
}
if(config_filesize == 0) {
comm_finalize();
exit(1);
}
err = broadcast_buffer(config_buffer, config_filesize);
err += parseConfig(config_buffer, config_filesize);
free(config_buffer);
if(MyRank == ROOT) {
if(PRINT_RARELY <= verbose_flag) { /* Print status info. */
printf("num_loads = %d\n", num_loads);
printf("num_workers = %d\n", num_workers);
printf("thermal_panic = %d\n", thermal_panic);
printf("thermal_relaxation_time = %d\n", thermal_relaxation_time);
printf("monitor_frequency = %d\n", monitor_frequency);
printf("monitor_output_frequency = %d\n", monitor_output_frequency);
printf("temperature_path = %s\n", temperature_path);
for(i = 0; i < num_loads; i++) {
printf("load_names[%d] = %s\n", i, load_names[i]);
}
printf("log_file = %s\n", log_file);
}
}
// num_loads = bcastConfig(num_loads); // Broadcast global variables from ROOT to all others.
if(num_loads <= 0) { // If there are no loads to run, exit the program.
comm_finalize();
exit(0);
}
/* Initialize the communication load if it is to be run */
if(comm_flag != 0) {
data pass;
pass.isize = 1;
pass.i = &comm_flag;
CommPlan = makeCommPlan(&pass);
} else {
CommPlan = 0;
}
if(CommPlan != 0) {
iflag = (CommPlan->fptr_initplan)(CommPlan);
}
/* Initialize the array of ThreadHandle structures for the workers. */
WorkerHandle = (ThreadHandle *)malloc(num_workers * sizeof(ThreadHandle));
if(WorkerHandle == NULL) {
EmitLog(MyRank, SCHEDULER_THREAD, "Aborting run - Insufficient memory for the WorkerHandle struct", -1, PRINT_ALWAYS);
comm_finalize();
exit(1);
}
errorFlags = initErrorFlags();
initWorkerFlags();
if(DO_PERF) {
performance_init();
} //DO_PERF
if(MyRank == ROOT) {
EmitLog(MyRank, SCHEDULER_THREAD, "Initialization complete. Beginning run.", -1, PRINT_ALWAYS);
}
StartMonitorThread();
StartWorkerThreads();
sleep(thermal_relaxation_time); /* idle for a baseline */
reduceTemps();
sleep(thermal_relaxation_time);
/*********************************************************************************
* The following code will subscribe plans and their sizes to worker threads.
* This is a temporary load distribution being that it comes from a single load.
* In the future we would like to have several loads lined up to distribute to the
* worker threads in a simlar fashion. This will allow us to run the benchmark
* for different time durations depending on the total load schedule.
*********************************************************************************/
nap = monitor_output_frequency / 4;
if(nap < 1) {
nap = 1;
}
for(i = 0; i < num_loads; i++) {
if(MyRank == ROOT) { // Pull load data from a load file.
assert(load_names);
load_filesize = initLoadOptions(load_names[i], &load_buffer);
if((load_filesize <= 0) || (load_buffer == NULL) ) {
// /* Redundant */fprintf(stderr, "This load file could not be opened/read... trying next (if available).\n");
EmitLog(MyRank, SCHEDULER_THREAD, "This load file could not be opened/read... trying next (if available).", -1, PRINT_ALWAYS);
load_filesize = 0;
if(load_buffer != NULL) {
free(load_buffer);
}
}
comm_broadcast_int(&load_filesize);
if(load_filesize == 0) {
continue;
}
} else {
comm_broadcast_int(&load_filesize);
if(load_filesize == 0) {
continue;
}
load_buffer = getFileBuffer(load_filesize);
}
err = broadcast_buffer(load_buffer, load_filesize);
err += parseLoad(load_buffer, &load_data);
free(load_buffer);
// err = bcastLoad(&load_data); // Broadcast the load structure to all nodes (processes).
err = WorkerSched(&load_data); // Assign the load to worker threads and check for errors
if(MyRank == ROOT) {
printLoad(&load_data); // Print the load data to the terminal.
}
if(err != ERR_CLEAN) {
errorFlags[SYSTEM + 1][err]++;
}
#define SB_CONTINUE 0x0
#define SB_LAST_TRIP 0x1
#define SB_DO_REDUCTIONS 0x2
if(MyRank == ROOT) { // ROOT notes when we start this load
gettimeofday(&StartTime, NULL);
last = StartTime.tv_sec;
}
do { // DELAY WHILE LOAD RUNS: loop while the load executes until ROOT's clock says stop. Sleep if CommPlan isn't valid.
if((comm_flag != 0) && (CommPlan) && (CommPlan->fptr_execplan) && (CommPlan->vptr)) {
iflag = (CommPlan->fptr_execplan)(CommPlan); // run an iteration of the comm plan if enabled
} else {
gettimeofday(&CurrentTime, NULL);
if(nap + CurrentTime.tv_sec < StartTime.tv_sec + load_data.runtime) {
sleep(nap);
} else {
sleep((StartTime.tv_sec + load_data.runtime)-CurrentTime.tv_sec);
}
}
if(MyRank == ROOT) {
gettimeofday(&CurrentTime, NULL);
pflag = ((CurrentTime.tv_sec > last + monitor_output_frequency) << 1) | (CurrentTime.tv_sec < StartTime.tv_sec + load_data.runtime);
}
comm_broadcast_int(&pflag);
if(pflag & SB_DO_REDUCTIONS) {
assert(errorFlags);
reduceFlags(errorFlags);
reduceTemps();
if(MyRank == ROOT) {
last = CurrentTime.tv_sec;
}
}
} while(pflag & SB_LAST_TRIP);
// LOAD COMPLETE
if(MyRank == ROOT) {
EmitLog(MyRank, SCHEDULER_THREAD, "Elapsed time for this load:", CurrentTime.tv_sec - StartTime.tv_sec, PRINT_ALWAYS);
}
freeLoad(&load_data);
}
sleep(thermal_relaxation_time);
reduceTemps();
StopWorkerThreads(); // tell them all to finish
// Clean up the Communication Plan
if((comm_flag != 0) && (CommPlan) && (CommPlan->vptr)) {
if(CommPlan->fptr_perfplan) {
iflag = (CommPlan->fptr_perfplan)(CommPlan);
}
if(CommPlan->fptr_killplan) {
CommPlan = (CommPlan->fptr_killplan)(CommPlan);
}
}
sleep(thermal_relaxation_time);
if(DO_PERF) {
sleep(30);
perf_table_print(LOCAL, PRINT_OFTEN);
perf_table_reduce();
perf_table_maxreduce();
perf_table_minreduce();
if(MyRank == ROOT) {
perf_table_print(GLOBAL, PRINT_ALWAYS);
}
} //DO_PERF
if(MyRank == ROOT) {
EmitLog(MyRank, SCHEDULER_THREAD, "Run Completed. Exiting.", -1, PRINT_ALWAYS);
}
comm_finalize();
exit(0);
} /* main */
