int MPI_Finalize()
{
PMPI_Barrier(MPI_COMM_WORLD);
EPLIB_finalize();
PMPI_Barrier(MPI_COMM_WORLD);
return PMPI_Finalize();
}
void client_init(int taskid, int num_tasks)
{
ASSERT(max_ep > 0);
/* Create data structure to track all clients */
MALLOC_ALIGN(client_table, sizeof(client_t) * max_ep, CACHELINE_SIZE);
PMPI_Barrier(MPI_COMM_WORLD);
/* Initialize command queues */
cqueue_init(max_ep);
/* Spawn server */
server_create();
/* Create client data-structure */
for (int clientid = 0; clientid < max_ep; clientid++)
{
client_t* myclient = &client_table[clientid];
myclient->clientid = clientid;
myclient->taskid = taskid;
/* Attach client to a command queue */
myclient->cqueue = cqueue_attach(clientid);
memory_set_cqueue(clientid, myclient->cqueue);
}
PMPI_Barrier(MPI_COMM_WORLD);
if (use_mem_hooks) set_mem_hooks = 1;
}
static int lock_test(void) { /* anonymnous collective */
FILE *fh;
int fd,rv,log_rank=-1;
double t0,t1,t2;
char fn[MAXSIZE_FILENAME];
sprintf(fn,"lockfile.%s",job.cookie);
if(!task.mpi_rank) {
unlink(fn);
}
PMPI_Barrier(MPI_COMM_WORLD);
IPM_TIME_GTOD(t0);
IPM_FILE_LOCK(fn,fd);
fh = fdopen(fd,"w+");
IPM_TIME_GTOD(t1);
rv = fscanf(fh,"%d", &log_rank);
if(rv == 1) {
log_rank ++;
} else {
log_rank = 0;
}
fseek(fh,0,SEEK_SET);
fprintf(fh, "%d ", log_rank);
IPM_FILE_UNLOCK(fn,fd);
IPM_TIME_GTOD(t2);
printf("IPM: %d lock_test log_rank=%d file=%s wait=%.6f got=%.6f done=%.6f\n", task.mpi_rank,
log_rank,
fn,
t0,
t1,
t2);
return 0;
}
/*
* compute global statistics for hashtable statistics
* - min of min
* - max of max
* - sum of sum
*/
void gstats_hent(ipm_hent_t hent, gstats_t *global)
{
#ifdef HAVE_MPI
IPM_REDUCE( &hent.t_tot, &(global->dmin), 1,
MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
IPM_REDUCE( &hent.t_tot, &(global->dmax), 1,
MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
IPM_REDUCE( &hent.t_tot, &(global->dsum), 1,
MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
IPM_REDUCE( &hent.count, &(global->nmin), 1,
IPM_COUNT_MPITYPE, MPI_MIN, 0, MPI_COMM_WORLD);
IPM_REDUCE( &hent.count, &(global->nmax), 1,
IPM_COUNT_MPITYPE, MPI_MAX, 0, MPI_COMM_WORLD);
IPM_REDUCE( &hent.count, &(global->nsum), 1,
IPM_COUNT_MPITYPE, MPI_SUM, 0, MPI_COMM_WORLD);
/* on the cray with IPM_REPORT =full this causes a message
flood and a crash - to prevent this put in a barrier
Nick 10 June 2010 */
PMPI_Barrier(MPI_COMM_WORLD);
#else
global->dmin = hent.t_tot;
global->dmax = hent.t_tot;
global->dsum = hent.t_tot;
global->nmin = hent.count;
global->nmax = hent.count;
global->nsum = hent.count;
#endif
}
int MPI_Barrier(MPI_Comm comm)
{
int ret;
cqueue_t* mycqueue = handle_get_cqueue(comm);
if (mycqueue != NULL)
ret = cqueue_barrier(mycqueue, comm);
else
ret = PMPI_Barrier(comm);
return ret;
}
void allocator_init()
{
memory_get_limits();
if (!use_allocator) return;
/* Initialize and register inter-process shmem manager */
int memid = -1;
memory_init();
memid = memory_register(NULL /* allocate new */, shm_size, uuid_str, 1 /* client */);
if (memid == -1)
ERROR("Client unable to create default shared memory region (%ld bytes)\n", shm_size);
PMPI_Barrier(MPI_COMM_WORLD);
}
int MPI_Comm_create_endpoints(MPI_Comm parent_comm, int num_endpoints, MPI_Info info, MPI_Comm out_comm_hdls[])
{
int* num_ep_list;
int tot_endpoints = 0;
int local_epid, global_epid = 0;
int i;
/* Return error if requested endpoints greater than servers */
if (num_endpoints > max_ep) return MPI_ERR_UNKNOWN;
/* Map endpoints to servers */
/* Step 1/3: Gather total number of endpoints requested by all
application tasks in the parent communicator parent_comm */
num_ep_list = (int*)malloc(num_tasks*sizeof(int));
ASSERT(num_ep_list != NULL);
PMPI_Allgather(&num_endpoints, 1, MPI_INT, (void*)num_ep_list, 1, MPI_INT, parent_comm);
/* Step 2/3: Create communicator handle objects */
for (i = 0; i < num_tasks; i++)
{
tot_endpoints += num_ep_list[i];
if (i < taskid)
global_epid += num_ep_list[i];
}
for (local_epid = 0; local_epid < num_endpoints; local_epid++)
{
out_comm_hdls[local_epid] = (MPI_Comm) handle_register(COMM_EP, MPI_COMM_NULL, parent_comm,
num_endpoints, local_epid, tot_endpoints,
global_epid, NULL, NULL);
global_epid++;
}
/* Step 3/3: */
client_multiplex_endpoints(max_ep, num_tasks, num_endpoints, num_ep_list, out_comm_hdls);
PMPI_Barrier(parent_comm);
free(num_ep_list);
return MPI_SUCCESS;
}
void
mpiPi_publishResults (int report_style)
{
FILE *fp = NULL;
static int printCount = 0;
if (mpiPi.collectorRank == mpiPi.rank)
{
/* Generate output filename, and open */
do
{
printCount++;
snprintf (mpiPi.oFilename, 256, "%s/%s.%d.%d.%d.mpiP",
mpiPi.outputDir, mpiPi.appName, mpiPi.size, mpiPi.procID,
printCount);
}
while (access (mpiPi.oFilename, F_OK) == 0);
fp = fopen (mpiPi.oFilename, "w");
if (fp == NULL)
{
mpiPi_msg_warn ("Could not open [%s], writing to stdout\n",
mpiPi.oFilename);
fp = stdout;
}
else
{
mpiPi_msg ("\n");
mpiPi_msg ("Storing mpiP output in [%s].\n", mpiPi.oFilename);
mpiPi_msg ("\n");
}
}
mpiPi_profile_print (fp, report_style);
PMPI_Barrier (MPI_COMM_WORLD);
if (fp != stdout && fp != NULL)
{
fclose (fp);
}
}
void ipm_log(void) { /* called by all tasks (or as many as possible) */
int i,ii,rv,icall,ibytes,irank,ireg,kreg;
int ipm_log_fd=-1, fan_out=0;
int log_rank=-1, search_offset=0, token=1;
FILE *ipm_mpi_log_fh;
MPI_File ipm_mpiio_log_fh;
DIR *ipm_mpi_log_dp;
struct dirent *de;
struct stat file_stat;
IPM_KEY_TYPE key,ikey;
char *cp, txt[MAXSIZE_TXTLINE];
char tmp_fname[MAXSIZE_FILENAME];
char tmp_pref[MAXSIZE_FILENAME];
char tmp_cmd[MAXSIZE_FILENAME];
double b_flops;
double stamp1, stamp2, stamp3, stamp4;
MPI_Status s[4];
MPI_Info outinfo;
if(task.flags & IPM_WROTELOG) return;
memset((void *)txt,0,(size_t)(MAXSIZE_TXTLINE*sizeof(char)));
task.flags |= IPM_WROTELOG;
/* only one chance, even if we fail at this point we should not return */
if(task.flags & DEBUG) {
printf("IPM: %d log enter job.cookie=%s username=%s \n",
task.mpi_rank,
job.cookie,
job.username);
fflush(stdout);
}
/*
** bail
*/
if(strcmp(job.log_dir, "/dev/null") == 0 ) {
if(task.flags & DEBUG) {
printf("IPM: %d log exit due to LOGDIR=/dev/null", task.mpi_rank);
}
return;
}
if(stat(job.log_dir,&file_stat)) {
if(!task.mpi_rank) {
printf("IPM: %d log IPMLOG_DIR %s not available using $CWD \n",
task.mpi_rank, job.log_dir);
}
sprintf(job.log_dir, "./");
}
/*
** Aggregation method #1 : Multiple Files - No Aggregation
IPM_LOG_USEMULTI
** Aggregation method #2 : Single File - Locks
IPM_LOG_USELOCKS
** Aggregation method #3 : Single File - SMP & /tmp
IPM_LOG_USETMPFS
** Aggregation method #4 : Single File - MPI - default
IPM_LOG_USEMPI
*/
#ifndef IPM_LOG_USEMULTI
#ifndef IPM_LOG_USELOCKS
#ifndef IPM_LOG_USETMPFS
#ifndef IPM_LOG_USEMPI
#endif
#endif
#endif
#endif
#ifdef IPM_LOG_USEMULTI
sprintf(job.log_fname,"%s/%s.%s.%d",
job.log_dir,
job.username,
job.cookie,
task.mpi_rank);
#else
if (!strcmp(job.log_fname,"unset")) {
sprintf(job.log_fname,"%s/%s.%s.%d",
job.log_dir,
job.username,
job.cookie,0);
}
else
{
sprintf(tmp_fname,"%s/%s",job.log_dir,job.log_fname);
sprintf(job.log_fname,"%s",tmp_fname);
}
#endif
if(task.flags & DEBUG) {
printf("IPM: %d log IPMLOG_DIR=%s FNAME=%s \n",
task.mpi_rank,
job.log_dir,
job.log_fname);
}
/*
** Aggregation method #1 : Multiple Files - No Aggregation {
*/
#ifdef IPM_LOG_USEMULTI
/* simplest case no locking just write each file. Parallel FS may
have metadata storm for N file creates */
ipm_mpi_log_fh = fopen(job.log_fname,"w");
if(ipm_mpi_log_fh == NULL) {
printf("IPM: %d log fopen failed fname=%s \n",
task.mpi_rank,
job.log_fname);
fflush(stdout);
}
rv = fprintf(ipm_mpi_log_fh,
"<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n<ipm>\n");
ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh);
rv = fprintf(ipm_mpi_log_fh,
"</ipm>\n");
fclose(ipm_mpi_log_fh);
chmod(job.log_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
return;
#endif
/* } */
/*
** Aggregation methods #2 : Single File - Locks {
*/
#ifdef IPM_LOG_USELOCKS
signal(SIGALRM, ipm_alarm_log_block);
alarm(30);
IPM_TIME_GTOD(stamp1);
if(task.flags & DEBUG) {
printf("IPM: %d log block_lock fname=%s fd=%d stamp=%f \n",
task.mpi_rank,
job.log_fname,
ipm_log_fd,
stamp1
);
fflush(stdout);
}
IPM_FILE_LOCK(job.log_fname,ipm_log_fd);
IPM_TIME_GTOD(stamp2);
if(task.flags & DEBUG) {
printf("IPM: %d log block_lock fname=%s fd=%d stamp=%12.6f delta=%.3e \n",
task.mpi_rank,
job.log_fname,
ipm_log_fd,
stamp2, stamp2-stamp1
);
fflush(stdout);
}
alarm(0);
signal(SIGALRM, SIG_DFL);
ipm_mpi_log_fh = fdopen(ipm_log_fd,"w+");
if(!ipm_mpi_log_fh || !ipm_log_fd) {
/* fail silently */
return;
}
/* got log fh */
fseek(ipm_mpi_log_fh,0,SEEK_END);
ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh);
IPM_TIME_GTOD(stamp3);
if(task.flags & DEBUG) {
printf("IPM: %d log write fname=%s fd=%d stamp=%12.6f delta=%.3e \n",
task.mpi_rank,
job.log_fname,
ipm_log_fd,
stamp3, stamp3-stamp2
);
fflush(stdout);
}
fflush(ipm_mpi_log_fh);
IPM_FILE_UNLOCK(job.log_fname,ipm_log_fd);
IPM_TIME_GTOD(stamp4);
if(task.flags & DEBUG) {
printf("IPM: %d log unlock fname=%s fd=%d stamp=%12.6f delta=%.3e \n",
task.mpi_rank,
job.log_fname,
ipm_log_fd,
stamp4, stamp4-stamp3
);
fflush(stdout);
}
#endif
/* } */
/*
** Aggregation method #3 : Single File - SMP & /tmp {
*/
#ifdef IPM_LOG_USETMPFS
if(task.flags & IPM_MPI_FINALIZING) {
if(task.mpi_size == 1) { /* special easy case now uneeded */
}
sprintf(tmp_fname,"/tmp/%s.%s.%d",
job.username,
job.cookie,
task.mpi_rank);
ipm_mpi_log_fh = fopen(tmp_fname,"w");
if(ipm_mpi_log_fh == NULL) {
printf("IPM: %d log fopen failed fname=%s \n",
task.mpi_rank,
tmp_fname);
fflush(stdout);
}
chmod(tmp_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh);
fclose(ipm_mpi_log_fh);
/* host local ring barrier so that /tmp is all good */
if(task.intra_size > 1) {
if(task.intra_root == task.mpi_rank) {
PMPI_Send(&i,1,MPI_INT,task.intra_right,0,MPI_COMM_WORLD);
PMPI_Recv(&i,1,MPI_INT,task.intra_left,0,MPI_COMM_WORLD,s);
} else {
PMPI_Recv(&i,1,MPI_INT,task.intra_left,0,MPI_COMM_WORLD,s);
PMPI_Send(&i,1,MPI_INT,task.intra_right,0,MPI_COMM_WORLD);
}
}
if(task.intra_root == task.mpi_rank) {
if(job.nhosts > 1 && task.mpi_rank) {
PMPI_Recv(&i,1,MPI_INT,task.inter_left,0,MPI_COMM_WORLD,s);
}
/* sh -c lacks PATH on some system so remove popen&system where possible */
sprintf(tmp_cmd, "/usr/bin/cat /tmp/%s.%s.* >> %s",
job.username,
job.cookie,
job.syslog_fname);
system(tmp_cmd);
chmod(job.syslog_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
sprintf(tmp_cmd, "/usr/bin/rm -f /tmp/%s.%s.* ",
job.username,
job.cookie);
system(tmp_cmd);
/* ugh! duplicating cat and rm is yucky
sprintf(tmp_pref,"%s.%s", job.username, job.cookie);
dp=opendir("/tmp");
if(dp) {
while(de=readdir(dp))!=NULL){
if(!strncmp(de->d_name,tmp_pref, strlen(tmp_fname))) {
sprintf(tmp_fname,"/tmp/%s", de->d_name);
fopen(tmp_fname,"r"
read in pieces and write them to the intra-node file
delete the .rank file
}
}
}
*/
if(job.nhosts > 1 && task.inter_right != 0) {
PMPI_Send(&i,1,MPI_INT,task.inter_right,0,MPI_COMM_WORLD);
}
}
return;
}
#endif
/* } */
/*
** Aggregation method #4 : Single File - MPI {
*/
#ifdef IPM_LOG_USEMPI
if (task.flags & PARALLEL_IO_LOG ) {
int buff_size=0;
MPI_Offset file_offset=0;
int64_t buff_sum=0;
int malloc_flag,malloc_sum;
char* buffer=NULL;
MPI_Info info;
/* measure size of buff required */
buff_size=ipm_log_write(&job, &task, txt, buffer,0,1);
malloc_flag=1;
buffer = (char*)malloc(buff_size+1);
if (buffer == NULL) {
malloc_flag=0;
} else {
rv=ipm_log_write(&job, &task, txt, buffer,buff_size,1);
}
/*see whether malloc suceeded across all mpi tasks */
PMPI_Allreduce(&malloc_flag,&malloc_sum,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
if (malloc_sum == task.mpi_size) {/* use parallel IO */
if(task.flags & DEBUG && !task.mpi_rank) {
printf("IPM: %d IPM report parallel IO used\n", task.mpi_rank);
}
PMPI_Info_create(&info);
#ifndef CRAY_GPFS_BUG
PMPI_Info_set(info,"access_style","write_once");
PMPI_Info_set(info,"collective_buffering","true");
PMPI_Info_set(info,"file_perm","0644");
PMPI_Info_set(info,"romio_cb_read","true");
PMPI_Info_set(info,"cb_align","2");
PMPI_Info_set(info,"romio_cb_write","true");
PMPI_Info_set(info,"cb_config_list","*:1");
PMPI_Info_set(info,"striping_factor","80");
PMPI_Info_set(info,"IBM_largeblock_io","true");
#endif
/* with allowing the user to choose the filename - can overwrite an old */
/* file - which would be fine if MPI-IO allowed TRUNC - but it doesn't */
/* so we just delete so that we don't end up with trailing garbage */
if (!task.mpi_rank) rv=PMPI_File_delete ( job.log_fname,MPI_INFO_NULL);
rv=PMPI_Barrier(MPI_COMM_WORLD);
rv = PMPI_File_open( MPI_COMM_WORLD, job.log_fname, MPI_MODE_WRONLY | MPI_MODE_CREATE,info, &ipm_mpiio_log_fh );
if (rv)
{
printf("IPM: %d syslog fopen failed fname=%s \n",
task.mpi_rank,
job.log_fname);
fflush(stdout);
return;
}
/* workaround for cases when MPI_INTEGER8 is not defined */
#ifndef MPI_INTEGER8
#define MPI_INTEGER8 MPI_LONG_LONG_INT
#endif
if (task.mpi_size > 1) {
if (task.mpi_rank == 0) {
buff_sum+=(int64_t)buff_size;
PMPI_Send (&buff_sum,1,MPI_INTEGER8,1,0,MPI_COMM_WORLD);
file_offset=0;
} else if (task.mpi_rank == (task.mpi_size-1)) {
PMPI_Recv (&buff_sum,1,MPI_INTEGER8,task.mpi_rank-1,0,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
file_offset=(MPI_Offset)buff_sum;
} else {
PMPI_Recv (&buff_sum,1,MPI_INTEGER8,task.mpi_rank-1,0,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
file_offset=(MPI_Offset)buff_sum;
buff_sum+=(int64_t)buff_size;
PMPI_Send (&buff_sum,1,MPI_INTEGER8,task.mpi_rank+1,0,MPI_COMM_WORLD);
}
}
rv=PMPI_File_set_view(ipm_mpiio_log_fh,file_offset,MPI_CHAR, MPI_CHAR,"native",info);
/*write info*/
rv=PMPI_File_write_all(ipm_mpiio_log_fh,buffer,buff_size,MPI_CHAR,MPI_STATUS_IGNORE);
rv = PMPI_File_close( &ipm_mpiio_log_fh );
PMPI_Barrier(MPI_COMM_WORLD);
/* Some MPI-IO implimentations (cray) permissions are not setable with hints */
if (task.mpi_rank == 0) chmod (job.log_fname,0744);
free (buffer);
return;
} else {
if (! task.mpi_rank) printf("IPM: %d Allocation of IO Buffer failed on one or more tasks\n",task.mpi_rank);
}
}
/*parallel IO failed */
if (! task.mpi_rank) printf("IPM: %d Using serial IO\n",task.mpi_rank);
/*************************************/
/* write log from rank zero using MPI*/
/*************************************/
if(task.mpi_rank==0) {
ipm_mpi_log_fh = fopen(job.log_fname,"w+");
if(ipm_mpi_log_fh == NULL) {
printf("IPM: %d syslog fopen failed fname=%s \n",
task.mpi_rank,
job.log_fname);
fflush(stdout);
}
chmod(job.log_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
ipm_log_write(&job, &task, txt,ipm_mpi_log_fh ,0,0);
/* we now pollute the local profile state irrevocably in the interest
of keeping a memory footprint which is a constant independent of
concurrency */
/* task 0 initiates a volley of Sends via a handshake */
for(i=1; i<job.ntasks; i++) {
PMPI_Send(&token,1,MPI_INT,i,0,MPI_COMM_WORLD);
PMPI_Recv(&job,sizeof(struct ipm_jobdata),MPI_BYTE,i,0,MPI_COMM_WORLD,s+0);
PMPI_Recv(&task,sizeof(struct ipm_taskdata),MPI_BYTE,i,1,MPI_COMM_WORLD,s+1);
PMPI_Recv(&(txt[0]),MAXSIZE_TXTLINE,MPI_CHAR,i,1,MPI_COMM_WORLD,s+1);
ipm_log_write(&job, &task, txt,ipm_mpi_log_fh ,0,0);
}
fclose(ipm_mpi_log_fh);
} else {
PMPI_Recv(&token,1,MPI_INT,0,0,MPI_COMM_WORLD,s+0);
PMPI_Send(&job,sizeof(struct ipm_jobdata),MPI_BYTE,0,0,MPI_COMM_WORLD);
PMPI_Send(&task,sizeof(struct ipm_taskdata),MPI_BYTE,0,1,MPI_COMM_WORLD);
PMPI_Send(&(txt[0]),MAXSIZE_TXTLINE,MPI_CHAR,0,1,MPI_COMM_WORLD);
}
PMPI_Barrier(MPI_COMM_WORLD);
return;
#endif
return;
}
void vt_sync(MPI_Comm comm, uint64_t* ltime, int64_t* offset)
{
VT_MPI_INT myrank, myrank_host, myrank_sync;
VT_MPI_INT numnodes;
uint64_t time;
MPI_Comm host_comm;
MPI_Comm sync_comm;
VT_SUSPEND_IO_TRACING(VT_CURRENT_THREAD);
/* mark begin of clock synchronization */
time = vt_pform_wtime();
vt_enter(VT_CURRENT_THREAD, &time, vt_trc_regid[VT__TRC_SYNCTIME]);
/* barrier at entry */
PMPI_Barrier(comm);
*offset = 0;
*ltime = vt_pform_wtime();
PMPI_Comm_rank(comm, &myrank);
/* create communicator containing all processes on the same node */
PMPI_Comm_split(comm, (vt_pform_node_id() & 0x7FFFFFFF), 0, &host_comm);
PMPI_Comm_rank(host_comm, &myrank_host);
/* create communicator containing all processes with rank zero in the
previously created communicators */
PMPI_Comm_split(comm, myrank_host, 0, &sync_comm);
PMPI_Comm_rank(sync_comm, &myrank_sync);
PMPI_Comm_size(sync_comm, &numnodes);
/* measure offsets between all nodes and the root node (rank 0 in sync_comm) */
if (myrank_host == 0)
{
VT_MPI_INT i;
for (i = 1; i < numnodes; i++)
{
PMPI_Barrier(sync_comm);
if (myrank_sync == i)
*offset = sync_slave(ltime, 0, sync_comm);
else if (myrank_sync == 0)
*offset = sync_master(ltime, i, sync_comm);
}
}
/* distribute offset and ltime across all processes on the same node */
PMPI_Bcast(offset, 1, MPI_LONG_LONG_INT, 0, host_comm);
PMPI_Bcast(ltime, 1, MPI_LONG_LONG_INT, 0, host_comm);
PMPI_Comm_free(&host_comm);
PMPI_Comm_free(&sync_comm);
/* barrier at exit */
PMPI_Barrier(comm);
/* mark end of clock synchronization */
time = vt_pform_wtime();
vt_exit(VT_CURRENT_THREAD, &time);
VT_RESUME_IO_TRACING(VT_CURRENT_THREAD);
}
void vt_esync(MPI_Comm comm)
{
uint64_t time, etime;
Sync_TsPerRun* temp_ts;
Sync_Map* temp_map;
VT_MPI_INT myrank;
VT_MPI_INT numnodes;
VT_MPI_INT partnerid, numslots;
VT_MPI_INT i;
VT_SUSPEND_IO_TRACING();
/* mark begin of clock synchronization */
time = vt_pform_wtime();
vt_enter(&time, vt_trc_regid[VT__TRC_SYNCTIME]);
/* ... also as comment for vtunify */
vt_comment(&time, "__ETIMESYNC__");
/* barrier at entry */
PMPI_Barrier(comm);
temp_ts = (Sync_TsPerRun*) malloc(sizeof(Sync_TsPerRun));
if (temp_ts == NULL) vt_error();
temp_ts->sync_phase = NULL;
temp_ts->next = NULL;
if (SyncTsPerRunFirst == NULL)
{
SyncTsPerRunFirst = temp_ts;
SyncTsPerRunLast = temp_ts;
}
else
{
SyncTsPerRunLast->next = temp_ts;
SyncTsPerRunLast = temp_ts;
}
/* measure time synchronization */
PMPI_Comm_rank(comm, &myrank);
PMPI_Comm_size(comm, &numnodes);
numslots = (VT_MPI_INT)ceil(log((double)(numnodes)) / log(2.0));
for(i = 0; i < numslots; i++)
{
partnerid = esync_commpartner(myrank, numnodes, i);
if( partnerid < numnodes )
{
if( myrank < partnerid )
{
esync_master(partnerid, comm, myrank);
}
else
{
esync_slave(partnerid, comm);
}
}
}
/* add myrank to list of map ids */
temp_map = (Sync_Map*)malloc(sizeof(Sync_Map));
if (temp_map == NULL) vt_error();
temp_map->id = myrank;
temp_map->time = time;
temp_map->duration = (uint32_t) 0;
temp_map->next = NULL;
if (SyncMapIdFirst == NULL)
{
SyncMapIdFirst = temp_map;
SyncMapIdLast = temp_map;
}
else
{
SyncMapIdLast->next = temp_map;
SyncMapIdLast = temp_map;
}
/* barrier at exit */
PMPI_Barrier(comm);
/* mark end of clock synchronization */
etime = vt_pform_wtime();
vt_exit(&etime);
/* increment number of sync. phases */
SyncRound++;
/* set timestamp of next synchronization if necessary */
if (SyncNext != (uint64_t)-1)
SyncNext = etime + SyncIntv;
/* calculate sync. duration */
SyncMapIdLast->duration = etime - time;
VT_RESUME_IO_TRACING();
}
int MPI_Barrier(MPI_Comm comm)
{
return PMPI_Barrier(comm);
}
void ipm_banner(FILE *f)
{
ipm_hent_t stats_mpi;
ipm_hent_t stats_pio;
ipm_hent_t stats_omp;
ipm_hent_t stats_ompi;
double wallt, gflops, mpip, piop, ompp;
ipm_hent_t stats_all[MAXSIZE_CALLTABLE];
int i, j;
for( i=0; i<MAXNUM_REGIONS; i++ ) {
banner.regions[i].valid=0;
banner.regions[i].name[0]='\0';
for( j=0; j<MAXNUM_REGNESTING; j++ ) {
banner.regions[i].nesting[j][0]='\0';
}
}
banner.flags=0;
#ifdef HAVE_MPI
banner.flags|=BANNER_HAVE_MPI;
#endif
#ifdef HAVE_POSIXIO
banner.flags|=BANNER_HAVE_POSIXIO;
#endif
#ifdef HAVE_OMPTRACEPOINTS
banner.flags|=BANNER_HAVE_OMP;
#endif
#ifdef HAVE_CUDA
banner.flags|=BANNER_HAVE_CUDA;
#endif
#ifdef HAVE_CUBLAS
banner.flags|=BANNER_HAVE_CUBLAS;
#endif
#ifdef HAVE_CUFFT
banner.flags|=BANNER_HAVE_CUFFT;
#endif
if( task.flags&FLAG_REPORT_FULL )
{
banner.flags|=BANNER_FULL;
for( i=0; i<MAXSIZE_CALLTABLE; i++ ) {
banner.calltable[i]=ipm_calltable[i].name;
}
}
gstats_double( task.procmem, &(banner.procmem) );
/* --- compute statistics for whole app --- */
clear_region_stats( &(banner.app) );
compute_region_stats(ipm_rstack->child, &(banner.app), 1);
for( j=2; j<MAXNUM_REGIONS; j++ ) {
region_t *reg=0;
region_t *tmp=0;
reg = rstack_find_region_by_id(ipm_rstack, j);
if( reg ) {
if( !((task.flags)&FLAG_NESTED_REGIONS) &&
reg->parent!=ipm_rstack->child ) {
continue;
}
banner.regions[j].valid=1;
strncpy(banner.regions[j].name, reg->name, MAXSIZE_REGLABEL);
/* record the nesting */
tmp=reg;
for( i=0; i<MAXNUM_REGNESTING; i++ ) {
if(!tmp || tmp==task.rstack)
break;
strncpy(banner.regions[j].nesting[i], tmp->name, MAXSIZE_REGLABEL);
tmp=tmp->parent;
}
clear_region_stats( &(banner.regions[j]) );
compute_region_stats(reg, &(banner.regions[j]), 1);
}
}
/* --- compute statistics for ipm_noregion --- */
clear_region_stats( &(banner.regions[1]) );
compute_region_stats(ipm_rstack->child, &(banner.regions[1]), 0);
sprintf(banner.regions[1].name, "ipm_noregion");
banner.regions[1].valid=1;
#ifdef HAVE_MPI
PMPI_Barrier(MPI_COMM_WORLD);
#endif
if( task.taskid==0 ) {
banner.app.valid=1;
banner.app.name[0]='\0';
/* --- prepare banner with data from task 0 --- */
banner.tstart = task.t_start;
banner.tstop = task.t_stop;
banner.ntasks = task.ntasks;
banner.nhosts = task.nhosts;
banner.nthreads = 1;
#ifdef HAVE_OMPTRACEPOINTS
banner.nthreads = maxthreads;
#endif /* HAVE_OMPTRACEPOINTS */
strcpy(banner.cmdline, (const char*)task.exec_cmdline);
strcpy(banner.hostname, task.hostname);
/* --- print it --- */
ipm_print_banner(f, &banner);
}
}