static void esync_master(VT_MPI_INT slave, MPI_Comm comm, VT_MPI_INT masterid)
{
int i;
uint64_t tsend, trecv, tslave;
uint64_t t1, t2, t3, t4;
MPI_Status stat;
MPI_Request req;
Sync_TsPerPhase* temp;
/* exchange LOOP_COUNT ping pong messages with the communication partner */
t1 = vt_pform_wtime();
PMPI_Isend( &t1, 1, MPI_LONG_LONG_INT, slave, 0, comm, &req );
PMPI_Recv( &t2, 1, MPI_LONG_LONG_INT, slave, 0, comm, &stat );
t4 = vt_pform_wtime();
t3 = t2;
PMPI_Waitall( 1, &req, &stat );
for( i = 1; i < LOOP_COUNT; i++ )
{
tsend = vt_pform_wtime();
/* message exchange */
PMPI_Isend(&tsend, 1, MPI_LONG_LONG_INT, slave, i, comm, &req);
PMPI_Recv(&tslave, 1, MPI_LONG_LONG_INT, slave, i, comm, &stat);
trecv = vt_pform_wtime();
PMPI_Waitall(1, &req, &stat);
/* select timestamps with minimum message delay in each direction */
if ( ( (int64_t)tslave - (int64_t)tsend ) < ( (int64_t)t2 - (int64_t)t1 ) )
{
t1 = tsend;
t2 = tslave;
}
if ( ( (int64_t)trecv - (int64_t)tslave ) < ( (int64_t)t4 - (int64_t)t3 ) )
{
t3 = tslave;
t4 = trecv;
}
}
/* save synchronization measurement data into internal data structure */
temp = (Sync_TsPerPhase*)malloc(sizeof(Sync_TsPerPhase));
if (!temp) vt_error();
temp->id1 = masterid;
temp->id2 = slave;
temp->t1 = t1;
temp->t2 = t2;
temp->t3 = t3;
temp->t4 = t4;
temp->next = SyncTsPerRunLast->sync_phase;
SyncTsPerRunLast->sync_phase = temp;
}
int MPI_Recv(void* buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status)
{
MPE_Log_event(START_RECV,0,"recv");
int wynik=PMPI_Recv(buf,count,datatype,source,tag,comm,status);
MPE_Log_event(END_RECV,0,"recv");
return wynik;
};
void Trace::recvTrace(int src){
int bufsize;
void *buf;
MPI_Comm comm = MPI_COMM_WORLD;
int position = 0;
PMPI_Recv(&bufsize, 1, MPI_INT, src, 0, comm, MPI_STATUS_IGNORE);
buf = malloc(bufsize);
if(!buf){
cerr << "Trace::recvTrace(): cannot allocate buffer" << endl;
exit(1);
}
PMPI_Recv(buf, bufsize, MPI_PACKED, src, 0, comm, MPI_STATUS_IGNORE);
unpack(buf, bufsize, &position, comm);
free(buf);
}
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status)
{
int err;
ALLOCATE_STATUS(newstatus,1)
err=PMPI_Recv(buf,count,datatype,source,tag,comm,newstatus);
COPY_STATUS(status,newstatus,1)
return err;
}
int MPI_Recv(void* buffer, int count, MPI_Datatype datatype,
int src, int tag, MPI_Comm comm, MPI_Status* status)
{
cqueue_t* mycqueue = handle_get_cqueue(comm);
if (mycqueue != NULL)
return cqueue_recv(mycqueue, buffer, count, datatype, src, tag, comm, status);
else
return PMPI_Recv(buffer, count, datatype, src, tag, comm, status);
}
void receive_keys(effort_data& effort_log, int src, MPI_Comm comm) {
MPI_Status status;
int bufsize;
PMPI_Recv(&bufsize, 1, MPI_INT, src, 0, comm, &status);
char buf[bufsize];
PMPI_Recv(buf, bufsize, MPI_PACKED, src, 0, comm, &status);
int position = 0;
ModuleId::id_map modules;
ModuleId::unpack_id_map(buf, bufsize, &position, modules, comm);
int num_keys;
PMPI_Unpack(buf, bufsize, &position, &num_keys, 1, MPI_INT, comm);
for (int i=0; i < num_keys; i++) {
effort_key key = effort_key::unpack(modules, buf, bufsize, &position, comm);
if (!effort_log.contains(key)) {
effort_log[key] = effort_record(effort_log.progress_count);
}
}
}
static int64_t sync_slave(uint64_t* ltime, VT_MPI_INT master, MPI_Comm comm)
{
VT_MPI_INT min;
MPI_Status stat;
uint64_t tsendrecv[LOOP_COUNT];
uint64_t sync_time;
int i;
for (i = 0; i < LOOP_COUNT; i++)
{
PMPI_Recv(NULL, 0, MPI_INT, master, 1, comm, &stat);
tsendrecv[i] = vt_pform_wtime();
PMPI_Send(NULL, 0, MPI_INT, master, 2, comm);
}
/* receive corresponding time together with its index from master */
PMPI_Recv(&min, 1, MPI_INT, master, 3, comm, &stat);
PMPI_Recv(&sync_time, 1, MPI_LONG_LONG_INT, master, 4, comm, &stat);
*ltime = tsendrecv[min];
return (int64_t)(sync_time - *ltime);
}
static void esync_slave(VT_MPI_INT master, MPI_Comm comm)
{
int i;
uint64_t t, tslave;
MPI_Status stat;
/* start communication with master */
for (i = 0; i < LOOP_COUNT; i++)
{
PMPI_Recv( &t, 1, MPI_LONG_LONG_INT, master, i, comm, &stat );
tslave = vt_pform_wtime();
PMPI_Send( &tslave, 1, MPI_LONG_LONG_INT, master, i, comm );
}
}
int MPI_Recv(void* buf, int num, MPI_Datatype dtype, int node,
int tag, MPI_Comm comm, MPI_Status *status)
{
int res;
printf("WRAPPER 4: Before recv\n");
fflush(stdout);
res=PMPI_Recv(buf,num,dtype,node,tag,comm,status);
printf("WRAPPER 4: After recv\n");
fflush(stdout);
return res;
}
int MPI_Recv(void *buf, int count, MPI_Datatype type, int source,
int tag, MPI_Comm comm, MPI_Status *status)
{
char typename[MPI_MAX_OBJECT_NAME], commname[MPI_MAX_OBJECT_NAME];
int len;
int rank;
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
PMPI_Type_get_name(type, typename, &len);
PMPI_Comm_get_name(comm, commname, &len);
fprintf(stderr, "MPI_RECV[%d]: buf %0" PRIxPTR " count %d datatype %s source %d tag %d comm %s\n",
rank, (uintptr_t) buf, count, typename, source, tag, commname);
fflush(stderr);
return PMPI_Recv(buf, count, type, source, tag, comm, status);
}
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source,
int tag, MPI_Comm comm, MPI_Status *status)
{
int returnVal;
int size;
returnVal = PMPI_Recv( buf, count, datatype, source, tag, comm, status );
if (source != MPI_PROC_NULL && returnVal == MPI_SUCCESS) {
MPI_Get_count( status, MPI_BYTE, &size );
prof_recv( procid_1, status->MPI_SOURCE,
status->MPI_TAG, size, "MPI_Recv" );
}
return returnVal;
}
static int64_t sync_master(uint64_t* ltime, VT_MPI_INT slave, MPI_Comm comm)
{
VT_MPI_INT min;
MPI_Status stat;
uint64_t tsend[LOOP_COUNT], trecv[LOOP_COUNT];
uint64_t pingpong_time, sync_time;
int i;
/* exchange LOOP_COUNT ping pong messages with slave */
for (i = 0; i < LOOP_COUNT; i++)
{
tsend[i] = vt_pform_wtime();
PMPI_Send(NULL, 0, MPI_INT, slave, 1, comm);
PMPI_Recv(NULL, 0, MPI_INT, slave, 2, comm, &stat);
trecv[i] = vt_pform_wtime();
}
/* select ping pong with shortest transfer time */
pingpong_time = trecv[0] - tsend[0];
min = 0;
for (i = 1; i < LOOP_COUNT; i++)
{
if ((trecv[i] - tsend[i]) < pingpong_time)
{
pingpong_time = (trecv[i] - tsend[i]);
min = (VT_MPI_INT)i;
}
}
sync_time = tsend[min] + (pingpong_time / 2);
/* send sync_time together with corresponding measurement index to slave */
PMPI_Send(&min, 1, MPI_INT, slave, 3, comm);
PMPI_Send(&sync_time, 1, MPI_LONG_LONG_INT, slave, 4, comm);
/* the process considered as the global clock returns 0 as offset */
*ltime = vt_pform_wtime();
return 0;
}
/* MPI_Bcast_user: This is our version of MPI function. */
int MPI_Bcast_user(void *buf, int count, MPI_Datatype datatype, int root,
MPI_Comm comm)
{
int i, rank, commsize;
PMPI_Comm_size(comm, &commsize);
PMPI_Comm_rank(comm, &rank);
/* Simple linear algorithm for broadcast */
if (rank == root) {
for (i = 0; i < commsize; i++) {
if (i != root)
PMPI_Send(buf, count, datatype, i, 4321, comm);
}
} else {
PMPI_Recv(buf, count, datatype, root, 4321, comm, MPI_STATUS_IGNORE);
}
return MPI_SUCCESS;
}
void ipm_log(void) { /* called by all tasks (or as many as possible) */
int i,ii,rv,icall,ibytes,irank,ireg,kreg;
int ipm_log_fd=-1, fan_out=0;
int log_rank=-1, search_offset=0, token=1;
FILE *ipm_mpi_log_fh;
MPI_File ipm_mpiio_log_fh;
DIR *ipm_mpi_log_dp;
struct dirent *de;
struct stat file_stat;
IPM_KEY_TYPE key,ikey;
char *cp, txt[MAXSIZE_TXTLINE];
char tmp_fname[MAXSIZE_FILENAME];
char tmp_pref[MAXSIZE_FILENAME];
char tmp_cmd[MAXSIZE_FILENAME];
double b_flops;
double stamp1, stamp2, stamp3, stamp4;
MPI_Status s[4];
MPI_Info outinfo;
if(task.flags & IPM_WROTELOG) return;
memset((void *)txt,0,(size_t)(MAXSIZE_TXTLINE*sizeof(char)));
task.flags |= IPM_WROTELOG;
/* only one chance, even if we fail at this point we should not return */
if(task.flags & DEBUG) {
printf("IPM: %d log enter job.cookie=%s username=%s \n",
task.mpi_rank,
job.cookie,
job.username);
fflush(stdout);
}
/*
** bail
*/
if(strcmp(job.log_dir, "/dev/null") == 0 ) {
if(task.flags & DEBUG) {
printf("IPM: %d log exit due to LOGDIR=/dev/null", task.mpi_rank);
}
return;
}
if(stat(job.log_dir,&file_stat)) {
if(!task.mpi_rank) {
printf("IPM: %d log IPMLOG_DIR %s not available using $CWD \n",
task.mpi_rank, job.log_dir);
}
sprintf(job.log_dir, "./");
}
/*
** Aggregation method #1 : Multiple Files - No Aggregation
IPM_LOG_USEMULTI
** Aggregation method #2 : Single File - Locks
IPM_LOG_USELOCKS
** Aggregation method #3 : Single File - SMP & /tmp
IPM_LOG_USETMPFS
** Aggregation method #4 : Single File - MPI - default
IPM_LOG_USEMPI
*/
#ifndef IPM_LOG_USEMULTI
#ifndef IPM_LOG_USELOCKS
#ifndef IPM_LOG_USETMPFS
#ifndef IPM_LOG_USEMPI
#endif
#endif
#endif
#endif
#ifdef IPM_LOG_USEMULTI
sprintf(job.log_fname,"%s/%s.%s.%d",
job.log_dir,
job.username,
job.cookie,
task.mpi_rank);
#else
if (!strcmp(job.log_fname,"unset")) {
sprintf(job.log_fname,"%s/%s.%s.%d",
job.log_dir,
job.username,
job.cookie,0);
}
else
{
sprintf(tmp_fname,"%s/%s",job.log_dir,job.log_fname);
sprintf(job.log_fname,"%s",tmp_fname);
}
#endif
if(task.flags & DEBUG) {
printf("IPM: %d log IPMLOG_DIR=%s FNAME=%s \n",
task.mpi_rank,
job.log_dir,
job.log_fname);
}
/*
** Aggregation method #1 : Multiple Files - No Aggregation {
*/
#ifdef IPM_LOG_USEMULTI
/* simplest case no locking just write each file. Parallel FS may
have metadata storm for N file creates */
ipm_mpi_log_fh = fopen(job.log_fname,"w");
if(ipm_mpi_log_fh == NULL) {
printf("IPM: %d log fopen failed fname=%s \n",
task.mpi_rank,
job.log_fname);
fflush(stdout);
}
rv = fprintf(ipm_mpi_log_fh,
"<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n<ipm>\n");
ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh);
rv = fprintf(ipm_mpi_log_fh,
"</ipm>\n");
fclose(ipm_mpi_log_fh);
chmod(job.log_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
return;
#endif
/* } */
/*
** Aggregation methods #2 : Single File - Locks {
*/
#ifdef IPM_LOG_USELOCKS
signal(SIGALRM, ipm_alarm_log_block);
alarm(30);
IPM_TIME_GTOD(stamp1);
if(task.flags & DEBUG) {
printf("IPM: %d log block_lock fname=%s fd=%d stamp=%f \n",
task.mpi_rank,
job.log_fname,
ipm_log_fd,
stamp1
);
fflush(stdout);
}
IPM_FILE_LOCK(job.log_fname,ipm_log_fd);
IPM_TIME_GTOD(stamp2);
if(task.flags & DEBUG) {
printf("IPM: %d log block_lock fname=%s fd=%d stamp=%12.6f delta=%.3e \n",
task.mpi_rank,
job.log_fname,
ipm_log_fd,
stamp2, stamp2-stamp1
);
fflush(stdout);
}
alarm(0);
signal(SIGALRM, SIG_DFL);
ipm_mpi_log_fh = fdopen(ipm_log_fd,"w+");
if(!ipm_mpi_log_fh || !ipm_log_fd) {
/* fail silently */
return;
}
/* got log fh */
fseek(ipm_mpi_log_fh,0,SEEK_END);
ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh);
IPM_TIME_GTOD(stamp3);
if(task.flags & DEBUG) {
printf("IPM: %d log write fname=%s fd=%d stamp=%12.6f delta=%.3e \n",
task.mpi_rank,
job.log_fname,
ipm_log_fd,
stamp3, stamp3-stamp2
);
fflush(stdout);
}
fflush(ipm_mpi_log_fh);
IPM_FILE_UNLOCK(job.log_fname,ipm_log_fd);
IPM_TIME_GTOD(stamp4);
if(task.flags & DEBUG) {
printf("IPM: %d log unlock fname=%s fd=%d stamp=%12.6f delta=%.3e \n",
task.mpi_rank,
job.log_fname,
ipm_log_fd,
stamp4, stamp4-stamp3
);
fflush(stdout);
}
#endif
/* } */
/*
** Aggregation method #3 : Single File - SMP & /tmp {
*/
#ifdef IPM_LOG_USETMPFS
if(task.flags & IPM_MPI_FINALIZING) {
if(task.mpi_size == 1) { /* special easy case now uneeded */
}
sprintf(tmp_fname,"/tmp/%s.%s.%d",
job.username,
job.cookie,
task.mpi_rank);
ipm_mpi_log_fh = fopen(tmp_fname,"w");
if(ipm_mpi_log_fh == NULL) {
printf("IPM: %d log fopen failed fname=%s \n",
task.mpi_rank,
tmp_fname);
fflush(stdout);
}
chmod(tmp_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh);
fclose(ipm_mpi_log_fh);
/* host local ring barrier so that /tmp is all good */
if(task.intra_size > 1) {
if(task.intra_root == task.mpi_rank) {
PMPI_Send(&i,1,MPI_INT,task.intra_right,0,MPI_COMM_WORLD);
PMPI_Recv(&i,1,MPI_INT,task.intra_left,0,MPI_COMM_WORLD,s);
} else {
PMPI_Recv(&i,1,MPI_INT,task.intra_left,0,MPI_COMM_WORLD,s);
PMPI_Send(&i,1,MPI_INT,task.intra_right,0,MPI_COMM_WORLD);
}
}
if(task.intra_root == task.mpi_rank) {
if(job.nhosts > 1 && task.mpi_rank) {
PMPI_Recv(&i,1,MPI_INT,task.inter_left,0,MPI_COMM_WORLD,s);
}
/* sh -c lacks PATH on some system so remove popen&system where possible */
sprintf(tmp_cmd, "/usr/bin/cat /tmp/%s.%s.* >> %s",
job.username,
job.cookie,
job.syslog_fname);
system(tmp_cmd);
chmod(job.syslog_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
sprintf(tmp_cmd, "/usr/bin/rm -f /tmp/%s.%s.* ",
job.username,
job.cookie);
system(tmp_cmd);
/* ugh! duplicating cat and rm is yucky
sprintf(tmp_pref,"%s.%s", job.username, job.cookie);
dp=opendir("/tmp");
if(dp) {
while(de=readdir(dp))!=NULL){
if(!strncmp(de->d_name,tmp_pref, strlen(tmp_fname))) {
sprintf(tmp_fname,"/tmp/%s", de->d_name);
fopen(tmp_fname,"r"
read in pieces and write them to the intra-node file
delete the .rank file
}
}
}
*/
if(job.nhosts > 1 && task.inter_right != 0) {
PMPI_Send(&i,1,MPI_INT,task.inter_right,0,MPI_COMM_WORLD);
}
}
return;
}
#endif
/* } */
/*
** Aggregation method #4 : Single File - MPI {
*/
#ifdef IPM_LOG_USEMPI
if (task.flags & PARALLEL_IO_LOG ) {
int buff_size=0;
MPI_Offset file_offset=0;
int64_t buff_sum=0;
int malloc_flag,malloc_sum;
char* buffer=NULL;
MPI_Info info;
/* measure size of buff required */
buff_size=ipm_log_write(&job, &task, txt, buffer,0,1);
malloc_flag=1;
buffer = (char*)malloc(buff_size+1);
if (buffer == NULL) {
malloc_flag=0;
} else {
rv=ipm_log_write(&job, &task, txt, buffer,buff_size,1);
}
/*see whether malloc suceeded across all mpi tasks */
PMPI_Allreduce(&malloc_flag,&malloc_sum,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
if (malloc_sum == task.mpi_size) {/* use parallel IO */
if(task.flags & DEBUG && !task.mpi_rank) {
printf("IPM: %d IPM report parallel IO used\n", task.mpi_rank);
}
PMPI_Info_create(&info);
#ifndef CRAY_GPFS_BUG
PMPI_Info_set(info,"access_style","write_once");
PMPI_Info_set(info,"collective_buffering","true");
PMPI_Info_set(info,"file_perm","0644");
PMPI_Info_set(info,"romio_cb_read","true");
PMPI_Info_set(info,"cb_align","2");
PMPI_Info_set(info,"romio_cb_write","true");
PMPI_Info_set(info,"cb_config_list","*:1");
PMPI_Info_set(info,"striping_factor","80");
PMPI_Info_set(info,"IBM_largeblock_io","true");
#endif
/* with allowing the user to choose the filename - can overwrite an old */
/* file - which would be fine if MPI-IO allowed TRUNC - but it doesn't */
/* so we just delete so that we don't end up with trailing garbage */
if (!task.mpi_rank) rv=PMPI_File_delete ( job.log_fname,MPI_INFO_NULL);
rv=PMPI_Barrier(MPI_COMM_WORLD);
rv = PMPI_File_open( MPI_COMM_WORLD, job.log_fname, MPI_MODE_WRONLY | MPI_MODE_CREATE,info, &ipm_mpiio_log_fh );
if (rv)
{
printf("IPM: %d syslog fopen failed fname=%s \n",
task.mpi_rank,
job.log_fname);
fflush(stdout);
return;
}
/* workaround for cases when MPI_INTEGER8 is not defined */
#ifndef MPI_INTEGER8
#define MPI_INTEGER8 MPI_LONG_LONG_INT
#endif
if (task.mpi_size > 1) {
if (task.mpi_rank == 0) {
buff_sum+=(int64_t)buff_size;
PMPI_Send (&buff_sum,1,MPI_INTEGER8,1,0,MPI_COMM_WORLD);
file_offset=0;
} else if (task.mpi_rank == (task.mpi_size-1)) {
PMPI_Recv (&buff_sum,1,MPI_INTEGER8,task.mpi_rank-1,0,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
file_offset=(MPI_Offset)buff_sum;
} else {
PMPI_Recv (&buff_sum,1,MPI_INTEGER8,task.mpi_rank-1,0,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
file_offset=(MPI_Offset)buff_sum;
buff_sum+=(int64_t)buff_size;
PMPI_Send (&buff_sum,1,MPI_INTEGER8,task.mpi_rank+1,0,MPI_COMM_WORLD);
}
}
rv=PMPI_File_set_view(ipm_mpiio_log_fh,file_offset,MPI_CHAR, MPI_CHAR,"native",info);
/*write info*/
rv=PMPI_File_write_all(ipm_mpiio_log_fh,buffer,buff_size,MPI_CHAR,MPI_STATUS_IGNORE);
rv = PMPI_File_close( &ipm_mpiio_log_fh );
PMPI_Barrier(MPI_COMM_WORLD);
/* Some MPI-IO implimentations (cray) permissions are not setable with hints */
if (task.mpi_rank == 0) chmod (job.log_fname,0744);
free (buffer);
return;
} else {
if (! task.mpi_rank) printf("IPM: %d Allocation of IO Buffer failed on one or more tasks\n",task.mpi_rank);
}
}
/*parallel IO failed */
if (! task.mpi_rank) printf("IPM: %d Using serial IO\n",task.mpi_rank);
/*************************************/
/* write log from rank zero using MPI*/
/*************************************/
if(task.mpi_rank==0) {
ipm_mpi_log_fh = fopen(job.log_fname,"w+");
if(ipm_mpi_log_fh == NULL) {
printf("IPM: %d syslog fopen failed fname=%s \n",
task.mpi_rank,
job.log_fname);
fflush(stdout);
}
chmod(job.log_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
ipm_log_write(&job, &task, txt,ipm_mpi_log_fh ,0,0);
/* we now pollute the local profile state irrevocably in the interest
of keeping a memory footprint which is a constant independent of
concurrency */
/* task 0 initiates a volley of Sends via a handshake */
for(i=1; i<job.ntasks; i++) {
PMPI_Send(&token,1,MPI_INT,i,0,MPI_COMM_WORLD);
PMPI_Recv(&job,sizeof(struct ipm_jobdata),MPI_BYTE,i,0,MPI_COMM_WORLD,s+0);
PMPI_Recv(&task,sizeof(struct ipm_taskdata),MPI_BYTE,i,1,MPI_COMM_WORLD,s+1);
PMPI_Recv(&(txt[0]),MAXSIZE_TXTLINE,MPI_CHAR,i,1,MPI_COMM_WORLD,s+1);
ipm_log_write(&job, &task, txt,ipm_mpi_log_fh ,0,0);
}
fclose(ipm_mpi_log_fh);
} else {
PMPI_Recv(&token,1,MPI_INT,0,0,MPI_COMM_WORLD,s+0);
PMPI_Send(&job,sizeof(struct ipm_jobdata),MPI_BYTE,0,0,MPI_COMM_WORLD);
PMPI_Send(&task,sizeof(struct ipm_taskdata),MPI_BYTE,0,1,MPI_COMM_WORLD);
PMPI_Send(&(txt[0]),MAXSIZE_TXTLINE,MPI_CHAR,0,1,MPI_COMM_WORLD);
}
PMPI_Barrier(MPI_COMM_WORLD);
return;
#endif
return;
}
void RECV_P2P_END(void *buf, int count, MPI_Datatype dt, int node, int tag,
MPI_Comm comm, int err, void **ptr, void **midptr, int type)
{
long checksum,incoming;
MPI_Status status;
int done;
PNMPIMOD_Datatype_Parameters_t ref;
char *b;
int l,s,i;
MPI_Datatype t;
if (comm!=MPI_COMM_NULL)
{
checksum=get_checksum(buf,count,dt);
PMPI_Recv(&incoming,1,MPI_LONG,node,tag,comm,&status);
if ((checksum==incoming) || ((type && PNMPIMOD_COMM_COLL_REDUCE)!=0))
{
if ((abs(verbosity_level) & 0xff)>=2)
{
printf("Checksum OK\n");
}
}
else
{
printf("Checksum Error - %li instead of %li (dt=%i)\n",checksum,incoming,dt);
if ((abs(verbosity_level) & 0xff)>=3)
{
printf("CSE - Got the following\n");
dt_get(buf, count, dt, &ref);
do
{
PNMPIMOD_Datatype_getItem(&ref,&b,&t,&l,&s,&done)
#ifdef USE_FUNCTIONS
;
#endif
printf("\t%i ",l);
switch (t)
{
case MPI_INT : printf("INT "); break;
case MPI_SHORT : printf("SHORT "); break;
case MPI_LONG : printf("LONG "); break;
case MPI_CHAR : printf("CHAR "); break;
case MPI_DOUBLE : printf("DOUBLE"); break;
case MPI_FLOAT : printf("FLOAT "); break;
default: printf("Other");
}
printf(" of size %i at buf %16p / %li == ",s,b,((long) b)-((long) buf));
for (i=0; i<l; i++)
{
switch (t)
{
case MPI_INT : printf(" %i",((int*)b)[i]); break;
case MPI_SHORT : printf(" %i",((short*)b)[i]); break;
case MPI_LONG : printf(" %li",((long*)b)[i]); break;
case MPI_CHAR : printf(" %c",((char*)b)[i]); break;
case MPI_DOUBLE : printf(" %f",((double*)b)[i]); break;
case MPI_FLOAT : printf(" %f",((float*)b)[i]); break;
}
}
printf("\n");
fflush(stdout);
}
while (!done);
dt_del(&ref);
}
}
if ((abs(verbosity_level) & 0xff)>=4)
{
PMPI_Recv(buf,count,dt,node,tag,comm,&status);
if (checksum!=incoming)
{
printf("CSE: Message should have been\n");
dt_get(buf, count, dt, &ref);
do
{
PNMPIMOD_Datatype_getItem(&ref,&b,&t,&l,&s,&done)
#ifdef USE_FUNCTIONS
;
#endif
printf("\t%i ",l);
switch (t)
{
case MPI_INT : printf("INT "); break;
case MPI_SHORT : printf("SHORT "); break;
case MPI_LONG : printf("LONG "); break;
case MPI_CHAR : printf("CHAR "); break;
case MPI_DOUBLE : printf("DOUBLE"); break;
case MPI_FLOAT : printf("FLOAT "); break;
default: printf("Other");
}
printf(" of size %i at buf %16p / %li == ",s,b,((long) b)-((long) buf));
for (i=0; i<l; i++)
{
switch (t)
{
case MPI_INT : printf(" %i",((int*)b)[i]); break;
case MPI_SHORT : printf(" %i",((short*)b)[i]); break;
case MPI_LONG : printf(" %li",((long*)b)[i]); break;
case MPI_CHAR : printf(" %c",((char*)b)[i]); break;
case MPI_DOUBLE : printf(" %f",((double*)b)[i]); break;
case MPI_FLOAT : printf(" %f",((float*)b)[i]); break;
}
}
printf("\n");
fflush(stdout);
}
while (!done);
dt_del(&ref);
}
}
if (checksum!=incoming)
{
if (verbosity_level==1)
{
while (0);
}
if (verbosity_level==-1)
{
*((int*) 0)=0;
}
}
}
}
int main(int argc, char *argv[]) {
int numproc, rank, len;
char hostname[MPI_MAX_PROCESSOR_NAME];
PMPI_Init(&argc, &argv);
PMPI_Comm_size(MPI_COMM_WORLD, &numproc);
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
PMPI_Get_processor_name(hostname, &len);
if (rank==0) {
int *freq,i,j;
freq=(int *)malloc(sizeof(int)*numproc);
char *temp;
temp=(char*)malloc(sizeof(char)*(numproc-1));
MPI_Status *stat, *stat1;
stat = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1));
stat1 = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1));
MPI_Request *req;
req = (MPI_Request *)malloc(sizeof(MPI_Request)*(numproc-1));
int N=numproc*numproc;
for(i=1; i<numproc; i++) {
PMPI_Recv(temp+i-1, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, stat+(i-1));//, req+(i-1)*2);
}
for(i=1; i<numproc; i++) {
PMPI_Recv(freq+i*numproc, numproc, MPI_INT, i, 1, MPI_COMM_WORLD,
stat1+(i-1));
}
printf("echo\n");
// MPI_Waitall((numproc-1), req, stat);
for (i=1; i<numproc; i++) {
printf("Rank %d ", i);
for (j=0; j<numproc; j++) {
if(j!=i) {
int loc = i*numproc+j;
printf("%d ",freq[loc]);
}
}
printf("\n");
}
}
else {
int i, *nsend;
char *rMsg, msg='x';
rMsg=(char*)malloc(sizeof(char));
nsend=(int*)malloc(sizeof(int)*numproc);
// msg=(char*)malloc(sizeof(char));
// memset(msg, 'z', sizeof(char));
memset(nsend, 0, sizeof(int)*numproc);
MPI_Request *req;
req = (MPI_Request *)malloc(sizeof(MPI_Request)*(numproc));
MPI_Status *stat;
stat = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1));
for (i=0; i<numproc; i++) {
if(i!=rank) {
*(nsend+i)+=*(nsend+i)+1;
PMPI_Isend(&msg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, &(req[i]));
}
}
// printf("Echo-1\n");
for (i=1; i<numproc; i++) {
if (i!=rank)
PMPI_Recv(rMsg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, stat+i-1);
}
// printf("Echo-2\n");
MPI_Isend(nsend, numproc, MPI_INT, 0, 1, MPI_COMM_WORLD, req+numproc);
// MPI_Isend(msg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, req+numproc);
// printf("Echo-3\n");
}
PMPI_Finalize();
return(0);
}
int MPI_Recv (void* message, int count, MPI_Datatype datatype, int source,
int tag, MPI_Comm comm, MPI_Status* status)
{
_MPI_COVERAGE();
return PMPI_Recv (message,count, datatype, source, tag, comm, status);
}
/******************************************************************
* *
* MPI Functions for Management *
* *
******************************************************************/
double E_MPI_Init(int * argc, char*** argv)
{
// assume all data files are existing
// users may run IMB manually.
// and copy datas to all machines manually
parse_loggpo("paras/cmp_para", &log_cmp);
parse_loggpo("paras/net_para", &log_net);
parse_loggpo("paras/smp_para", &log_smp);
parse_imb("paras/coll_para", &imb);
// get self location HOSTNAME:CORE
char proc_file_name[50];
sprintf(proc_file_name, "/proc/%d/stat", getpid());
FILE* proc_file = fopen(proc_file_name, "r");
if (! proc_file) {
printf("Proc File %s Open Failed!\n", proc_file_name);
}
int core;
if (1 != fscanf(proc_file, "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %*u %*u %*d %*d %*d %*d %*d %*d %*u %*u %*d %*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*d %d %*u %*u %*u",&core))
printf("Read Core ID Failed!\n");
char hostname[40];
gethostname(hostname,40);
// Send their info to rank:0
int gsize;
PMPI_Comm_size( MPI_COMM_WORLD, &gsize);
location = (pLocation)malloc(gsize*sizeof(Location));
int myrank;
PMPI_Comm_rank(MPI_COMM_WORLD, &myrank);
if (myrank != 0) {
char sendbuf[100];
sprintf(sendbuf, "%s %d",hostname,core);
PMPI_Send( sendbuf, 100, MPI_CHAR, 0, myrank, MPI_COMM_WORLD);
}
else {
char rbuf[100];
MPI_Status ms;
char** ls = (char**)malloc(gsize*sizeof(char*));
int cnt = 0;
ls[0] = (char*)malloc(40);
if (ls[0] == strcpy(ls[0],hostname))
++ cnt;
location[0].node = 0;
location[0].core = core;
char r_hn[40];
int r_core;
for (int rank = 1; rank < gsize; ++ rank) {
PMPI_Recv(rbuf, 100, MPI_CHAR, rank, rank, MPI_COMM_WORLD, &ms);
sscanf(rbuf,"%s %d",r_hn,&r_core);
int i;
for (i = 0; i < cnt; ++ i) {
if(strcmp(ls[i],r_hn) == 0) {
location[rank].node = i;
location[rank].core = r_core;
break;
}
}
if (i == cnt) {
ls[i] = (char*)malloc(40);
if (ls[i] == strcpy(ls[i],r_hn))
++ cnt;
location[rank].node = i;
location[rank].core = r_core;
}
}
for (int i = 0; i < cnt; ++ i)
free(ls[i]);
free(ls);
#if 0
printf("from RANK 0\n");
for (int i = 0; i < gsize; ++ i) {
printf ("rank:%d, node:%d, core:%d\n", i, location[i].node, location[i].core );
}
#endif
}
// boardcast to all MPI ranks
PMPI_Bcast(location, 2*gsize, MPI_INT, 0, MPI_COMM_WORLD);
#if 0
if (myrank == 20) {
printf("from RANK 20\n");
for (int i = 0; i < gsize; ++ i) {
printf ("rank:%d, node:%d, core:%d\n", i, location[i].node, location[i].core );
}
}
#endif
req_list.len = 0;
req_list.head = NULL;
return 0;
}
void RECV_P2P_END(void *buf, int count, MPI_Datatype dt, int node, int tag,
MPI_Comm comm, int err, void **ptr, void **midptr, int type)
{
int remote_timestamp, old_timestamp, ncritpath, rcritpath, redge, ledge;
int pbdata[2];
MPI_Status status;
mytime_t timing_diff;
timing_diff =
mytime_convertToSec(mytime_timeDiff(timing_start, mytime_getTimeStamp()));
/* get piggyback data */
PMPI_Recv(pbdata, 2, MPI_INT, node, tag, comm, &status);
/* create graph */
remote_timestamp = (pbdata[0] & 0xFFFFFFFE) / 2;
rcritpath = (pbdata[0] & 1);
if (timing_diff <= WAITTHRESHHOLD)
{
ncritpath = critpath;
redge = 0;
ledge = critpath;
}
else
{
ncritpath = rcritpath;
ledge = 0;
redge = rcritpath;
}
old_timestamp = local_timestamp;
if (remote_timestamp >= local_timestamp)
local_timestamp = remote_timestamp + 1;
else
local_timestamp++;
cp_setnode(critpath_myid, local_timestamp, timing_diff, ncritpath);
if (after_barrier)
{
if ((ledge) || (verbosity_level))
cp_setedge(-1, old_timestamp, critpath_myid, local_timestamp, ledge);
}
else
{
if ((ledge) || (verbosity_level))
cp_setedge(critpath_myid, old_timestamp, critpath_myid, local_timestamp,
ledge);
}
if ((redge) || (verbosity_level))
{
cp_setnode(node, remote_timestamp, -1, rcritpath);
cp_setedge(node, remote_timestamp, critpath_myid, local_timestamp, redge);
}
critpath = ncritpath;
after_barrier = 0;
}
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag,
MPI_Comm comm, MPI_Status * status)
{
return PMPI_Recv(buf, count, datatype, src, tag, comm, status);
}
/*==========================================================================*/
int PMPI_Wait(MPI_Request* request, MPI_Status* status) {
int retval;
MPI_Status recv_status;
if ( !request ) {
_MPI_ERR_ROUTINE(MPI_ERR_REQUEST,"Request pointer is null");
return _MPI_NOT_OK;
}
/* ----------------------------------------------- */
/* A null request requires no wait */
/* ----------------------------------------------- */
if ((*request) == MPI_REQUEST_NULL) return MPI_SUCCESS;
/* ----------------------------------------------- */
/* Send requests are always ready (eager) */
/* We may have to actually do a recv() here if it */
/* is not a send request. */
/* ----------------------------------------------- */
if (!(*request)->send) {
retval = PMPI_Recv((*request)->buffer,
(*request)->count,
(*request)->type,
_MPI_RANK,
(*request)->tag,
(*request)->comm,
&recv_status);
if ( retval == MPI_ERR_TAG && (*request)->cancel )
{
/* no matching send and the recv request has been cancelled */
_MPI_Req_Invalid((*request));
*request = MPI_REQUEST_NULL;
return MPI_SUCCESS;
}
else if (retval != MPI_SUCCESS) {
return retval;
}
}
/* Copy in the status */
if ( status && status != MPI_STATUS_IGNORE) {
status->MPI_SOURCE = _MPI_RANK;
status->MPI_TAG = (*request)->tag;
status->MPI_ERROR = MPI_SUCCESS;
if ((*request)->send) {
status->__count = _MPI_calculateSize((*request)->count, (*request)->type);
} else {
status->__count = recv_status.__count;
}
}
/* ----------------------------------------------- */
/* Mark the request available in the pool and then */
/* write REQUEST_NULL back into the original req */
/* so that subsequent requests will immediately */
/* succeed. */
/* ----------------------------------------------- */
_MPI_Req_Invalid((*request));
*request = MPI_REQUEST_NULL;
return MPI_SUCCESS;
}
int main(int argc, char **argv) {
/* Validate arguments */
if (argc != 5) {
fprintf(stderr, "Usage: %s [input file] [output file] [grid size] [iterations]\n", argv[0]);
return 1;
}
double t1 = MPI_Wtime();
/* Initialize MPI */
int tasks, rank;
MPI_Init(NULL, NULL);
MPI_Comm_size(MPI_COMM_WORLD, &tasks);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
double t_init = MPI_Wtime();
/* Declare global variables */
double *initial = NULL;
int N = atoi(argv[3]);
int T = atoi(argv[4]);
/*****************************************************************/
/* READ DATA */
/*****************************************************************/
if (rank == 0) {
if (DEBUG) {
fprintf(stderr, "\n-------------------------------------------\n");
fprintf(stderr, "Starting heat.c with %d processes\n", tasks);
}
/* Read data */
FILE *in = fopen(argv[1], "r");
initial = malloc(N * N * sizeof(double));
if (DEBUG) {
for (int i = 1; i <= N; i++) {
for (int j = 1; j <= N; j++) {
double val = (double) (i * (N - i - 1) * j * (N - j - 1));
initial[(i - 1) * N + (j - 1)] = val;
}
}
} else {
for (int i = 0; i < N * N; i++) {
int x, y;
double z;
fscanf(in, "%d %d %lf\n", &x, &y, &z);
initial[(x - 1) * N + (y - 1)] = z;
}
}
fclose(in);
}
double t_read = MPI_Wtime();
/*****************************************************************/
/* DISTRIBUTE DATA */
/*****************************************************************/
/* Preliminaries */
int rowsPerWorker = N / tasks;
int extra = N % tasks;
/* Determine neighboring workers */
int pred = rank - 1;
int succ = rank + 1;
/* Determine how many values each worker will get */
int offset = 0;
int offsets[tasks];
int items[tasks];
for (int i = 0; i < tasks; i++) {
items[i] = rowsPerWorker * N;
if (i < extra) items[i] += N;
offsets[i] = offset;
offset += items[i];
}
/* Scatter the rows appropriately */
int myrows = items[rank] / N;
/* Allocate an extra row of padding on either end */
double *current = calloc((items[rank] + 2 * N), sizeof(double));
double *old = calloc((items[rank] + 2 * N), sizeof(double));
if (DEBUG && rank == 0) fprintf(stderr, "scattering...\n");
PMPI_Scatterv(initial, items, offsets, MPI_DOUBLE,
current + N, items[rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
double t_scatter = MPI_Wtime();
/*****************************************************************/
/* CALCULATE */
/*****************************************************************/
MPI_Request req;
double t_net = 0;
for (int t = 0; t < T; t++) {
if (DEBUG) fprintf(stderr, "Beginning iteration %d: rank %d\n", t, rank);
/* Swap old and current so we can overwrite current */
double *temp = current;
current = old;
old = temp;
/* Hold onto some useful pointers into old */
double *succrow = old + N * (myrows + 1);
double *predrow = old;
double *firstrow = old + N;
double *lastrow = old + myrows;
double t_temp = MPI_Wtime();
/* Send last row to succ and receive it from pred if eligible */
if (succ < tasks) {
PMPI_Isend(lastrow, N, MPI_DOUBLE,
succ, 0, MPI_COMM_WORLD, &req);
}
if (pred >= 0) {
PMPI_Recv(predrow, N, MPI_DOUBLE, pred, 0, MPI_COMM_WORLD, 0);
}
/* Send first row to pred and receive it from succ if eligible */
if (pred >= 0) {
PMPI_Isend(firstrow, N, MPI_DOUBLE, pred, 0, MPI_COMM_WORLD, &req);
}
if (succ < tasks) {
PMPI_Recv(succrow, N, MPI_DOUBLE, succ, 0, MPI_COMM_WORLD, 0);
}
t_net += MPI_Wtime() - t_temp;
/* Determine current from old, predrow, and succrow */
for (int j = 1; j <= myrows; j++) {
for (int k = 0; k < N; k++) {
/* Determine adjacent cells */
double left = 0, right = 0;
if (k > 0 ) left = old[j * N + k - 1];
if (k < N - 1) right = old[j * N + k + 1];
double top = old[(j - 1) * N + k];
double bottom = old[(j + 1) * N + k];
double focus = old[j * N + k];
/* Calculate the new cell value */
current[j * N + k] = focus + .1 * (top + bottom - 2 * focus)
+ .1 * (left + right - 2 * focus);
}
}
}
free(old);
double t_work = MPI_Wtime();
/*****************************************************************/
/* WRITE THE OUTPUT */
/*****************************************************************/
PMPI_Gatherv(current + N, items[rank], MPI_DOUBLE, initial,
items, offsets, MPI_DOUBLE, 0, MPI_COMM_WORLD);
double t_gather = MPI_Wtime();
free(current);
if (rank == 0 && !DEBUG) {
FILE *out = fopen(argv[2], "w");
for (int i = 0; i < N; i++) {
for (int j = 0; j < N; j++) {
fprintf(out, "%d %d %lf\n", i, j, initial[i * N + j]);
}
}
fclose(out);
}
double t2 = MPI_Wtime();
if (rank == 0) {
fprintf(stderr, "-----------------------------------------\n");
fprintf(stderr, "TIMING INFORMATION \n");
fprintf(stderr, "-----------------------------------------\n");
fprintf(stderr, "RANK INIT READ SCATTER WORK GATHER WRITE TOTAL NET\n");
}
MPI_Barrier(MPI_COMM_WORLD);
fprintf(stderr, "%4.2d %8.2f %8.2f %8.2f %8.2f %8.2f %8.2f %8.2f %8.2f\n",
rank, t_init - t1, t_read - t_init, t_scatter - t_read, t_work - t_scatter,
t_gather - t_work, t2 - t_gather, t2 - t1, t_net);
free(initial);
MPI_Finalize();
return 0;
}
int
mpiPi_mergeResults ()
{
int ac;
callsite_stats_t **av;
int totalCount = 0;
int maxCount = 0;
int retval = 1, sendval;
/* gather local task data */
h_gather_data (mpiPi.task_callsite_stats, &ac, (void ***) &av);
/* determine size of space necessary on collector */
PMPI_Allreduce (&ac, &totalCount, 1, MPI_INT, MPI_SUM, mpiPi.comm);
PMPI_Reduce (&ac, &maxCount, 1, MPI_INT, MPI_MAX, mpiPi.collectorRank,
mpiPi.comm);
if (totalCount < 1)
{
mpiPi_msg_warn
("Collector found no records to merge. Omitting report.\n");
return 0;
}
/* gather global data at collector */
if (mpiPi.rank == mpiPi.collectorRank)
{
int i;
int ndx = 0;
#ifdef ENABLE_BFD
if (mpiPi.appFullName != NULL)
{
if (open_bfd_executable (mpiPi.appFullName) == 0)
mpiPi.do_lookup = 0;
}
#elif defined(USE_LIBDWARF)
if (mpiPi.appFullName != NULL)
{
if (open_dwarf_executable (mpiPi.appFullName) == 0)
mpiPi.do_lookup = 0;
}
#endif
#if defined(ENABLE_BFD) || defined(USE_LIBDWARF)
else
{
mpiPi_msg_warn ("Failed to open executable\n");
mpiPi.do_lookup = 0;
}
#endif
/* convert data to src line; merge, if nec */
mpiPi.global_callsite_stats = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_src_hashkey,
mpiPi_callsite_stats_src_comparator);
mpiPi.global_callsite_stats_agg = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_src_id_hashkey,
mpiPi_callsite_stats_src_id_comparator);
if (callsite_pc_cache == NULL)
{
callsite_pc_cache = h_open (mpiPi.tableSize,
callsite_pc_cache_hashkey,
callsite_pc_cache_comparator);
}
if (callsite_src_id_cache == NULL)
{
callsite_src_id_cache = h_open (mpiPi.tableSize,
callsite_src_id_cache_hashkey,
callsite_src_id_cache_comparator);
}
/* Try to allocate space for max count of callsite info from all tasks */
mpiPi.rawCallsiteData =
(callsite_stats_t *) calloc (maxCount, sizeof (callsite_stats_t));
if (mpiPi.rawCallsiteData == NULL)
{
mpiPi_msg_warn
("Failed to allocate memory to collect callsite info");
retval = 0;
}
/* Clear global_mpi_time and global_mpi_size before accumulation in mpiPi_insert_callsite_records */
mpiPi.global_mpi_time = 0.0;
mpiPi.global_mpi_size = 0.0;
if (retval == 1)
{
/* Insert collector callsite data into global and task-specific hash tables */
for (ndx = 0; ndx < ac; ndx++)
{
mpiPi_insert_callsite_records (av[ndx]);
}
ndx = 0;
for (i = 1; i < mpiPi.size; i++) /* n-1 */
{
MPI_Status status;
int count;
int j;
/* okay in any order */
PMPI_Probe (MPI_ANY_SOURCE, mpiPi.tag, mpiPi.comm, &status);
PMPI_Get_count (&status, MPI_CHAR, &count);
PMPI_Recv (&(mpiPi.rawCallsiteData[ndx]), count, MPI_CHAR,
status.MPI_SOURCE, mpiPi.tag, mpiPi.comm, &status);
count /= sizeof (callsite_stats_t);
for (j = 0; j < count; j++)
{
mpiPi_insert_callsite_records (&(mpiPi.rawCallsiteData[j]));
}
}
free (mpiPi.rawCallsiteData);
}
}
else
{
int ndx;
char *sbuf = (char *) malloc (ac * sizeof (callsite_stats_t));
for (ndx = 0; ndx < ac; ndx++)
{
bcopy (av[ndx],
&(sbuf[ndx * sizeof (callsite_stats_t)]),
sizeof (callsite_stats_t));
}
PMPI_Send (sbuf, ac * sizeof (callsite_stats_t),
MPI_CHAR, mpiPi.collectorRank, mpiPi.tag, mpiPi.comm);
free (sbuf);
}
if (mpiPi.rank == mpiPi.collectorRank && retval == 1)
{
if (mpiPi.collective_report == 0)
mpiPi_msg_debug
("MEMORY : Allocated for global_callsite_stats : %13ld\n",
h_count (mpiPi.global_callsite_stats) * sizeof (callsite_stats_t));
mpiPi_msg_debug
("MEMORY : Allocated for global_callsite_stats_agg : %13ld\n",
h_count (mpiPi.global_callsite_stats_agg) *
sizeof (callsite_stats_t));
}
/* TODO: need to free all these pointers as well. */
free (av);
if (mpiPi.rank == mpiPi.collectorRank)
{
if (mpiPi.do_lookup == 1)
{
#ifdef ENABLE_BFD
/* clean up */
close_bfd_executable ();
#elif defined(USE_LIBDWARF)
close_dwarf_executable ();
#endif
}
}
/* Quadrics MPI does not appear to support MPI_IN_PLACE */
sendval = retval;
PMPI_Allreduce (&sendval, &retval, 1, MPI_INT, MPI_MIN, mpiPi.comm);
return retval;
}
/*
* If MPICH is built with the --enable-debugger option, MPI_Init and
* MPI_Init_thread will call MPIR_WaitForDebugger. This ensures both that
* the debugger can gather information on the MPI job before the MPI_Init
* returns to the user and that the necessary symbols for providing
* information such as message queues is available.
*
* In addition, the environment variable MPIEXEC_DEBUG, if set, will cause
* all MPI processes to wait in this routine until the variable
* MPIR_debug_gate is set to 1.
*/
void MPIR_WaitForDebugger( void )
{
#ifdef MPIU_PROCTABLE_NEEDED
int rank = MPIR_Process.comm_world->rank;
#if defined(FINEGRAIN_MPI)
int size = MPIR_Process.comm_world->num_osprocs;
#else
int size = MPIR_Process.comm_world->local_size;
#endif
int i, maxsize;
/* FIXME: In MPICH, the executables may not have the information
on the other processes; this is part of the Process Manager Interface
(PMI). We need another way to provide this information to
a debugger */
/* The process manager probably has all of this data - the MPI2
debugger interface API provides (at least originally) a way
to access this. */
/* Also, to avoid scaling problems, we only populate the first 64
entries (default) */
maxsize = MPIR_CVAR_PROCTABLE_SIZE;
if (maxsize > size) maxsize = size;
if (rank == 0) {
char hostname[MPI_MAX_PROCESSOR_NAME+1];
int hostlen;
int val;
MPIR_proctable = (MPIR_PROCDESC *)MPIU_Malloc(
size * sizeof(MPIR_PROCDESC) );
for (i=0; i<size; i++) {
/* Initialize the proctable */
MPIR_proctable[i].host_name = 0;
MPIR_proctable[i].executable_name = 0;
MPIR_proctable[i].pid = -1;
}
PMPI_Get_processor_name( hostname, &hostlen );
MPIR_proctable[0].host_name = (char *)MPIU_Strdup( hostname );
MPIR_proctable[0].executable_name = 0;
MPIR_proctable[0].pid = getpid();
for (i=1; i<maxsize; i++) {
int msg[2];
PMPI_Recv( msg, 2, MPI_INT, i, 0, MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPIR_proctable[i].pid = msg[1];
MPIR_proctable[i].host_name = (char *)MPIU_Malloc( msg[0] + 1 );
PMPI_Recv( MPIR_proctable[i].host_name, msg[0]+1, MPI_CHAR,
i, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
MPIR_proctable[i].host_name[msg[0]] = 0;
}
MPIR_proctable_size = size;
/* Debugging hook */
if (MPIR_CVAR_PROCTABLE_PRINT) {
for (i=0; i<maxsize; i++) {
printf( "PT[%d].pid = %d, .host_name = %s\n",
i, MPIR_proctable[i].pid, MPIR_proctable[i].host_name );
}
fflush( stdout );
}
MPIR_Add_finalize( MPIR_FreeProctable, MPIR_proctable, 0 );
}
else {
char hostname[MPI_MAX_PROCESSOR_NAME+1];
int hostlen;
int mypid = getpid();
int msg[2];
if (rank < maxsize) {
PMPI_Get_processor_name( hostname, &hostlen );
msg[0] = hostlen;
msg[1] = mypid;
/* Deliver to the root process the proctable information */
PMPI_Ssend( msg, 2, MPI_INT, 0, 0, MPI_COMM_WORLD );
PMPI_Ssend( hostname, hostlen, MPI_CHAR, 0, 0, MPI_COMM_WORLD );
}
}
#endif /* MPIU_PROCTABLE_NEEDED */
/* Put the breakpoint after setting up the proctable */
MPIR_debug_state = MPIR_DEBUG_SPAWNED;
#ifdef MPIU_BREAKPOINT_NEEDED
(void)MPIR_Breakpoint();
#endif
/* After we exit the MPIR_Breakpoint routine, the debugger may have
set variables such as MPIR_being_debugged */
/* Initialize the sendq support */
SendqInit();
if (getenv("MPIEXEC_DEBUG")) {
while (!MPIR_debug_gate) ;
}
}
void
mpiPi_recv_pt2pt_stats(int ac, pt2pt_stats_t** av)
{
int i;
int pt2pt_size = sizeof(pt2pt_stats_t);
int nsenders = 0;
/* Count number of senders, receiver will wait for them */
/* i = 0 is copied locally */
for(i = 1; i < mpiPi.size; i++)
{
if (mpiPi.accumulatedPt2ptCounts[i])
nsenders++;
}
mpiPi_msg_debug("(%d) Waiting for %d senders\n",mpiPi.rank,nsenders);
/* Allocate a pointer for each rank */
mpiPi.accumulatedPt2ptData = (pt2pt_stats_t **) calloc (mpiPi.size, sizeof(pt2pt_stats_t*));
if (mpiPi.accumulatedPt2ptData == NULL)
{
mpiPi_msg_warn
("Failed to allocate memory to collect point to point info");
assert(0);
}
/* Copy Data for collector rank */
if (ac)
{
mpiPi.accumulatedPt2ptData[mpiPi.rank] = *av;
}
i = 0;
/* Insert pt2pt data into aggregate array indexed by rank */
while(i < nsenders)
{
MPI_Status status;
int count;
pt2pt_stats_t* ptp;
unsigned src_rank;
/* okay in any order */
PMPI_Probe (MPI_ANY_SOURCE, mpiPi.tag, mpiPi.comm, &status);
PMPI_Get_count (&status, MPI_CHAR, &count);
src_rank = status.MPI_SOURCE;
/* Allocate space for count number of pt2pt_stat_t structs */
ptp = (pt2pt_stats_t*) calloc(count, pt2pt_size);
mpiPi_msg_debug("(%d): Receiving %d bytes in pt2pt records from %d...\n",
mpiPi.rank, count, src_rank);
PMPI_Recv (ptp, count, MPI_CHAR, src_rank,
mpiPi.tag, mpiPi.comm, &status);
mpiPi_msg_debug("(%d): Received\n",mpiPi.rank);
count /= pt2pt_size;
assert(src_rank < mpiPi.size);
assert(mpiPi.accumulatedPt2ptCounts[src_rank] == count);
mpiPi.accumulatedPt2ptData[src_rank] = ptp;
i++;
}
}
