static void esync_master(VT_MPI_INT slave, MPI_Comm comm, VT_MPI_INT masterid) { int i; uint64_t tsend, trecv, tslave; uint64_t t1, t2, t3, t4; MPI_Status stat; MPI_Request req; Sync_TsPerPhase* temp; /* exchange LOOP_COUNT ping pong messages with the communication partner */ t1 = vt_pform_wtime(); PMPI_Isend( &t1, 1, MPI_LONG_LONG_INT, slave, 0, comm, &req ); PMPI_Recv( &t2, 1, MPI_LONG_LONG_INT, slave, 0, comm, &stat ); t4 = vt_pform_wtime(); t3 = t2; PMPI_Waitall( 1, &req, &stat ); for( i = 1; i < LOOP_COUNT; i++ ) { tsend = vt_pform_wtime(); /* message exchange */ PMPI_Isend(&tsend, 1, MPI_LONG_LONG_INT, slave, i, comm, &req); PMPI_Recv(&tslave, 1, MPI_LONG_LONG_INT, slave, i, comm, &stat); trecv = vt_pform_wtime(); PMPI_Waitall(1, &req, &stat); /* select timestamps with minimum message delay in each direction */ if ( ( (int64_t)tslave - (int64_t)tsend ) < ( (int64_t)t2 - (int64_t)t1 ) ) { t1 = tsend; t2 = tslave; } if ( ( (int64_t)trecv - (int64_t)tslave ) < ( (int64_t)t4 - (int64_t)t3 ) ) { t3 = tslave; t4 = trecv; } } /* save synchronization measurement data into internal data structure */ temp = (Sync_TsPerPhase*)malloc(sizeof(Sync_TsPerPhase)); if (!temp) vt_error(); temp->id1 = masterid; temp->id2 = slave; temp->t1 = t1; temp->t2 = t2; temp->t3 = t3; temp->t4 = t4; temp->next = SyncTsPerRunLast->sync_phase; SyncTsPerRunLast->sync_phase = temp; }
int MPI_Recv(void* buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status) { MPE_Log_event(START_RECV,0,"recv"); int wynik=PMPI_Recv(buf,count,datatype,source,tag,comm,status); MPE_Log_event(END_RECV,0,"recv"); return wynik; };
void Trace::recvTrace(int src){ int bufsize; void *buf; MPI_Comm comm = MPI_COMM_WORLD; int position = 0; PMPI_Recv(&bufsize, 1, MPI_INT, src, 0, comm, MPI_STATUS_IGNORE); buf = malloc(bufsize); if(!buf){ cerr << "Trace::recvTrace(): cannot allocate buffer" << endl; exit(1); } PMPI_Recv(buf, bufsize, MPI_PACKED, src, 0, comm, MPI_STATUS_IGNORE); unpack(buf, bufsize, &position, comm); free(buf); }
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status) { int err; ALLOCATE_STATUS(newstatus,1) err=PMPI_Recv(buf,count,datatype,source,tag,comm,newstatus); COPY_STATUS(status,newstatus,1) return err; }
int MPI_Recv(void* buffer, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status* status) { cqueue_t* mycqueue = handle_get_cqueue(comm); if (mycqueue != NULL) return cqueue_recv(mycqueue, buffer, count, datatype, src, tag, comm, status); else return PMPI_Recv(buffer, count, datatype, src, tag, comm, status); }
void receive_keys(effort_data& effort_log, int src, MPI_Comm comm) { MPI_Status status; int bufsize; PMPI_Recv(&bufsize, 1, MPI_INT, src, 0, comm, &status); char buf[bufsize]; PMPI_Recv(buf, bufsize, MPI_PACKED, src, 0, comm, &status); int position = 0; ModuleId::id_map modules; ModuleId::unpack_id_map(buf, bufsize, &position, modules, comm); int num_keys; PMPI_Unpack(buf, bufsize, &position, &num_keys, 1, MPI_INT, comm); for (int i=0; i < num_keys; i++) { effort_key key = effort_key::unpack(modules, buf, bufsize, &position, comm); if (!effort_log.contains(key)) { effort_log[key] = effort_record(effort_log.progress_count); } } }
static int64_t sync_slave(uint64_t* ltime, VT_MPI_INT master, MPI_Comm comm) { VT_MPI_INT min; MPI_Status stat; uint64_t tsendrecv[LOOP_COUNT]; uint64_t sync_time; int i; for (i = 0; i < LOOP_COUNT; i++) { PMPI_Recv(NULL, 0, MPI_INT, master, 1, comm, &stat); tsendrecv[i] = vt_pform_wtime(); PMPI_Send(NULL, 0, MPI_INT, master, 2, comm); } /* receive corresponding time together with its index from master */ PMPI_Recv(&min, 1, MPI_INT, master, 3, comm, &stat); PMPI_Recv(&sync_time, 1, MPI_LONG_LONG_INT, master, 4, comm, &stat); *ltime = tsendrecv[min]; return (int64_t)(sync_time - *ltime); }
static void esync_slave(VT_MPI_INT master, MPI_Comm comm) { int i; uint64_t t, tslave; MPI_Status stat; /* start communication with master */ for (i = 0; i < LOOP_COUNT; i++) { PMPI_Recv( &t, 1, MPI_LONG_LONG_INT, master, i, comm, &stat ); tslave = vt_pform_wtime(); PMPI_Send( &tslave, 1, MPI_LONG_LONG_INT, master, i, comm ); } }
int MPI_Recv(void* buf, int num, MPI_Datatype dtype, int node, int tag, MPI_Comm comm, MPI_Status *status) { int res; printf("WRAPPER 4: Before recv\n"); fflush(stdout); res=PMPI_Recv(buf,num,dtype,node,tag,comm,status); printf("WRAPPER 4: After recv\n"); fflush(stdout); return res; }
int MPI_Recv(void *buf, int count, MPI_Datatype type, int source, int tag, MPI_Comm comm, MPI_Status *status) { char typename[MPI_MAX_OBJECT_NAME], commname[MPI_MAX_OBJECT_NAME]; int len; int rank; PMPI_Comm_rank(MPI_COMM_WORLD, &rank); PMPI_Type_get_name(type, typename, &len); PMPI_Comm_get_name(comm, commname, &len); fprintf(stderr, "MPI_RECV[%d]: buf %0" PRIxPTR " count %d datatype %s source %d tag %d comm %s\n", rank, (uintptr_t) buf, count, typename, source, tag, commname); fflush(stderr); return PMPI_Recv(buf, count, type, source, tag, comm, status); }
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status) { int returnVal; int size; returnVal = PMPI_Recv( buf, count, datatype, source, tag, comm, status ); if (source != MPI_PROC_NULL && returnVal == MPI_SUCCESS) { MPI_Get_count( status, MPI_BYTE, &size ); prof_recv( procid_1, status->MPI_SOURCE, status->MPI_TAG, size, "MPI_Recv" ); } return returnVal; }
static int64_t sync_master(uint64_t* ltime, VT_MPI_INT slave, MPI_Comm comm) { VT_MPI_INT min; MPI_Status stat; uint64_t tsend[LOOP_COUNT], trecv[LOOP_COUNT]; uint64_t pingpong_time, sync_time; int i; /* exchange LOOP_COUNT ping pong messages with slave */ for (i = 0; i < LOOP_COUNT; i++) { tsend[i] = vt_pform_wtime(); PMPI_Send(NULL, 0, MPI_INT, slave, 1, comm); PMPI_Recv(NULL, 0, MPI_INT, slave, 2, comm, &stat); trecv[i] = vt_pform_wtime(); } /* select ping pong with shortest transfer time */ pingpong_time = trecv[0] - tsend[0]; min = 0; for (i = 1; i < LOOP_COUNT; i++) { if ((trecv[i] - tsend[i]) < pingpong_time) { pingpong_time = (trecv[i] - tsend[i]); min = (VT_MPI_INT)i; } } sync_time = tsend[min] + (pingpong_time / 2); /* send sync_time together with corresponding measurement index to slave */ PMPI_Send(&min, 1, MPI_INT, slave, 3, comm); PMPI_Send(&sync_time, 1, MPI_LONG_LONG_INT, slave, 4, comm); /* the process considered as the global clock returns 0 as offset */ *ltime = vt_pform_wtime(); return 0; }
/* MPI_Bcast_user: This is our version of MPI function. */ int MPI_Bcast_user(void *buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm) { int i, rank, commsize; PMPI_Comm_size(comm, &commsize); PMPI_Comm_rank(comm, &rank); /* Simple linear algorithm for broadcast */ if (rank == root) { for (i = 0; i < commsize; i++) { if (i != root) PMPI_Send(buf, count, datatype, i, 4321, comm); } } else { PMPI_Recv(buf, count, datatype, root, 4321, comm, MPI_STATUS_IGNORE); } return MPI_SUCCESS; }
void ipm_log(void) { /* called by all tasks (or as many as possible) */ int i,ii,rv,icall,ibytes,irank,ireg,kreg; int ipm_log_fd=-1, fan_out=0; int log_rank=-1, search_offset=0, token=1; FILE *ipm_mpi_log_fh; MPI_File ipm_mpiio_log_fh; DIR *ipm_mpi_log_dp; struct dirent *de; struct stat file_stat; IPM_KEY_TYPE key,ikey; char *cp, txt[MAXSIZE_TXTLINE]; char tmp_fname[MAXSIZE_FILENAME]; char tmp_pref[MAXSIZE_FILENAME]; char tmp_cmd[MAXSIZE_FILENAME]; double b_flops; double stamp1, stamp2, stamp3, stamp4; MPI_Status s[4]; MPI_Info outinfo; if(task.flags & IPM_WROTELOG) return; memset((void *)txt,0,(size_t)(MAXSIZE_TXTLINE*sizeof(char))); task.flags |= IPM_WROTELOG; /* only one chance, even if we fail at this point we should not return */ if(task.flags & DEBUG) { printf("IPM: %d log enter job.cookie=%s username=%s \n", task.mpi_rank, job.cookie, job.username); fflush(stdout); } /* ** bail */ if(strcmp(job.log_dir, "/dev/null") == 0 ) { if(task.flags & DEBUG) { printf("IPM: %d log exit due to LOGDIR=/dev/null", task.mpi_rank); } return; } if(stat(job.log_dir,&file_stat)) { if(!task.mpi_rank) { printf("IPM: %d log IPMLOG_DIR %s not available using $CWD \n", task.mpi_rank, job.log_dir); } sprintf(job.log_dir, "./"); } /* ** Aggregation method #1 : Multiple Files - No Aggregation IPM_LOG_USEMULTI ** Aggregation method #2 : Single File - Locks IPM_LOG_USELOCKS ** Aggregation method #3 : Single File - SMP & /tmp IPM_LOG_USETMPFS ** Aggregation method #4 : Single File - MPI - default IPM_LOG_USEMPI */ #ifndef IPM_LOG_USEMULTI #ifndef IPM_LOG_USELOCKS #ifndef IPM_LOG_USETMPFS #ifndef IPM_LOG_USEMPI #endif #endif #endif #endif #ifdef IPM_LOG_USEMULTI sprintf(job.log_fname,"%s/%s.%s.%d", job.log_dir, job.username, job.cookie, task.mpi_rank); #else if (!strcmp(job.log_fname,"unset")) { sprintf(job.log_fname,"%s/%s.%s.%d", job.log_dir, job.username, job.cookie,0); } else { sprintf(tmp_fname,"%s/%s",job.log_dir,job.log_fname); sprintf(job.log_fname,"%s",tmp_fname); } #endif if(task.flags & DEBUG) { printf("IPM: %d log IPMLOG_DIR=%s FNAME=%s \n", task.mpi_rank, job.log_dir, job.log_fname); } /* ** Aggregation method #1 : Multiple Files - No Aggregation { */ #ifdef IPM_LOG_USEMULTI /* simplest case no locking just write each file. Parallel FS may have metadata storm for N file creates */ ipm_mpi_log_fh = fopen(job.log_fname,"w"); if(ipm_mpi_log_fh == NULL) { printf("IPM: %d log fopen failed fname=%s \n", task.mpi_rank, job.log_fname); fflush(stdout); } rv = fprintf(ipm_mpi_log_fh, "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n<ipm>\n"); ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh); rv = fprintf(ipm_mpi_log_fh, "</ipm>\n"); fclose(ipm_mpi_log_fh); chmod(job.log_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); return; #endif /* } */ /* ** Aggregation methods #2 : Single File - Locks { */ #ifdef IPM_LOG_USELOCKS signal(SIGALRM, ipm_alarm_log_block); alarm(30); IPM_TIME_GTOD(stamp1); if(task.flags & DEBUG) { printf("IPM: %d log block_lock fname=%s fd=%d stamp=%f \n", task.mpi_rank, job.log_fname, ipm_log_fd, stamp1 ); fflush(stdout); } IPM_FILE_LOCK(job.log_fname,ipm_log_fd); IPM_TIME_GTOD(stamp2); if(task.flags & DEBUG) { printf("IPM: %d log block_lock fname=%s fd=%d stamp=%12.6f delta=%.3e \n", task.mpi_rank, job.log_fname, ipm_log_fd, stamp2, stamp2-stamp1 ); fflush(stdout); } alarm(0); signal(SIGALRM, SIG_DFL); ipm_mpi_log_fh = fdopen(ipm_log_fd,"w+"); if(!ipm_mpi_log_fh || !ipm_log_fd) { /* fail silently */ return; } /* got log fh */ fseek(ipm_mpi_log_fh,0,SEEK_END); ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh); IPM_TIME_GTOD(stamp3); if(task.flags & DEBUG) { printf("IPM: %d log write fname=%s fd=%d stamp=%12.6f delta=%.3e \n", task.mpi_rank, job.log_fname, ipm_log_fd, stamp3, stamp3-stamp2 ); fflush(stdout); } fflush(ipm_mpi_log_fh); IPM_FILE_UNLOCK(job.log_fname,ipm_log_fd); IPM_TIME_GTOD(stamp4); if(task.flags & DEBUG) { printf("IPM: %d log unlock fname=%s fd=%d stamp=%12.6f delta=%.3e \n", task.mpi_rank, job.log_fname, ipm_log_fd, stamp4, stamp4-stamp3 ); fflush(stdout); } #endif /* } */ /* ** Aggregation method #3 : Single File - SMP & /tmp { */ #ifdef IPM_LOG_USETMPFS if(task.flags & IPM_MPI_FINALIZING) { if(task.mpi_size == 1) { /* special easy case now uneeded */ } sprintf(tmp_fname,"/tmp/%s.%s.%d", job.username, job.cookie, task.mpi_rank); ipm_mpi_log_fh = fopen(tmp_fname,"w"); if(ipm_mpi_log_fh == NULL) { printf("IPM: %d log fopen failed fname=%s \n", task.mpi_rank, tmp_fname); fflush(stdout); } chmod(tmp_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); ipm_log_write_task(&job, &task, txt, ipm_mpi_log_fh); fclose(ipm_mpi_log_fh); /* host local ring barrier so that /tmp is all good */ if(task.intra_size > 1) { if(task.intra_root == task.mpi_rank) { PMPI_Send(&i,1,MPI_INT,task.intra_right,0,MPI_COMM_WORLD); PMPI_Recv(&i,1,MPI_INT,task.intra_left,0,MPI_COMM_WORLD,s); } else { PMPI_Recv(&i,1,MPI_INT,task.intra_left,0,MPI_COMM_WORLD,s); PMPI_Send(&i,1,MPI_INT,task.intra_right,0,MPI_COMM_WORLD); } } if(task.intra_root == task.mpi_rank) { if(job.nhosts > 1 && task.mpi_rank) { PMPI_Recv(&i,1,MPI_INT,task.inter_left,0,MPI_COMM_WORLD,s); } /* sh -c lacks PATH on some system so remove popen&system where possible */ sprintf(tmp_cmd, "/usr/bin/cat /tmp/%s.%s.* >> %s", job.username, job.cookie, job.syslog_fname); system(tmp_cmd); chmod(job.syslog_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); sprintf(tmp_cmd, "/usr/bin/rm -f /tmp/%s.%s.* ", job.username, job.cookie); system(tmp_cmd); /* ugh! duplicating cat and rm is yucky sprintf(tmp_pref,"%s.%s", job.username, job.cookie); dp=opendir("/tmp"); if(dp) { while(de=readdir(dp))!=NULL){ if(!strncmp(de->d_name,tmp_pref, strlen(tmp_fname))) { sprintf(tmp_fname,"/tmp/%s", de->d_name); fopen(tmp_fname,"r" read in pieces and write them to the intra-node file delete the .rank file } } } */ if(job.nhosts > 1 && task.inter_right != 0) { PMPI_Send(&i,1,MPI_INT,task.inter_right,0,MPI_COMM_WORLD); } } return; } #endif /* } */ /* ** Aggregation method #4 : Single File - MPI { */ #ifdef IPM_LOG_USEMPI if (task.flags & PARALLEL_IO_LOG ) { int buff_size=0; MPI_Offset file_offset=0; int64_t buff_sum=0; int malloc_flag,malloc_sum; char* buffer=NULL; MPI_Info info; /* measure size of buff required */ buff_size=ipm_log_write(&job, &task, txt, buffer,0,1); malloc_flag=1; buffer = (char*)malloc(buff_size+1); if (buffer == NULL) { malloc_flag=0; } else { rv=ipm_log_write(&job, &task, txt, buffer,buff_size,1); } /*see whether malloc suceeded across all mpi tasks */ PMPI_Allreduce(&malloc_flag,&malloc_sum,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); if (malloc_sum == task.mpi_size) {/* use parallel IO */ if(task.flags & DEBUG && !task.mpi_rank) { printf("IPM: %d IPM report parallel IO used\n", task.mpi_rank); } PMPI_Info_create(&info); #ifndef CRAY_GPFS_BUG PMPI_Info_set(info,"access_style","write_once"); PMPI_Info_set(info,"collective_buffering","true"); PMPI_Info_set(info,"file_perm","0644"); PMPI_Info_set(info,"romio_cb_read","true"); PMPI_Info_set(info,"cb_align","2"); PMPI_Info_set(info,"romio_cb_write","true"); PMPI_Info_set(info,"cb_config_list","*:1"); PMPI_Info_set(info,"striping_factor","80"); PMPI_Info_set(info,"IBM_largeblock_io","true"); #endif /* with allowing the user to choose the filename - can overwrite an old */ /* file - which would be fine if MPI-IO allowed TRUNC - but it doesn't */ /* so we just delete so that we don't end up with trailing garbage */ if (!task.mpi_rank) rv=PMPI_File_delete ( job.log_fname,MPI_INFO_NULL); rv=PMPI_Barrier(MPI_COMM_WORLD); rv = PMPI_File_open( MPI_COMM_WORLD, job.log_fname, MPI_MODE_WRONLY | MPI_MODE_CREATE,info, &ipm_mpiio_log_fh ); if (rv) { printf("IPM: %d syslog fopen failed fname=%s \n", task.mpi_rank, job.log_fname); fflush(stdout); return; } /* workaround for cases when MPI_INTEGER8 is not defined */ #ifndef MPI_INTEGER8 #define MPI_INTEGER8 MPI_LONG_LONG_INT #endif if (task.mpi_size > 1) { if (task.mpi_rank == 0) { buff_sum+=(int64_t)buff_size; PMPI_Send (&buff_sum,1,MPI_INTEGER8,1,0,MPI_COMM_WORLD); file_offset=0; } else if (task.mpi_rank == (task.mpi_size-1)) { PMPI_Recv (&buff_sum,1,MPI_INTEGER8,task.mpi_rank-1,0,MPI_COMM_WORLD,MPI_STATUS_IGNORE); file_offset=(MPI_Offset)buff_sum; } else { PMPI_Recv (&buff_sum,1,MPI_INTEGER8,task.mpi_rank-1,0,MPI_COMM_WORLD,MPI_STATUS_IGNORE); file_offset=(MPI_Offset)buff_sum; buff_sum+=(int64_t)buff_size; PMPI_Send (&buff_sum,1,MPI_INTEGER8,task.mpi_rank+1,0,MPI_COMM_WORLD); } } rv=PMPI_File_set_view(ipm_mpiio_log_fh,file_offset,MPI_CHAR, MPI_CHAR,"native",info); /*write info*/ rv=PMPI_File_write_all(ipm_mpiio_log_fh,buffer,buff_size,MPI_CHAR,MPI_STATUS_IGNORE); rv = PMPI_File_close( &ipm_mpiio_log_fh ); PMPI_Barrier(MPI_COMM_WORLD); /* Some MPI-IO implimentations (cray) permissions are not setable with hints */ if (task.mpi_rank == 0) chmod (job.log_fname,0744); free (buffer); return; } else { if (! task.mpi_rank) printf("IPM: %d Allocation of IO Buffer failed on one or more tasks\n",task.mpi_rank); } } /*parallel IO failed */ if (! task.mpi_rank) printf("IPM: %d Using serial IO\n",task.mpi_rank); /*************************************/ /* write log from rank zero using MPI*/ /*************************************/ if(task.mpi_rank==0) { ipm_mpi_log_fh = fopen(job.log_fname,"w+"); if(ipm_mpi_log_fh == NULL) { printf("IPM: %d syslog fopen failed fname=%s \n", task.mpi_rank, job.log_fname); fflush(stdout); } chmod(job.log_fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); ipm_log_write(&job, &task, txt,ipm_mpi_log_fh ,0,0); /* we now pollute the local profile state irrevocably in the interest of keeping a memory footprint which is a constant independent of concurrency */ /* task 0 initiates a volley of Sends via a handshake */ for(i=1; i<job.ntasks; i++) { PMPI_Send(&token,1,MPI_INT,i,0,MPI_COMM_WORLD); PMPI_Recv(&job,sizeof(struct ipm_jobdata),MPI_BYTE,i,0,MPI_COMM_WORLD,s+0); PMPI_Recv(&task,sizeof(struct ipm_taskdata),MPI_BYTE,i,1,MPI_COMM_WORLD,s+1); PMPI_Recv(&(txt[0]),MAXSIZE_TXTLINE,MPI_CHAR,i,1,MPI_COMM_WORLD,s+1); ipm_log_write(&job, &task, txt,ipm_mpi_log_fh ,0,0); } fclose(ipm_mpi_log_fh); } else { PMPI_Recv(&token,1,MPI_INT,0,0,MPI_COMM_WORLD,s+0); PMPI_Send(&job,sizeof(struct ipm_jobdata),MPI_BYTE,0,0,MPI_COMM_WORLD); PMPI_Send(&task,sizeof(struct ipm_taskdata),MPI_BYTE,0,1,MPI_COMM_WORLD); PMPI_Send(&(txt[0]),MAXSIZE_TXTLINE,MPI_CHAR,0,1,MPI_COMM_WORLD); } PMPI_Barrier(MPI_COMM_WORLD); return; #endif return; }
void RECV_P2P_END(void *buf, int count, MPI_Datatype dt, int node, int tag, MPI_Comm comm, int err, void **ptr, void **midptr, int type) { long checksum,incoming; MPI_Status status; int done; PNMPIMOD_Datatype_Parameters_t ref; char *b; int l,s,i; MPI_Datatype t; if (comm!=MPI_COMM_NULL) { checksum=get_checksum(buf,count,dt); PMPI_Recv(&incoming,1,MPI_LONG,node,tag,comm,&status); if ((checksum==incoming) || ((type && PNMPIMOD_COMM_COLL_REDUCE)!=0)) { if ((abs(verbosity_level) & 0xff)>=2) { printf("Checksum OK\n"); } } else { printf("Checksum Error - %li instead of %li (dt=%i)\n",checksum,incoming,dt); if ((abs(verbosity_level) & 0xff)>=3) { printf("CSE - Got the following\n"); dt_get(buf, count, dt, &ref); do { PNMPIMOD_Datatype_getItem(&ref,&b,&t,&l,&s,&done) #ifdef USE_FUNCTIONS ; #endif printf("\t%i ",l); switch (t) { case MPI_INT : printf("INT "); break; case MPI_SHORT : printf("SHORT "); break; case MPI_LONG : printf("LONG "); break; case MPI_CHAR : printf("CHAR "); break; case MPI_DOUBLE : printf("DOUBLE"); break; case MPI_FLOAT : printf("FLOAT "); break; default: printf("Other"); } printf(" of size %i at buf %16p / %li == ",s,b,((long) b)-((long) buf)); for (i=0; i<l; i++) { switch (t) { case MPI_INT : printf(" %i",((int*)b)[i]); break; case MPI_SHORT : printf(" %i",((short*)b)[i]); break; case MPI_LONG : printf(" %li",((long*)b)[i]); break; case MPI_CHAR : printf(" %c",((char*)b)[i]); break; case MPI_DOUBLE : printf(" %f",((double*)b)[i]); break; case MPI_FLOAT : printf(" %f",((float*)b)[i]); break; } } printf("\n"); fflush(stdout); } while (!done); dt_del(&ref); } } if ((abs(verbosity_level) & 0xff)>=4) { PMPI_Recv(buf,count,dt,node,tag,comm,&status); if (checksum!=incoming) { printf("CSE: Message should have been\n"); dt_get(buf, count, dt, &ref); do { PNMPIMOD_Datatype_getItem(&ref,&b,&t,&l,&s,&done) #ifdef USE_FUNCTIONS ; #endif printf("\t%i ",l); switch (t) { case MPI_INT : printf("INT "); break; case MPI_SHORT : printf("SHORT "); break; case MPI_LONG : printf("LONG "); break; case MPI_CHAR : printf("CHAR "); break; case MPI_DOUBLE : printf("DOUBLE"); break; case MPI_FLOAT : printf("FLOAT "); break; default: printf("Other"); } printf(" of size %i at buf %16p / %li == ",s,b,((long) b)-((long) buf)); for (i=0; i<l; i++) { switch (t) { case MPI_INT : printf(" %i",((int*)b)[i]); break; case MPI_SHORT : printf(" %i",((short*)b)[i]); break; case MPI_LONG : printf(" %li",((long*)b)[i]); break; case MPI_CHAR : printf(" %c",((char*)b)[i]); break; case MPI_DOUBLE : printf(" %f",((double*)b)[i]); break; case MPI_FLOAT : printf(" %f",((float*)b)[i]); break; } } printf("\n"); fflush(stdout); } while (!done); dt_del(&ref); } } if (checksum!=incoming) { if (verbosity_level==1) { while (0); } if (verbosity_level==-1) { *((int*) 0)=0; } } } }
int main(int argc, char *argv[]) { int numproc, rank, len; char hostname[MPI_MAX_PROCESSOR_NAME]; PMPI_Init(&argc, &argv); PMPI_Comm_size(MPI_COMM_WORLD, &numproc); PMPI_Comm_rank(MPI_COMM_WORLD, &rank); PMPI_Get_processor_name(hostname, &len); if (rank==0) { int *freq,i,j; freq=(int *)malloc(sizeof(int)*numproc); char *temp; temp=(char*)malloc(sizeof(char)*(numproc-1)); MPI_Status *stat, *stat1; stat = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1)); stat1 = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1)); MPI_Request *req; req = (MPI_Request *)malloc(sizeof(MPI_Request)*(numproc-1)); int N=numproc*numproc; for(i=1; i<numproc; i++) { PMPI_Recv(temp+i-1, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, stat+(i-1));//, req+(i-1)*2); } for(i=1; i<numproc; i++) { PMPI_Recv(freq+i*numproc, numproc, MPI_INT, i, 1, MPI_COMM_WORLD, stat1+(i-1)); } printf("echo\n"); // MPI_Waitall((numproc-1), req, stat); for (i=1; i<numproc; i++) { printf("Rank %d ", i); for (j=0; j<numproc; j++) { if(j!=i) { int loc = i*numproc+j; printf("%d ",freq[loc]); } } printf("\n"); } } else { int i, *nsend; char *rMsg, msg='x'; rMsg=(char*)malloc(sizeof(char)); nsend=(int*)malloc(sizeof(int)*numproc); // msg=(char*)malloc(sizeof(char)); // memset(msg, 'z', sizeof(char)); memset(nsend, 0, sizeof(int)*numproc); MPI_Request *req; req = (MPI_Request *)malloc(sizeof(MPI_Request)*(numproc)); MPI_Status *stat; stat = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1)); for (i=0; i<numproc; i++) { if(i!=rank) { *(nsend+i)+=*(nsend+i)+1; PMPI_Isend(&msg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, &(req[i])); } } // printf("Echo-1\n"); for (i=1; i<numproc; i++) { if (i!=rank) PMPI_Recv(rMsg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, stat+i-1); } // printf("Echo-2\n"); MPI_Isend(nsend, numproc, MPI_INT, 0, 1, MPI_COMM_WORLD, req+numproc); // MPI_Isend(msg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, req+numproc); // printf("Echo-3\n"); } PMPI_Finalize(); return(0); }
int MPI_Recv (void* message, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status* status) { _MPI_COVERAGE(); return PMPI_Recv (message,count, datatype, source, tag, comm, status); }
/****************************************************************** * * * MPI Functions for Management * * * ******************************************************************/ double E_MPI_Init(int * argc, char*** argv) { // assume all data files are existing // users may run IMB manually. // and copy datas to all machines manually parse_loggpo("paras/cmp_para", &log_cmp); parse_loggpo("paras/net_para", &log_net); parse_loggpo("paras/smp_para", &log_smp); parse_imb("paras/coll_para", &imb); // get self location HOSTNAME:CORE char proc_file_name[50]; sprintf(proc_file_name, "/proc/%d/stat", getpid()); FILE* proc_file = fopen(proc_file_name, "r"); if (! proc_file) { printf("Proc File %s Open Failed!\n", proc_file_name); } int core; if (1 != fscanf(proc_file, "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %*u %*u %*d %*d %*d %*d %*d %*d %*u %*u %*d %*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*d %d %*u %*u %*u",&core)) printf("Read Core ID Failed!\n"); char hostname[40]; gethostname(hostname,40); // Send their info to rank:0 int gsize; PMPI_Comm_size( MPI_COMM_WORLD, &gsize); location = (pLocation)malloc(gsize*sizeof(Location)); int myrank; PMPI_Comm_rank(MPI_COMM_WORLD, &myrank); if (myrank != 0) { char sendbuf[100]; sprintf(sendbuf, "%s %d",hostname,core); PMPI_Send( sendbuf, 100, MPI_CHAR, 0, myrank, MPI_COMM_WORLD); } else { char rbuf[100]; MPI_Status ms; char** ls = (char**)malloc(gsize*sizeof(char*)); int cnt = 0; ls[0] = (char*)malloc(40); if (ls[0] == strcpy(ls[0],hostname)) ++ cnt; location[0].node = 0; location[0].core = core; char r_hn[40]; int r_core; for (int rank = 1; rank < gsize; ++ rank) { PMPI_Recv(rbuf, 100, MPI_CHAR, rank, rank, MPI_COMM_WORLD, &ms); sscanf(rbuf,"%s %d",r_hn,&r_core); int i; for (i = 0; i < cnt; ++ i) { if(strcmp(ls[i],r_hn) == 0) { location[rank].node = i; location[rank].core = r_core; break; } } if (i == cnt) { ls[i] = (char*)malloc(40); if (ls[i] == strcpy(ls[i],r_hn)) ++ cnt; location[rank].node = i; location[rank].core = r_core; } } for (int i = 0; i < cnt; ++ i) free(ls[i]); free(ls); #if 0 printf("from RANK 0\n"); for (int i = 0; i < gsize; ++ i) { printf ("rank:%d, node:%d, core:%d\n", i, location[i].node, location[i].core ); } #endif } // boardcast to all MPI ranks PMPI_Bcast(location, 2*gsize, MPI_INT, 0, MPI_COMM_WORLD); #if 0 if (myrank == 20) { printf("from RANK 20\n"); for (int i = 0; i < gsize; ++ i) { printf ("rank:%d, node:%d, core:%d\n", i, location[i].node, location[i].core ); } } #endif req_list.len = 0; req_list.head = NULL; return 0; }
void RECV_P2P_END(void *buf, int count, MPI_Datatype dt, int node, int tag, MPI_Comm comm, int err, void **ptr, void **midptr, int type) { int remote_timestamp, old_timestamp, ncritpath, rcritpath, redge, ledge; int pbdata[2]; MPI_Status status; mytime_t timing_diff; timing_diff = mytime_convertToSec(mytime_timeDiff(timing_start, mytime_getTimeStamp())); /* get piggyback data */ PMPI_Recv(pbdata, 2, MPI_INT, node, tag, comm, &status); /* create graph */ remote_timestamp = (pbdata[0] & 0xFFFFFFFE) / 2; rcritpath = (pbdata[0] & 1); if (timing_diff <= WAITTHRESHHOLD) { ncritpath = critpath; redge = 0; ledge = critpath; } else { ncritpath = rcritpath; ledge = 0; redge = rcritpath; } old_timestamp = local_timestamp; if (remote_timestamp >= local_timestamp) local_timestamp = remote_timestamp + 1; else local_timestamp++; cp_setnode(critpath_myid, local_timestamp, timing_diff, ncritpath); if (after_barrier) { if ((ledge) || (verbosity_level)) cp_setedge(-1, old_timestamp, critpath_myid, local_timestamp, ledge); } else { if ((ledge) || (verbosity_level)) cp_setedge(critpath_myid, old_timestamp, critpath_myid, local_timestamp, ledge); } if ((redge) || (verbosity_level)) { cp_setnode(node, remote_timestamp, -1, rcritpath); cp_setedge(node, remote_timestamp, critpath_myid, local_timestamp, redge); } critpath = ncritpath; after_barrier = 0; }
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status * status) { return PMPI_Recv(buf, count, datatype, src, tag, comm, status); }
/*==========================================================================*/ int PMPI_Wait(MPI_Request* request, MPI_Status* status) { int retval; MPI_Status recv_status; if ( !request ) { _MPI_ERR_ROUTINE(MPI_ERR_REQUEST,"Request pointer is null"); return _MPI_NOT_OK; } /* ----------------------------------------------- */ /* A null request requires no wait */ /* ----------------------------------------------- */ if ((*request) == MPI_REQUEST_NULL) return MPI_SUCCESS; /* ----------------------------------------------- */ /* Send requests are always ready (eager) */ /* We may have to actually do a recv() here if it */ /* is not a send request. */ /* ----------------------------------------------- */ if (!(*request)->send) { retval = PMPI_Recv((*request)->buffer, (*request)->count, (*request)->type, _MPI_RANK, (*request)->tag, (*request)->comm, &recv_status); if ( retval == MPI_ERR_TAG && (*request)->cancel ) { /* no matching send and the recv request has been cancelled */ _MPI_Req_Invalid((*request)); *request = MPI_REQUEST_NULL; return MPI_SUCCESS; } else if (retval != MPI_SUCCESS) { return retval; } } /* Copy in the status */ if ( status && status != MPI_STATUS_IGNORE) { status->MPI_SOURCE = _MPI_RANK; status->MPI_TAG = (*request)->tag; status->MPI_ERROR = MPI_SUCCESS; if ((*request)->send) { status->__count = _MPI_calculateSize((*request)->count, (*request)->type); } else { status->__count = recv_status.__count; } } /* ----------------------------------------------- */ /* Mark the request available in the pool and then */ /* write REQUEST_NULL back into the original req */ /* so that subsequent requests will immediately */ /* succeed. */ /* ----------------------------------------------- */ _MPI_Req_Invalid((*request)); *request = MPI_REQUEST_NULL; return MPI_SUCCESS; }
int main(int argc, char **argv) { /* Validate arguments */ if (argc != 5) { fprintf(stderr, "Usage: %s [input file] [output file] [grid size] [iterations]\n", argv[0]); return 1; } double t1 = MPI_Wtime(); /* Initialize MPI */ int tasks, rank; MPI_Init(NULL, NULL); MPI_Comm_size(MPI_COMM_WORLD, &tasks); MPI_Comm_rank(MPI_COMM_WORLD, &rank); double t_init = MPI_Wtime(); /* Declare global variables */ double *initial = NULL; int N = atoi(argv[3]); int T = atoi(argv[4]); /*****************************************************************/ /* READ DATA */ /*****************************************************************/ if (rank == 0) { if (DEBUG) { fprintf(stderr, "\n-------------------------------------------\n"); fprintf(stderr, "Starting heat.c with %d processes\n", tasks); } /* Read data */ FILE *in = fopen(argv[1], "r"); initial = malloc(N * N * sizeof(double)); if (DEBUG) { for (int i = 1; i <= N; i++) { for (int j = 1; j <= N; j++) { double val = (double) (i * (N - i - 1) * j * (N - j - 1)); initial[(i - 1) * N + (j - 1)] = val; } } } else { for (int i = 0; i < N * N; i++) { int x, y; double z; fscanf(in, "%d %d %lf\n", &x, &y, &z); initial[(x - 1) * N + (y - 1)] = z; } } fclose(in); } double t_read = MPI_Wtime(); /*****************************************************************/ /* DISTRIBUTE DATA */ /*****************************************************************/ /* Preliminaries */ int rowsPerWorker = N / tasks; int extra = N % tasks; /* Determine neighboring workers */ int pred = rank - 1; int succ = rank + 1; /* Determine how many values each worker will get */ int offset = 0; int offsets[tasks]; int items[tasks]; for (int i = 0; i < tasks; i++) { items[i] = rowsPerWorker * N; if (i < extra) items[i] += N; offsets[i] = offset; offset += items[i]; } /* Scatter the rows appropriately */ int myrows = items[rank] / N; /* Allocate an extra row of padding on either end */ double *current = calloc((items[rank] + 2 * N), sizeof(double)); double *old = calloc((items[rank] + 2 * N), sizeof(double)); if (DEBUG && rank == 0) fprintf(stderr, "scattering...\n"); PMPI_Scatterv(initial, items, offsets, MPI_DOUBLE, current + N, items[rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); double t_scatter = MPI_Wtime(); /*****************************************************************/ /* CALCULATE */ /*****************************************************************/ MPI_Request req; double t_net = 0; for (int t = 0; t < T; t++) { if (DEBUG) fprintf(stderr, "Beginning iteration %d: rank %d\n", t, rank); /* Swap old and current so we can overwrite current */ double *temp = current; current = old; old = temp; /* Hold onto some useful pointers into old */ double *succrow = old + N * (myrows + 1); double *predrow = old; double *firstrow = old + N; double *lastrow = old + myrows; double t_temp = MPI_Wtime(); /* Send last row to succ and receive it from pred if eligible */ if (succ < tasks) { PMPI_Isend(lastrow, N, MPI_DOUBLE, succ, 0, MPI_COMM_WORLD, &req); } if (pred >= 0) { PMPI_Recv(predrow, N, MPI_DOUBLE, pred, 0, MPI_COMM_WORLD, 0); } /* Send first row to pred and receive it from succ if eligible */ if (pred >= 0) { PMPI_Isend(firstrow, N, MPI_DOUBLE, pred, 0, MPI_COMM_WORLD, &req); } if (succ < tasks) { PMPI_Recv(succrow, N, MPI_DOUBLE, succ, 0, MPI_COMM_WORLD, 0); } t_net += MPI_Wtime() - t_temp; /* Determine current from old, predrow, and succrow */ for (int j = 1; j <= myrows; j++) { for (int k = 0; k < N; k++) { /* Determine adjacent cells */ double left = 0, right = 0; if (k > 0 ) left = old[j * N + k - 1]; if (k < N - 1) right = old[j * N + k + 1]; double top = old[(j - 1) * N + k]; double bottom = old[(j + 1) * N + k]; double focus = old[j * N + k]; /* Calculate the new cell value */ current[j * N + k] = focus + .1 * (top + bottom - 2 * focus) + .1 * (left + right - 2 * focus); } } } free(old); double t_work = MPI_Wtime(); /*****************************************************************/ /* WRITE THE OUTPUT */ /*****************************************************************/ PMPI_Gatherv(current + N, items[rank], MPI_DOUBLE, initial, items, offsets, MPI_DOUBLE, 0, MPI_COMM_WORLD); double t_gather = MPI_Wtime(); free(current); if (rank == 0 && !DEBUG) { FILE *out = fopen(argv[2], "w"); for (int i = 0; i < N; i++) { for (int j = 0; j < N; j++) { fprintf(out, "%d %d %lf\n", i, j, initial[i * N + j]); } } fclose(out); } double t2 = MPI_Wtime(); if (rank == 0) { fprintf(stderr, "-----------------------------------------\n"); fprintf(stderr, "TIMING INFORMATION \n"); fprintf(stderr, "-----------------------------------------\n"); fprintf(stderr, "RANK INIT READ SCATTER WORK GATHER WRITE TOTAL NET\n"); } MPI_Barrier(MPI_COMM_WORLD); fprintf(stderr, "%4.2d %8.2f %8.2f %8.2f %8.2f %8.2f %8.2f %8.2f %8.2f\n", rank, t_init - t1, t_read - t_init, t_scatter - t_read, t_work - t_scatter, t_gather - t_work, t2 - t_gather, t2 - t1, t_net); free(initial); MPI_Finalize(); return 0; }
int mpiPi_mergeResults () { int ac; callsite_stats_t **av; int totalCount = 0; int maxCount = 0; int retval = 1, sendval; /* gather local task data */ h_gather_data (mpiPi.task_callsite_stats, &ac, (void ***) &av); /* determine size of space necessary on collector */ PMPI_Allreduce (&ac, &totalCount, 1, MPI_INT, MPI_SUM, mpiPi.comm); PMPI_Reduce (&ac, &maxCount, 1, MPI_INT, MPI_MAX, mpiPi.collectorRank, mpiPi.comm); if (totalCount < 1) { mpiPi_msg_warn ("Collector found no records to merge. Omitting report.\n"); return 0; } /* gather global data at collector */ if (mpiPi.rank == mpiPi.collectorRank) { int i; int ndx = 0; #ifdef ENABLE_BFD if (mpiPi.appFullName != NULL) { if (open_bfd_executable (mpiPi.appFullName) == 0) mpiPi.do_lookup = 0; } #elif defined(USE_LIBDWARF) if (mpiPi.appFullName != NULL) { if (open_dwarf_executable (mpiPi.appFullName) == 0) mpiPi.do_lookup = 0; } #endif #if defined(ENABLE_BFD) || defined(USE_LIBDWARF) else { mpiPi_msg_warn ("Failed to open executable\n"); mpiPi.do_lookup = 0; } #endif /* convert data to src line; merge, if nec */ mpiPi.global_callsite_stats = h_open (mpiPi.tableSize, mpiPi_callsite_stats_src_hashkey, mpiPi_callsite_stats_src_comparator); mpiPi.global_callsite_stats_agg = h_open (mpiPi.tableSize, mpiPi_callsite_stats_src_id_hashkey, mpiPi_callsite_stats_src_id_comparator); if (callsite_pc_cache == NULL) { callsite_pc_cache = h_open (mpiPi.tableSize, callsite_pc_cache_hashkey, callsite_pc_cache_comparator); } if (callsite_src_id_cache == NULL) { callsite_src_id_cache = h_open (mpiPi.tableSize, callsite_src_id_cache_hashkey, callsite_src_id_cache_comparator); } /* Try to allocate space for max count of callsite info from all tasks */ mpiPi.rawCallsiteData = (callsite_stats_t *) calloc (maxCount, sizeof (callsite_stats_t)); if (mpiPi.rawCallsiteData == NULL) { mpiPi_msg_warn ("Failed to allocate memory to collect callsite info"); retval = 0; } /* Clear global_mpi_time and global_mpi_size before accumulation in mpiPi_insert_callsite_records */ mpiPi.global_mpi_time = 0.0; mpiPi.global_mpi_size = 0.0; if (retval == 1) { /* Insert collector callsite data into global and task-specific hash tables */ for (ndx = 0; ndx < ac; ndx++) { mpiPi_insert_callsite_records (av[ndx]); } ndx = 0; for (i = 1; i < mpiPi.size; i++) /* n-1 */ { MPI_Status status; int count; int j; /* okay in any order */ PMPI_Probe (MPI_ANY_SOURCE, mpiPi.tag, mpiPi.comm, &status); PMPI_Get_count (&status, MPI_CHAR, &count); PMPI_Recv (&(mpiPi.rawCallsiteData[ndx]), count, MPI_CHAR, status.MPI_SOURCE, mpiPi.tag, mpiPi.comm, &status); count /= sizeof (callsite_stats_t); for (j = 0; j < count; j++) { mpiPi_insert_callsite_records (&(mpiPi.rawCallsiteData[j])); } } free (mpiPi.rawCallsiteData); } } else { int ndx; char *sbuf = (char *) malloc (ac * sizeof (callsite_stats_t)); for (ndx = 0; ndx < ac; ndx++) { bcopy (av[ndx], &(sbuf[ndx * sizeof (callsite_stats_t)]), sizeof (callsite_stats_t)); } PMPI_Send (sbuf, ac * sizeof (callsite_stats_t), MPI_CHAR, mpiPi.collectorRank, mpiPi.tag, mpiPi.comm); free (sbuf); } if (mpiPi.rank == mpiPi.collectorRank && retval == 1) { if (mpiPi.collective_report == 0) mpiPi_msg_debug ("MEMORY : Allocated for global_callsite_stats : %13ld\n", h_count (mpiPi.global_callsite_stats) * sizeof (callsite_stats_t)); mpiPi_msg_debug ("MEMORY : Allocated for global_callsite_stats_agg : %13ld\n", h_count (mpiPi.global_callsite_stats_agg) * sizeof (callsite_stats_t)); } /* TODO: need to free all these pointers as well. */ free (av); if (mpiPi.rank == mpiPi.collectorRank) { if (mpiPi.do_lookup == 1) { #ifdef ENABLE_BFD /* clean up */ close_bfd_executable (); #elif defined(USE_LIBDWARF) close_dwarf_executable (); #endif } } /* Quadrics MPI does not appear to support MPI_IN_PLACE */ sendval = retval; PMPI_Allreduce (&sendval, &retval, 1, MPI_INT, MPI_MIN, mpiPi.comm); return retval; }
/* * If MPICH is built with the --enable-debugger option, MPI_Init and * MPI_Init_thread will call MPIR_WaitForDebugger. This ensures both that * the debugger can gather information on the MPI job before the MPI_Init * returns to the user and that the necessary symbols for providing * information such as message queues is available. * * In addition, the environment variable MPIEXEC_DEBUG, if set, will cause * all MPI processes to wait in this routine until the variable * MPIR_debug_gate is set to 1. */ void MPIR_WaitForDebugger( void ) { #ifdef MPIU_PROCTABLE_NEEDED int rank = MPIR_Process.comm_world->rank; #if defined(FINEGRAIN_MPI) int size = MPIR_Process.comm_world->num_osprocs; #else int size = MPIR_Process.comm_world->local_size; #endif int i, maxsize; /* FIXME: In MPICH, the executables may not have the information on the other processes; this is part of the Process Manager Interface (PMI). We need another way to provide this information to a debugger */ /* The process manager probably has all of this data - the MPI2 debugger interface API provides (at least originally) a way to access this. */ /* Also, to avoid scaling problems, we only populate the first 64 entries (default) */ maxsize = MPIR_CVAR_PROCTABLE_SIZE; if (maxsize > size) maxsize = size; if (rank == 0) { char hostname[MPI_MAX_PROCESSOR_NAME+1]; int hostlen; int val; MPIR_proctable = (MPIR_PROCDESC *)MPIU_Malloc( size * sizeof(MPIR_PROCDESC) ); for (i=0; i<size; i++) { /* Initialize the proctable */ MPIR_proctable[i].host_name = 0; MPIR_proctable[i].executable_name = 0; MPIR_proctable[i].pid = -1; } PMPI_Get_processor_name( hostname, &hostlen ); MPIR_proctable[0].host_name = (char *)MPIU_Strdup( hostname ); MPIR_proctable[0].executable_name = 0; MPIR_proctable[0].pid = getpid(); for (i=1; i<maxsize; i++) { int msg[2]; PMPI_Recv( msg, 2, MPI_INT, i, 0, MPI_COMM_WORLD,MPI_STATUS_IGNORE); MPIR_proctable[i].pid = msg[1]; MPIR_proctable[i].host_name = (char *)MPIU_Malloc( msg[0] + 1 ); PMPI_Recv( MPIR_proctable[i].host_name, msg[0]+1, MPI_CHAR, i, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE ); MPIR_proctable[i].host_name[msg[0]] = 0; } MPIR_proctable_size = size; /* Debugging hook */ if (MPIR_CVAR_PROCTABLE_PRINT) { for (i=0; i<maxsize; i++) { printf( "PT[%d].pid = %d, .host_name = %s\n", i, MPIR_proctable[i].pid, MPIR_proctable[i].host_name ); } fflush( stdout ); } MPIR_Add_finalize( MPIR_FreeProctable, MPIR_proctable, 0 ); } else { char hostname[MPI_MAX_PROCESSOR_NAME+1]; int hostlen; int mypid = getpid(); int msg[2]; if (rank < maxsize) { PMPI_Get_processor_name( hostname, &hostlen ); msg[0] = hostlen; msg[1] = mypid; /* Deliver to the root process the proctable information */ PMPI_Ssend( msg, 2, MPI_INT, 0, 0, MPI_COMM_WORLD ); PMPI_Ssend( hostname, hostlen, MPI_CHAR, 0, 0, MPI_COMM_WORLD ); } } #endif /* MPIU_PROCTABLE_NEEDED */ /* Put the breakpoint after setting up the proctable */ MPIR_debug_state = MPIR_DEBUG_SPAWNED; #ifdef MPIU_BREAKPOINT_NEEDED (void)MPIR_Breakpoint(); #endif /* After we exit the MPIR_Breakpoint routine, the debugger may have set variables such as MPIR_being_debugged */ /* Initialize the sendq support */ SendqInit(); if (getenv("MPIEXEC_DEBUG")) { while (!MPIR_debug_gate) ; } }
void mpiPi_recv_pt2pt_stats(int ac, pt2pt_stats_t** av) { int i; int pt2pt_size = sizeof(pt2pt_stats_t); int nsenders = 0; /* Count number of senders, receiver will wait for them */ /* i = 0 is copied locally */ for(i = 1; i < mpiPi.size; i++) { if (mpiPi.accumulatedPt2ptCounts[i]) nsenders++; } mpiPi_msg_debug("(%d) Waiting for %d senders\n",mpiPi.rank,nsenders); /* Allocate a pointer for each rank */ mpiPi.accumulatedPt2ptData = (pt2pt_stats_t **) calloc (mpiPi.size, sizeof(pt2pt_stats_t*)); if (mpiPi.accumulatedPt2ptData == NULL) { mpiPi_msg_warn ("Failed to allocate memory to collect point to point info"); assert(0); } /* Copy Data for collector rank */ if (ac) { mpiPi.accumulatedPt2ptData[mpiPi.rank] = *av; } i = 0; /* Insert pt2pt data into aggregate array indexed by rank */ while(i < nsenders) { MPI_Status status; int count; pt2pt_stats_t* ptp; unsigned src_rank; /* okay in any order */ PMPI_Probe (MPI_ANY_SOURCE, mpiPi.tag, mpiPi.comm, &status); PMPI_Get_count (&status, MPI_CHAR, &count); src_rank = status.MPI_SOURCE; /* Allocate space for count number of pt2pt_stat_t structs */ ptp = (pt2pt_stats_t*) calloc(count, pt2pt_size); mpiPi_msg_debug("(%d): Receiving %d bytes in pt2pt records from %d...\n", mpiPi.rank, count, src_rank); PMPI_Recv (ptp, count, MPI_CHAR, src_rank, mpiPi.tag, mpiPi.comm, &status); mpiPi_msg_debug("(%d): Received\n",mpiPi.rank); count /= pt2pt_size; assert(src_rank < mpiPi.size); assert(mpiPi.accumulatedPt2ptCounts[src_rank] == count); mpiPi.accumulatedPt2ptData[src_rank] = ptp; i++; } }