/* -------------------------------------------------------- *\ DDI_GDLBRest() ============== Reset the global dynamic load-balance counter. \* -------------------------------------------------------- */ void DDI_GDLBReset() { int np,me; # if !defined USE_SYSV int remote_id = 0; DDI_Patch Patch; # endif if(DDI_WORKING_COMM != DDI_COMM_WORLD) { fprintf(stdout," DDI Error: group dlb counter must be reset in DDI_WORLD scope.\n"); Fatal_error(911); } DDI_NProc(&np,&me); DDI_Sync(3041); #if defined DDI_ARMCI DDI_ARMCI_GDLBReset(); #else if(me == 0) { # if defined USE_SYSV DDI_GDLBReset_local(); # else Patch.oper = DDI_GDLBRESET; DDI_Send_request(&Patch,&remote_id,NULL); # endif } #endif DDI_Sync(3042); }
/* ------------------------------------- *\ FORTRAN Wrapper for DDI_Create_custom \* ------------------------------------- */ void F77_Create_custom(int_f77 *idim, int_f77 *jdim, int_f77 *jcols, int_f77 *handle) { int ihandle,i,np,me; int ijcols[MAX_PROCESSORS]; DDI_NProc(&np,&me); for(i=0; i<np; i++) ijcols[i] = (int) *jcols; DDI_Create_custom( (int) *idim, (int) *jdim, ijcols, &ihandle ); *handle = (int_f77) ihandle; }
static void Init_soc(int argc,char** argv) { /* --------------- *\ Local Variables \* --------------- */ int i,j,iproc,nodearg,me; int kickport,nodeid,procid,nn,np,nsocks; int *ports,*sockets; char kickhost[256]; int not_it; int ncpus,cpus,nics; char tmpstr[256]; DDI_Comm *comm = (DDI_Comm *) &gv(ddi_base_comm); /* ---------------------- *\ Assign signal handlers \* ---------------------- */ signal(SIGTERM, Fatal_error); /* handles a terminate signal */ signal(SIGFPE, Fatal_error); /* handles a floating point error in the DDI process */ signal(SIGABRT, Fatal_error); signal(SIGSEGV, Fatal_error); signal(SIGINT, Fatal_error); signal(SIGILL, Fatal_error); signal(SIGQUIT, Fatal_error); signal(SIGPIPE, Fatal_error); signal(SIGXCPU, Fatal_error); signal(SIGURG, _sigurg_hndlr); /* handles a OOB (out-of-band) message */ # if defined DDI_MPI signal(SIGURG, Fatal_error); # endif /* ---------------------------------- *\ Find where the DDI arguments begin \* ---------------------------------- */ # if defined DDI_SOC && !defined DDI_MPI for(i=0,not_it=1; i<argc && not_it; i++) not_it = strcmp(argv[i],"-ddi"); if(i==argc) { fprintf(stdout," DDI: Invalid command-line arguments!\n Are you using the ddikick program??\n"); Fatal_error(911); } strcpy(kickhost,argv[i++]); kickport = atoi(argv[i++]); /* kickoff port number */ nodeid = atoi(argv[i++]); /* rank of current node */ procid = atoi(argv[i++]); /* rank of current proc */ nn = atoi(argv[i++]); /* number of nodes */ np = atoi(argv[i++]); /* number of procs */ /* --------------------------------------------------------------- *\ * Initialize DDI_COMM_WORLD communicator. This is not a complete * initialization; the remaining bits and pieces will be set up in * Comm_init(). \* --------------------------------------------------------------- */ comm->id = DDI_COMM_WORLD; comm->nn = nn; comm->my = nodeid; comm->np = np; comm->me = procid; /* -------------------------- *\ Initialize Local Variables \* -------------------------- */ me = procid; nodearg = i; nsocks = np; if(USING_DATA_SERVERS()) nsocks *= 2; DEBUG_OUT(LVL1,(stdout," DDI Process %i: Execution begun on node %i.\n",procid,nodeid)) DEBUG_OUT(LVL3,(stdout," The master kickoff program responds on %s:%i\n",kickhost,kickport)) /* ------------------------------------------------ *\ Parse command-line arguments for DDI information \* ------------------------------------------------ */ if(Parse_node_args(argc,argv,nodearg,nn,gv(ddinodes)) < 0 ) { fprintf(stderr,"%s: Error while parsing node arguments.\n",DDI_Id()); Fatal_error(911); } /* ------------------------------------------------ *\ * Parse command-line for scratch/working directory \* ------------------------------------------------ */ Init_scratch(argc,argv); /* ----------------------------------------------------------- *\ * Using parsed information, initialize global data structures \* ----------------------------------------------------------- */ for(i=0,iproc=0,ncpus=0; i<nn; i++) { gv(ddinodes)[i].nodemaster = iproc; cpus = gv(ddinodes)[i].cpus; nics = gv(ddinodes)[i].nics; gv(ddinodes)[i].myds = (me % cpus) + ncpus + np; DEBUG_OUT(LVL4,(stdout,"%s: ddinodes[%i] = { nodemaster=%i cpus=%i nics=%i }\n",DDI_Id(),i,iproc,cpus,nics)) ncpus += cpus; # if !defined USE_SYSV if(cpus > 1 && comm->me == 0) { if(comm->me == 0) fprintf(stdout,"%s %s\n%s %s\n", "Error: detected :cpus in the ddi argument list, however," "DDI was not compiled using -DUSE_SYSV. The program is" "confused. Either recompile DDI using shared-memory or" "generate a proper argument list."); Fatal_error(911); } # endif for(j=0; j<cpus; j++,iproc++) { gv(ddiprocs)[iproc].node = i; gv(ddiprocs)[iproc+np].node = i; strcpy(tmpstr,gv(ddinodes)[i].hostname); if(nics) { strcat(tmpstr,gv(ddinodes)[i].netext[j%nics]); } DEBUG_OUT(LVL4,(stdout,"%s: ddiprocs[%i] = { node=%i hostname=%s }\n",DDI_Id(),iproc,i,tmpstr)) gv(ddiprocs)[iproc].hostname = (char *) strdup(tmpstr); # if FULL_SMP if(nn == 1) continue; # endif gv(ddiprocs)[iproc+np].hostname = (char *) strdup(tmpstr); } } # else /* --------------------------------------------------------------- *\ When mixing MPI and TCP/IP, MPI has been previously initialized \* --------------------------------------------------------------- */ DDI_NProc(&np,&me); nsocks = np; if(USING_DATA_SERVERS()) nsocks += np; # endif /* ----------------------------------- *\ Allocate memory for sockets & ports \* ----------------------------------- */ sockets = (int *) Malloc(nsocks*sizeof(int)); ports = (int *) Malloc(nsocks*sizeof(int)); gv(sockets) = sockets; for(i=0; i<nsocks; i++) { sockets[i] = -1; ports[i] = -1; } /* ----------------------------------------------------------------- *\ Create a server socket for accepting connections to DDI processes \* ----------------------------------------------------------------- */ gv(serversock) = SOC_Create(SERVER_SOCKET,&gv(serverport),NULL); DEBUG_OUT(LVL3,(stdout,"%s: Receiving connections on port %d\n",DDI_Id(),gv(serverport))) /* --------------------------------------------- *\ Connect to Master Kickoff Program. Socket ownership is required to handle SIGURG Use Premium network if available. \* ---------------------------------------------- */ # if defined DDI_SOC && !defined DDI_MPI gv(kickoffsock) = SOC_Create(CLIENT_SOCKET,&kickport,kickhost); /* ---------------------------------- *\ Report information back to ddikick \* ---------------------------------- */ Send(gv(kickoffsock),&comm->me,sizeof(int),0); Send(gv(kickoffsock),&gv(serverport),sizeof(int),0); /* ------------------------------------------------- *\ Wait until all DDI processes have checked in. The master ddikick.x will broadcast the ddi info. \* ------------------------------------------------- */ Recv(gv(kickoffsock),ports,nsocks*sizeof(int),0); DEBUG_OUT(LVL3,(stdout,"%s: Received the list of server ports.\n",DDI_Id())) # else DEBUG_OUT(LVL3,(stdout,"%s: using MPI to gather TCP/IP port info.\n",DDI_Id())) MPI_Allgather(&gv(serverport),1,MPI_INT,ports,1,MPI_INT,comm->world_comm); # endif /* ------------------------------------------------------- *\ Now starts the process of connecting the DDI processes: Each DDI compute process actively connects to all DDI processes whose ranks are greater than its own, but only after it accepts connections from lower ranks. \* ------------------------------------------------------- */ Init_soc_accept(np,me,gv(serversock),sockets); Init_soc_create(np,me,nsocks,ports,sockets); }
/* ---------------------------------------------------- *\ DDI_Init(argc,argv) =================== [IN] int argc -- number of arguments [IN] char **argv -- list of arguments Initialize message-passing library and DDI variables \* ---------------------------------------------------- */ void DDI_Init(int argc,char **argv) { char *header = NULL; # if defined DDI_SOC || defined DDI_MPI int np,me; /* ---------------------------------- *\ Initialize Message Passing Library \* ---------------------------------- */ #if defined DDI_ARMCI DDI_ARMCI_Init(argc,argv); #else # if defined DDI_SHMEM shmem_init(); # endif # if defined DDI_MPI Init_mpi(argc,argv); # endif # if defined DDI_SOC Init_soc(argc,argv); # endif # if defined DDI_LAPI Init_lapi(); # endif # if defined USE_SYSV Init_smp(); # endif # endif /* --------------------------------- *\ Determine Rank and Number of CPUs \* --------------------------------- */ DDI_NProc(&np,&me); /* --------------------------- *\ Initialize Profile Counters \* --------------------------- */ # if defined DDI_COUNTERS Init_counters(); # endif /* ------------------------ *\ * Initialize communicators \* ------------------------ */ Comm_init(); /* ----------------------- *\ Initialize Data Servers \* ----------------------- */ if(me >= np) DDI_Server(); #if defined DDI_BGL || defined DDI_BGP /* Print runtime header if requested. This works on any POSIX system. */ header = getenv("DDI_HEADER"); if(me == 0 && header != NULL) { if (strcmp(header,"RUNTIME") == 0) { DDI_Runtime_print(stdout); fprintf(stdout,"\n"); } fflush(stdout); } #endif # if defined CRAY_MPI /* DDI_SMP_NProc(&np,&me); if(me == 1) DS_Thread_init(); */ # endif /* ----------------------------- *\ Synchronize Compute Processes \* ----------------------------- */ DDI_Sync(3061); # endif }
int main(int argc,char *argv[]) { int i,np,me,ncols; int nn,my; int handle; size_t counter; double a[10]; DDI_Patch patch; DDI_Init(argc,argv); // DebugOutput(0); DDI_Memory(50); DDI_NProc(&np,&me); DDI_NNode(&nn,&my); DDI_Create(10,np,&handle); DDI_DistribP(handle,me,&patch); ncols = patch.jhi-patch.jlo+1; // if(me == 0) Comm_patch_print(&patch); for(i=0; i<10; i++) a[i] = 93*1.0; for(i=patch.jlo; i<=patch.jhi; i++) { DDI_Put(handle,patch.ilo,patch.ihi,i,i,&a); } for(i=0; i<10; i++) a[i] = -1.0; counter = -1; DDI_DLBReset(); do { DDI_DLBNext(&counter); if(counter % 200 == 0) { fprintf(stdout,"%s: counter=%i\n",DDI_Id(),counter); fflush(stdout); } if(my == 0) usleep(20); } while (counter < 30000); // MPI_Barrier(MPI_COMM_WORLD); // if(me) sleep(2); // else DS_Thread_init(); // MPI_Barrier(MPI_COMM_WORLD); // fprintf(stdout,"%s: first get\n",DDI_Id()); // fflush(stdout); DDI_Sync(10); if(me == 0) printf("done with dlb test\n"); fflush(stdout); DDI_Sync(11); DDI_Get(handle,patch.ilo,patch.ihi,0,0,a); for(i=0; i<10; i++) { if(a[i] != 93.0) fprintf(stdout," %i: a[%i]=%lf != 93\n",me,i,a[i]); a[i] = 1.0; fflush(stdout); } if(me == 0) printf("%s: done with get\n",DDI_Id()); fflush(stdout); DDI_Sync(10); if(me == 0) fprintf(stdout,"%s: starting acc\n",DDI_Id()); fflush(stdout); DDI_Sync(12); DDI_Acc(handle,patch.ilo,patch.ihi,0,0,a); if(me == 0) fprintf(stdout,"%s: finished acc; syncing\n",DDI_Id()); fflush(stdout); DDI_Sync(14); if(me==0) fprintf(stdout,"%s: finished acc; new get\n",DDI_Id()); fflush(stdout); DDI_Sync(20); DDI_Get(handle,patch.ilo,patch.ihi,0,0,a); for(i=0; i<10; i++) { if(a[i] != 93.0+np) fprintf(stdout," %i: a[%i]=%lf !=%d \n",me,i,a[i],np+93); a[i] = 1.0; fflush(stdout); } DDI_Sync(15); if(me==0) fprintf(stdout,"%i: tested get\n",me); fflush(stdout); DDI_Sync(20); // DDI_Put(handle,patch.ilo,patch.ihi,0,np-1,a); if(me==0) fprintf(stdout,"%s: finished global put\n",DDI_Id()); fflush(stdout); DDI_Destroy(handle); // fprintf(stdout,"%s: after destroy\n",DDI_Id()); // fflush(stdout); DDI_Finalize(); return 0; }
/* ---------------------------------------------------------- *\ DDI_GDLBNext(counter) ==================== [OUT] counter - value of the load balance counter returned to the calling process. An atomic operation that sets the value of counter to the value of the global load-balance counter, then increments the global counter. \* --------------------------------------------------------- */ void DDI_GDLBNext(size_t *counter) { int np,me,nn,my,tmp_scope,remote_id=0; DDI_Patch Patch; # if defined DDI_LAPI lapi_cntr_t org_cntr; uint tgt = gv(lapi_map)[0]; int *tgt_var = gv(lapi_gdlb_cntr_addr)[tgt]; int in_val = 1; int prev_tgt_val = -1; # endif # if defined DDI_ARMCI DDI_ARMCI_GDLBNext(counter); return; # endif DDI_NProc(&np,&me); if(me == 0) { /* ---------------------------------- *\ We need to work in the world scope \* ---------------------------------- */ tmp_scope = DDI_WORKING_COMM; gv(ddi_working_comm) = DDI_COMM_WORLD; DDI_NProc(&np,&me); DDI_NNode(&nn,&my); # if FULL_SMP # if defined DDI_LAPI if(LAPI_Setcntr(gv(lapi_hnd),&org_cntr,0) != LAPI_SUCCESS) { fprintf(stdout,"%s: LAPI_Setcntr failed in DDI_GDLBNext.\n",DDI_Id()); Fatal_error(911); } if(LAPI_Rmw(gv(lapi_hnd),FETCH_AND_ADD,tgt,tgt_var,&in_val, &prev_tgt_val,&org_cntr) != LAPI_SUCCESS) { fprintf(stdout,"%s: LAPI_Rmw failed in DDI_GDLBNext.\n",DDI_Id()); Fatal_error(911); } if(LAPI_Waitcntr(gv(lapi_hnd),&org_cntr,1,NULL) != LAPI_SUCCESS) { fprintf(stdout,"%s: LAPI_Waitcntr failed in DDI_GDLBNext.\n",DDI_Id()); Fatal_error(911); } if(prev_tgt_val == -1) { fprintf(stdout,"%s: LAPI version of DDI_GDLBNext is not working correctly.\n",DDI_Id()); Fatal_error(911); } else { *counter = (size_t) prev_tgt_val; } # else if(my == 0) { DDI_GDLBNext_local(counter); } else { Patch.oper = DDI_GDLBNEXT; DDI_Send_request(&Patch,&remote_id,NULL); DDI_Recv(counter,sizeof(size_t),remote_id); } # endif # else Patch.oper = DDI_GDLBNEXT; DDI_Send_request(&Patch,&remote_id,NULL); DDI_Recv(counter,sizeof(size_t),remote_id); # endif /* --------------------------- *\ Return to the working scope \* --------------------------- */ gv(ddi_working_comm) = tmp_scope; } DDI_BCast(counter,sizeof(size_t),0); }
/* -------------------------------------------------------------- *\ DDI_GetAccP(handle,patch,buff) ============================ [IN] handle - Handle of the distributed array to be accessed. [IN] patch - structure containing ilo, ihi, jlo, jhi, etc. [IN] buff - Data segment to be operated on. \* -------------------------------------------------------------- */ void DDI_GetAccP(int handle,DDI_Patch *patch,void *buff) { /* --------------- *\ Local Variables \* --------------- */ char ack=57; int i,np,me,nn,my,remote_id,nsubp; int ranks[MAX_NODES]; DDI_Patch subp[MAX_NODES]; char *working_buffer = (char *) buff; # if defined DDI_LAPI DDI_Patch *local_patch = NULL; lapi_cntr_t cntr[MAX_NODES]; # endif STD_DEBUG((stdout,"%s: Entering DDI_GetAccP.\n",DDI_Id())) /* -------------------- *\ Process OR Node Rank \* -------------------- */ DDI_NProc(&np,&me); DDI_NNode(&nn,&my); /* ------------------------------------- *\ Ensure the patch has the correct info \* ------------------------------------- */ patch->oper = DDI_GETACC; patch->handle = handle; /* ---------------------------------- *\ Check calling arguments for errors \* ---------------------------------- */ # if defined DDI_CHECK_ARGS if(handle < 0 || handle >= gv(ndda)) { fprintf(stdout,"%s: Invalid handle [%i] in DDI_GetAcc.\n",DDI_Id(),handle); Fatal_error(911); } if(patch->ilo > patch->ihi || patch->ilo < 0 || patch->ihi >= gv(nrow)[handle]) { fprintf(stdout,"%s: Invalid row dimensions during DDI_GetAcc => ilo=%i ihi=%i.\n",DDI_Id(),patch->ilo,patch->ihi); Fatal_error(911); } if(patch->jlo > patch->jhi || patch->jlo < 0 || patch->jhi >= gv(ncol)[handle]) { fprintf(stdout,"%s: Invalid colum dimensions during DDI_GetAcc => jlo=%i jhi=%i.\n",DDI_Id(),patch->jlo,patch->jhi); Fatal_error(911); } # endif /* ------------------------------ *\ Log some simple profiling info \* ------------------------------ */ # if defined DDI_COUNTERS gv(acc_profile).ncalls++; gv(acc_profile).nbytes += DDI_Patch_sizeof(patch); # endif /* ------------------------------------------------------- *\ Determine where the pieces of the requested patch exist \* ------------------------------------------------------- */ DDI_Subpatch(handle,patch,&nsubp,ranks,subp); MAX_DEBUG((stdout,"%s: %i subpatches.\n",DDI_Id(),nsubp)) /* ------------------------------------------------------------------- *\ Send data requests for all non-local pieces of the requested patch. Operate immediately to GetAcc a local portion of the patch. \* ------------------------------------------------------------------- */ for(i=0; i<nsubp; i++) { ULTRA_DEBUG((stdout,"%s: GetAccumulating subpatch %i.\n",DDI_Id(),i)) /* ------------------------------------------------------------- *\ Using SysV, take advantage of shared-memory for a local patch \* ------------------------------------------------------------- */ # if defined USE_SYSV /* ------------------------------------------------ *\ Determine if the ith patch is local to 'my' node \* ------------------------------------------------ */ if(ranks[i] == my) { MAX_DEBUG((stdout,"%s: Subpatch %i is local.\n",DDI_Id(),i)) /* ---------------------------------------------------- *\ Using LAPI, perform the local Getacc after all the data requests have been sent ==> maximize concurrency. \* ---------------------------------------------------- */ # if defined DDI_LAPI local_patch = &subp[i]; local_patch->cp_buffer_addr = working_buffer; # else /* --------------------------------------------- *\ Otherwise, perform the local Getacc immediately. \* --------------------------------------------- */ DDI_GetAcc_local(&subp[i],working_buffer); # endif /* ------------------------------------------------------- *\ Move the working buffer to the next patch and continue. \* ------------------------------------------------------- */ working_buffer += subp[i].size; continue; } # endif /* --------------------------------- *\ If the current patch is NOT local \* --------------------------------- */ remote_id = ranks[i]; /* ----------------------------------------------- *\ Using LAPI, then include some extra information \* ----------------------------------------------- */ # if defined DDI_LAPI subp[i].cp_lapi_id = gv(lapi_map)[me]; subp[i].cp_lapi_cntr = (void *) &cntr[i]; subp[i].cp_buffer_addr = (void *) working_buffer; LAPI_Setcntr(gv(lapi_hnd),&cntr[i],0); ULTRA_DEBUG((stdout,"%s: cp_lapi_id=%i.\n",DDI_Id(),gv(lapi_map)[me])) ULTRA_DEBUG((stdout,"%s: cp_lapi_cntr=%x.\n",DDI_Id(),&cntr[i])) ULTRA_DEBUG((stdout,"%s: cp_buffer_addr=%x.\n",DDI_Id(),working_buffer)) # endif /* -------------------------------- *\ Send data request for subpatch i \* -------------------------------- */ MAX_DEBUG((stdout,"%s: Sending data request to node %i.\n",DDI_Id(),remote_id)) DDI_Send_request(&subp[i],&remote_id,NULL); MAX_DEBUG((stdout,"%s: data request sent to global process %i.\n",DDI_Id(),remote_id)) /* ------------------------------------------------------------ *\ Receive an acknowledgement that the data server has raised a fence that will protect the distributed array from get or put access until all accumulates have finished. This block- ing receive ensures that the current process executing this accumulate can *NOT* finish, until the fence has been raised \* ------------------------------------------------------------ */ # if !defined DDI_LAPI # if defined USE_SYSV MAX_DEBUG((stdout,"%s: Receiving remote fence ACK.\n",DDI_Id())) DDI_Recv(&ack,1,remote_id); # endif /* ---------------------------- *\ Recv subpatch from remote_id \* ---------------------------- */ MAX_DEBUG((stdout,"%s: Sending subpatch %i to %i.\n",DDI_Id(),i,remote_id)) DDI_Send(working_buffer,subp[i].size,remote_id); DDI_Recv(working_buffer,subp[i].size,remote_id); # endif /* ------------ *\ Shift buffer \* ------------ */ working_buffer += subp[i].size; } /* ----------------------------------------------------------- *\ Using LAPI, perform the local Getaccumulate (if needed) as the remote processes are getting the data to Getaccumulate on the target processes. Then wait for all the data to be copied out of the buffer before returning. \* ----------------------------------------------------------- */ # if defined DDI_LAPI /* ------------------------------------ *\ GetAccumulating local patch (if exists) \* ------------------------------------ */ if(local_patch) DDI_GetAcc_local(local_patch,local_patch->cp_buffer_addr); /* ---------------------------------------------------------- *\ Wait for all remote LAPI_Gets to finish copying local data \* ---------------------------------------------------------- */ for(i=0; i<nsubp; i++) { if(subp[i].cp_lapi_cntr) { ULTRA_DEBUG((stdout,"%s: Wait for subpatch %i to be copied.\n",DDI_Id(),i)) LAPI_Waitcntr(gv(lapi_hnd),&cntr[i],3,NULL); ULTRA_DEBUG((stdout,"%s: Subpatch %i copy completed.\n",DDI_Id(),i)) } }
/* ----------------------------- *\ FORTRAN Wrapper for DDI_NProc \* ----------------------------- */ void F77_NProc(int_f77* np, int_f77* me) { int ddinp,ddime; DDI_NProc(&ddinp,&ddime); *np = (int_f77) ddinp; *me = (int_f77) ddime; }
/* -------------------------------------------------------------------- *\ DDI_Timer_output() ================== Synchronous barrier on compute processes, but also collects total cpu time from each compute process and prints the totals stdout. \* -------------------------------------------------------------------- */ void DDI_Timer_output() { int i,me,np; struct rusage mycputime; struct rusage *timings = NULL; struct timeval cpu_total; struct timeval wall_total; DDI_NProc(&np,&me); DDI_Sync(3081); if(me == 0) { timings = (struct rusage *) Malloc(np*sizeof(struct rusage)); getrusage(RUSAGE_SELF,timings); gettimeofday(&wall_total,NULL); } else { getrusage(RUSAGE_SELF,&mycputime); timings = &mycputime; } timings->ru_utime.tv_sec -= gv(cpu_timer).ru_utime.tv_sec; timings->ru_utime.tv_usec -= gv(cpu_timer).ru_utime.tv_usec; if(timings->ru_utime.tv_usec < 0) { timings->ru_utime.tv_sec--; timings->ru_utime.tv_usec += 1000000; } timings->ru_stime.tv_sec -= gv(cpu_timer).ru_stime.tv_sec; timings->ru_stime.tv_usec -= gv(cpu_timer).ru_stime.tv_usec; if(timings->ru_stime.tv_usec < 0) { timings->ru_stime.tv_sec--; timings->ru_stime.tv_usec += 1000000; } wall_total.tv_sec -= gv(wall_timer).tv_sec; wall_total.tv_usec -= gv(wall_timer).tv_usec; if(wall_total.tv_usec < 0) { wall_total.tv_sec--; wall_total.tv_usec += 1000000; } if(me == 0) { for(i=1; i<np; i++) DDI_Recv(&timings[i],sizeof(struct rusage),i); fprintf(stdout,"\n ------------------------------------------------"); fprintf(stdout,"\n CPU timing information for all compute processes"); fprintf(stdout,"\n ================================================"); for(i=0; i<np; i++) { cpu_total.tv_sec = timings[i].ru_utime.tv_sec + timings[i].ru_stime.tv_sec; cpu_total.tv_usec = timings[i].ru_utime.tv_usec + timings[i].ru_stime.tv_usec; if(cpu_total.tv_usec > 1000000) { cpu_total.tv_sec++; cpu_total.tv_usec -= 1000000; } fprintf(stdout,"\n %4i: %d.%.6d + %d.%.6d = %d.%.6d",i, (int)timings[i].ru_utime.tv_sec,(int)timings[i].ru_utime.tv_usec, (int)timings[i].ru_stime.tv_sec,(int)timings[i].ru_stime.tv_usec, (int)cpu_total.tv_sec,(int)cpu_total.tv_usec); } fprintf(stdout,"\n Wall: %d.%.6d", (int) wall_total.tv_sec, (int) wall_total.tv_usec); fprintf(stdout,"\n ================================================\n\n"); fflush(stdout); free(timings); } else { DDI_Send(&mycputime,sizeof(struct rusage),0); } DDI_Sync(3082); }