/* -------------------------------------------------------- *\
DDI_GDLBRest()
==============
Reset the global dynamic load-balance counter.
\* -------------------------------------------------------- */
void DDI_GDLBReset() {
int np,me;
# if !defined USE_SYSV
int remote_id = 0;
DDI_Patch Patch;
# endif
if(DDI_WORKING_COMM != DDI_COMM_WORLD) {
fprintf(stdout," DDI Error: group dlb counter must be reset in DDI_WORLD scope.\n");
Fatal_error(911);
}
DDI_NProc(&np,&me);
DDI_Sync(3041);
#if defined DDI_ARMCI
DDI_ARMCI_GDLBReset();
#else
if(me == 0) {
# if defined USE_SYSV
DDI_GDLBReset_local();
# else
Patch.oper = DDI_GDLBRESET;
DDI_Send_request(&Patch,&remote_id,NULL);
# endif
}
#endif
DDI_Sync(3042);
}
/* ---------------------------------------------------- *\
DDI_Init(argc,argv)
===================
[IN] int argc -- number of arguments
[IN] char **argv -- list of arguments
Initialize message-passing library and DDI variables
\* ---------------------------------------------------- */
void DDI_Init(int argc,char **argv) {
char *header = NULL;
# if defined DDI_SOC || defined DDI_MPI
int np,me;
/* ---------------------------------- *\
Initialize Message Passing Library
\* ---------------------------------- */
#if defined DDI_ARMCI
DDI_ARMCI_Init(argc,argv);
#else
# if defined DDI_SHMEM
shmem_init();
# endif
# if defined DDI_MPI
Init_mpi(argc,argv);
# endif
# if defined DDI_SOC
Init_soc(argc,argv);
# endif
# if defined DDI_LAPI
Init_lapi();
# endif
# if defined USE_SYSV
Init_smp();
# endif
# endif
/* --------------------------------- *\
Determine Rank and Number of CPUs
\* --------------------------------- */
DDI_NProc(&np,&me);
/* --------------------------- *\
Initialize Profile Counters
\* --------------------------- */
# if defined DDI_COUNTERS
Init_counters();
# endif
/* ------------------------ *\
* Initialize communicators
\* ------------------------ */
Comm_init();
/* ----------------------- *\
Initialize Data Servers
\* ----------------------- */
if(me >= np) DDI_Server();
#if defined DDI_BGL || defined DDI_BGP
/* Print runtime header if requested.
This works on any POSIX system. */
header = getenv("DDI_HEADER");
if(me == 0 && header != NULL) {
if (strcmp(header,"RUNTIME") == 0) {
DDI_Runtime_print(stdout);
fprintf(stdout,"\n");
}
fflush(stdout);
}
#endif
# if defined CRAY_MPI
/*
DDI_SMP_NProc(&np,&me);
if(me == 1) DS_Thread_init();
*/
# endif
/* ----------------------------- *\
Synchronize Compute Processes
\* ----------------------------- */
DDI_Sync(3061);
# endif
}
int main(int argc,char *argv[]) {
int i,np,me,ncols;
int nn,my;
int handle;
size_t counter;
double a[10];
DDI_Patch patch;
DDI_Init(argc,argv);
// DebugOutput(0);
DDI_Memory(50);
DDI_NProc(&np,&me);
DDI_NNode(&nn,&my);
DDI_Create(10,np,&handle);
DDI_DistribP(handle,me,&patch);
ncols = patch.jhi-patch.jlo+1;
// if(me == 0) Comm_patch_print(&patch);
for(i=0; i<10; i++) a[i] = 93*1.0;
for(i=patch.jlo; i<=patch.jhi; i++) {
DDI_Put(handle,patch.ilo,patch.ihi,i,i,&a);
}
for(i=0; i<10; i++) a[i] = -1.0;
counter = -1;
DDI_DLBReset();
do {
DDI_DLBNext(&counter);
if(counter % 200 == 0) {
fprintf(stdout,"%s: counter=%i\n",DDI_Id(),counter);
fflush(stdout);
}
if(my == 0) usleep(20);
} while (counter < 30000);
// MPI_Barrier(MPI_COMM_WORLD);
// if(me) sleep(2);
// else DS_Thread_init();
// MPI_Barrier(MPI_COMM_WORLD);
// fprintf(stdout,"%s: first get\n",DDI_Id());
// fflush(stdout);
DDI_Sync(10);
if(me == 0) printf("done with dlb test\n");
fflush(stdout);
DDI_Sync(11);
DDI_Get(handle,patch.ilo,patch.ihi,0,0,a);
for(i=0; i<10; i++) {
if(a[i] != 93.0) fprintf(stdout," %i: a[%i]=%lf != 93\n",me,i,a[i]);
a[i] = 1.0;
fflush(stdout);
}
if(me == 0) printf("%s: done with get\n",DDI_Id());
fflush(stdout);
DDI_Sync(10);
if(me == 0) fprintf(stdout,"%s: starting acc\n",DDI_Id());
fflush(stdout);
DDI_Sync(12);
DDI_Acc(handle,patch.ilo,patch.ihi,0,0,a);
if(me == 0) fprintf(stdout,"%s: finished acc; syncing\n",DDI_Id());
fflush(stdout);
DDI_Sync(14);
if(me==0) fprintf(stdout,"%s: finished acc; new get\n",DDI_Id());
fflush(stdout);
DDI_Sync(20);
DDI_Get(handle,patch.ilo,patch.ihi,0,0,a);
for(i=0; i<10; i++) {
if(a[i] != 93.0+np) fprintf(stdout," %i: a[%i]=%lf !=%d \n",me,i,a[i],np+93);
a[i] = 1.0;
fflush(stdout);
}
DDI_Sync(15);
if(me==0) fprintf(stdout,"%i: tested get\n",me);
fflush(stdout);
DDI_Sync(20);
// DDI_Put(handle,patch.ilo,patch.ihi,0,np-1,a);
if(me==0) fprintf(stdout,"%s: finished global put\n",DDI_Id());
fflush(stdout);
DDI_Destroy(handle);
// fprintf(stdout,"%s: after destroy\n",DDI_Id());
// fflush(stdout);
DDI_Finalize();
return 0;
}
/* ---------------------------- *\
FORTRAN Wrapper for DDI_Sync
\* ---------------------------- */
void F77_Sync(int_f77 *msgtag) {
int tag = (int) *msgtag;
DDI_Sync(tag);
}
/* -------------------------------------------------------------------- *\
DDI_Timer_output()
==================
Synchronous barrier on compute processes, but also collects total
cpu time from each compute process and prints the totals stdout.
\* -------------------------------------------------------------------- */
void DDI_Timer_output() {
int i,me,np;
struct rusage mycputime;
struct rusage *timings = NULL;
struct timeval cpu_total;
struct timeval wall_total;
DDI_NProc(&np,&me);
DDI_Sync(3081);
if(me == 0) {
timings = (struct rusage *) Malloc(np*sizeof(struct rusage));
getrusage(RUSAGE_SELF,timings);
gettimeofday(&wall_total,NULL);
} else {
getrusage(RUSAGE_SELF,&mycputime);
timings = &mycputime;
}
timings->ru_utime.tv_sec -= gv(cpu_timer).ru_utime.tv_sec;
timings->ru_utime.tv_usec -= gv(cpu_timer).ru_utime.tv_usec;
if(timings->ru_utime.tv_usec < 0) {
timings->ru_utime.tv_sec--;
timings->ru_utime.tv_usec += 1000000;
}
timings->ru_stime.tv_sec -= gv(cpu_timer).ru_stime.tv_sec;
timings->ru_stime.tv_usec -= gv(cpu_timer).ru_stime.tv_usec;
if(timings->ru_stime.tv_usec < 0) {
timings->ru_stime.tv_sec--;
timings->ru_stime.tv_usec += 1000000;
}
wall_total.tv_sec -= gv(wall_timer).tv_sec;
wall_total.tv_usec -= gv(wall_timer).tv_usec;
if(wall_total.tv_usec < 0) {
wall_total.tv_sec--;
wall_total.tv_usec += 1000000;
}
if(me == 0) {
for(i=1; i<np; i++) DDI_Recv(&timings[i],sizeof(struct rusage),i);
fprintf(stdout,"\n ------------------------------------------------");
fprintf(stdout,"\n CPU timing information for all compute processes");
fprintf(stdout,"\n ================================================");
for(i=0; i<np; i++) {
cpu_total.tv_sec = timings[i].ru_utime.tv_sec + timings[i].ru_stime.tv_sec;
cpu_total.tv_usec = timings[i].ru_utime.tv_usec + timings[i].ru_stime.tv_usec;
if(cpu_total.tv_usec > 1000000) {
cpu_total.tv_sec++;
cpu_total.tv_usec -= 1000000;
}
fprintf(stdout,"\n %4i: %d.%.6d + %d.%.6d = %d.%.6d",i,
(int)timings[i].ru_utime.tv_sec,(int)timings[i].ru_utime.tv_usec,
(int)timings[i].ru_stime.tv_sec,(int)timings[i].ru_stime.tv_usec,
(int)cpu_total.tv_sec,(int)cpu_total.tv_usec);
}
fprintf(stdout,"\n Wall: %d.%.6d",
(int) wall_total.tv_sec, (int) wall_total.tv_usec);
fprintf(stdout,"\n ================================================\n\n");
fflush(stdout);
free(timings);
} else {
DDI_Send(&mycputime,sizeof(struct rusage),0);
}
DDI_Sync(3082);
}
