int main(int argc, char *argv[])
{
struct Zoltan_Timer *zt1, *zt2, *zt3, *zt4;
int i, me;
const int MAINLOOP=20;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &me);
zt1 = Zoltan_Timer_Create(ZOLTAN_TIME_WALL);
zt2 = Zoltan_Timer_Create(ZOLTAN_TIME_USER);
zt3 = Zoltan_Timer_Create(ZOLTAN_TIME_WALL);
for (i = 0; i < MAINLOOP; i++) {
if (me == 0) printf("\n\n\t****Beginning first test****\n");
first_test(zt1);
if (me == 0) printf("\n\n\t****Beginning second test****\n");
second_test(zt2);
}
if (me == 0) printf("\n\nFINAL RESULTS -- FIRST TEST:\n");
Zoltan_Timer_PrintAll(zt1, 0, MPI_COMM_WORLD, stdout);
if (me == 0) printf("\n\nFINAL RESULTS -- SECOND TEST:\n");
Zoltan_Timer_PrintAll(zt2, 0, MPI_COMM_WORLD, stdout);
/* Copy tests */
Zoltan_Timer_Copy_To(&zt3, zt1);
zt4 = Zoltan_Timer_Copy(zt2);
for (i = 0; i < MAINLOOP; i++) {
if (me == 0) printf("\n\n\t****Beginning first copy test****\n");
first_test(zt3);
if (me == 0) printf("\n\n\t****Beginning second copy test****\n");
second_test(zt4);
}
if (me == 0) printf("\n\nFINAL RESULTS -- FIRST COPY TEST:\n");
Zoltan_Timer_PrintAll(zt3, 0, MPI_COMM_WORLD, stdout);
if (me == 0) printf("\n\nFINAL RESULTS -- SECOND COPY TEST:\n");
Zoltan_Timer_PrintAll(zt4, 0, MPI_COMM_WORLD, stdout);
/* Test printing while timer is still running. */
if (me == 0) printf("\n\n\t****Intermediate print test****\n");
third_test(zt1);
if (me == 0) printf("\n\nFINAL RESULTS -- INTERMEDIATE PRINT TEST:\n");
Zoltan_Timer_PrintAll(zt1, 0, MPI_COMM_WORLD, stdout);
if (me == 0) printf("\n\nTHE END\n");
Zoltan_Timer_Destroy(&zt1);
Zoltan_Timer_Destroy(&zt2);
MPI_Finalize();
return 0;
}
static VCycle *newVCycle(ZZ *zz, HGraph *hg, Partition part, VCycle *finer,
int vcycle_timing)
{
VCycle *vcycle;
if (!(vcycle = (VCycle*) ZOLTAN_MALLOC (sizeof(VCycle))))
return NULL;
vcycle->finer = finer;
vcycle->Part = part;
vcycle->LevelMap = NULL;
vcycle->LevelData = NULL;
vcycle->LevelCnt = 0;
vcycle->LevelSndCnt = 0;
vcycle->comm_plan = NULL;
if (vcycle_timing)
vcycle->timer = Zoltan_Timer_Create(zz->Timer);
else
vcycle->timer = NULL;
vcycle->timer_match = -1;
vcycle->timer_coarse = -1;
vcycle->timer_refine = -1;
vcycle->timer_project = -1;
vcycle->hg = hg ? hg : (HGraph*) ZOLTAN_MALLOC (sizeof(HGraph));
if (!vcycle->hg) {
ZOLTAN_FREE (&vcycle);
return NULL;
}
if (hg && (allocVCycle(vcycle) != ZOLTAN_OK)) {
ZOLTAN_FREE (&vcycle->hg);
ZOLTAN_FREE (&vcycle);
}
return vcycle;
}
int main (int argc, char *argv[])
{
Zoltan_DD_Directory *dd;
ZTIMER *zt;
int myproc; /* MPI rank, my processor's MPI ID */
int nproc; /* MPI size, number of processors */
ZOLTAN_ID_PTR glist = NULL; /* list of GIDs */
ZOLTAN_ID_PTR llist = NULL; /* list of LIDs */
ZOLTAN_ID_PTR ulist = NULL; /* list of user data of type
ZOLTAN_ID_TYPE */
int *plist = NULL; /* list of partitions */
int *olist = NULL; /* list of owners */
Param param ; /* program's adjustable parameters */
char *store = NULL; /* non directory storage of test data */
Data *data = NULL; /* pointer into store */
/* these are all temporary variables */
int new_owner;
int count;
int i;
char *p;
int err;
int errcount;
int loop;
char str[100]; /* for building output messages */
static int timer[7] = {-1,-1,-1,-1,-1,-1,-1};
char *yo = "DD_Main";
/* initialize MPI communications, ignore errors */
MPI_Init (&argc, &argv);
MPI_Comm_rank (MPI_COMM_WORLD, &myproc);
MPI_Comm_size (MPI_COMM_WORLD, &nproc);
get_params (¶m); /* read input parameters */
zt = Zoltan_Timer_Create(1);
MACRO_TIMER_START(5, "Total time", 0);
MACRO_TIMER_START(0, "DD_Create time", 0);
err = Zoltan_DD_Create (&dd, MPI_COMM_WORLD, param.glen, param.llen,
param.ulen, param.tlen, param.debug_level);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Create");
MACRO_TIMER_STOP(0);
param.slen = sizeof (Data) + sizeof(ZOLTAN_ID_TYPE)
* (param.glen + param.llen + param.ulen);
param.slen = Zoltan_Align(param.slen);
store = (char*) ZOLTAN_MALLOC (param.count * param.slen);
/* allocate storage for various lists */
glist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC(sizeof(ZOLTAN_ID_TYPE) * param.count
* param.glen);
plist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count);
olist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count);
if (param.llen)
llist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE)
* param.count * param.llen);
if (param.ulen)
ulist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE)
* param.count * param.ulen) ;
if (store == NULL || glist == NULL || (param.llen != 0 && llist == NULL)
|| (param.ulen != 0 && ulist == NULL) || plist == NULL || olist == NULL) {
ZOLTAN_PRINT_ERROR (myproc, yo, "Unable to malloc storage lists");
return ZOLTAN_MEMERR;
}
initialize_data (¶m, store, nproc);
/* create & update directory with myproc's initial simulated GIDs */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data *)p)->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data*)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data *)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen,
((Data *)p)->id + (param.glen + param.llen));
plist [count] = ((Data *)p)->partition;
++count;
}
MACRO_TIMER_START (1, "DD_Update timer", 0);
err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
i = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen) {
ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id);
++i;
}
MACRO_TIMER_START(2, "DD_Find timer", 0);
err = Zoltan_DD_Find (dd, glist, llist, ulist, plist, param.count, olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
errcount = 0;
for (i = 0; i < param.count; i++)
if (olist[i] != ((Data *) (store + i * param.slen))->new_owner)
++errcount;
if (errcount)
sprintf (str, "FIRST TEST FAILED, errcount is %d", errcount);
else
sprintf (str, "FIRST TEST SUCCESSFUL");
ZOLTAN_PRINT_INFO (myproc, yo, str);
/* repeatedly simulate moving "dots" around the system */
for (loop = 0; loop < param.nloops; loop++)
{
for (p = store; p < store + param.count * param.slen; p += param.slen)
((Data*) p)->old_owner = ((Data*) p)->new_owner;
/* randomly exchange some dots and randomly reassign others */
for (i = 0; i < (param.pmove * param.count)/100; i++) {
Data *d1, *d2;
d1 = (Data*) (store + param.slen * (rand() % param.count));
d2 = (Data*) (store + param.slen * (rand() % param.count));
new_owner = d1->new_owner;
d1->new_owner = d2->new_owner;
d2->new_owner = new_owner;
}
for (i = 0; i < (param.count * param.pscatter)/100; i++)
((Data*) (store + param.slen *(rand() % param.count)))->new_owner
= rand() % nproc;
/* determine which new GIDs myproc gained, and update directory */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == myproc) {
((Data*)p)->id[param.glen] = count; /* set LID */
ZOLTAN_SET_ID (param.glen, glist + count * param.glen,
((Data *)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data*)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen,
((Data*)p)->id + (param.glen + param.llen));
plist [count] = ((Data *)p)->partition;
++count;
}
MACRO_TIMER_START(1, "DD_Update", 0);
err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
/* use directory to "find" GIDs */
i = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen) {
ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id);
++i;
}
MACRO_TIMER_START(2, "DD_Find timer", 0);
if (loop % 5 == 0)
err = Zoltan_DD_Find(dd,glist,NULL,ulist,plist,param.count,olist);
else if (loop % 7 == 0)
err = Zoltan_DD_Find(dd,glist,llist,NULL,plist,param.count,olist);
else if (loop % 9 == 0)
err = Zoltan_DD_Find(dd,glist,llist,ulist,NULL,param.count,olist);
else if (loop % 2 == 0)
err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,NULL);
else
err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
if (loop % 2) {
errcount = 0 ;
for (i = 0; i < param.count; i++)
if (olist[i] != ((Data *)(store + i * param.slen))->new_owner)
++errcount;
if (errcount)
sprintf (str,"LOOP %d TEST FAILED, errcount is %d",loop,errcount);
else
sprintf (str, "LOOP %d TEST SUCCESSFUL", loop);
ZOLTAN_PRINT_INFO (myproc, yo, str) ;
}
else {
sprintf (str, "LOOP %d Completed", loop);
ZOLTAN_PRINT_INFO (myproc, yo, str);
}
/* randomly remove a percentage of GIDs from the directory */
count = 0;
for (i = 0; i < (param.count * param.pdelete)/100; i++) {
data = (Data*) (store + param.slen * (rand() % param.count));
if (data->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id);
++count;
}
data->new_owner = NO_PROC;
}
MACRO_TIMER_START(3, "DD_Remove timer", 0);
err = Zoltan_DD_Remove (dd, glist, count);
MACRO_TIMER_STOP(3);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Remove");
/* update directory (put directory entries back in) */
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == NO_PROC)
((Data*)p)->new_owner = loop % nproc; /* place in new location */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == myproc) {
ZOLTAN_SET_ID(param.glen,glist+count*param.glen,((Data*)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data*)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID(param.ulen, ulist + count * param.ulen,
((Data *)p)->id + (param.glen + param.llen));
plist [count] = ((Data*)p)->partition;
++count;
}
MACRO_TIMER_START(1, "DD_Update timer", 0);
err = Zoltan_DD_Update (dd, glist, NULL, NULL, NULL, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
}
/* now Find directory info for GIDs myproc now owns and validate */
count = 0;
for (i = 0; i < param.count; i++) {
data = (Data *) (store + i * param.slen);
if (data->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id);
++count;
}
}
MACRO_TIMER_START(2, "DD_Find", 0);
err = Zoltan_DD_Find (dd, glist, NULL, NULL, NULL, count, olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
errcount = 0;
for (i = 0; i < count; i++)
if (olist[i] != myproc)
++errcount;
if (errcount) {
sprintf (str, "errcount is %d", errcount);
ZOLTAN_PRINT_ERROR (myproc, yo, str);
}
else
ZOLTAN_PRINT_INFO (myproc, yo, "TEST SUCCESSFUL");
Zoltan_DD_Stats (dd);
/* done, now free memory, stop MPI & directory, return */
ZOLTAN_FREE (&store);
ZOLTAN_FREE (&glist);
ZOLTAN_FREE (&plist);
ZOLTAN_FREE (&olist);
if (param.llen) ZOLTAN_FREE (&llist);
if (param.ulen) ZOLTAN_FREE (&ulist);
ZOLTAN_PRINT_INFO (myproc, yo, "Completing program");
MACRO_TIMER_START(4, "DD_Destroy", 0);
Zoltan_DD_Destroy (&dd);
MACRO_TIMER_STOP(4);
MACRO_TIMER_STOP(5);
Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stderr);
MPI_Finalize ();
return ZOLTAN_OK;
}
ZZ *Zoltan_Create(MPI_Comm communicator)
{
/*
* Function to create a Zoltan structure. May want more than one
* structure if using different decompositions with different techniques.
* This function allocates and initializes the structure.
* Output:
* ZZ * -- Pointer to a Zoltan structure.
*
*/
char *yo = "Zoltan_Create";
ZZ *zz;
/*
* Allocate storage for the Zoltan structure.
*/
zz = (ZZ *) ZOLTAN_MALLOC(sizeof(ZZ));
if (!zz) {
int proc;
MPI_Comm_rank(communicator, &proc);
ZOLTAN_PRINT_ERROR(proc, yo, "Insufficient memory to create structure.");
return NULL;
}
Zoltan_Init(zz);
/*
* Set MPI values for zz:
*/
if (communicator == MPI_COMM_NULL) {
/*
* The processor is not in the communicator for the load-balancing
* structure. Set zz->Communicator to MPI_COMM_NULL and give dummy
* values to zz->Proc and zz->Num_Proc.
*/
zz->Communicator = MPI_COMM_NULL;
zz->Proc = -1;
zz->Num_Proc = 0;
}
else {
/*
* Set Communicator to the communicator passed in.
*/
MPI_Comm_dup(communicator, &(zz->Communicator));
MPI_Comm_size(zz->Communicator, &(zz->Num_Proc));
MPI_Comm_rank(zz->Communicator, &(zz->Proc));
}
Zoltan_LB_Init(&(zz->LB), zz->Num_Proc);
Zoltan_Migrate_Init(&(zz->Migrate));
#ifdef ZOLTAN_DRUM
/* Initialize DRUM-related structure field */
Zoltan_Drum_Init_Struct(&(zz->Drum));
#endif
zz->ZTime = Zoltan_Timer_Create(ZOLTAN_TIMER_DEF);
/* Test that size_t is uniform on all processors */
if (communicator != MPI_COMM_NULL) {
int my_sizet = sizeof(size_t);
int max_sizet, min_sizet;
MPI_Allreduce(&my_sizet, &max_sizet, 1, MPI_INT, MPI_MAX, zz->Communicator);
MPI_Allreduce(&my_sizet, &min_sizet, 1, MPI_INT, MPI_MIN, zz->Communicator);
if (min_sizet != max_sizet) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"min sizeof(size_t) != max sizeof(size_t)");
Zoltan_Destroy(&zz);
}
}
return(zz);
}
