/*
* void Zoltan_Oct_setcildren(pOctant octant, pOctant children, int *cpids)
*
* sets the child pointers to point to the children
*/
void Zoltan_Oct_setchildren(pOctant oct, pOctant *children, int *cpids) {
int i;
for(i = 0; i < 8; i++) {
Zoltan_Oct_setchild(oct, i, children[i]);
Zoltan_Oct_setCpid(oct, i, cpids[i]);
}
}
/*
* void Zoltan_Oct_gen_tree_from_input_data()
*
* This function will create a root node of on each processor which will
* then be used to create an octree with regions associated with it. The
* tree will then be balanced and the output used to balance "mesh regions"
* on several processors.
*/
static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag,
int *c1, int *c2, int *c3,
float *c0, int createpartree)
{
char *yo = "Zoltan_Oct_gen_tree_from_input_data";
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
COORD min, /* min coord bounds of objects */
max; /* max coord bounds of objects */
int num_extra; /* number of orphaned objects */
int num_objs; /* total number of local objects */
pRegion ptr, /* pointer to iterate trough region list */
ptr1; /* pointer to iterate trough region list */
pOctant root; /* root of the partition tree */
int i; /* index counter */
int count, /* count for leaf nodes in partition tree */
proc, /* proc leaf node of parition tree belongs to */
extra, /* extra leaf node flag, if not evenly divisible */
remainder; /* remainder of node, or processors to fill */
pOctant cursor, /* cursor to iterate through octant list */
cursor2, /* another cursor to iterate through octant list */
parent; /* parent of an octant */
int level, /* number of levels of refinement */
n, /* index counter */
part; /* partition counter */
Map *array; /* map of which processors own which octants */
int hold; /* used for calculating partition divisions */
int ierr = 0;
#ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX};
COORD global_min, global_max;
#endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */
int nroots = 0;
/*test*/
/* COORD gmin,gmax; */
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* If there are no objects on this processor, do not create a root octant.
* The partitioner will probably assign objects to this processor
*/
if(zz->Get_Num_Obj == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register ZOLTAN_NUM_OBJ_FN function");
abort();
}
*c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr) {
fprintf(stderr, "OCT [%d] %s: Error returned from user defined "
"Get_Num_Obj function.\n", zz->Proc, yo);
exit (-1);
}
ptr1 = NULL;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds");
/* Need A Function To Get The Bounds Of The Local Objects */
Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0);
#ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
/* For now, don't want to add the new query function to Zoltan. */
/* Zoltan_Oct_get_bounds appears to compute the global min and max from */
/* the object input. */
vector_set(OCT_info->OCT_gmin, min);
vector_set(OCT_info->OCT_gmax, max);
#else
/*test*/
/*getMaxBounds(&gmin, &gmax);*/
if(zz->Get_Bounds_Geom == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register Get_Bounds_Geom function");
abort();
}
zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr);
MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3,
MPI_DOUBLE, MPI_MIN, zz->Communicator);
MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3,
MPI_DOUBLE, MPI_MAX, zz->Communicator);
vector_set(OCT_info->OCT_gmin, global_min);
vector_set(OCT_info->OCT_gmax, global_max);
#endif
/*
* the following code segment was added to create a pseudo global octree
* needed for the partitioner. The basic idea is to regroup all the
* regions into something close to an octree partitioning and build the
* tree from that.
* NOTE: This way of doing things is very costly, especially when calling
* this for the first time on a mesh not partitioned in an octree style
* partitioning.
*/
level = 0; /* initialize level count */
/*
* if more than 1 processor, need to find what level of refinement needed
* to initially partition bounding box among the processors
*/
if(zz->Num_Proc > 1) {
n = zz->Num_Proc;
if(OCT_info->OCT_dimension == 2)
hold = 4;
else
hold = 8;
remainder = hold;
for(; remainder > 0; level++) {
int pr = (int)POW(hold, level);
remainder = n - pr;
}
level--;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree");
if(createpartree) {
/* create the global root octant */
root = Zoltan_Oct_POct_new(OCT_info);
Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax);
/* Zoltan_Oct_setOrientation(root, 0); */
/* subdivide to as many levels as calculated */
for(i=0; i<level; i++) {
cursor = root;
while(cursor != NULL) {
if(Zoltan_Oct_isTerminal(cursor)) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
Zoltan_Oct_terminal_refine(zz, cursor, 0);
cursor = cursor2;
}
else
cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
}
}
#if 0
if(zz->Proc == 0)
for(i=0; i<8; i++)
if(Zoltan_Oct_child(root, i) == NULL)
fprintf(stderr,"NULL child pointer\n");
else
fprintf(stderr, "child %d exists\n", i);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array");
/* this part creates the map array */
if(OCT_info->OCT_dimension == 2) {
hold = (int)POW(4, level); /* ignoring the z+ octants */
if(hold == 0)
hold = 1;
}
else
hold = (int)POW(8, level);
part = hold / zz->Num_Proc; /* how many octants per partition */
remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */
extra = zz->Num_Proc - remainder;/* where to start adding extra octants */
array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */
if(array == NULL) {
fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n",
zz->Proc);
abort();
}
/* initialize variables */
proc = 0;
count = 0;
i = 0;
cursor = root;
while(cursor != NULL) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) {
if(proc == extra) {
part++;
extra = -1;
}
if(count != part) {
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
count++;
}
else {
count = 1;
proc++;
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
}
if(proc == zz->Proc) {
array[i].npid = -1;
/* KDDKDD Added RL_freeList below. The
* KDDKDD implementation from RPI leaked memory because the
* KDDKDD test cases for setting array[i].list were not mutually
* KDDKDD exclusive. Freeing the list produces the result we got
* KDDKDD before, without the memory leak.
*/
/* LGG -- it seems to me that this array[i].list assignment is
* not really necessary. It looks as though it has already been
* assigned with the same information from the prev if-else
* commented out RL_freeList(), and RL_initRootList()
*/
/*RL_freeList(&(array[i].list));*/
/* KDDKDD End addition */
/*array[i].list = RL_initRootList();*/
parent = Zoltan_Oct_parent(cursor);
if(parent != NULL)
Zoltan_Oct_setchild(parent, cursor->which, NULL);
/* octant into local root list */
Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1);
Zoltan_Oct_setMapIdx(cursor, i);
nroots++;
/* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc);
octant into local root list */
}
i++;
}
cursor = cursor2;
}
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RootOct = RL_nextRootOctant(&RootList);
if(RootOct != root) {
/* KDDKDDFREE changed root to &root to allow root to be reset to NULL */
Zoltan_Oct_POct_delTree(OCT_info,&root);
}
OCT_info->map = array;
OCT_info->mapsize = hold;
}
/*
* attach the regions to the root... Zoltan_Oct_fix will create the octree
* starting with the root and subdividing as needed
*/
num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans");
Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map,
c1, c2);
/* ZOLTAN_FREE(&array); */
while(ptr1 != NULL) {
ptr = ptr1->next;
ZOLTAN_FREE(&(ptr1->Global_ID));
ZOLTAN_FREE(&(ptr1->Local_ID));
ZOLTAN_FREE(&ptr1);
ptr1 = ptr;
}
ZOLTAN_TRACE_EXIT(zz, yo);
}
/*
* Zoltan_Oct_terminal_refine(oct)
*
* subdivide a terminal octant and divvy up
* its regions to the 8 children; recurse if
* necessary to satisfy MAXOCTREGIONS
*
*/
static void Zoltan_Oct_terminal_refine(ZZ *zz, pOctant oct,int count)
{
COORD min, /* coordinates of minimum bounds of region */
max, /* coordinates of maximum bounds of region */
origin; /* origin of region */
pOctant child[8]; /* array of child octants */
int cnum; /* child number */
int i; /* index counter */
pRegion region; /* a region to be associated to an octant */
pRegion entry;
COORD cmin[8], cmax[8];
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
for(i=0;i<3;i++)
min[i] = max[i] = 0;
/* upper limit of refinement levels */
/* ATTN: may not be used anymore, but can be put back in if necessary */
if (count>=20) {
fprintf(stderr, "OCT ERROR: Zoltan_Oct_terminal_refine: bailing out at "
"10 levels\n");
abort();
}
oct_nref++; /* increment refinement counter */
/* octant should be terminal in order to be refined (subdivided) */
if (!Zoltan_Oct_isTerminal(oct)) {
fprintf(stderr,"OCT ref_octant: oct not terminal\n");
abort();
}
/* get the bounds of an octant */
Zoltan_Oct_bounds(oct,min,max);
/* calculate the origin from the bounds */
Zoltan_Oct_bounds_to_origin(min,max,origin);
region = Zoltan_Oct_regionlist(oct); /* Get list while still terminal */
oct->list = NULL; /* remove regions from octant, it won't be terminal */
/* create the children and set their id's */
Zoltan_Oct_child_bounds_wrapper(OCT_info,oct, cmin, cmax);
for (i=0; i<8; i++) {
if(OCT_info->OCT_dimension == 2) {
/* KDDKDD 3/01 see changes to Zoltan_Oct_child_bounds_wrapper that allow this
* KDDKDD 3/01 test to work for GRAY and HILBERT mappings.
*/
if(cmin[i][2] > OCT_info->OCT_gmin[2]) { /* ignore the z+ octants */
child[i] = NULL;
continue;
}
}
child[i]=Zoltan_Oct_POct_new(OCT_info); /* create a new octant */
child[i]->dir = Zoltan_Oct_get_child_dir(OCT_info, oct->dir, i);
/* create a new octant */
/* set the child->parent link */
Zoltan_Oct_POct_setparent(OCT_info, child[i], oct, zz->Proc);
Zoltan_Oct_setchildnum(child[i], i); /* which child of the parent */
Zoltan_Oct_setchild(oct, i, child[i]); /* set the parent->child link */
#ifdef LGG_MIGOCT
Zoltan_Oct_setID(child[i], Zoltan_Oct_nextId()); /* set child id num */
#endif /* LGG_MIGOCT */
Zoltan_Oct_setbounds(child[i], cmin[i], cmax[i]); /* set child bounds */
Zoltan_Oct_setCpid(oct, i, zz->Proc); /* set child to be a local oct */
/* Zoltan_Oct_setOrientation(child[i],
Zoltan_Oct_child_orientation(oct->orientation, oct->which)); */
}
/* assign newly created children to child array*/
if(OCT_info->OCT_dimension == 3) {
if(Zoltan_Oct_children(oct, child) != 8) {
/*
* if subdivision of oct was successful, oct should have 8 children;
* thus a return value of 0 here is a fatal error
*/
fprintf(stderr, "OCT ref_octant: subdivide failed, %d children.\n",
Zoltan_Oct_children(oct, child));
abort();
}
}
else
if(Zoltan_Oct_children(oct, child) != 4) {
/*
* if subdivision of oct was successful, oct should have 4 children;
* thus a return value of 0 here is a fatal error
*/
fprintf(stderr,
"OCT ref_octant:subdivide failed, %d children, expected 4\n",
Zoltan_Oct_children(oct, child));
abort();
}
/* iterate through and find which child each region should belong to */
while(region != NULL) {
entry = region->next;
cnum=Zoltan_Oct_child_which_wrapper(OCT_info,oct, region->Coord);
/* add region to octant's regionlist */
Zoltan_Oct_addRegion(zz, child[cnum], region);
ZOLTAN_FREE(&(region->Global_ID));
ZOLTAN_FREE(&(region->Local_ID));
ZOLTAN_FREE(®ion);
region = entry;
}
for (i=0; i<8; i++) /* Recursion */
if(child[i] != NULL)
/* KDDKDD Replaced the following to allow multiple regions with the
* KDDKDD same coordinates to be placed in the same octant.
if (Zoltan_Oct_nRegions(child[i]) > MAXOCTREGIONS) {
*/
if (Zoltan_Oct_nUniqueRegions(OCT_info,child[i]) > MAXOCTREGIONS) {
Zoltan_Oct_terminal_refine(zz, child[i],count+1);
}
}
static int Zoltan_Oct_Update_Connections(
ZZ *zz,
pOctant *octs, /* octs[nocts] */
int *newpids, /* newpids[nocts] */
pOctant *newocts, /* newocts[nocts] */
int nocts) /* number of octants leaving this processor */
{
int i, j;
int nsends;
int nreceives;
pOctant parent;
pOctant child;
int ppid;
int cpid;
int childnum;
int *despid = NULL;
Update_msg umsg;
Update_msg *localumsg = NULL;
Update_msg *remoteumsg = NULL;
Update_msg *rcv_umsg = NULL;
int localcount;
int remotecount;
int ierr = ZOLTAN_OK;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
char *yo = "Zoltan_Oct_Update_Connections";
OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure;
localcount=0;
remotecount=0;
/* count number of sends */
nsends = 0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc)
nsends++;
if(nocts > 0) {
if((remoteumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((localumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
return ZOLTAN_MEMERR;
}
if((despid = (int *) ZOLTAN_MALLOC((nocts+1) * sizeof(int)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
return ZOLTAN_MEMERR;
}
}
else {
remoteumsg = NULL;
localumsg = NULL;
despid = NULL;
}
localcount = 0;
remotecount = 0;
for (i=0; i<nocts; i++) /* Send connection updates */
if (newpids[i]!=zz->Proc) {
parent = Zoltan_Oct_parent(octs[i]);
ppid = Zoltan_Oct_Ppid(octs[i]);
childnum = Zoltan_Oct_childnum(octs[i]);
if (parent) { /* Let parent of oct[i] know that it's moving */
if (ppid==zz->Proc) {
FILLUPDATEMSG(localumsg[localcount], parent, childnum, newocts[i], newpids[i]);
localcount++;
}
else {
FILLUPDATEMSG(remoteumsg[remotecount], parent, childnum, newocts[i], newpids[i]);
despid[remotecount++] = ppid;
}
}
for (j=0; j<8; j++) {
child = Zoltan_Oct_child(octs[i],j);
cpid = octs[i]->cpid[j];
/* Tell child of oct[i] that it is moving */
if (child) {
if (cpid==zz->Proc) {
/* NOTE: -1 signals PARENT */
FILLUPDATEMSG(localumsg[localcount], child, -1, newocts[i], newpids[i]);
localcount++;
}
else {
/* NOTE: -1 signals PARENT */
FILLUPDATEMSG(remoteumsg[remotecount], child, -1, newocts[i], newpids[i]);
despid[remotecount++] = cpid;
}
}
}
}
ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator,
MigUpdCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
return (ierr);
}
/* if(nreceives > 0) { */
if((rcv_umsg = (Update_msg *) ZOLTAN_MALLOC((nreceives +1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
return ZOLTAN_MEMERR;
}
ierr = Zoltan_Comm_Do(comm_plan, MigUpdCommDo, (char *) remoteumsg,
sizeof(Update_msg), (char *) rcv_umsg);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_umsg);
return (ierr);
}
/* } */
/* else { */
/* rcv_umsg = NULL; */
/* } */
/* update new octants */
for (i=0; i< (localcount+nreceives); i++) {
if (i<localcount)
umsg=localumsg[i];
else
umsg=rcv_umsg[i-localcount];
if (umsg.childnum>=0) {
Zoltan_Oct_setchild(umsg.oct,umsg.childnum,umsg.newptr);
Zoltan_Oct_setCpid(umsg.oct,umsg.childnum,umsg.newpid);
}
else {
if((Zoltan_Oct_data_newpid(umsg.oct) == OCT_info->OCT_localpid) ||
((Zoltan_Oct_data_newpid(umsg.oct) != OCT_info->OCT_localpid) && (umsg.newpid == OCT_info->OCT_localpid)))
Zoltan_Oct_POct_setparent(OCT_info, umsg.oct,umsg.newptr,umsg.newpid);
else {
umsg.oct->ppid = umsg.newpid;
umsg.oct->parent = umsg.newptr;
}
}
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_umsg);
return (ierr);
}
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&rcv_umsg);
ZOLTAN_FREE(&despid);
return ierr;
}