/*
* void Zoltan_Oct_POct_init(int processor_id)
*
* sets up global variables for the octree partitioner
*/
OCT_Global_Info *Zoltan_Oct_POct_init(ZZ *zz, int pid, int dim) {
char *yo = "Zoltan_Oct_POct_init";
OCT_Global_Info *OCT_info;
if((OCT_info = (OCT_Global_Info *) ZOLTAN_MALLOC(sizeof(OCT_Global_Info))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return NULL;
}
zz->LB.Data_Structure = (void *) OCT_info;
OCT_info->OCT_localpid=pid;
OCT_info->OCT_rootlist=RL_initRootList();
Zoltan_Oct_initCounters();
if((dim > 3) || (dim < 2)) {
fprintf(stderr,"WARNING: illegal dimension, using default (3D).\n");
OCT_info->OCT_dimension = 3;
}
else
OCT_info->OCT_dimension = dim;
return(OCT_info);
}
/*
* void Zoltan_Oct_gen_tree_from_input_data()
*
* This function will create a root node of on each processor which will
* then be used to create an octree with regions associated with it. The
* tree will then be balanced and the output used to balance "mesh regions"
* on several processors.
*/
static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag,
int *c1, int *c2, int *c3,
float *c0, int createpartree)
{
char *yo = "Zoltan_Oct_gen_tree_from_input_data";
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
COORD min, /* min coord bounds of objects */
max; /* max coord bounds of objects */
int num_extra; /* number of orphaned objects */
int num_objs; /* total number of local objects */
pRegion ptr, /* pointer to iterate trough region list */
ptr1; /* pointer to iterate trough region list */
pOctant root; /* root of the partition tree */
int i; /* index counter */
int count, /* count for leaf nodes in partition tree */
proc, /* proc leaf node of parition tree belongs to */
extra, /* extra leaf node flag, if not evenly divisible */
remainder; /* remainder of node, or processors to fill */
pOctant cursor, /* cursor to iterate through octant list */
cursor2, /* another cursor to iterate through octant list */
parent; /* parent of an octant */
int level, /* number of levels of refinement */
n, /* index counter */
part; /* partition counter */
Map *array; /* map of which processors own which octants */
int hold; /* used for calculating partition divisions */
int ierr = 0;
#ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX};
COORD global_min, global_max;
#endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */
int nroots = 0;
/*test*/
/* COORD gmin,gmax; */
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* If there are no objects on this processor, do not create a root octant.
* The partitioner will probably assign objects to this processor
*/
if(zz->Get_Num_Obj == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register ZOLTAN_NUM_OBJ_FN function");
abort();
}
*c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr) {
fprintf(stderr, "OCT [%d] %s: Error returned from user defined "
"Get_Num_Obj function.\n", zz->Proc, yo);
exit (-1);
}
ptr1 = NULL;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds");
/* Need A Function To Get The Bounds Of The Local Objects */
Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0);
#ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
/* For now, don't want to add the new query function to Zoltan. */
/* Zoltan_Oct_get_bounds appears to compute the global min and max from */
/* the object input. */
vector_set(OCT_info->OCT_gmin, min);
vector_set(OCT_info->OCT_gmax, max);
#else
/*test*/
/*getMaxBounds(&gmin, &gmax);*/
if(zz->Get_Bounds_Geom == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register Get_Bounds_Geom function");
abort();
}
zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr);
MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3,
MPI_DOUBLE, MPI_MIN, zz->Communicator);
MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3,
MPI_DOUBLE, MPI_MAX, zz->Communicator);
vector_set(OCT_info->OCT_gmin, global_min);
vector_set(OCT_info->OCT_gmax, global_max);
#endif
/*
* the following code segment was added to create a pseudo global octree
* needed for the partitioner. The basic idea is to regroup all the
* regions into something close to an octree partitioning and build the
* tree from that.
* NOTE: This way of doing things is very costly, especially when calling
* this for the first time on a mesh not partitioned in an octree style
* partitioning.
*/
level = 0; /* initialize level count */
/*
* if more than 1 processor, need to find what level of refinement needed
* to initially partition bounding box among the processors
*/
if(zz->Num_Proc > 1) {
n = zz->Num_Proc;
if(OCT_info->OCT_dimension == 2)
hold = 4;
else
hold = 8;
remainder = hold;
for(; remainder > 0; level++) {
int pr = (int)POW(hold, level);
remainder = n - pr;
}
level--;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree");
if(createpartree) {
/* create the global root octant */
root = Zoltan_Oct_POct_new(OCT_info);
Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax);
/* Zoltan_Oct_setOrientation(root, 0); */
/* subdivide to as many levels as calculated */
for(i=0; i<level; i++) {
cursor = root;
while(cursor != NULL) {
if(Zoltan_Oct_isTerminal(cursor)) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
Zoltan_Oct_terminal_refine(zz, cursor, 0);
cursor = cursor2;
}
else
cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
}
}
#if 0
if(zz->Proc == 0)
for(i=0; i<8; i++)
if(Zoltan_Oct_child(root, i) == NULL)
fprintf(stderr,"NULL child pointer\n");
else
fprintf(stderr, "child %d exists\n", i);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array");
/* this part creates the map array */
if(OCT_info->OCT_dimension == 2) {
hold = (int)POW(4, level); /* ignoring the z+ octants */
if(hold == 0)
hold = 1;
}
else
hold = (int)POW(8, level);
part = hold / zz->Num_Proc; /* how many octants per partition */
remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */
extra = zz->Num_Proc - remainder;/* where to start adding extra octants */
array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */
if(array == NULL) {
fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n",
zz->Proc);
abort();
}
/* initialize variables */
proc = 0;
count = 0;
i = 0;
cursor = root;
while(cursor != NULL) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) {
if(proc == extra) {
part++;
extra = -1;
}
if(count != part) {
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
count++;
}
else {
count = 1;
proc++;
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
}
if(proc == zz->Proc) {
array[i].npid = -1;
/* KDDKDD Added RL_freeList below. The
* KDDKDD implementation from RPI leaked memory because the
* KDDKDD test cases for setting array[i].list were not mutually
* KDDKDD exclusive. Freeing the list produces the result we got
* KDDKDD before, without the memory leak.
*/
/* LGG -- it seems to me that this array[i].list assignment is
* not really necessary. It looks as though it has already been
* assigned with the same information from the prev if-else
* commented out RL_freeList(), and RL_initRootList()
*/
/*RL_freeList(&(array[i].list));*/
/* KDDKDD End addition */
/*array[i].list = RL_initRootList();*/
parent = Zoltan_Oct_parent(cursor);
if(parent != NULL)
Zoltan_Oct_setchild(parent, cursor->which, NULL);
/* octant into local root list */
Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1);
Zoltan_Oct_setMapIdx(cursor, i);
nroots++;
/* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc);
octant into local root list */
}
i++;
}
cursor = cursor2;
}
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RootOct = RL_nextRootOctant(&RootList);
if(RootOct != root) {
/* KDDKDDFREE changed root to &root to allow root to be reset to NULL */
Zoltan_Oct_POct_delTree(OCT_info,&root);
}
OCT_info->map = array;
OCT_info->mapsize = hold;
}
/*
* attach the regions to the root... Zoltan_Oct_fix will create the octree
* starting with the root and subdividing as needed
*/
num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans");
Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map,
c1, c2);
/* ZOLTAN_FREE(&array); */
while(ptr1 != NULL) {
ptr = ptr1->next;
ZOLTAN_FREE(&(ptr1->Global_ID));
ZOLTAN_FREE(&(ptr1->Local_ID));
ZOLTAN_FREE(&ptr1);
ptr1 = ptr;
}
ZOLTAN_TRACE_EXIT(zz, yo);
}
