/*
* pOctant oct_localTree_nextDfs(pOctant oct, void **temp)
* (previously: pOctant oct_global_nextDfs(pParoct oct, void **temp))
*
* do a dfs of the local octree on each processor.
* subtrees are traversed in the order of the local roots
* list, and each subtree is done in dfs order.
*
* Undefined action will take place if you modify the
* tree while traversing.
*
*/
pOctant oct_localTree_nextDfs(pOctant oct, void **temp)
{
if (*temp==NULL) /* Start of traversal */
return(PList_next(OCT_info->OCT_rootlist,temp));
if (oct=Zoltan_Oct_POct_nextDfs(oct, 0))
return(oct);
return(PList_next(OCT_info->OCT_rootlist,temp));
}
void Zoltan_Oct_Plots(ZZ *zz) {
pRList RootList; /* list of all local roots */
pOctant RootOct;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
FILE *fp;
pRegion tmp;
Zoltan_Print_Sync_Start(zz->Communicator, 1);
if (zz->Proc == 0)
fp = fopen("octants.gnu", "w");
else
fp = fopen("octants.gnu", "a");
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
/* Octant is terminal; surf the region list and print regions. */
tmp = RootOct->list;
while (tmp != NULL) {
printf("%d PLOTREG %f %f\n", zz->Proc, tmp->Coord[0], tmp->Coord[1]);
tmp = tmp->next;
}
/* Print the octant midpoint */
printf("%d PLOTOCT %f %f\n", zz->Proc,
0.5 * (RootOct->min[0] + RootOct->max[0]),
0.5 * (RootOct->min[1] + RootOct->max[1]));
/* Print the octant bounding box */
fprintf(fp, "%f %f\n", RootOct->min[0], RootOct->min[1]);
fprintf(fp, "%f %f\n", RootOct->max[0], RootOct->min[1]);
fprintf(fp, "%f %f\n", RootOct->max[0], RootOct->max[1]);
fprintf(fp, "%f %f\n", RootOct->min[0], RootOct->max[1]);
fprintf(fp, "%f %f\n\n", RootOct->min[0], RootOct->min[1]);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
fclose(fp);
Zoltan_Print_Sync_End(zz->Communicator, 1);
}
/*
* oct_global_clear()
*
* delete all regions from all octants on the local processor
*
*/
static void Zoltan_Oct_global_clear(OCT_Global_Info * OCT_info)
{
pRList RootList; /* list of all local roots */
pOctant Oct;
/*
* iterate through the list of local roots
* traverse down the subtree
* delete regions associated with octant
*/
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while ((Oct = RL_nextRootOctant(&RootList))) {
while(Oct) {
if(Zoltan_Oct_isTerminal(Oct))
Zoltan_Oct_clearRegions(Oct);
Oct = Zoltan_Oct_POct_nextDfs(OCT_info, Oct);
}
}
}
/*
* void Zoltan_Oct_gen_tree_from_input_data()
*
* This function will create a root node of on each processor which will
* then be used to create an octree with regions associated with it. The
* tree will then be balanced and the output used to balance "mesh regions"
* on several processors.
*/
static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag,
int *c1, int *c2, int *c3,
float *c0, int createpartree)
{
char *yo = "Zoltan_Oct_gen_tree_from_input_data";
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
COORD min, /* min coord bounds of objects */
max; /* max coord bounds of objects */
int num_extra; /* number of orphaned objects */
int num_objs; /* total number of local objects */
pRegion ptr, /* pointer to iterate trough region list */
ptr1; /* pointer to iterate trough region list */
pOctant root; /* root of the partition tree */
int i; /* index counter */
int count, /* count for leaf nodes in partition tree */
proc, /* proc leaf node of parition tree belongs to */
extra, /* extra leaf node flag, if not evenly divisible */
remainder; /* remainder of node, or processors to fill */
pOctant cursor, /* cursor to iterate through octant list */
cursor2, /* another cursor to iterate through octant list */
parent; /* parent of an octant */
int level, /* number of levels of refinement */
n, /* index counter */
part; /* partition counter */
Map *array; /* map of which processors own which octants */
int hold; /* used for calculating partition divisions */
int ierr = 0;
#ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX};
COORD global_min, global_max;
#endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */
int nroots = 0;
/*test*/
/* COORD gmin,gmax; */
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* If there are no objects on this processor, do not create a root octant.
* The partitioner will probably assign objects to this processor
*/
if(zz->Get_Num_Obj == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register ZOLTAN_NUM_OBJ_FN function");
abort();
}
*c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr) {
fprintf(stderr, "OCT [%d] %s: Error returned from user defined "
"Get_Num_Obj function.\n", zz->Proc, yo);
exit (-1);
}
ptr1 = NULL;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds");
/* Need A Function To Get The Bounds Of The Local Objects */
Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0);
#ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
/* For now, don't want to add the new query function to Zoltan. */
/* Zoltan_Oct_get_bounds appears to compute the global min and max from */
/* the object input. */
vector_set(OCT_info->OCT_gmin, min);
vector_set(OCT_info->OCT_gmax, max);
#else
/*test*/
/*getMaxBounds(&gmin, &gmax);*/
if(zz->Get_Bounds_Geom == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register Get_Bounds_Geom function");
abort();
}
zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr);
MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3,
MPI_DOUBLE, MPI_MIN, zz->Communicator);
MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3,
MPI_DOUBLE, MPI_MAX, zz->Communicator);
vector_set(OCT_info->OCT_gmin, global_min);
vector_set(OCT_info->OCT_gmax, global_max);
#endif
/*
* the following code segment was added to create a pseudo global octree
* needed for the partitioner. The basic idea is to regroup all the
* regions into something close to an octree partitioning and build the
* tree from that.
* NOTE: This way of doing things is very costly, especially when calling
* this for the first time on a mesh not partitioned in an octree style
* partitioning.
*/
level = 0; /* initialize level count */
/*
* if more than 1 processor, need to find what level of refinement needed
* to initially partition bounding box among the processors
*/
if(zz->Num_Proc > 1) {
n = zz->Num_Proc;
if(OCT_info->OCT_dimension == 2)
hold = 4;
else
hold = 8;
remainder = hold;
for(; remainder > 0; level++) {
int pr = (int)POW(hold, level);
remainder = n - pr;
}
level--;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree");
if(createpartree) {
/* create the global root octant */
root = Zoltan_Oct_POct_new(OCT_info);
Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax);
/* Zoltan_Oct_setOrientation(root, 0); */
/* subdivide to as many levels as calculated */
for(i=0; i<level; i++) {
cursor = root;
while(cursor != NULL) {
if(Zoltan_Oct_isTerminal(cursor)) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
Zoltan_Oct_terminal_refine(zz, cursor, 0);
cursor = cursor2;
}
else
cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
}
}
#if 0
if(zz->Proc == 0)
for(i=0; i<8; i++)
if(Zoltan_Oct_child(root, i) == NULL)
fprintf(stderr,"NULL child pointer\n");
else
fprintf(stderr, "child %d exists\n", i);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array");
/* this part creates the map array */
if(OCT_info->OCT_dimension == 2) {
hold = (int)POW(4, level); /* ignoring the z+ octants */
if(hold == 0)
hold = 1;
}
else
hold = (int)POW(8, level);
part = hold / zz->Num_Proc; /* how many octants per partition */
remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */
extra = zz->Num_Proc - remainder;/* where to start adding extra octants */
array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */
if(array == NULL) {
fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n",
zz->Proc);
abort();
}
/* initialize variables */
proc = 0;
count = 0;
i = 0;
cursor = root;
while(cursor != NULL) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) {
if(proc == extra) {
part++;
extra = -1;
}
if(count != part) {
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
count++;
}
else {
count = 1;
proc++;
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
}
if(proc == zz->Proc) {
array[i].npid = -1;
/* KDDKDD Added RL_freeList below. The
* KDDKDD implementation from RPI leaked memory because the
* KDDKDD test cases for setting array[i].list were not mutually
* KDDKDD exclusive. Freeing the list produces the result we got
* KDDKDD before, without the memory leak.
*/
/* LGG -- it seems to me that this array[i].list assignment is
* not really necessary. It looks as though it has already been
* assigned with the same information from the prev if-else
* commented out RL_freeList(), and RL_initRootList()
*/
/*RL_freeList(&(array[i].list));*/
/* KDDKDD End addition */
/*array[i].list = RL_initRootList();*/
parent = Zoltan_Oct_parent(cursor);
if(parent != NULL)
Zoltan_Oct_setchild(parent, cursor->which, NULL);
/* octant into local root list */
Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1);
Zoltan_Oct_setMapIdx(cursor, i);
nroots++;
/* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc);
octant into local root list */
}
i++;
}
cursor = cursor2;
}
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RootOct = RL_nextRootOctant(&RootList);
if(RootOct != root) {
/* KDDKDDFREE changed root to &root to allow root to be reset to NULL */
Zoltan_Oct_POct_delTree(OCT_info,&root);
}
OCT_info->map = array;
OCT_info->mapsize = hold;
}
/*
* attach the regions to the root... Zoltan_Oct_fix will create the octree
* starting with the root and subdividing as needed
*/
num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans");
Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map,
c1, c2);
/* ZOLTAN_FREE(&array); */
while(ptr1 != NULL) {
ptr = ptr1->next;
ZOLTAN_FREE(&(ptr1->Global_ID));
ZOLTAN_FREE(&(ptr1->Local_ID));
ZOLTAN_FREE(&ptr1);
ptr1 = ptr;
}
ZOLTAN_TRACE_EXIT(zz, yo);
}
/*
* void lb_oct_init();
*
* initialize the calls needed to start the octree load balancing rounties
*/
static int lb_oct_init(
ZZ *zz, /* The Zoltan structure with info for
the OCTPART balancer. */
int *num_export, /* Number of non-local objs assigned to this
processor in the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition. */
int **export_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition. */
int **export_to_part, /* Returned value: array of partitions to
which objects are imported.
KDDKDD Assume #parts==#procs. */
int oct_dim, /* Dimension of method (2D or 3D) */
int oct_method, /* Flag specifying curve to be used. */
int oct_maxoctregions, /* max # of objects in leaves of octree. */
int oct_minoctregions, /* min # of objects in leaves of octree. */
int oct_output_level, /* Flag specifying amount of output. */
int oct_wgtflag, /* Flag specifying use of object weights. */
float *part_sizes /* Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
)
{
char *yo = "lb_oct_init";
OCT_Global_Info *OCT_info;
int nsentags; /* number of tags being sent */
pRegion export_regs; /* */
int nrectags; /* number of tags received */
int kk;
double time1,time2; /* timers */
double timestart,timestop; /* timers */
double timers[4]; /* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
int counters[6]; /* diagnostic counts
0 = # of median iterations
1 = # of objects sent
2 = # of objects received
3 = most objects this proc ever owns
*/
float c[4];
int createpartree = 0;
/*int num_gid_entries = zz->Num_GID;*/
/*int num_lid_entries = zz->Num_LID;*/
ZOLTAN_TRACE_ENTER(zz, yo);
MPI_Barrier(zz->Communicator);
timestart = MPI_Wtime();
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = 0;
counters[4] = 0;
counters[5] = 0;
c[0] = 0;
c[1] = 0;
c[2] = 0;
c[3] = 0;
timers[1] = 0.0;
timers[2] = 0.0;
timers[3] = 0.0;
nsentags = nrectags = 0;
if(zz->LB.Data_Structure == NULL) {
OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim);
Zoltan_Oct_set_method(OCT_info, oct_method);
Zoltan_Oct_set_maxregions(oct_maxoctregions);
Zoltan_Oct_set_minregions(oct_minoctregions);
createpartree = 1;
}
else {
OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
}
/* create the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data");
Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1],
&counters[2], &counters[3], &c[0],
createpartree);
time2 = MPI_Wtime();
timers[0] = time2 - time1; /* time took to create octree */
/* Zoltan_Oct_POct_printResults(OCT_info); */
/* partition the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition");
/* old call to dfs_paritition: */
#if 0
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]);
#else
/***************************
if(zz->Proc == 0) {
int debug_i;
for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) {
fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]);
}
}
****************************/
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes);
#endif
time2 = MPI_Wtime();
timers[1] = time2 - time1; /* time took to partition octree */
if (oct_output_level > 2) {
Zoltan_Oct_Plots(zz);
}
/* set up tags for migrations */
time1 = MPI_Wtime();
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate");
Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags,
&c[2], &c[3], &counters[3], &counters[5]);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags");
if (zz->LB.Return_Lists) {
*num_export = nrectags;
if (nrectags > 0)
Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids,
export_procs, export_to_part, nrectags,
export_regs);
}
time2 = MPI_Wtime();
timers[2] = time2 - time1; /* time took to setup migration */
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
/* count the number of objects on this processor */
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
counters[4] = nsentags;
MPI_Barrier(zz->Communicator);
timestop = MPI_Wtime();
if (oct_output_level > 0) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats");
Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c,
oct_output_level);
}
for (kk = 0; kk < nrectags; kk++) {
ZOLTAN_FREE(&(export_regs[kk].Global_ID));
ZOLTAN_FREE(&(export_regs[kk].Local_ID));
}
ZOLTAN_FREE(&export_regs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear");
Zoltan_Oct_global_clear(OCT_info);
/* KDDKDD Don't understand how re-used octree will work, especially without
* KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything;
* KDDKDD we can move back to saving some of the tree later.
*/
Zoltan_Oct_Free_Structure(zz);
/* KDDKDD END */
/* Temporary return value until error codes are fully implemented. */
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
/*
* void Zoltan_Oct_dfs_migrate()
*
* sets up information so the migrate octant routines can create the
* proper export_tags and import_tags arrays
*/
void Zoltan_Oct_dfs_migrate(ZZ *zz, int *nsentags,
pRegion *import_regs, int *nrectags,
float *c2, float *c3, int *counter3, int *counter4)
{
pRList RootList; /* list of the local roots */
pOctant oct; /* octree octant */
pOctant *docts = NULL; /* array of octants being sent */
int *dpids = NULL; /* array of octant pids */
int dcount; /* count of octants being sent */
int pid; /* processor id */
int nrecocts;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
char *yo = "Zoltan_Oct_dfs_migrate";
if(Zoltan_Oct_nOctants()) { /* allocate space for octants being migrated */
docts = (pOctant *)ZOLTAN_MALLOC(Zoltan_Oct_nOctants() * sizeof(pOctant));
if(!docts) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "cannot allocate arrays.");
abort();
}
dpids = (int *) ZOLTAN_MALLOC(Zoltan_Oct_nOctants() * sizeof(int));
if(!dpids) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "cannot allocate arrays.");
ZOLTAN_FREE(&docts);
abort();
}
}
dcount=0;
/* go through the local octants and make sure each has a valid npid */
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((oct = RL_nextRootOctant(&RootList)))
while(oct) {
pid = Zoltan_Oct_data_newpid(oct);
if (pid<0 || pid>=zz->Num_Proc) {
fprintf(stderr,"%d Zoltan_Oct_dfs_migrate: bad dest pid %d\n", zz->Proc, pid);
abort();
}
if (dcount<Zoltan_Oct_nOctants()) {
docts[dcount]=oct;
dpids[dcount]=pid;
}
oct=Zoltan_Oct_POct_nextDfs(OCT_info, oct);
dcount++;
}
if (dcount!=Zoltan_Oct_nOctants()) {
fprintf(stderr, "ERROR: in Zoltan_Oct_dfs_migrate, octant count mismatch (I counted %d but there should be %d)\n",dcount, Zoltan_Oct_nOctants());
/* dcount=Zoltan_Oct_nOctants(); */
/* abort(); */
}
/* setup the import_regs */
Zoltan_Oct_migrate_objects(zz, docts, dpids, dcount, nsentags,
import_regs, nrectags, c2, c3, counter3, counter4);
Zoltan_Oct_migrate_octants(zz, dpids, docts, dcount, &nrecocts);
ZOLTAN_FREE(&docts);
ZOLTAN_FREE(&dpids);
}