/*
* pOctant Zoltan_Oct_global_find(COORD centroid)
*
* Return the octant in the local octree that contains the
* given point, if it exists. Otherwise, return NULL.
*
*/
static pOctant Zoltan_Oct_global_find(OCT_Global_Info *OCT_info,COORD point)
{
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
pOctant oct; /* octree octant */
COORD min, /* minimum bounds coordinates */
max; /* maximum bounds coordinates */
/* get a list of all local roots */
RootList = Zoltan_Oct_POct_localroots(OCT_info);
oct = NULL;
/* iterate through root list to find if point lies inside root's bounds */
while ((!oct) && (RootOct = RL_nextRootOctant(&RootList)) ) {
Zoltan_Oct_bounds(RootOct,min,max);
/* ATTN: Zoltan_Oct_in_box may need adjusting for the lower bounds */
/* check if point fits inside */
if (Zoltan_Oct_in_box(OCT_info,point,min,max)) {
/* find the exact octant for point */
oct=Zoltan_Oct_findOctant(OCT_info,RootOct,point);
}
}
return(oct);
}
void Zoltan_Oct_Free_Structure(ZZ *zz)
{
/*
* Deallocate the persistent OCT data structures in zz->Structure.
*/
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
pRList RootList; /* list of all local roots */
pOctant RootOct;
int i;
if (OCT_info != NULL) {
/* free octree */
/* KDDKDD In distribution from RPI, the following frees were commented out.
* KDDKDD I do not think they should be commented out, as Debug_Memory
* KDDKDD reports lots of memory leaks.
* KDDKDD I have un-commented them. 12/2000
*/
/* KDDKDDFREE Rearranged the frees because the global octants were trying to
* KDDKDDFREE access children that were already deleted in Zoltan_Oct_POct_delTree.
* KDDKDDFREE Now delete global octants first, then the tree. 1/2001
*/
/* free global octants */
for(i = 0; i < OCT_info->mapsize; i++) {
RootList = OCT_info->map[i].list;
while((RootOct = RL_nextRootOctant(&RootList))) {
Zoltan_Oct_free(OCT_info, &RootOct);
/* KDDKDDFREE Set RootList's oct ptr to NULL as the octant is deleted.*/
RootList->oct = NULL;
}
RL_freeList(&(OCT_info->map[i].list));
}
/* free map array */
ZOLTAN_FREE(&(OCT_info->map));
/* Free the octree */
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while ((RootOct = RL_nextRootOctant(&RootList))){
/* KDDKDDFREE The Zoltan_Oct_POct_delTree also frees RootList, since
* KDDKDDFREE Zoltan_Oct_POct_free frees the root list entry for root octants.
* KDDKDDFREE Need to reset RootList in each iteration. */
Zoltan_Oct_POct_delTree(OCT_info, &RootOct);
RootList = Zoltan_Oct_POct_localroots(OCT_info);
}
RL_freeList(&(OCT_info->OCT_rootlist));
/* KDDKDD End of previously commented-out section. 12/2000 */
ZOLTAN_FREE(&(zz->LB.Data_Structure));
}
}
/*
* void Zoltan_Oct_insert_orphan(pRegion region)
*
* Insert orphan regions migrated from off processors, or to insert
* regions that lie on the boundary.
*/
static int Zoltan_Oct_insert_orphan(ZZ *zz, Region reg) {
pRList RootList; /* list of all local roots */
pOctant RootOct;
int rflag; /* flag to indicate region fits in octant */
int i, j; /* index counters */
double upper, /* upper bounds of the octant */
lower; /* lower bounds of the octant */
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
char *yo = "Zoltan_Oct_insert_orphan";
int ierr = ZOLTAN_OK;
if (OCT_info->OCT_dimension == 2)
i = 2; /* ignore z coordinates */
else
i = 3;
rflag = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info); /* get a list all local roots */
while((RootOct = RL_nextRootOctant(&RootList))) {
rflag = 1;
for (j=0; j<i; j++) {
lower = RootOct->min[j];
upper = RootOct->max[j];
if (reg.Coord[j]<lower || reg.Coord[j]>upper) {
/* if region coord lie outside bounds, then cannot fit */
rflag = 0;
break;
}
}
if(rflag == 1) {
/* region fits inside octant */
/* found a place to insert region */
Zoltan_Oct_subtree_insert(zz, RootOct, ®);
return ierr;
}
}
ierr = ZOLTAN_WARN;
ZOLTAN_TRACE_DETAIL(zz, yo, "could not insert region");
fprintf(stderr,"%s failed to insert %f %f %f on proc %d\n",yo, reg.Coord[0], reg.Coord[1], reg.Coord[2], zz->Proc);
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RL_printRootOctants(RootList);
return ierr;
}
/*
* Zoltan_Oct_global_dref()
*
* refine and derefine octree as necessary by number of
* regions in each octant
*
*/
static void Zoltan_Oct_global_dref(ZZ *zz, OCT_Global_Info *OCT_info)
{
pRList RootList; /* list of all local roots */
pOctant RootOct;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while ((RootOct = RL_nextRootOctant(&RootList)))
Zoltan_Oct_subtree_dref(zz, OCT_info, RootOct);
}
/*
* void Zoltan_Oct_visit_all_subtrees()
*
* visits each of the subtrees that are on the local processor
*/
static void Zoltan_Oct_visit_all_subtrees(ZZ *zz, float *part_sizes) {
pRList RootList; /* list of all local roots */
pOctant RootOct;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
/* get the list of all the local roots */
/* iterate through each root in localroot list */
/* mark each subtree as being Zoltan_Oct_visited */
/* and free the costs */
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while ((RootOct = RL_nextRootOctant(&RootList))) {
Zoltan_Oct_visit(zz, RootOct, part_sizes);
Zoltan_Oct_costs_free(OCT_info, RootOct);
}
}
/*
* floatg Zoltan_Oct_costs_global_compute() (was: all_subtree_costs())
*
* return costs of all subtrees.
*
* Note: MUST set sequence numbers first, otherwise
* it will not know what order to traverse the different
* subtrees in.
*/
float Zoltan_Oct_costs_global_compute(OCT_Global_Info *OCT_info) {
pRList RootList; /* list of all local roots */
pOctant RootOct;
int seq = 0; /* sequencing number */
float totcost= 0; /* total cost local octree */
/* initialize octants for COST and NPID data tags */
/* and calculate cost of all the subtree */
/* Zoltan_Oct_POct_printResults(OCT_info); */
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
Zoltan_Oct_costs_init(OCT_info, RootOct);
totcost+=Zoltan_Oct_costs_subtree_compute(OCT_info, RootOct, &seq);
}
return(totcost);
}
void Zoltan_Oct_Plots(ZZ *zz) {
pRList RootList; /* list of all local roots */
pOctant RootOct;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
FILE *fp;
pRegion tmp;
Zoltan_Print_Sync_Start(zz->Communicator, 1);
if (zz->Proc == 0)
fp = fopen("octants.gnu", "w");
else
fp = fopen("octants.gnu", "a");
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
/* Octant is terminal; surf the region list and print regions. */
tmp = RootOct->list;
while (tmp != NULL) {
printf("%d PLOTREG %f %f\n", zz->Proc, tmp->Coord[0], tmp->Coord[1]);
tmp = tmp->next;
}
/* Print the octant midpoint */
printf("%d PLOTOCT %f %f\n", zz->Proc,
0.5 * (RootOct->min[0] + RootOct->max[0]),
0.5 * (RootOct->min[1] + RootOct->max[1]));
/* Print the octant bounding box */
fprintf(fp, "%f %f\n", RootOct->min[0], RootOct->min[1]);
fprintf(fp, "%f %f\n", RootOct->max[0], RootOct->min[1]);
fprintf(fp, "%f %f\n", RootOct->max[0], RootOct->max[1]);
fprintf(fp, "%f %f\n", RootOct->min[0], RootOct->max[1]);
fprintf(fp, "%f %f\n\n", RootOct->min[0], RootOct->min[1]);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
fclose(fp);
Zoltan_Print_Sync_End(zz->Communicator, 1);
}
/*
* oct_global_clear()
*
* delete all regions from all octants on the local processor
*
*/
static void Zoltan_Oct_global_clear(OCT_Global_Info * OCT_info)
{
pRList RootList; /* list of all local roots */
pOctant Oct;
/*
* iterate through the list of local roots
* traverse down the subtree
* delete regions associated with octant
*/
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while ((Oct = RL_nextRootOctant(&RootList))) {
while(Oct) {
if(Zoltan_Oct_isTerminal(Oct))
Zoltan_Oct_clearRegions(Oct);
Oct = Zoltan_Oct_POct_nextDfs(OCT_info, Oct);
}
}
}
/*
* Zoltan_Oct_POct_setparent(pOctant octant, pOctant parent, int parent_processor_id)
*
* sets the parent of the octant. If the parent is offprocessor, then
* add octant to the local root list
*/
void Zoltan_Oct_POct_setparent(OCT_Global_Info *OCT_info, pOctant oct, pOctant parent, int ppid) {
pRList RootList = Zoltan_Oct_POct_localroots(OCT_info);
if(Zoltan_Oct_Ppid(oct) == OCT_info->OCT_localpid) {
if(ppid != OCT_info->OCT_localpid) {
RL_addRootOctant(RootList, oct); /* was local -- now nonlocal */
}
}
else {
if(ppid == OCT_info->OCT_localpid) {
/* KDDKDDFREE Now passing pointer to OCT_rootlist so that, if
* KDDKDDFREE head of list is deleted, this pointer can be updated
* KDDKDDFREE appropriately (i.e., no longer points to deleted entry). */
RL_delRootOctant(OCT_info, &(OCT_info->OCT_rootlist), oct); /* was foreign -- now local */
}
}
oct->ppid=ppid;
oct->parent=parent;
if(parent && (ppid == OCT_info->OCT_localpid) && (oct->mapidx < 0))
oct->mapidx=parent->mapidx;
}
/*
* void Zoltan_Oct_gen_tree_from_input_data()
*
* This function will create a root node of on each processor which will
* then be used to create an octree with regions associated with it. The
* tree will then be balanced and the output used to balance "mesh regions"
* on several processors.
*/
static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag,
int *c1, int *c2, int *c3,
float *c0, int createpartree)
{
char *yo = "Zoltan_Oct_gen_tree_from_input_data";
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
COORD min, /* min coord bounds of objects */
max; /* max coord bounds of objects */
int num_extra; /* number of orphaned objects */
int num_objs; /* total number of local objects */
pRegion ptr, /* pointer to iterate trough region list */
ptr1; /* pointer to iterate trough region list */
pOctant root; /* root of the partition tree */
int i; /* index counter */
int count, /* count for leaf nodes in partition tree */
proc, /* proc leaf node of parition tree belongs to */
extra, /* extra leaf node flag, if not evenly divisible */
remainder; /* remainder of node, or processors to fill */
pOctant cursor, /* cursor to iterate through octant list */
cursor2, /* another cursor to iterate through octant list */
parent; /* parent of an octant */
int level, /* number of levels of refinement */
n, /* index counter */
part; /* partition counter */
Map *array; /* map of which processors own which octants */
int hold; /* used for calculating partition divisions */
int ierr = 0;
#ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX};
COORD global_min, global_max;
#endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */
int nroots = 0;
/*test*/
/* COORD gmin,gmax; */
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* If there are no objects on this processor, do not create a root octant.
* The partitioner will probably assign objects to this processor
*/
if(zz->Get_Num_Obj == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register ZOLTAN_NUM_OBJ_FN function");
abort();
}
*c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr) {
fprintf(stderr, "OCT [%d] %s: Error returned from user defined "
"Get_Num_Obj function.\n", zz->Proc, yo);
exit (-1);
}
ptr1 = NULL;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds");
/* Need A Function To Get The Bounds Of The Local Objects */
Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0);
#ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
/* For now, don't want to add the new query function to Zoltan. */
/* Zoltan_Oct_get_bounds appears to compute the global min and max from */
/* the object input. */
vector_set(OCT_info->OCT_gmin, min);
vector_set(OCT_info->OCT_gmax, max);
#else
/*test*/
/*getMaxBounds(&gmin, &gmax);*/
if(zz->Get_Bounds_Geom == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register Get_Bounds_Geom function");
abort();
}
zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr);
MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3,
MPI_DOUBLE, MPI_MIN, zz->Communicator);
MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3,
MPI_DOUBLE, MPI_MAX, zz->Communicator);
vector_set(OCT_info->OCT_gmin, global_min);
vector_set(OCT_info->OCT_gmax, global_max);
#endif
/*
* the following code segment was added to create a pseudo global octree
* needed for the partitioner. The basic idea is to regroup all the
* regions into something close to an octree partitioning and build the
* tree from that.
* NOTE: This way of doing things is very costly, especially when calling
* this for the first time on a mesh not partitioned in an octree style
* partitioning.
*/
level = 0; /* initialize level count */
/*
* if more than 1 processor, need to find what level of refinement needed
* to initially partition bounding box among the processors
*/
if(zz->Num_Proc > 1) {
n = zz->Num_Proc;
if(OCT_info->OCT_dimension == 2)
hold = 4;
else
hold = 8;
remainder = hold;
for(; remainder > 0; level++) {
int pr = (int)POW(hold, level);
remainder = n - pr;
}
level--;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree");
if(createpartree) {
/* create the global root octant */
root = Zoltan_Oct_POct_new(OCT_info);
Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax);
/* Zoltan_Oct_setOrientation(root, 0); */
/* subdivide to as many levels as calculated */
for(i=0; i<level; i++) {
cursor = root;
while(cursor != NULL) {
if(Zoltan_Oct_isTerminal(cursor)) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
Zoltan_Oct_terminal_refine(zz, cursor, 0);
cursor = cursor2;
}
else
cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
}
}
#if 0
if(zz->Proc == 0)
for(i=0; i<8; i++)
if(Zoltan_Oct_child(root, i) == NULL)
fprintf(stderr,"NULL child pointer\n");
else
fprintf(stderr, "child %d exists\n", i);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array");
/* this part creates the map array */
if(OCT_info->OCT_dimension == 2) {
hold = (int)POW(4, level); /* ignoring the z+ octants */
if(hold == 0)
hold = 1;
}
else
hold = (int)POW(8, level);
part = hold / zz->Num_Proc; /* how many octants per partition */
remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */
extra = zz->Num_Proc - remainder;/* where to start adding extra octants */
array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */
if(array == NULL) {
fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n",
zz->Proc);
abort();
}
/* initialize variables */
proc = 0;
count = 0;
i = 0;
cursor = root;
while(cursor != NULL) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) {
if(proc == extra) {
part++;
extra = -1;
}
if(count != part) {
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
count++;
}
else {
count = 1;
proc++;
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
}
if(proc == zz->Proc) {
array[i].npid = -1;
/* KDDKDD Added RL_freeList below. The
* KDDKDD implementation from RPI leaked memory because the
* KDDKDD test cases for setting array[i].list were not mutually
* KDDKDD exclusive. Freeing the list produces the result we got
* KDDKDD before, without the memory leak.
*/
/* LGG -- it seems to me that this array[i].list assignment is
* not really necessary. It looks as though it has already been
* assigned with the same information from the prev if-else
* commented out RL_freeList(), and RL_initRootList()
*/
/*RL_freeList(&(array[i].list));*/
/* KDDKDD End addition */
/*array[i].list = RL_initRootList();*/
parent = Zoltan_Oct_parent(cursor);
if(parent != NULL)
Zoltan_Oct_setchild(parent, cursor->which, NULL);
/* octant into local root list */
Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1);
Zoltan_Oct_setMapIdx(cursor, i);
nroots++;
/* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc);
octant into local root list */
}
i++;
}
cursor = cursor2;
}
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RootOct = RL_nextRootOctant(&RootList);
if(RootOct != root) {
/* KDDKDDFREE changed root to &root to allow root to be reset to NULL */
Zoltan_Oct_POct_delTree(OCT_info,&root);
}
OCT_info->map = array;
OCT_info->mapsize = hold;
}
/*
* attach the regions to the root... Zoltan_Oct_fix will create the octree
* starting with the root and subdividing as needed
*/
num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans");
Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map,
c1, c2);
/* ZOLTAN_FREE(&array); */
while(ptr1 != NULL) {
ptr = ptr1->next;
ZOLTAN_FREE(&(ptr1->Global_ID));
ZOLTAN_FREE(&(ptr1->Local_ID));
ZOLTAN_FREE(&ptr1);
ptr1 = ptr;
}
ZOLTAN_TRACE_EXIT(zz, yo);
}
/*
* void lb_oct_init();
*
* initialize the calls needed to start the octree load balancing rounties
*/
static int lb_oct_init(
ZZ *zz, /* The Zoltan structure with info for
the OCTPART balancer. */
int *num_export, /* Number of non-local objs assigned to this
processor in the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition. */
int **export_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition. */
int **export_to_part, /* Returned value: array of partitions to
which objects are imported.
KDDKDD Assume #parts==#procs. */
int oct_dim, /* Dimension of method (2D or 3D) */
int oct_method, /* Flag specifying curve to be used. */
int oct_maxoctregions, /* max # of objects in leaves of octree. */
int oct_minoctregions, /* min # of objects in leaves of octree. */
int oct_output_level, /* Flag specifying amount of output. */
int oct_wgtflag, /* Flag specifying use of object weights. */
float *part_sizes /* Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
)
{
char *yo = "lb_oct_init";
OCT_Global_Info *OCT_info;
int nsentags; /* number of tags being sent */
pRegion export_regs; /* */
int nrectags; /* number of tags received */
int kk;
double time1,time2; /* timers */
double timestart,timestop; /* timers */
double timers[4]; /* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
int counters[6]; /* diagnostic counts
0 = # of median iterations
1 = # of objects sent
2 = # of objects received
3 = most objects this proc ever owns
*/
float c[4];
int createpartree = 0;
/*int num_gid_entries = zz->Num_GID;*/
/*int num_lid_entries = zz->Num_LID;*/
ZOLTAN_TRACE_ENTER(zz, yo);
MPI_Barrier(zz->Communicator);
timestart = MPI_Wtime();
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = 0;
counters[4] = 0;
counters[5] = 0;
c[0] = 0;
c[1] = 0;
c[2] = 0;
c[3] = 0;
timers[1] = 0.0;
timers[2] = 0.0;
timers[3] = 0.0;
nsentags = nrectags = 0;
if(zz->LB.Data_Structure == NULL) {
OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim);
Zoltan_Oct_set_method(OCT_info, oct_method);
Zoltan_Oct_set_maxregions(oct_maxoctregions);
Zoltan_Oct_set_minregions(oct_minoctregions);
createpartree = 1;
}
else {
OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
}
/* create the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data");
Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1],
&counters[2], &counters[3], &c[0],
createpartree);
time2 = MPI_Wtime();
timers[0] = time2 - time1; /* time took to create octree */
/* Zoltan_Oct_POct_printResults(OCT_info); */
/* partition the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition");
/* old call to dfs_paritition: */
#if 0
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]);
#else
/***************************
if(zz->Proc == 0) {
int debug_i;
for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) {
fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]);
}
}
****************************/
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes);
#endif
time2 = MPI_Wtime();
timers[1] = time2 - time1; /* time took to partition octree */
if (oct_output_level > 2) {
Zoltan_Oct_Plots(zz);
}
/* set up tags for migrations */
time1 = MPI_Wtime();
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate");
Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags,
&c[2], &c[3], &counters[3], &counters[5]);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags");
if (zz->LB.Return_Lists) {
*num_export = nrectags;
if (nrectags > 0)
Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids,
export_procs, export_to_part, nrectags,
export_regs);
}
time2 = MPI_Wtime();
timers[2] = time2 - time1; /* time took to setup migration */
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
/* count the number of objects on this processor */
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
counters[4] = nsentags;
MPI_Barrier(zz->Communicator);
timestop = MPI_Wtime();
if (oct_output_level > 0) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats");
Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c,
oct_output_level);
}
for (kk = 0; kk < nrectags; kk++) {
ZOLTAN_FREE(&(export_regs[kk].Global_ID));
ZOLTAN_FREE(&(export_regs[kk].Local_ID));
}
ZOLTAN_FREE(&export_regs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear");
Zoltan_Oct_global_clear(OCT_info);
/* KDDKDD Don't understand how re-used octree will work, especially without
* KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything;
* KDDKDD we can move back to saving some of the tree later.
*/
Zoltan_Oct_Free_Structure(zz);
/* KDDKDD END */
/* Temporary return value until error codes are fully implemented. */
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
/*
* void Zoltan_Oct_dfs_partition()
*
* This function calls the different subfunctions to partition the octree
*/
void Zoltan_Oct_dfs_partition(ZZ *zz, int *counter, float *c1,
float *part_sizes) {
float mycost; /* cost of the octant */
/*float globalcost;*/ /* costs of all the octants */
float prefcost; /* sum of costs from previous processors */
int nprevoct; /* the number of previous octants */
pRList RootList; /* list of the local roots */
pOctant RootOct;
float prevwork;
float pastwork;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
DFS_Part_Count = 0;
*c1 = mycost = Zoltan_Oct_costs_global_compute(OCT_info);
/* gets the number of octants from the previous processors */
nprevoct=Zoltan_Oct_msg_int_scan(zz->Communicator, zz->Proc,
Zoltan_Oct_nOctants());
/* iterate through, and advance each id by nprevocts */
/* this is trying to make all the octant id's to be unquie globally */
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList)))
Zoltan_Oct_dfs_SetIds(OCT_info, RootOct, nprevoct);
/* Sum a value from each processor, and return sum to all processors */
MPI_Allreduce(&mycost,&globalcost,1,MPI_FLOAT,MPI_SUM,zz->Communicator);
prefcost=Zoltan_Oct_msg_float_scan(zz->Communicator, zz->Proc, mycost);
/* Initialize static vars */
optcost=globalcost/zz->Num_Proc; /* Optimal partition size */
#if 0
if(optcost > 0)
partition=(int)(prefcost/optcost); /* Start work on this partition */
else
partition=0;
if (partition==zz->Num_Proc)
partition=zz->Num_Proc-1;
#else
tmpcost=0;
prevwork = prefcost;
if(optcost > 0) {
partition=(int)(prefcost/optcost); /* Start work on this partition */
partition = 0;
optsize = part_sizes[partition]*globalcost;
pastwork = part_sizes[partition]*globalcost;
while(pastwork < prevwork) {
tmpcost += part_sizes[partition];
partition++;
optsize = part_sizes[partition]*globalcost;
pastwork += (part_sizes[partition]*globalcost);
}
if((pastwork == prevwork) && (partition < (zz->Num_Proc - 1))){
tmpcost += part_sizes[partition];
partition++;
optsize = part_sizes[partition]*globalcost;
pastwork += (part_sizes[partition]*globalcost);
}
}
else
partition=0;
while((part_sizes[partition] == 0) && (partition < (zz->Num_Proc - 1))) {
partition++;
optsize = part_sizes[partition]*globalcost;
}
#endif
/*optcost = part_sizes[partition]*globalcost;*/
/*total=partition*optcost;*/ /* Total cost of all previous parts */
total=tmpcost*globalcost; /* Total cost of all previous parts */
/*pcost=prefcost-(partition*optcost);*/ /* Current partition cost */
pcost=prefcost-(tmpcost*globalcost); /* Current partition cost */
pmass=0.0; /* initialize octant volume variable */
vector_set_comp(pcoord,0,0,0);
Zoltan_Oct_visit_all_subtrees(zz, part_sizes);
(*counter) = DFS_Part_Count;
}
/*
* void Zoltan_Oct_dfs_migrate()
*
* sets up information so the migrate octant routines can create the
* proper export_tags and import_tags arrays
*/
void Zoltan_Oct_dfs_migrate(ZZ *zz, int *nsentags,
pRegion *import_regs, int *nrectags,
float *c2, float *c3, int *counter3, int *counter4)
{
pRList RootList; /* list of the local roots */
pOctant oct; /* octree octant */
pOctant *docts = NULL; /* array of octants being sent */
int *dpids = NULL; /* array of octant pids */
int dcount; /* count of octants being sent */
int pid; /* processor id */
int nrecocts;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
char *yo = "Zoltan_Oct_dfs_migrate";
if(Zoltan_Oct_nOctants()) { /* allocate space for octants being migrated */
docts = (pOctant *)ZOLTAN_MALLOC(Zoltan_Oct_nOctants() * sizeof(pOctant));
if(!docts) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "cannot allocate arrays.");
abort();
}
dpids = (int *) ZOLTAN_MALLOC(Zoltan_Oct_nOctants() * sizeof(int));
if(!dpids) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "cannot allocate arrays.");
ZOLTAN_FREE(&docts);
abort();
}
}
dcount=0;
/* go through the local octants and make sure each has a valid npid */
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((oct = RL_nextRootOctant(&RootList)))
while(oct) {
pid = Zoltan_Oct_data_newpid(oct);
if (pid<0 || pid>=zz->Num_Proc) {
fprintf(stderr,"%d Zoltan_Oct_dfs_migrate: bad dest pid %d\n", zz->Proc, pid);
abort();
}
if (dcount<Zoltan_Oct_nOctants()) {
docts[dcount]=oct;
dpids[dcount]=pid;
}
oct=Zoltan_Oct_POct_nextDfs(OCT_info, oct);
dcount++;
}
if (dcount!=Zoltan_Oct_nOctants()) {
fprintf(stderr, "ERROR: in Zoltan_Oct_dfs_migrate, octant count mismatch (I counted %d but there should be %d)\n",dcount, Zoltan_Oct_nOctants());
/* dcount=Zoltan_Oct_nOctants(); */
/* abort(); */
}
/* setup the import_regs */
Zoltan_Oct_migrate_objects(zz, docts, dpids, dcount, nsentags,
import_regs, nrectags, c2, c3, counter3, counter4);
Zoltan_Oct_migrate_octants(zz, dpids, docts, dcount, &nrecocts);
ZOLTAN_FREE(&docts);
ZOLTAN_FREE(&dpids);
}
static int Zoltan_Oct_build_global_rootlist(ZZ *zz,Migrate_msg **ret_rmsg, int *size) {
int j, k = 0;
int *despid = NULL;
int nroots, nreceives;
pRList RootList; /* list of the local roots */
pOctant RootOct;
Migrate_msg *snd_rmsg = NULL;
Migrate_msg *rcv_rmsg = NULL;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
/*Map *array = OCT_info->map;*/
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_Oct_build_global_rootlist";
nroots = RL_numRootOctants(Zoltan_Oct_POct_localroots(OCT_info));
if (nroots > 0) { /* KDDKDD -- Added test to prevent departure before comm */
if((despid = (int *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(int))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((snd_rmsg = (Migrate_msg *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
return ZOLTAN_MEMERR;
}
k = 0;
for (j=0; j<zz->Num_Proc; j++) {
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
/* if(array[Zoltan_Oct_mapidx(RootOct)].npid > 0) { */
FILLMIGRATEMSG(RootOct, RootOct, snd_rmsg[k], zz->Proc);
despid[k] = j;
k++;
/* } */
}
}
} /* KDDKDD */
ierr = Zoltan_Comm_Create(&comm_plan, k, despid, zz->Communicator,
RootListCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
return (ierr);
}
if (nreceives > 0) {
if((rcv_rmsg = (Migrate_msg *) ZOLTAN_MALLOC(nreceives * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
return ZOLTAN_MEMERR;
}
}
ierr = Zoltan_Comm_Do(comm_plan, RootListCommDo, (char *) snd_rmsg, sizeof(Migrate_msg), (char *) rcv_rmsg);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
ZOLTAN_FREE(&rcv_rmsg);
return (ierr);
}
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
*ret_rmsg = rcv_rmsg;
*size = nreceives;
return ierr;
}
