/*
* oct_global_clear()
*
* delete all regions from all octants on the local processor
*
*/
static void Zoltan_Oct_global_clear(OCT_Global_Info * OCT_info)
{
pRList RootList; /* list of all local roots */
pOctant Oct;
/*
* iterate through the list of local roots
* traverse down the subtree
* delete regions associated with octant
*/
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while ((Oct = RL_nextRootOctant(&RootList))) {
while(Oct) {
if(Zoltan_Oct_isTerminal(Oct))
Zoltan_Oct_clearRegions(Oct);
Oct = Zoltan_Oct_POct_nextDfs(OCT_info, Oct);
}
}
}
/*
* void Zoltan_Oct_tag_subtree(pOctant octant, int partition_number)
*
* marks all the octants within the subtree to be in the current partition
*/
static void Zoltan_Oct_tag_subtree(OCT_Global_Info *OCT_info,pOctant octant,
int part) {
pOctant children[8]; /* children of octant */
int i; /* index counter */
/* modify NPID so octant know where to migrate to */
Zoltan_Oct_modify_newpid(octant, part);
if (Zoltan_Oct_isTerminal(octant))
return;
/* if octant has children, have to tag them too */
Zoltan_Oct_children(octant,children);
for (i=0; i<8; i++) /* Simple - just visit in order */
/* if (children[i] && Zoltan_Oct_local(OCT_info, children[i])) */
if(children[i] && Zoltan_Oct_POct_local(OCT_info, octant,i))
Zoltan_Oct_tag_subtree(OCT_info,children[i],part);
}
/*
* Zoltan_Oct_findOctant(oct, coord)
* (replaces : PO_findOctant(oct,coord))
*
*
* find the octant in a subtree containing coord (if coord
* is not in the subtree, returns the closest octant in the subtree).
* NOTE: return NULL if we hit an off-processor link
*
*/
static pOctant Zoltan_Oct_findOctant(OCT_Global_Info *OCT_info,pOctant oct,
COORD coord)
{
pOctant child; /* child of an octant */
int cnum; /* child number */
/* if octant is terminal, then this is the right octant */
if (Zoltan_Oct_isTerminal(oct))
return(oct);
/* find closest child to coord */
cnum = Zoltan_Oct_child_which_wrapper(OCT_info,oct,coord);
child = Zoltan_Oct_child(oct, cnum); /* get that child */
/* ATTN: are these local checks necessary? */
/* if ( !Zoltan_Oct_POct_local(child) ) */
if(!Zoltan_Oct_POct_local(OCT_info,oct, cnum)) /* make sure oct is local */
return(NULL);
/* recursivly search down the tree */
return(Zoltan_Oct_findOctant(OCT_info,child,coord));
}
/*
* int Zoltan_Oct_dfs_SetIds(pOctant octant, int number_of_previous_octants)
*
* sets the ids of all the octants so that there is a global numbering
*/
static int Zoltan_Oct_dfs_SetIds(OCT_Global_Info *OCT_info, pOctant oct,
int nprevoct) {
int id, /* the id number of an octant */
i; /* index counter */
int pid;
pOctant child; /* ith child of an octant */
if(Zoltan_Oct_isTerminal(oct)) {
id = Zoltan_Oct_id(oct);
Zoltan_Oct_setID(oct, id+nprevoct); /* now have global id's */
}
else {
for(i=0; i<8; i++) {
child = Zoltan_Oct_child(oct, i);
pid = Zoltan_Oct_Cpid(oct,i);
if ((pid == OCT_info->OCT_localpid) && child != NULL)
Zoltan_Oct_dfs_SetIds(OCT_info,child, nprevoct);
}
id = Zoltan_Oct_id(oct);
Zoltan_Oct_setID(oct, id+nprevoct); /* now have global id's */
}
return 0;
}
/*
* Zoltan_Oct_subtree_insert(oct,region)
*
* Insert region in oct, carrying out multiple refinement if necessary
*/
int Zoltan_Oct_subtree_insert(ZZ *zz, pOctant oct, pRegion region)
{
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
/* if oct is not terminal, find leaf node the centroid can be attahced to */
if (!Zoltan_Oct_isTerminal(oct)) {
oct=Zoltan_Oct_findOctant(OCT_info, oct,region->Coord);
}
if (!oct)
return(0);
/* add the region to the octant */
Zoltan_Oct_addRegion(zz, oct, region);
/* check if octant has too many regions and needs to be refined */
/* KDDKDD Replaced the following to allow multiple regions with the
* KDDKDD same coordinates to be placed in a single octant.
if(Zoltan_Oct_nRegions(oct) > MAXOCTREGIONS)
*/
if(Zoltan_Oct_nUniqueRegions(OCT_info, oct) > MAXOCTREGIONS)
Zoltan_Oct_terminal_refine(zz, oct,0); /* After, dest may be nonterm */
return(1);
}
/*
* float Zoltan_Oct_costs_subtree_compute(pOctant octant, int sequence_number)
*
* Do a DFS on the octree, calculating the costs for any given subtree.
* (Subtree is defined as any octant and all its descendants.)
*
* Tag every octant visited with a sequence number, automatically
* incrementing the sequence number.
*
* NOTE: must call Zoltan_Oct_costs_init() first
*/
static float Zoltan_Oct_costs_subtree_compute(OCT_Global_Info *OCT_info,pOctant octant, int *seq) {
pOctant children[8]; /* the children of the octant */
float c = 0; /* cost of each subtree */
int i = 0; /* index counter */
/* #ifdef LGG_MIGOCT */
Zoltan_Oct_setID(octant,(*seq)++); /* set new ID for local ordering */
/* #endif */ /* LGG_MIGOCT */
if (!Zoltan_Oct_isTerminal(octant)) {
/* get the children of each octant */
Zoltan_Oct_children(octant,children);
/* sum the cost for each child to get octant's cost */
for (i=0; i<8; i++)
if(children[i] && Zoltan_Oct_POct_local(OCT_info, octant, i))
c += Zoltan_Oct_costs_subtree_compute(OCT_info,children[i], seq);
}
else /* terminal */
c=Zoltan_Oct_costs_weight(octant);
/* set the cost data to the octant */
Zoltan_Oct_modify_cost(octant, c);
return(c);
}
/*
* void tag_regions()
* Iterates through the list of octants on the processor and finds which
* are to be migrated. It then looks at the region list for those octants
* and stores the migrating regions into the export_tags array.
*/
static int tag_regions(ZZ *zz,
pOctant *octs,
int *newpids,
int nocts,
Region **exported_tags,
ZOLTAN_ID_PTR *exported_gids,
ZOLTAN_ID_PTR *exported_lids,
int *nsentags,
int **tag_pids,
Region **p_tags,
ZOLTAN_ID_PTR *p_gids,
ZOLTAN_ID_PTR *p_lids,
int *npimtags,
float *c2,
int *max_objs)
{
char *yo = "tag_regions";
int i; /* index counter */
pRegion regionlist; /* list of region on this processor */
int index; /* index counter */
int index2; /* yet another index counter */
int count; /* count of objects exported form this processor */
int count2; /* count of objects that are kept on processor */
int count3;
int *exported_pids; /* array of pids where regions are being exported to */
pRegion mtags; /* object tags of objects to be migrated */
pRegion ptags; /* tags of objects that were previously migrated */
float ex_load;
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
ex_load = 0;
(*max_objs) = 0;
if (!nsentags)
return ierr;
/* find how many objects have been exported */
count = 0;
/* find number of local objs to export */
count2 = 0;
count3 = 0;
for (i=0; i<nocts; i++) {
if(Zoltan_Oct_isTerminal(octs[i])) {
(*max_objs) += Zoltan_Oct_nRegions(octs[i]);
regionlist = Zoltan_Oct_regionlist(octs[i]);
while(regionlist != NULL) {
count3++;
if(regionlist->Proc != zz->Proc) {
count++;
if(newpids[i] != zz->Proc)
regionlist->newProc = newpids[i];
else
regionlist->newProc = zz->Proc;
}
else {
if(newpids[i] != zz->Proc) {
count2++;
regionlist->newProc = newpids[i];
}
else
regionlist->newProc = zz->Proc;
}
regionlist = regionlist->next; /* get next region */
}
}
}
#if 0
{
{
if (newpids[i]!=zz->Proc) {
count+=Zoltan_Oct_nRegions(octs[i]);
}
else {
pRegion regions;
regions = Zoltan_Oct_regionlist(octs[i]);
while(regions != NULL) {
if(regions->Proc != zz->Proc)
count2++;
regions = regions->next;
}
}
}
}
#endif
/* set up the return pointers */
*nsentags = count;
*npimtags = count2;
if (!exported_tags) {
return ierr;
}
if (count > 0) {
/* allocate some space */
if((mtags=(pRegion)ZOLTAN_MALLOC((unsigned)count*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((exported_pids = (int *)ZOLTAN_MALLOC((unsigned)count*sizeof(int))) ==
NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
return ZOLTAN_MEMERR;
}
*exported_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count);
*exported_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count);
if(!(*exported_gids) || (num_lid_entries && !(*exported_lids))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
return ZOLTAN_MEMERR;
}
}
else {
mtags = NULL;
exported_pids = NULL;
*exported_gids = NULL;
*exported_lids = NULL;
}
/* set up return pointers */
*exported_tags=mtags;
*tag_pids = exported_pids;
if (count2 > 0) {
/* allocate some space */
if((ptags=(pRegion)ZOLTAN_MALLOC((unsigned)count2*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
ZOLTAN_FREE(exported_gids);
ZOLTAN_FREE(exported_lids);
return ZOLTAN_MEMERR;
}
*p_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count2);
*p_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count2);
if(!(*p_gids) || (num_lid_entries && !(*p_lids))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
ZOLTAN_FREE(exported_gids);
ZOLTAN_FREE(exported_lids);
ZOLTAN_FREE(&ptags);
ZOLTAN_FREE(p_gids);
ZOLTAN_FREE(p_lids);
return ZOLTAN_MEMERR;
}
}
else {
ptags = NULL;
*p_gids = NULL;
*p_lids = NULL;
}
/* set up return pointers */
*p_tags=ptags;
index = index2 = 0;
for (i=0; i<nocts; i++) {
if(Zoltan_Oct_isTerminal(octs[i])) {
regionlist = Zoltan_Oct_regionlist(octs[i]);
while(regionlist != NULL) {
if(regionlist->Proc != zz->Proc) {
/* place information in the appropritate array */
mtags[index] = *regionlist;
ZOLTAN_SET_GID(zz, &((*exported_gids)[index*num_gid_entries]),
regionlist->Global_ID);
ZOLTAN_SET_LID(zz, &((*exported_lids)[index*num_lid_entries]),
regionlist->Local_ID);
/*ex_load += (float)(regionlist->Weight);*/
exported_pids[index] = regionlist->Proc;
index++; /* increment counter */
}
else if(newpids[i] != zz->Proc) {
ptags[index2] = *regionlist; /* get region information */
ZOLTAN_SET_GID(zz, &((*p_gids)[index2*num_gid_entries]),
regionlist->Global_ID);
ZOLTAN_SET_LID(zz, &((*p_lids)[index2*num_lid_entries]),
regionlist->Local_ID);
index2++; /* increment counter */
}
regionlist = regionlist->next; /* get next region */
}
}
}
if (index!=count) { /* error check */
ZOLTAN_TRACE_DETAIL(zz, yo,
"Fatal error, inconsistent number of regions.\n");
return ZOLTAN_FATAL;
}
*c2 = ex_load;
return ierr;
}
/*
* void Zoltan_Oct_gen_tree_from_input_data()
*
* This function will create a root node of on each processor which will
* then be used to create an octree with regions associated with it. The
* tree will then be balanced and the output used to balance "mesh regions"
* on several processors.
*/
static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag,
int *c1, int *c2, int *c3,
float *c0, int createpartree)
{
char *yo = "Zoltan_Oct_gen_tree_from_input_data";
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
COORD min, /* min coord bounds of objects */
max; /* max coord bounds of objects */
int num_extra; /* number of orphaned objects */
int num_objs; /* total number of local objects */
pRegion ptr, /* pointer to iterate trough region list */
ptr1; /* pointer to iterate trough region list */
pOctant root; /* root of the partition tree */
int i; /* index counter */
int count, /* count for leaf nodes in partition tree */
proc, /* proc leaf node of parition tree belongs to */
extra, /* extra leaf node flag, if not evenly divisible */
remainder; /* remainder of node, or processors to fill */
pOctant cursor, /* cursor to iterate through octant list */
cursor2, /* another cursor to iterate through octant list */
parent; /* parent of an octant */
int level, /* number of levels of refinement */
n, /* index counter */
part; /* partition counter */
Map *array; /* map of which processors own which octants */
int hold; /* used for calculating partition divisions */
int ierr = 0;
#ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX};
COORD global_min, global_max;
#endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */
int nroots = 0;
/*test*/
/* COORD gmin,gmax; */
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* If there are no objects on this processor, do not create a root octant.
* The partitioner will probably assign objects to this processor
*/
if(zz->Get_Num_Obj == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register ZOLTAN_NUM_OBJ_FN function");
abort();
}
*c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr) {
fprintf(stderr, "OCT [%d] %s: Error returned from user defined "
"Get_Num_Obj function.\n", zz->Proc, yo);
exit (-1);
}
ptr1 = NULL;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds");
/* Need A Function To Get The Bounds Of The Local Objects */
Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0);
#ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
/* For now, don't want to add the new query function to Zoltan. */
/* Zoltan_Oct_get_bounds appears to compute the global min and max from */
/* the object input. */
vector_set(OCT_info->OCT_gmin, min);
vector_set(OCT_info->OCT_gmax, max);
#else
/*test*/
/*getMaxBounds(&gmin, &gmax);*/
if(zz->Get_Bounds_Geom == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register Get_Bounds_Geom function");
abort();
}
zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr);
MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3,
MPI_DOUBLE, MPI_MIN, zz->Communicator);
MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3,
MPI_DOUBLE, MPI_MAX, zz->Communicator);
vector_set(OCT_info->OCT_gmin, global_min);
vector_set(OCT_info->OCT_gmax, global_max);
#endif
/*
* the following code segment was added to create a pseudo global octree
* needed for the partitioner. The basic idea is to regroup all the
* regions into something close to an octree partitioning and build the
* tree from that.
* NOTE: This way of doing things is very costly, especially when calling
* this for the first time on a mesh not partitioned in an octree style
* partitioning.
*/
level = 0; /* initialize level count */
/*
* if more than 1 processor, need to find what level of refinement needed
* to initially partition bounding box among the processors
*/
if(zz->Num_Proc > 1) {
n = zz->Num_Proc;
if(OCT_info->OCT_dimension == 2)
hold = 4;
else
hold = 8;
remainder = hold;
for(; remainder > 0; level++) {
int pr = (int)POW(hold, level);
remainder = n - pr;
}
level--;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree");
if(createpartree) {
/* create the global root octant */
root = Zoltan_Oct_POct_new(OCT_info);
Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax);
/* Zoltan_Oct_setOrientation(root, 0); */
/* subdivide to as many levels as calculated */
for(i=0; i<level; i++) {
cursor = root;
while(cursor != NULL) {
if(Zoltan_Oct_isTerminal(cursor)) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
Zoltan_Oct_terminal_refine(zz, cursor, 0);
cursor = cursor2;
}
else
cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
}
}
#if 0
if(zz->Proc == 0)
for(i=0; i<8; i++)
if(Zoltan_Oct_child(root, i) == NULL)
fprintf(stderr,"NULL child pointer\n");
else
fprintf(stderr, "child %d exists\n", i);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array");
/* this part creates the map array */
if(OCT_info->OCT_dimension == 2) {
hold = (int)POW(4, level); /* ignoring the z+ octants */
if(hold == 0)
hold = 1;
}
else
hold = (int)POW(8, level);
part = hold / zz->Num_Proc; /* how many octants per partition */
remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */
extra = zz->Num_Proc - remainder;/* where to start adding extra octants */
array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */
if(array == NULL) {
fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n",
zz->Proc);
abort();
}
/* initialize variables */
proc = 0;
count = 0;
i = 0;
cursor = root;
while(cursor != NULL) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) {
if(proc == extra) {
part++;
extra = -1;
}
if(count != part) {
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
count++;
}
else {
count = 1;
proc++;
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
}
if(proc == zz->Proc) {
array[i].npid = -1;
/* KDDKDD Added RL_freeList below. The
* KDDKDD implementation from RPI leaked memory because the
* KDDKDD test cases for setting array[i].list were not mutually
* KDDKDD exclusive. Freeing the list produces the result we got
* KDDKDD before, without the memory leak.
*/
/* LGG -- it seems to me that this array[i].list assignment is
* not really necessary. It looks as though it has already been
* assigned with the same information from the prev if-else
* commented out RL_freeList(), and RL_initRootList()
*/
/*RL_freeList(&(array[i].list));*/
/* KDDKDD End addition */
/*array[i].list = RL_initRootList();*/
parent = Zoltan_Oct_parent(cursor);
if(parent != NULL)
Zoltan_Oct_setchild(parent, cursor->which, NULL);
/* octant into local root list */
Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1);
Zoltan_Oct_setMapIdx(cursor, i);
nroots++;
/* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc);
octant into local root list */
}
i++;
}
cursor = cursor2;
}
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RootOct = RL_nextRootOctant(&RootList);
if(RootOct != root) {
/* KDDKDDFREE changed root to &root to allow root to be reset to NULL */
Zoltan_Oct_POct_delTree(OCT_info,&root);
}
OCT_info->map = array;
OCT_info->mapsize = hold;
}
/*
* attach the regions to the root... Zoltan_Oct_fix will create the octree
* starting with the root and subdividing as needed
*/
num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans");
Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map,
c1, c2);
/* ZOLTAN_FREE(&array); */
while(ptr1 != NULL) {
ptr = ptr1->next;
ZOLTAN_FREE(&(ptr1->Global_ID));
ZOLTAN_FREE(&(ptr1->Local_ID));
ZOLTAN_FREE(&ptr1);
ptr1 = ptr;
}
ZOLTAN_TRACE_EXIT(zz, yo);
}
/*
* void lb_oct_init();
*
* initialize the calls needed to start the octree load balancing rounties
*/
static int lb_oct_init(
ZZ *zz, /* The Zoltan structure with info for
the OCTPART balancer. */
int *num_export, /* Number of non-local objs assigned to this
processor in the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition. */
int **export_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition. */
int **export_to_part, /* Returned value: array of partitions to
which objects are imported.
KDDKDD Assume #parts==#procs. */
int oct_dim, /* Dimension of method (2D or 3D) */
int oct_method, /* Flag specifying curve to be used. */
int oct_maxoctregions, /* max # of objects in leaves of octree. */
int oct_minoctregions, /* min # of objects in leaves of octree. */
int oct_output_level, /* Flag specifying amount of output. */
int oct_wgtflag, /* Flag specifying use of object weights. */
float *part_sizes /* Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
)
{
char *yo = "lb_oct_init";
OCT_Global_Info *OCT_info;
int nsentags; /* number of tags being sent */
pRegion export_regs; /* */
int nrectags; /* number of tags received */
int kk;
double time1,time2; /* timers */
double timestart,timestop; /* timers */
double timers[4]; /* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
int counters[6]; /* diagnostic counts
0 = # of median iterations
1 = # of objects sent
2 = # of objects received
3 = most objects this proc ever owns
*/
float c[4];
int createpartree = 0;
/*int num_gid_entries = zz->Num_GID;*/
/*int num_lid_entries = zz->Num_LID;*/
ZOLTAN_TRACE_ENTER(zz, yo);
MPI_Barrier(zz->Communicator);
timestart = MPI_Wtime();
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = 0;
counters[4] = 0;
counters[5] = 0;
c[0] = 0;
c[1] = 0;
c[2] = 0;
c[3] = 0;
timers[1] = 0.0;
timers[2] = 0.0;
timers[3] = 0.0;
nsentags = nrectags = 0;
if(zz->LB.Data_Structure == NULL) {
OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim);
Zoltan_Oct_set_method(OCT_info, oct_method);
Zoltan_Oct_set_maxregions(oct_maxoctregions);
Zoltan_Oct_set_minregions(oct_minoctregions);
createpartree = 1;
}
else {
OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
}
/* create the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data");
Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1],
&counters[2], &counters[3], &c[0],
createpartree);
time2 = MPI_Wtime();
timers[0] = time2 - time1; /* time took to create octree */
/* Zoltan_Oct_POct_printResults(OCT_info); */
/* partition the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition");
/* old call to dfs_paritition: */
#if 0
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]);
#else
/***************************
if(zz->Proc == 0) {
int debug_i;
for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) {
fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]);
}
}
****************************/
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes);
#endif
time2 = MPI_Wtime();
timers[1] = time2 - time1; /* time took to partition octree */
if (oct_output_level > 2) {
Zoltan_Oct_Plots(zz);
}
/* set up tags for migrations */
time1 = MPI_Wtime();
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate");
Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags,
&c[2], &c[3], &counters[3], &counters[5]);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags");
if (zz->LB.Return_Lists) {
*num_export = nrectags;
if (nrectags > 0)
Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids,
export_procs, export_to_part, nrectags,
export_regs);
}
time2 = MPI_Wtime();
timers[2] = time2 - time1; /* time took to setup migration */
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
/* count the number of objects on this processor */
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
counters[4] = nsentags;
MPI_Barrier(zz->Communicator);
timestop = MPI_Wtime();
if (oct_output_level > 0) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats");
Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c,
oct_output_level);
}
for (kk = 0; kk < nrectags; kk++) {
ZOLTAN_FREE(&(export_regs[kk].Global_ID));
ZOLTAN_FREE(&(export_regs[kk].Local_ID));
}
ZOLTAN_FREE(&export_regs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear");
Zoltan_Oct_global_clear(OCT_info);
/* KDDKDD Don't understand how re-used octree will work, especially without
* KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything;
* KDDKDD we can move back to saving some of the tree later.
*/
Zoltan_Oct_Free_Structure(zz);
/* KDDKDD END */
/* Temporary return value until error codes are fully implemented. */
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
/*
* Zoltan_Oct_terminal_refine(oct)
*
* subdivide a terminal octant and divvy up
* its regions to the 8 children; recurse if
* necessary to satisfy MAXOCTREGIONS
*
*/
static void Zoltan_Oct_terminal_refine(ZZ *zz, pOctant oct,int count)
{
COORD min, /* coordinates of minimum bounds of region */
max, /* coordinates of maximum bounds of region */
origin; /* origin of region */
pOctant child[8]; /* array of child octants */
int cnum; /* child number */
int i; /* index counter */
pRegion region; /* a region to be associated to an octant */
pRegion entry;
COORD cmin[8], cmax[8];
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
for(i=0;i<3;i++)
min[i] = max[i] = 0;
/* upper limit of refinement levels */
/* ATTN: may not be used anymore, but can be put back in if necessary */
if (count>=20) {
fprintf(stderr, "OCT ERROR: Zoltan_Oct_terminal_refine: bailing out at "
"10 levels\n");
abort();
}
oct_nref++; /* increment refinement counter */
/* octant should be terminal in order to be refined (subdivided) */
if (!Zoltan_Oct_isTerminal(oct)) {
fprintf(stderr,"OCT ref_octant: oct not terminal\n");
abort();
}
/* get the bounds of an octant */
Zoltan_Oct_bounds(oct,min,max);
/* calculate the origin from the bounds */
Zoltan_Oct_bounds_to_origin(min,max,origin);
region = Zoltan_Oct_regionlist(oct); /* Get list while still terminal */
oct->list = NULL; /* remove regions from octant, it won't be terminal */
/* create the children and set their id's */
Zoltan_Oct_child_bounds_wrapper(OCT_info,oct, cmin, cmax);
for (i=0; i<8; i++) {
if(OCT_info->OCT_dimension == 2) {
/* KDDKDD 3/01 see changes to Zoltan_Oct_child_bounds_wrapper that allow this
* KDDKDD 3/01 test to work for GRAY and HILBERT mappings.
*/
if(cmin[i][2] > OCT_info->OCT_gmin[2]) { /* ignore the z+ octants */
child[i] = NULL;
continue;
}
}
child[i]=Zoltan_Oct_POct_new(OCT_info); /* create a new octant */
child[i]->dir = Zoltan_Oct_get_child_dir(OCT_info, oct->dir, i);
/* create a new octant */
/* set the child->parent link */
Zoltan_Oct_POct_setparent(OCT_info, child[i], oct, zz->Proc);
Zoltan_Oct_setchildnum(child[i], i); /* which child of the parent */
Zoltan_Oct_setchild(oct, i, child[i]); /* set the parent->child link */
#ifdef LGG_MIGOCT
Zoltan_Oct_setID(child[i], Zoltan_Oct_nextId()); /* set child id num */
#endif /* LGG_MIGOCT */
Zoltan_Oct_setbounds(child[i], cmin[i], cmax[i]); /* set child bounds */
Zoltan_Oct_setCpid(oct, i, zz->Proc); /* set child to be a local oct */
/* Zoltan_Oct_setOrientation(child[i],
Zoltan_Oct_child_orientation(oct->orientation, oct->which)); */
}
/* assign newly created children to child array*/
if(OCT_info->OCT_dimension == 3) {
if(Zoltan_Oct_children(oct, child) != 8) {
/*
* if subdivision of oct was successful, oct should have 8 children;
* thus a return value of 0 here is a fatal error
*/
fprintf(stderr, "OCT ref_octant: subdivide failed, %d children.\n",
Zoltan_Oct_children(oct, child));
abort();
}
}
else
if(Zoltan_Oct_children(oct, child) != 4) {
/*
* if subdivision of oct was successful, oct should have 4 children;
* thus a return value of 0 here is a fatal error
*/
fprintf(stderr,
"OCT ref_octant:subdivide failed, %d children, expected 4\n",
Zoltan_Oct_children(oct, child));
abort();
}
/* iterate through and find which child each region should belong to */
while(region != NULL) {
entry = region->next;
cnum=Zoltan_Oct_child_which_wrapper(OCT_info,oct, region->Coord);
/* add region to octant's regionlist */
Zoltan_Oct_addRegion(zz, child[cnum], region);
ZOLTAN_FREE(&(region->Global_ID));
ZOLTAN_FREE(&(region->Local_ID));
ZOLTAN_FREE(®ion);
region = entry;
}
for (i=0; i<8; i++) /* Recursion */
if(child[i] != NULL)
/* KDDKDD Replaced the following to allow multiple regions with the
* KDDKDD same coordinates to be placed in the same octant.
if (Zoltan_Oct_nRegions(child[i]) > MAXOCTREGIONS) {
*/
if (Zoltan_Oct_nUniqueRegions(OCT_info,child[i]) > MAXOCTREGIONS) {
Zoltan_Oct_terminal_refine(zz, child[i],count+1);
}
}
/*
* pRegion Zoltan_Oct_regionlist(pOctant octant)
* get a copy of the octant's region list
*/
pRegion Zoltan_Oct_regionlist(pOctant oct) {
if (!Zoltan_Oct_isTerminal(oct))
abort();
return(oct->list);
}
/*
* void Zoltan_Oct_visit(pOctant octant)
*
* This routine references the following (static) global variables:
*
* partition - (RW) number of the partition we are currently working on
* total - (RW) total cost of all *previous* partitions
* pcost - (RW) partition cost for current partition
* optcost - (RO) optimal partition cost
*/
static void Zoltan_Oct_visit(ZZ *zz, pOctant octant, float *part_sizes) {
float cost; /* Cost of this octant */
float togo; /* Remaining room in current partition */
float behind; /* How many to make up for from all prev parts */
pOctant children[8]; /* children of the octant */
int i; /* index counter */
COORD origin; /* center of the octant */
double volume; /* volume of the octant */
double prod[3]; /* product of octant origin and its volume */
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
DFS_Part_Count++;
cost = Zoltan_Oct_costs_value(octant); /* get the cost of the octant */
/*behind = partition * optcost - total;*/ /* calcuate how much behind */
behind = (tmpcost*globalcost) - total; /* calcuate how much behind */
if(0)
fprintf(stderr,"LGG[%d] pc=%f, c=%f, ps=%f, b=%f\n", partition, pcost,
cost, optsize, behind);
/* If octant does not overflow the current partition, then use it. */
/*if( cost==0 || (pcost+cost) <= (optcost+behind)) {*/
if(cost==0 || ((pcost+cost) <= (optsize+behind))) {
Zoltan_Oct_tag_subtree(OCT_info,octant,partition);
/*fprintf(stderr,"LGG[%d] pc=%f, c=%f, ps=%f, b=%f\n", partition, pcost,
cost, optsize, behind);*/
pcost+=cost;
Zoltan_Oct_origin_volume(octant, origin, &volume);
vector_cmult(prod,volume,origin);
pmass+=volume;
vector_add(pcoord,pcoord,prod);
return;
}
/*
* Can't use entire octant because it is too big. If it has suboctants,
* visit them.
*/
if (!Zoltan_Oct_isTerminal(octant)) {
Zoltan_Oct_modify_newpid(octant, partition); /* Nonterm */
Zoltan_Oct_children(octant,children);
for (i=0; i<8; i++) /* Simple - just visit in order */
if(children[i] && Zoltan_Oct_POct_local(OCT_info, octant,i))
Zoltan_Oct_visit(zz,children[i],part_sizes);
return;
}
/*
* No suboctants!
* We've hit bottom - have to decide whether to add to
* the current partition or start a new one.
*/
togo = behind + optsize - pcost;
/*printf("proc=%d, part=%d, b=%f, pcost=%f, cost=%f, os=%f\n",
zz->Proc, partition, behind, pcost, cost, optsize);*/
if ((cost-togo) >= togo) {
/*printf("proc=%d, part=%d, togo=%f, pcost=%f, cost=%f, g=%f\n",
zz->Proc, partition, togo, pcost, cost, globalcost);*/
/*
* End current part and start new one. We are more "over" than "under"
*/
tmpcost += part_sizes[partition];
partition++; /* Move on to next partition */
while((part_sizes[partition] == 0) && (partition < (zz->Num_Proc - 1)))
partition++;
optsize = part_sizes[partition]*globalcost;
total += pcost;
pcost = 0;
pmass = 0;
vector_set_comp(pcoord,0,0,0);
}
/*** Add terminal octant to current partition */
Zoltan_Oct_modify_newpid(octant, partition);
pcost += cost;
Zoltan_Oct_origin_volume(octant, origin, &volume);
vector_cmult(prod,volume,origin);
pmass += volume;
vector_add(pcoord,pcoord,prod);
}
