/*
* Zoltan_Oct_subtree_dref(oct)
*
* Coarsen octree so that leaf octants do not have less than MINOCTREGIONS
* regions. Refinement takes precedence, so coarsening will not take place
* unless all subtrees agree on it.
*/
static int Zoltan_Oct_subtree_dref(ZZ *zz, OCT_Global_Info *OCT_info,
pOctant oct)
{
pOctant child; /* child of an octant */
int coarsen; /* flag to indicate need to coarsen */
int i; /* index counter */
int nregions; /* number of regions */
int total; /* total number of regions */
/* if terminal octant, cannot coarsen on own */
if (Zoltan_Oct_isTerminal(oct)) {
nregions=Zoltan_Oct_nRegions(oct);
if (nregions > (MAXOCTREGIONS*2) ) {
fprintf(stderr,
"OCT Zoltan_Oct_subtree_dref: warning: too many (%d) regions "
"in oct (id=%d)\n",Zoltan_Oct_nRegions(oct),Zoltan_Oct_id(oct));
return(-1);
}
else
if (nregions < MINOCTREGIONS)
return(nregions);
else
return(-1);
}
coarsen=1; /* assume to be coarsen */
total=0;
/* look at each child, see if they need to be coarsened */
for (i=0; coarsen && i<8; i++) {
child = Zoltan_Oct_child(oct,i);
/* if child is off processor cannot coarsen */
/* if (!Zoltan_Oct_local(child) ) */
if(!Zoltan_Oct_POct_local(OCT_info, oct, i))
coarsen=0;
else {
/* get the number of region of the child */
nregions=Zoltan_Oct_subtree_dref(zz, OCT_info,child);
if (nregions<0)
coarsen=0;
else
total+=nregions; /* total the region count */
}
}
/* check if octant can be coarsened */
if (coarsen && total<MAXOCTREGIONS) {
Zoltan_Oct_terminal_coarsen(zz, OCT_info,oct);
if (total < MINOCTREGIONS)
return(total);
else
return(-1);
}
return(-1);
}
/* KDDKDDFREE Changed root to *root to allow NULL from ZOLTAN_FREE to propagate
* KDDKDDFREE back to calling routine. */
int Zoltan_Oct_POct_delTree(OCT_Global_Info *OCT_info, pOctant *root) {
int i; /* index counter */
pOctant child; /* child of an octant */
if(*root == NULL)
return 1;
if(Zoltan_Oct_isTerminal(*root)) {
if(Zoltan_Oct_nRegions(*root))
Zoltan_Oct_clearRegions(*root);
Zoltan_Oct_POct_free(OCT_info, root);
}
else {
for(i=0; i<8; i++) {
child = Zoltan_Oct_child(*root, i);
if(child != NULL && Zoltan_Oct_POct_local(OCT_info,*root, i)) {
Zoltan_Oct_POct_delTree(OCT_info,&child);
/* KDDKDDFREE propagate NULL from Zoltan_Oct_POct_delTree to root->child */
(*root)->child[i] = NULL;
(*root)->cpid[i] = -1;
}
/* KDDKDDFREE Added this condition so that tests (in other parts of the
* KDDKDDFREE code) for NULL children work */
else if (child != NULL) {
(*root)->child[i] = NULL;
(*root)->cpid[i] = -1;
}
/* END KDDKDDFREE */
}
Zoltan_Oct_POct_free(OCT_info, root);
}
return 1;
}
/*
* void tag_regions()
* Iterates through the list of octants on the processor and finds which
* are to be migrated. It then looks at the region list for those octants
* and stores the migrating regions into the export_tags array.
*/
static int tag_regions(ZZ *zz,
pOctant *octs,
int *newpids,
int nocts,
Region **exported_tags,
ZOLTAN_ID_PTR *exported_gids,
ZOLTAN_ID_PTR *exported_lids,
int *nsentags,
int **tag_pids,
Region **p_tags,
ZOLTAN_ID_PTR *p_gids,
ZOLTAN_ID_PTR *p_lids,
int *npimtags,
float *c2,
int *max_objs)
{
char *yo = "tag_regions";
int i; /* index counter */
pRegion regionlist; /* list of region on this processor */
int index; /* index counter */
int index2; /* yet another index counter */
int count; /* count of objects exported form this processor */
int count2; /* count of objects that are kept on processor */
int count3;
int *exported_pids; /* array of pids where regions are being exported to */
pRegion mtags; /* object tags of objects to be migrated */
pRegion ptags; /* tags of objects that were previously migrated */
float ex_load;
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
ex_load = 0;
(*max_objs) = 0;
if (!nsentags)
return ierr;
/* find how many objects have been exported */
count = 0;
/* find number of local objs to export */
count2 = 0;
count3 = 0;
for (i=0; i<nocts; i++) {
if(Zoltan_Oct_isTerminal(octs[i])) {
(*max_objs) += Zoltan_Oct_nRegions(octs[i]);
regionlist = Zoltan_Oct_regionlist(octs[i]);
while(regionlist != NULL) {
count3++;
if(regionlist->Proc != zz->Proc) {
count++;
if(newpids[i] != zz->Proc)
regionlist->newProc = newpids[i];
else
regionlist->newProc = zz->Proc;
}
else {
if(newpids[i] != zz->Proc) {
count2++;
regionlist->newProc = newpids[i];
}
else
regionlist->newProc = zz->Proc;
}
regionlist = regionlist->next; /* get next region */
}
}
}
#if 0
{
{
if (newpids[i]!=zz->Proc) {
count+=Zoltan_Oct_nRegions(octs[i]);
}
else {
pRegion regions;
regions = Zoltan_Oct_regionlist(octs[i]);
while(regions != NULL) {
if(regions->Proc != zz->Proc)
count2++;
regions = regions->next;
}
}
}
}
#endif
/* set up the return pointers */
*nsentags = count;
*npimtags = count2;
if (!exported_tags) {
return ierr;
}
if (count > 0) {
/* allocate some space */
if((mtags=(pRegion)ZOLTAN_MALLOC((unsigned)count*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((exported_pids = (int *)ZOLTAN_MALLOC((unsigned)count*sizeof(int))) ==
NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
return ZOLTAN_MEMERR;
}
*exported_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count);
*exported_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count);
if(!(*exported_gids) || (num_lid_entries && !(*exported_lids))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
return ZOLTAN_MEMERR;
}
}
else {
mtags = NULL;
exported_pids = NULL;
*exported_gids = NULL;
*exported_lids = NULL;
}
/* set up return pointers */
*exported_tags=mtags;
*tag_pids = exported_pids;
if (count2 > 0) {
/* allocate some space */
if((ptags=(pRegion)ZOLTAN_MALLOC((unsigned)count2*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
ZOLTAN_FREE(exported_gids);
ZOLTAN_FREE(exported_lids);
return ZOLTAN_MEMERR;
}
*p_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count2);
*p_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count2);
if(!(*p_gids) || (num_lid_entries && !(*p_lids))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
ZOLTAN_FREE(exported_gids);
ZOLTAN_FREE(exported_lids);
ZOLTAN_FREE(&ptags);
ZOLTAN_FREE(p_gids);
ZOLTAN_FREE(p_lids);
return ZOLTAN_MEMERR;
}
}
else {
ptags = NULL;
*p_gids = NULL;
*p_lids = NULL;
}
/* set up return pointers */
*p_tags=ptags;
index = index2 = 0;
for (i=0; i<nocts; i++) {
if(Zoltan_Oct_isTerminal(octs[i])) {
regionlist = Zoltan_Oct_regionlist(octs[i]);
while(regionlist != NULL) {
if(regionlist->Proc != zz->Proc) {
/* place information in the appropritate array */
mtags[index] = *regionlist;
ZOLTAN_SET_GID(zz, &((*exported_gids)[index*num_gid_entries]),
regionlist->Global_ID);
ZOLTAN_SET_LID(zz, &((*exported_lids)[index*num_lid_entries]),
regionlist->Local_ID);
/*ex_load += (float)(regionlist->Weight);*/
exported_pids[index] = regionlist->Proc;
index++; /* increment counter */
}
else if(newpids[i] != zz->Proc) {
ptags[index2] = *regionlist; /* get region information */
ZOLTAN_SET_GID(zz, &((*p_gids)[index2*num_gid_entries]),
regionlist->Global_ID);
ZOLTAN_SET_LID(zz, &((*p_lids)[index2*num_lid_entries]),
regionlist->Local_ID);
index2++; /* increment counter */
}
regionlist = regionlist->next; /* get next region */
}
}
}
if (index!=count) { /* error check */
ZOLTAN_TRACE_DETAIL(zz, yo,
"Fatal error, inconsistent number of regions.\n");
return ZOLTAN_FATAL;
}
*c2 = ex_load;
return ierr;
}
/*
* void lb_oct_init();
*
* initialize the calls needed to start the octree load balancing rounties
*/
static int lb_oct_init(
ZZ *zz, /* The Zoltan structure with info for
the OCTPART balancer. */
int *num_export, /* Number of non-local objs assigned to this
processor in the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition. */
int **export_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition. */
int **export_to_part, /* Returned value: array of partitions to
which objects are imported.
KDDKDD Assume #parts==#procs. */
int oct_dim, /* Dimension of method (2D or 3D) */
int oct_method, /* Flag specifying curve to be used. */
int oct_maxoctregions, /* max # of objects in leaves of octree. */
int oct_minoctregions, /* min # of objects in leaves of octree. */
int oct_output_level, /* Flag specifying amount of output. */
int oct_wgtflag, /* Flag specifying use of object weights. */
float *part_sizes /* Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
)
{
char *yo = "lb_oct_init";
OCT_Global_Info *OCT_info;
int nsentags; /* number of tags being sent */
pRegion export_regs; /* */
int nrectags; /* number of tags received */
int kk;
double time1,time2; /* timers */
double timestart,timestop; /* timers */
double timers[4]; /* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
int counters[6]; /* diagnostic counts
0 = # of median iterations
1 = # of objects sent
2 = # of objects received
3 = most objects this proc ever owns
*/
float c[4];
int createpartree = 0;
/*int num_gid_entries = zz->Num_GID;*/
/*int num_lid_entries = zz->Num_LID;*/
ZOLTAN_TRACE_ENTER(zz, yo);
MPI_Barrier(zz->Communicator);
timestart = MPI_Wtime();
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = 0;
counters[4] = 0;
counters[5] = 0;
c[0] = 0;
c[1] = 0;
c[2] = 0;
c[3] = 0;
timers[1] = 0.0;
timers[2] = 0.0;
timers[3] = 0.0;
nsentags = nrectags = 0;
if(zz->LB.Data_Structure == NULL) {
OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim);
Zoltan_Oct_set_method(OCT_info, oct_method);
Zoltan_Oct_set_maxregions(oct_maxoctregions);
Zoltan_Oct_set_minregions(oct_minoctregions);
createpartree = 1;
}
else {
OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
}
/* create the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data");
Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1],
&counters[2], &counters[3], &c[0],
createpartree);
time2 = MPI_Wtime();
timers[0] = time2 - time1; /* time took to create octree */
/* Zoltan_Oct_POct_printResults(OCT_info); */
/* partition the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition");
/* old call to dfs_paritition: */
#if 0
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]);
#else
/***************************
if(zz->Proc == 0) {
int debug_i;
for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) {
fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]);
}
}
****************************/
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes);
#endif
time2 = MPI_Wtime();
timers[1] = time2 - time1; /* time took to partition octree */
if (oct_output_level > 2) {
Zoltan_Oct_Plots(zz);
}
/* set up tags for migrations */
time1 = MPI_Wtime();
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate");
Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags,
&c[2], &c[3], &counters[3], &counters[5]);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags");
if (zz->LB.Return_Lists) {
*num_export = nrectags;
if (nrectags > 0)
Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids,
export_procs, export_to_part, nrectags,
export_regs);
}
time2 = MPI_Wtime();
timers[2] = time2 - time1; /* time took to setup migration */
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
/* count the number of objects on this processor */
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
counters[4] = nsentags;
MPI_Barrier(zz->Communicator);
timestop = MPI_Wtime();
if (oct_output_level > 0) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats");
Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c,
oct_output_level);
}
for (kk = 0; kk < nrectags; kk++) {
ZOLTAN_FREE(&(export_regs[kk].Global_ID));
ZOLTAN_FREE(&(export_regs[kk].Local_ID));
}
ZOLTAN_FREE(&export_regs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear");
Zoltan_Oct_global_clear(OCT_info);
/* KDDKDD Don't understand how re-used octree will work, especially without
* KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything;
* KDDKDD we can move back to saving some of the tree later.
*/
Zoltan_Oct_Free_Structure(zz);
/* KDDKDD END */
/* Temporary return value until error codes are fully implemented. */
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
