/* * oct_global_clear() * * delete all regions from all octants on the local processor * */ static void Zoltan_Oct_global_clear(OCT_Global_Info * OCT_info) { pRList RootList; /* list of all local roots */ pOctant Oct; /* * iterate through the list of local roots * traverse down the subtree * delete regions associated with octant */ RootList = Zoltan_Oct_POct_localroots(OCT_info); while ((Oct = RL_nextRootOctant(&RootList))) { while(Oct) { if(Zoltan_Oct_isTerminal(Oct)) Zoltan_Oct_clearRegions(Oct); Oct = Zoltan_Oct_POct_nextDfs(OCT_info, Oct); } } }
/* * void Zoltan_Oct_tag_subtree(pOctant octant, int partition_number) * * marks all the octants within the subtree to be in the current partition */ static void Zoltan_Oct_tag_subtree(OCT_Global_Info *OCT_info,pOctant octant, int part) { pOctant children[8]; /* children of octant */ int i; /* index counter */ /* modify NPID so octant know where to migrate to */ Zoltan_Oct_modify_newpid(octant, part); if (Zoltan_Oct_isTerminal(octant)) return; /* if octant has children, have to tag them too */ Zoltan_Oct_children(octant,children); for (i=0; i<8; i++) /* Simple - just visit in order */ /* if (children[i] && Zoltan_Oct_local(OCT_info, children[i])) */ if(children[i] && Zoltan_Oct_POct_local(OCT_info, octant,i)) Zoltan_Oct_tag_subtree(OCT_info,children[i],part); }
/* * Zoltan_Oct_findOctant(oct, coord) * (replaces : PO_findOctant(oct,coord)) * * * find the octant in a subtree containing coord (if coord * is not in the subtree, returns the closest octant in the subtree). * NOTE: return NULL if we hit an off-processor link * */ static pOctant Zoltan_Oct_findOctant(OCT_Global_Info *OCT_info,pOctant oct, COORD coord) { pOctant child; /* child of an octant */ int cnum; /* child number */ /* if octant is terminal, then this is the right octant */ if (Zoltan_Oct_isTerminal(oct)) return(oct); /* find closest child to coord */ cnum = Zoltan_Oct_child_which_wrapper(OCT_info,oct,coord); child = Zoltan_Oct_child(oct, cnum); /* get that child */ /* ATTN: are these local checks necessary? */ /* if ( !Zoltan_Oct_POct_local(child) ) */ if(!Zoltan_Oct_POct_local(OCT_info,oct, cnum)) /* make sure oct is local */ return(NULL); /* recursivly search down the tree */ return(Zoltan_Oct_findOctant(OCT_info,child,coord)); }
/* * int Zoltan_Oct_dfs_SetIds(pOctant octant, int number_of_previous_octants) * * sets the ids of all the octants so that there is a global numbering */ static int Zoltan_Oct_dfs_SetIds(OCT_Global_Info *OCT_info, pOctant oct, int nprevoct) { int id, /* the id number of an octant */ i; /* index counter */ int pid; pOctant child; /* ith child of an octant */ if(Zoltan_Oct_isTerminal(oct)) { id = Zoltan_Oct_id(oct); Zoltan_Oct_setID(oct, id+nprevoct); /* now have global id's */ } else { for(i=0; i<8; i++) { child = Zoltan_Oct_child(oct, i); pid = Zoltan_Oct_Cpid(oct,i); if ((pid == OCT_info->OCT_localpid) && child != NULL) Zoltan_Oct_dfs_SetIds(OCT_info,child, nprevoct); } id = Zoltan_Oct_id(oct); Zoltan_Oct_setID(oct, id+nprevoct); /* now have global id's */ } return 0; }
/* * Zoltan_Oct_subtree_insert(oct,region) * * Insert region in oct, carrying out multiple refinement if necessary */ int Zoltan_Oct_subtree_insert(ZZ *zz, pOctant oct, pRegion region) { OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); /* if oct is not terminal, find leaf node the centroid can be attahced to */ if (!Zoltan_Oct_isTerminal(oct)) { oct=Zoltan_Oct_findOctant(OCT_info, oct,region->Coord); } if (!oct) return(0); /* add the region to the octant */ Zoltan_Oct_addRegion(zz, oct, region); /* check if octant has too many regions and needs to be refined */ /* KDDKDD Replaced the following to allow multiple regions with the * KDDKDD same coordinates to be placed in a single octant. if(Zoltan_Oct_nRegions(oct) > MAXOCTREGIONS) */ if(Zoltan_Oct_nUniqueRegions(OCT_info, oct) > MAXOCTREGIONS) Zoltan_Oct_terminal_refine(zz, oct,0); /* After, dest may be nonterm */ return(1); }
/* * float Zoltan_Oct_costs_subtree_compute(pOctant octant, int sequence_number) * * Do a DFS on the octree, calculating the costs for any given subtree. * (Subtree is defined as any octant and all its descendants.) * * Tag every octant visited with a sequence number, automatically * incrementing the sequence number. * * NOTE: must call Zoltan_Oct_costs_init() first */ static float Zoltan_Oct_costs_subtree_compute(OCT_Global_Info *OCT_info,pOctant octant, int *seq) { pOctant children[8]; /* the children of the octant */ float c = 0; /* cost of each subtree */ int i = 0; /* index counter */ /* #ifdef LGG_MIGOCT */ Zoltan_Oct_setID(octant,(*seq)++); /* set new ID for local ordering */ /* #endif */ /* LGG_MIGOCT */ if (!Zoltan_Oct_isTerminal(octant)) { /* get the children of each octant */ Zoltan_Oct_children(octant,children); /* sum the cost for each child to get octant's cost */ for (i=0; i<8; i++) if(children[i] && Zoltan_Oct_POct_local(OCT_info, octant, i)) c += Zoltan_Oct_costs_subtree_compute(OCT_info,children[i], seq); } else /* terminal */ c=Zoltan_Oct_costs_weight(octant); /* set the cost data to the octant */ Zoltan_Oct_modify_cost(octant, c); return(c); }
/* * void tag_regions() * Iterates through the list of octants on the processor and finds which * are to be migrated. It then looks at the region list for those octants * and stores the migrating regions into the export_tags array. */ static int tag_regions(ZZ *zz, pOctant *octs, int *newpids, int nocts, Region **exported_tags, ZOLTAN_ID_PTR *exported_gids, ZOLTAN_ID_PTR *exported_lids, int *nsentags, int **tag_pids, Region **p_tags, ZOLTAN_ID_PTR *p_gids, ZOLTAN_ID_PTR *p_lids, int *npimtags, float *c2, int *max_objs) { char *yo = "tag_regions"; int i; /* index counter */ pRegion regionlist; /* list of region on this processor */ int index; /* index counter */ int index2; /* yet another index counter */ int count; /* count of objects exported form this processor */ int count2; /* count of objects that are kept on processor */ int count3; int *exported_pids; /* array of pids where regions are being exported to */ pRegion mtags; /* object tags of objects to be migrated */ pRegion ptags; /* tags of objects that were previously migrated */ float ex_load; int ierr = ZOLTAN_OK; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; ex_load = 0; (*max_objs) = 0; if (!nsentags) return ierr; /* find how many objects have been exported */ count = 0; /* find number of local objs to export */ count2 = 0; count3 = 0; for (i=0; i<nocts; i++) { if(Zoltan_Oct_isTerminal(octs[i])) { (*max_objs) += Zoltan_Oct_nRegions(octs[i]); regionlist = Zoltan_Oct_regionlist(octs[i]); while(regionlist != NULL) { count3++; if(regionlist->Proc != zz->Proc) { count++; if(newpids[i] != zz->Proc) regionlist->newProc = newpids[i]; else regionlist->newProc = zz->Proc; } else { if(newpids[i] != zz->Proc) { count2++; regionlist->newProc = newpids[i]; } else regionlist->newProc = zz->Proc; } regionlist = regionlist->next; /* get next region */ } } } #if 0 { { if (newpids[i]!=zz->Proc) { count+=Zoltan_Oct_nRegions(octs[i]); } else { pRegion regions; regions = Zoltan_Oct_regionlist(octs[i]); while(regions != NULL) { if(regions->Proc != zz->Proc) count2++; regions = regions->next; } } } } #endif /* set up the return pointers */ *nsentags = count; *npimtags = count2; if (!exported_tags) { return ierr; } if (count > 0) { /* allocate some space */ if((mtags=(pRegion)ZOLTAN_MALLOC((unsigned)count*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((exported_pids = (int *)ZOLTAN_MALLOC((unsigned)count*sizeof(int))) == NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); return ZOLTAN_MEMERR; } *exported_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count); *exported_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count); if(!(*exported_gids) || (num_lid_entries && !(*exported_lids))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); return ZOLTAN_MEMERR; } } else { mtags = NULL; exported_pids = NULL; *exported_gids = NULL; *exported_lids = NULL; } /* set up return pointers */ *exported_tags=mtags; *tag_pids = exported_pids; if (count2 > 0) { /* allocate some space */ if((ptags=(pRegion)ZOLTAN_MALLOC((unsigned)count2*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); ZOLTAN_FREE(exported_gids); ZOLTAN_FREE(exported_lids); return ZOLTAN_MEMERR; } *p_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count2); *p_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count2); if(!(*p_gids) || (num_lid_entries && !(*p_lids))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); ZOLTAN_FREE(exported_gids); ZOLTAN_FREE(exported_lids); ZOLTAN_FREE(&ptags); ZOLTAN_FREE(p_gids); ZOLTAN_FREE(p_lids); return ZOLTAN_MEMERR; } } else { ptags = NULL; *p_gids = NULL; *p_lids = NULL; } /* set up return pointers */ *p_tags=ptags; index = index2 = 0; for (i=0; i<nocts; i++) { if(Zoltan_Oct_isTerminal(octs[i])) { regionlist = Zoltan_Oct_regionlist(octs[i]); while(regionlist != NULL) { if(regionlist->Proc != zz->Proc) { /* place information in the appropritate array */ mtags[index] = *regionlist; ZOLTAN_SET_GID(zz, &((*exported_gids)[index*num_gid_entries]), regionlist->Global_ID); ZOLTAN_SET_LID(zz, &((*exported_lids)[index*num_lid_entries]), regionlist->Local_ID); /*ex_load += (float)(regionlist->Weight);*/ exported_pids[index] = regionlist->Proc; index++; /* increment counter */ } else if(newpids[i] != zz->Proc) { ptags[index2] = *regionlist; /* get region information */ ZOLTAN_SET_GID(zz, &((*p_gids)[index2*num_gid_entries]), regionlist->Global_ID); ZOLTAN_SET_LID(zz, &((*p_lids)[index2*num_lid_entries]), regionlist->Local_ID); index2++; /* increment counter */ } regionlist = regionlist->next; /* get next region */ } } } if (index!=count) { /* error check */ ZOLTAN_TRACE_DETAIL(zz, yo, "Fatal error, inconsistent number of regions.\n"); return ZOLTAN_FATAL; } *c2 = ex_load; return ierr; }
/* * void Zoltan_Oct_gen_tree_from_input_data() * * This function will create a root node of on each processor which will * then be used to create an octree with regions associated with it. The * tree will then be balanced and the output used to balance "mesh regions" * on several processors. */ static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag, int *c1, int *c2, int *c3, float *c0, int createpartree) { char *yo = "Zoltan_Oct_gen_tree_from_input_data"; pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ COORD min, /* min coord bounds of objects */ max; /* max coord bounds of objects */ int num_extra; /* number of orphaned objects */ int num_objs; /* total number of local objects */ pRegion ptr, /* pointer to iterate trough region list */ ptr1; /* pointer to iterate trough region list */ pOctant root; /* root of the partition tree */ int i; /* index counter */ int count, /* count for leaf nodes in partition tree */ proc, /* proc leaf node of parition tree belongs to */ extra, /* extra leaf node flag, if not evenly divisible */ remainder; /* remainder of node, or processors to fill */ pOctant cursor, /* cursor to iterate through octant list */ cursor2, /* another cursor to iterate through octant list */ parent; /* parent of an octant */ int level, /* number of levels of refinement */ n, /* index counter */ part; /* partition counter */ Map *array; /* map of which processors own which octants */ int hold; /* used for calculating partition divisions */ int ierr = 0; #ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX}; COORD global_min, global_max; #endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */ int nroots = 0; /*test*/ /* COORD gmin,gmax; */ OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); ZOLTAN_TRACE_ENTER(zz, yo); /* * If there are no objects on this processor, do not create a root octant. * The partitioner will probably assign objects to this processor */ if(zz->Get_Num_Obj == NULL) { fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:", "Must register ZOLTAN_NUM_OBJ_FN function"); abort(); } *c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr) { fprintf(stderr, "OCT [%d] %s: Error returned from user defined " "Get_Num_Obj function.\n", zz->Proc, yo); exit (-1); } ptr1 = NULL; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds"); /* Need A Function To Get The Bounds Of The Local Objects */ Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0); #ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN /* For now, don't want to add the new query function to Zoltan. */ /* Zoltan_Oct_get_bounds appears to compute the global min and max from */ /* the object input. */ vector_set(OCT_info->OCT_gmin, min); vector_set(OCT_info->OCT_gmax, max); #else /*test*/ /*getMaxBounds(&gmin, &gmax);*/ if(zz->Get_Bounds_Geom == NULL) { fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:", "Must register Get_Bounds_Geom function"); abort(); } zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr); MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3, MPI_DOUBLE, MPI_MIN, zz->Communicator); MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3, MPI_DOUBLE, MPI_MAX, zz->Communicator); vector_set(OCT_info->OCT_gmin, global_min); vector_set(OCT_info->OCT_gmax, global_max); #endif /* * the following code segment was added to create a pseudo global octree * needed for the partitioner. The basic idea is to regroup all the * regions into something close to an octree partitioning and build the * tree from that. * NOTE: This way of doing things is very costly, especially when calling * this for the first time on a mesh not partitioned in an octree style * partitioning. */ level = 0; /* initialize level count */ /* * if more than 1 processor, need to find what level of refinement needed * to initially partition bounding box among the processors */ if(zz->Num_Proc > 1) { n = zz->Num_Proc; if(OCT_info->OCT_dimension == 2) hold = 4; else hold = 8; remainder = hold; for(; remainder > 0; level++) { int pr = (int)POW(hold, level); remainder = n - pr; } level--; } ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree"); if(createpartree) { /* create the global root octant */ root = Zoltan_Oct_POct_new(OCT_info); Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax); /* Zoltan_Oct_setOrientation(root, 0); */ /* subdivide to as many levels as calculated */ for(i=0; i<level; i++) { cursor = root; while(cursor != NULL) { if(Zoltan_Oct_isTerminal(cursor)) { cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); Zoltan_Oct_terminal_refine(zz, cursor, 0); cursor = cursor2; } else cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); } } #if 0 if(zz->Proc == 0) for(i=0; i<8; i++) if(Zoltan_Oct_child(root, i) == NULL) fprintf(stderr,"NULL child pointer\n"); else fprintf(stderr, "child %d exists\n", i); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array"); /* this part creates the map array */ if(OCT_info->OCT_dimension == 2) { hold = (int)POW(4, level); /* ignoring the z+ octants */ if(hold == 0) hold = 1; } else hold = (int)POW(8, level); part = hold / zz->Num_Proc; /* how many octants per partition */ remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */ extra = zz->Num_Proc - remainder;/* where to start adding extra octants */ array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */ if(array == NULL) { fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n", zz->Proc); abort(); } /* initialize variables */ proc = 0; count = 0; i = 0; cursor = root; while(cursor != NULL) { cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) { if(proc == extra) { part++; extra = -1; } if(count != part) { array[i].npid = proc; array[i].list = RL_initRootList(); Zoltan_Oct_bounds(cursor, min, max); vector_set(array[i].min, min); vector_set(array[i].max, max); count++; } else { count = 1; proc++; array[i].npid = proc; array[i].list = RL_initRootList(); Zoltan_Oct_bounds(cursor, min, max); vector_set(array[i].min, min); vector_set(array[i].max, max); } if(proc == zz->Proc) { array[i].npid = -1; /* KDDKDD Added RL_freeList below. The * KDDKDD implementation from RPI leaked memory because the * KDDKDD test cases for setting array[i].list were not mutually * KDDKDD exclusive. Freeing the list produces the result we got * KDDKDD before, without the memory leak. */ /* LGG -- it seems to me that this array[i].list assignment is * not really necessary. It looks as though it has already been * assigned with the same information from the prev if-else * commented out RL_freeList(), and RL_initRootList() */ /*RL_freeList(&(array[i].list));*/ /* KDDKDD End addition */ /*array[i].list = RL_initRootList();*/ parent = Zoltan_Oct_parent(cursor); if(parent != NULL) Zoltan_Oct_setchild(parent, cursor->which, NULL); /* octant into local root list */ Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1); Zoltan_Oct_setMapIdx(cursor, i); nroots++; /* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc); octant into local root list */ } i++; } cursor = cursor2; } RootList = Zoltan_Oct_POct_localroots(OCT_info); RootOct = RL_nextRootOctant(&RootList); if(RootOct != root) { /* KDDKDDFREE changed root to &root to allow root to be reset to NULL */ Zoltan_Oct_POct_delTree(OCT_info,&root); } OCT_info->map = array; OCT_info->mapsize = hold; } /* * attach the regions to the root... Zoltan_Oct_fix will create the octree * starting with the root and subdividing as needed */ num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans"); Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map, c1, c2); /* ZOLTAN_FREE(&array); */ while(ptr1 != NULL) { ptr = ptr1->next; ZOLTAN_FREE(&(ptr1->Global_ID)); ZOLTAN_FREE(&(ptr1->Local_ID)); ZOLTAN_FREE(&ptr1); ptr1 = ptr; } ZOLTAN_TRACE_EXIT(zz, yo); }
/* * void lb_oct_init(); * * initialize the calls needed to start the octree load balancing rounties */ static int lb_oct_init( ZZ *zz, /* The Zoltan structure with info for the OCTPART balancer. */ int *num_export, /* Number of non-local objs assigned to this processor in the new decomposition. */ ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for non-local objects in this processor's new decomposition. */ ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for non-local objects in this processor's new decomposition. */ int **export_procs, /* Returned value: array of processor IDs for processors owning the non-local objects in this processor's new decomposition. */ int **export_to_part, /* Returned value: array of partitions to which objects are imported. KDDKDD Assume #parts==#procs. */ int oct_dim, /* Dimension of method (2D or 3D) */ int oct_method, /* Flag specifying curve to be used. */ int oct_maxoctregions, /* max # of objects in leaves of octree. */ int oct_minoctregions, /* min # of objects in leaves of octree. */ int oct_output_level, /* Flag specifying amount of output. */ int oct_wgtflag, /* Flag specifying use of object weights. */ float *part_sizes /* Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ ) { char *yo = "lb_oct_init"; OCT_Global_Info *OCT_info; int nsentags; /* number of tags being sent */ pRegion export_regs; /* */ int nrectags; /* number of tags received */ int kk; double time1,time2; /* timers */ double timestart,timestop; /* timers */ double timers[4]; /* diagnostic timers 0 = start-up time before recursion 1 = time before median iterations 2 = time in median iterations 3 = communication time */ int counters[6]; /* diagnostic counts 0 = # of median iterations 1 = # of objects sent 2 = # of objects received 3 = most objects this proc ever owns */ float c[4]; int createpartree = 0; /*int num_gid_entries = zz->Num_GID;*/ /*int num_lid_entries = zz->Num_LID;*/ ZOLTAN_TRACE_ENTER(zz, yo); MPI_Barrier(zz->Communicator); timestart = MPI_Wtime(); /* initialize timers and counters */ counters[0] = 0; counters[1] = 0; counters[2] = 0; counters[3] = 0; counters[4] = 0; counters[5] = 0; c[0] = 0; c[1] = 0; c[2] = 0; c[3] = 0; timers[1] = 0.0; timers[2] = 0.0; timers[3] = 0.0; nsentags = nrectags = 0; if(zz->LB.Data_Structure == NULL) { OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim); Zoltan_Oct_set_method(OCT_info, oct_method); Zoltan_Oct_set_maxregions(oct_maxoctregions); Zoltan_Oct_set_minregions(oct_minoctregions); createpartree = 1; } else { OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); } /* create the octree structure */ time1 = MPI_Wtime(); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data"); Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1], &counters[2], &counters[3], &c[0], createpartree); time2 = MPI_Wtime(); timers[0] = time2 - time1; /* time took to create octree */ /* Zoltan_Oct_POct_printResults(OCT_info); */ /* partition the octree structure */ time1 = MPI_Wtime(); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition"); /* old call to dfs_paritition: */ #if 0 Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]); #else /*************************** if(zz->Proc == 0) { int debug_i; for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) { fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]); } } ****************************/ Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes); #endif time2 = MPI_Wtime(); timers[1] = time2 - time1; /* time took to partition octree */ if (oct_output_level > 2) { Zoltan_Oct_Plots(zz); } /* set up tags for migrations */ time1 = MPI_Wtime(); #if 0 /* KDDKDD -- Count is never used; why is it computed? */ { pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ int count = 0; RootList = Zoltan_Oct_POct_localroots(OCT_info); while((RootOct = RL_nextRootOctant(&RootList))) { while(RootOct) { if(Zoltan_Oct_isTerminal(RootOct)) { count += Zoltan_Oct_nRegions(RootOct); } RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct); } } } #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate"); Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags, &c[2], &c[3], &counters[3], &counters[5]); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags"); if (zz->LB.Return_Lists) { *num_export = nrectags; if (nrectags > 0) Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids, export_procs, export_to_part, nrectags, export_regs); } time2 = MPI_Wtime(); timers[2] = time2 - time1; /* time took to setup migration */ #if 0 /* KDDKDD -- Count is never used; why is it computed? */ { /* count the number of objects on this processor */ pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ int count = 0; RootList = Zoltan_Oct_POct_localroots(OCT_info); while((RootOct = RL_nextRootOctant(&RootList))) { while(RootOct) { if(Zoltan_Oct_isTerminal(RootOct)) { count += Zoltan_Oct_nRegions(RootOct); } RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct); } } } #endif counters[4] = nsentags; MPI_Barrier(zz->Communicator); timestop = MPI_Wtime(); if (oct_output_level > 0) { ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats"); Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c, oct_output_level); } for (kk = 0; kk < nrectags; kk++) { ZOLTAN_FREE(&(export_regs[kk].Global_ID)); ZOLTAN_FREE(&(export_regs[kk].Local_ID)); } ZOLTAN_FREE(&export_regs); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear"); Zoltan_Oct_global_clear(OCT_info); /* KDDKDD Don't understand how re-used octree will work, especially without * KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything; * KDDKDD we can move back to saving some of the tree later. */ Zoltan_Oct_Free_Structure(zz); /* KDDKDD END */ /* Temporary return value until error codes are fully implemented. */ ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_OK); }
/* * Zoltan_Oct_terminal_refine(oct) * * subdivide a terminal octant and divvy up * its regions to the 8 children; recurse if * necessary to satisfy MAXOCTREGIONS * */ static void Zoltan_Oct_terminal_refine(ZZ *zz, pOctant oct,int count) { COORD min, /* coordinates of minimum bounds of region */ max, /* coordinates of maximum bounds of region */ origin; /* origin of region */ pOctant child[8]; /* array of child octants */ int cnum; /* child number */ int i; /* index counter */ pRegion region; /* a region to be associated to an octant */ pRegion entry; COORD cmin[8], cmax[8]; OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); for(i=0;i<3;i++) min[i] = max[i] = 0; /* upper limit of refinement levels */ /* ATTN: may not be used anymore, but can be put back in if necessary */ if (count>=20) { fprintf(stderr, "OCT ERROR: Zoltan_Oct_terminal_refine: bailing out at " "10 levels\n"); abort(); } oct_nref++; /* increment refinement counter */ /* octant should be terminal in order to be refined (subdivided) */ if (!Zoltan_Oct_isTerminal(oct)) { fprintf(stderr,"OCT ref_octant: oct not terminal\n"); abort(); } /* get the bounds of an octant */ Zoltan_Oct_bounds(oct,min,max); /* calculate the origin from the bounds */ Zoltan_Oct_bounds_to_origin(min,max,origin); region = Zoltan_Oct_regionlist(oct); /* Get list while still terminal */ oct->list = NULL; /* remove regions from octant, it won't be terminal */ /* create the children and set their id's */ Zoltan_Oct_child_bounds_wrapper(OCT_info,oct, cmin, cmax); for (i=0; i<8; i++) { if(OCT_info->OCT_dimension == 2) { /* KDDKDD 3/01 see changes to Zoltan_Oct_child_bounds_wrapper that allow this * KDDKDD 3/01 test to work for GRAY and HILBERT mappings. */ if(cmin[i][2] > OCT_info->OCT_gmin[2]) { /* ignore the z+ octants */ child[i] = NULL; continue; } } child[i]=Zoltan_Oct_POct_new(OCT_info); /* create a new octant */ child[i]->dir = Zoltan_Oct_get_child_dir(OCT_info, oct->dir, i); /* create a new octant */ /* set the child->parent link */ Zoltan_Oct_POct_setparent(OCT_info, child[i], oct, zz->Proc); Zoltan_Oct_setchildnum(child[i], i); /* which child of the parent */ Zoltan_Oct_setchild(oct, i, child[i]); /* set the parent->child link */ #ifdef LGG_MIGOCT Zoltan_Oct_setID(child[i], Zoltan_Oct_nextId()); /* set child id num */ #endif /* LGG_MIGOCT */ Zoltan_Oct_setbounds(child[i], cmin[i], cmax[i]); /* set child bounds */ Zoltan_Oct_setCpid(oct, i, zz->Proc); /* set child to be a local oct */ /* Zoltan_Oct_setOrientation(child[i], Zoltan_Oct_child_orientation(oct->orientation, oct->which)); */ } /* assign newly created children to child array*/ if(OCT_info->OCT_dimension == 3) { if(Zoltan_Oct_children(oct, child) != 8) { /* * if subdivision of oct was successful, oct should have 8 children; * thus a return value of 0 here is a fatal error */ fprintf(stderr, "OCT ref_octant: subdivide failed, %d children.\n", Zoltan_Oct_children(oct, child)); abort(); } } else if(Zoltan_Oct_children(oct, child) != 4) { /* * if subdivision of oct was successful, oct should have 4 children; * thus a return value of 0 here is a fatal error */ fprintf(stderr, "OCT ref_octant:subdivide failed, %d children, expected 4\n", Zoltan_Oct_children(oct, child)); abort(); } /* iterate through and find which child each region should belong to */ while(region != NULL) { entry = region->next; cnum=Zoltan_Oct_child_which_wrapper(OCT_info,oct, region->Coord); /* add region to octant's regionlist */ Zoltan_Oct_addRegion(zz, child[cnum], region); ZOLTAN_FREE(&(region->Global_ID)); ZOLTAN_FREE(&(region->Local_ID)); ZOLTAN_FREE(®ion); region = entry; } for (i=0; i<8; i++) /* Recursion */ if(child[i] != NULL) /* KDDKDD Replaced the following to allow multiple regions with the * KDDKDD same coordinates to be placed in the same octant. if (Zoltan_Oct_nRegions(child[i]) > MAXOCTREGIONS) { */ if (Zoltan_Oct_nUniqueRegions(OCT_info,child[i]) > MAXOCTREGIONS) { Zoltan_Oct_terminal_refine(zz, child[i],count+1); } }
/* * pRegion Zoltan_Oct_regionlist(pOctant octant) * get a copy of the octant's region list */ pRegion Zoltan_Oct_regionlist(pOctant oct) { if (!Zoltan_Oct_isTerminal(oct)) abort(); return(oct->list); }
/* * void Zoltan_Oct_visit(pOctant octant) * * This routine references the following (static) global variables: * * partition - (RW) number of the partition we are currently working on * total - (RW) total cost of all *previous* partitions * pcost - (RW) partition cost for current partition * optcost - (RO) optimal partition cost */ static void Zoltan_Oct_visit(ZZ *zz, pOctant octant, float *part_sizes) { float cost; /* Cost of this octant */ float togo; /* Remaining room in current partition */ float behind; /* How many to make up for from all prev parts */ pOctant children[8]; /* children of the octant */ int i; /* index counter */ COORD origin; /* center of the octant */ double volume; /* volume of the octant */ double prod[3]; /* product of octant origin and its volume */ OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); DFS_Part_Count++; cost = Zoltan_Oct_costs_value(octant); /* get the cost of the octant */ /*behind = partition * optcost - total;*/ /* calcuate how much behind */ behind = (tmpcost*globalcost) - total; /* calcuate how much behind */ if(0) fprintf(stderr,"LGG[%d] pc=%f, c=%f, ps=%f, b=%f\n", partition, pcost, cost, optsize, behind); /* If octant does not overflow the current partition, then use it. */ /*if( cost==0 || (pcost+cost) <= (optcost+behind)) {*/ if(cost==0 || ((pcost+cost) <= (optsize+behind))) { Zoltan_Oct_tag_subtree(OCT_info,octant,partition); /*fprintf(stderr,"LGG[%d] pc=%f, c=%f, ps=%f, b=%f\n", partition, pcost, cost, optsize, behind);*/ pcost+=cost; Zoltan_Oct_origin_volume(octant, origin, &volume); vector_cmult(prod,volume,origin); pmass+=volume; vector_add(pcoord,pcoord,prod); return; } /* * Can't use entire octant because it is too big. If it has suboctants, * visit them. */ if (!Zoltan_Oct_isTerminal(octant)) { Zoltan_Oct_modify_newpid(octant, partition); /* Nonterm */ Zoltan_Oct_children(octant,children); for (i=0; i<8; i++) /* Simple - just visit in order */ if(children[i] && Zoltan_Oct_POct_local(OCT_info, octant,i)) Zoltan_Oct_visit(zz,children[i],part_sizes); return; } /* * No suboctants! * We've hit bottom - have to decide whether to add to * the current partition or start a new one. */ togo = behind + optsize - pcost; /*printf("proc=%d, part=%d, b=%f, pcost=%f, cost=%f, os=%f\n", zz->Proc, partition, behind, pcost, cost, optsize);*/ if ((cost-togo) >= togo) { /*printf("proc=%d, part=%d, togo=%f, pcost=%f, cost=%f, g=%f\n", zz->Proc, partition, togo, pcost, cost, globalcost);*/ /* * End current part and start new one. We are more "over" than "under" */ tmpcost += part_sizes[partition]; partition++; /* Move on to next partition */ while((part_sizes[partition] == 0) && (partition < (zz->Num_Proc - 1))) partition++; optsize = part_sizes[partition]*globalcost; total += pcost; pcost = 0; pmass = 0; vector_set_comp(pcoord,0,0,0); } /*** Add terminal octant to current partition */ Zoltan_Oct_modify_newpid(octant, partition); pcost += cost; Zoltan_Oct_origin_volume(octant, origin, &volume); vector_cmult(prod,volume,origin); pmass += volume; vector_add(pcoord,pcoord,prod); }