int Zoltan_Oct_child_which_wrapper(OCT_Global_Info *OCT_info,pOctant oct, COORD point) { COORD min,max; COORD origin; Zoltan_Oct_bounds(oct,min,max); /* get the bounds of the octant */ Zoltan_Oct_bounds_to_origin(min,max,origin); /* convert to bound to origin */ return Zoltan_Oct_convert_idx_from_map(OCT_info, oct->dir, Zoltan_Oct_child_which(OCT_info,origin, point)); }
/* * pOctant Zoltan_Oct_global_find(COORD centroid) * * Return the octant in the local octree that contains the * given point, if it exists. Otherwise, return NULL. * */ static pOctant Zoltan_Oct_global_find(OCT_Global_Info *OCT_info,COORD point) { pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ pOctant oct; /* octree octant */ COORD min, /* minimum bounds coordinates */ max; /* maximum bounds coordinates */ /* get a list of all local roots */ RootList = Zoltan_Oct_POct_localroots(OCT_info); oct = NULL; /* iterate through root list to find if point lies inside root's bounds */ while ((!oct) && (RootOct = RL_nextRootOctant(&RootList)) ) { Zoltan_Oct_bounds(RootOct,min,max); /* ATTN: Zoltan_Oct_in_box may need adjusting for the lower bounds */ /* check if point fits inside */ if (Zoltan_Oct_in_box(OCT_info,point,min,max)) { /* find the exact octant for point */ oct=Zoltan_Oct_findOctant(OCT_info,RootOct,point); } } return(oct); }
/* * void Zoltan_Oct_origin_volume(pOctant oct, COORD origin, double *volume) * * gets the origin and volume of the octant */ void Zoltan_Oct_origin_volume(pOctant oct, COORD origin, double *volume) { COORD min, /* octant minimum bound */ max; /* octant maximum bound */ double size[3]; /* size of the octant */ Zoltan_Oct_bounds(oct,min,max); Zoltan_Oct_bounds_to_origin_size(min,max,origin,size); *volume=size[0]*size[1]*size[2]; }
int RL_printRootOctants(pRList rlist) { pOctant rootoct; COORD rmin, rmax; if((rlist == NULL) || (rlist->oct != NULL)) return -1; while((rootoct = RL_nextRootOctant(&rlist))) { Zoltan_Oct_bounds(rootoct,rmin,rmax); fprintf(stderr, "RL_printRootOctants ppid %d id %d area %f npid %d ", rootoct->ppid, rootoct->id, rootoct->area, rootoct->npid); fprintf(stderr,"min box %f %f %f ",rmin[0], rmin[1], rmin[2]); fprintf(stderr,"max box %f %f %f\n",rmax[0], rmax[1], rmax[2]); } return 0; }
void Zoltan_Oct_child_bounds_wrapper(OCT_Global_Info *OCT_info, pOctant oct, COORD cmin[], COORD cmax[]) { int i, j; COORD min, max; COORD origin; /* get the bounds of an octant */ Zoltan_Oct_bounds(oct,min,max); /* calculate the origin from the bounds */ Zoltan_Oct_bounds_to_origin(min,max,origin); /* KDDKDD 3/01 Added special cases depending on OCT_info->OCT_dimension. * KDDKDD 3/01 When OCT_dimension == 2, Zoltan_Oct_convert_idx_to_map was called * KDDKDD 3/01 with values >= 4, which are not supported in the GRAY and * KDDKDD 3/01 HILBERT maps used in Zoltan_Oct_convert_idx_to_map. * KDDKDD 3/01 This change calls Zoltan_Oct_convert_idx_to_map with only valid * KDDKDD 3/01 values, and sets cmin and cmax so that the loop following * KDDKDD 3/01 the call to Zoltan_Oct_child_bounds_wrapper "continues" for children * KDDKDD 3/01 4-7, rather than initializing them. */ if (OCT_info->OCT_dimension == 3) { for(i=0; i<8; i++) { Zoltan_Oct_child_bounds(min, max, origin, Zoltan_Oct_convert_idx_to_map(OCT_info, oct->dir, i), cmin[i], cmax[i]); } } else if (OCT_info->OCT_dimension == 2) { for(i=0; i<4; i++) { Zoltan_Oct_child_bounds(min, max, origin, Zoltan_Oct_convert_idx_to_map(OCT_info, oct->dir, i), cmin[i], cmax[i]); } for(i=4; i<8; i++) { for(j=0;j<3;j++) { cmin[i][j] = DBL_MAX; cmax[i][j] = -DBL_MAX; } } } else { fprintf(stderr, "Zoltan_Oct_child_bounds_wrapper: Invalid OCT_dimension\n"); abort(); } }
/* * void Zoltan_Oct_gen_tree_from_input_data() * * This function will create a root node of on each processor which will * then be used to create an octree with regions associated with it. The * tree will then be balanced and the output used to balance "mesh regions" * on several processors. */ static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag, int *c1, int *c2, int *c3, float *c0, int createpartree) { char *yo = "Zoltan_Oct_gen_tree_from_input_data"; pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ COORD min, /* min coord bounds of objects */ max; /* max coord bounds of objects */ int num_extra; /* number of orphaned objects */ int num_objs; /* total number of local objects */ pRegion ptr, /* pointer to iterate trough region list */ ptr1; /* pointer to iterate trough region list */ pOctant root; /* root of the partition tree */ int i; /* index counter */ int count, /* count for leaf nodes in partition tree */ proc, /* proc leaf node of parition tree belongs to */ extra, /* extra leaf node flag, if not evenly divisible */ remainder; /* remainder of node, or processors to fill */ pOctant cursor, /* cursor to iterate through octant list */ cursor2, /* another cursor to iterate through octant list */ parent; /* parent of an octant */ int level, /* number of levels of refinement */ n, /* index counter */ part; /* partition counter */ Map *array; /* map of which processors own which octants */ int hold; /* used for calculating partition divisions */ int ierr = 0; #ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX}; COORD global_min, global_max; #endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */ int nroots = 0; /*test*/ /* COORD gmin,gmax; */ OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); ZOLTAN_TRACE_ENTER(zz, yo); /* * If there are no objects on this processor, do not create a root octant. * The partitioner will probably assign objects to this processor */ if(zz->Get_Num_Obj == NULL) { fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:", "Must register ZOLTAN_NUM_OBJ_FN function"); abort(); } *c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr) { fprintf(stderr, "OCT [%d] %s: Error returned from user defined " "Get_Num_Obj function.\n", zz->Proc, yo); exit (-1); } ptr1 = NULL; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds"); /* Need A Function To Get The Bounds Of The Local Objects */ Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0); #ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN /* For now, don't want to add the new query function to Zoltan. */ /* Zoltan_Oct_get_bounds appears to compute the global min and max from */ /* the object input. */ vector_set(OCT_info->OCT_gmin, min); vector_set(OCT_info->OCT_gmax, max); #else /*test*/ /*getMaxBounds(&gmin, &gmax);*/ if(zz->Get_Bounds_Geom == NULL) { fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:", "Must register Get_Bounds_Geom function"); abort(); } zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr); MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3, MPI_DOUBLE, MPI_MIN, zz->Communicator); MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3, MPI_DOUBLE, MPI_MAX, zz->Communicator); vector_set(OCT_info->OCT_gmin, global_min); vector_set(OCT_info->OCT_gmax, global_max); #endif /* * the following code segment was added to create a pseudo global octree * needed for the partitioner. The basic idea is to regroup all the * regions into something close to an octree partitioning and build the * tree from that. * NOTE: This way of doing things is very costly, especially when calling * this for the first time on a mesh not partitioned in an octree style * partitioning. */ level = 0; /* initialize level count */ /* * if more than 1 processor, need to find what level of refinement needed * to initially partition bounding box among the processors */ if(zz->Num_Proc > 1) { n = zz->Num_Proc; if(OCT_info->OCT_dimension == 2) hold = 4; else hold = 8; remainder = hold; for(; remainder > 0; level++) { int pr = (int)POW(hold, level); remainder = n - pr; } level--; } ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree"); if(createpartree) { /* create the global root octant */ root = Zoltan_Oct_POct_new(OCT_info); Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax); /* Zoltan_Oct_setOrientation(root, 0); */ /* subdivide to as many levels as calculated */ for(i=0; i<level; i++) { cursor = root; while(cursor != NULL) { if(Zoltan_Oct_isTerminal(cursor)) { cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); Zoltan_Oct_terminal_refine(zz, cursor, 0); cursor = cursor2; } else cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); } } #if 0 if(zz->Proc == 0) for(i=0; i<8; i++) if(Zoltan_Oct_child(root, i) == NULL) fprintf(stderr,"NULL child pointer\n"); else fprintf(stderr, "child %d exists\n", i); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array"); /* this part creates the map array */ if(OCT_info->OCT_dimension == 2) { hold = (int)POW(4, level); /* ignoring the z+ octants */ if(hold == 0) hold = 1; } else hold = (int)POW(8, level); part = hold / zz->Num_Proc; /* how many octants per partition */ remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */ extra = zz->Num_Proc - remainder;/* where to start adding extra octants */ array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */ if(array == NULL) { fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n", zz->Proc); abort(); } /* initialize variables */ proc = 0; count = 0; i = 0; cursor = root; while(cursor != NULL) { cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) { if(proc == extra) { part++; extra = -1; } if(count != part) { array[i].npid = proc; array[i].list = RL_initRootList(); Zoltan_Oct_bounds(cursor, min, max); vector_set(array[i].min, min); vector_set(array[i].max, max); count++; } else { count = 1; proc++; array[i].npid = proc; array[i].list = RL_initRootList(); Zoltan_Oct_bounds(cursor, min, max); vector_set(array[i].min, min); vector_set(array[i].max, max); } if(proc == zz->Proc) { array[i].npid = -1; /* KDDKDD Added RL_freeList below. The * KDDKDD implementation from RPI leaked memory because the * KDDKDD test cases for setting array[i].list were not mutually * KDDKDD exclusive. Freeing the list produces the result we got * KDDKDD before, without the memory leak. */ /* LGG -- it seems to me that this array[i].list assignment is * not really necessary. It looks as though it has already been * assigned with the same information from the prev if-else * commented out RL_freeList(), and RL_initRootList() */ /*RL_freeList(&(array[i].list));*/ /* KDDKDD End addition */ /*array[i].list = RL_initRootList();*/ parent = Zoltan_Oct_parent(cursor); if(parent != NULL) Zoltan_Oct_setchild(parent, cursor->which, NULL); /* octant into local root list */ Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1); Zoltan_Oct_setMapIdx(cursor, i); nroots++; /* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc); octant into local root list */ } i++; } cursor = cursor2; } RootList = Zoltan_Oct_POct_localroots(OCT_info); RootOct = RL_nextRootOctant(&RootList); if(RootOct != root) { /* KDDKDDFREE changed root to &root to allow root to be reset to NULL */ Zoltan_Oct_POct_delTree(OCT_info,&root); } OCT_info->map = array; OCT_info->mapsize = hold; } /* * attach the regions to the root... Zoltan_Oct_fix will create the octree * starting with the root and subdividing as needed */ num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans"); Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map, c1, c2); /* ZOLTAN_FREE(&array); */ while(ptr1 != NULL) { ptr = ptr1->next; ZOLTAN_FREE(&(ptr1->Global_ID)); ZOLTAN_FREE(&(ptr1->Local_ID)); ZOLTAN_FREE(&ptr1); ptr1 = ptr; } ZOLTAN_TRACE_EXIT(zz, yo); }
/* * Zoltan_Oct_terminal_refine(oct) * * subdivide a terminal octant and divvy up * its regions to the 8 children; recurse if * necessary to satisfy MAXOCTREGIONS * */ static void Zoltan_Oct_terminal_refine(ZZ *zz, pOctant oct,int count) { COORD min, /* coordinates of minimum bounds of region */ max, /* coordinates of maximum bounds of region */ origin; /* origin of region */ pOctant child[8]; /* array of child octants */ int cnum; /* child number */ int i; /* index counter */ pRegion region; /* a region to be associated to an octant */ pRegion entry; COORD cmin[8], cmax[8]; OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); for(i=0;i<3;i++) min[i] = max[i] = 0; /* upper limit of refinement levels */ /* ATTN: may not be used anymore, but can be put back in if necessary */ if (count>=20) { fprintf(stderr, "OCT ERROR: Zoltan_Oct_terminal_refine: bailing out at " "10 levels\n"); abort(); } oct_nref++; /* increment refinement counter */ /* octant should be terminal in order to be refined (subdivided) */ if (!Zoltan_Oct_isTerminal(oct)) { fprintf(stderr,"OCT ref_octant: oct not terminal\n"); abort(); } /* get the bounds of an octant */ Zoltan_Oct_bounds(oct,min,max); /* calculate the origin from the bounds */ Zoltan_Oct_bounds_to_origin(min,max,origin); region = Zoltan_Oct_regionlist(oct); /* Get list while still terminal */ oct->list = NULL; /* remove regions from octant, it won't be terminal */ /* create the children and set their id's */ Zoltan_Oct_child_bounds_wrapper(OCT_info,oct, cmin, cmax); for (i=0; i<8; i++) { if(OCT_info->OCT_dimension == 2) { /* KDDKDD 3/01 see changes to Zoltan_Oct_child_bounds_wrapper that allow this * KDDKDD 3/01 test to work for GRAY and HILBERT mappings. */ if(cmin[i][2] > OCT_info->OCT_gmin[2]) { /* ignore the z+ octants */ child[i] = NULL; continue; } } child[i]=Zoltan_Oct_POct_new(OCT_info); /* create a new octant */ child[i]->dir = Zoltan_Oct_get_child_dir(OCT_info, oct->dir, i); /* create a new octant */ /* set the child->parent link */ Zoltan_Oct_POct_setparent(OCT_info, child[i], oct, zz->Proc); Zoltan_Oct_setchildnum(child[i], i); /* which child of the parent */ Zoltan_Oct_setchild(oct, i, child[i]); /* set the parent->child link */ #ifdef LGG_MIGOCT Zoltan_Oct_setID(child[i], Zoltan_Oct_nextId()); /* set child id num */ #endif /* LGG_MIGOCT */ Zoltan_Oct_setbounds(child[i], cmin[i], cmax[i]); /* set child bounds */ Zoltan_Oct_setCpid(oct, i, zz->Proc); /* set child to be a local oct */ /* Zoltan_Oct_setOrientation(child[i], Zoltan_Oct_child_orientation(oct->orientation, oct->which)); */ } /* assign newly created children to child array*/ if(OCT_info->OCT_dimension == 3) { if(Zoltan_Oct_children(oct, child) != 8) { /* * if subdivision of oct was successful, oct should have 8 children; * thus a return value of 0 here is a fatal error */ fprintf(stderr, "OCT ref_octant: subdivide failed, %d children.\n", Zoltan_Oct_children(oct, child)); abort(); } } else if(Zoltan_Oct_children(oct, child) != 4) { /* * if subdivision of oct was successful, oct should have 4 children; * thus a return value of 0 here is a fatal error */ fprintf(stderr, "OCT ref_octant:subdivide failed, %d children, expected 4\n", Zoltan_Oct_children(oct, child)); abort(); } /* iterate through and find which child each region should belong to */ while(region != NULL) { entry = region->next; cnum=Zoltan_Oct_child_which_wrapper(OCT_info,oct, region->Coord); /* add region to octant's regionlist */ Zoltan_Oct_addRegion(zz, child[cnum], region); ZOLTAN_FREE(&(region->Global_ID)); ZOLTAN_FREE(&(region->Local_ID)); ZOLTAN_FREE(®ion); region = entry; } for (i=0; i<8; i++) /* Recursion */ if(child[i] != NULL) /* KDDKDD Replaced the following to allow multiple regions with the * KDDKDD same coordinates to be placed in the same octant. if (Zoltan_Oct_nRegions(child[i]) > MAXOCTREGIONS) { */ if (Zoltan_Oct_nUniqueRegions(OCT_info,child[i]) > MAXOCTREGIONS) { Zoltan_Oct_terminal_refine(zz, child[i],count+1); } }
int Zoltan_Oct_migreg_migrate_orphans(ZZ *zz, pRegion RegionList, int nregions, int level, Map *array, int *c1, int *c2) { int i, j, k; /* index counters */ pRegion ptr; /* region in the mesh */ COORD origin; /* centroid coordinate information */ pRegion *regions = NULL; /* an array of regions */ int *npids = NULL; Region *regions2 = NULL; /* an array of regions */ int *npids2 = NULL; int nreg; /* number of regions */ COORD min, /* minimum bounds of an octant */ max; /* maximum bounds of an octant */ COORD cmin, /* minimum bounds of a child octant */ cmax; /* maximum bounds of a child octant */ COORD rmin, /* minimum bounds of a remote octant */ rmax; /* maximum bounds of a remote octant */ int new_num; int n; int dir = 0; pRList RootList; pOctant RootOct; OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); char *yo = "Zoltan_Oct_migreg_migrate_orphans_static"; int ierr = ZOLTAN_OK; ZOLTAN_ID_PTR gids2, lids2; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; if(nregions > 0) { /* create the array of messages to be sent to other processors */ /* Array = (Message *) ZOLTAN_MALLOC(nregions * sizeof(Message)); */ if((regions = (pRegion *) ZOLTAN_MALLOC(nregions * sizeof(pRegion))) == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((npids = (int *) ZOLTAN_MALLOC(nregions * sizeof(int))) == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(®ions); return ZOLTAN_MEMERR; } } ptr = RegionList; n = nreg = 0; while((ptr != NULL) && (nregions > 0)) { if(ptr->attached == 1) { /* if region already attached to an octant, then skip to next region */ ptr = ptr->next; continue; } /* region not attached, have to find which processor to send to */ j=0; dir = 0; vector_set(min, OCT_info->OCT_gmin); vector_set(max, OCT_info->OCT_gmax); /* * for each level of refinement, find which child region belongs to. * translate which child to which entry in map array. */ for(i=0; i<level; i++) { Zoltan_Oct_bounds_to_origin(min, max, origin); if(OCT_info->OCT_dimension == 2) j = j * 4; else j = j * 8; k = Zoltan_Oct_child_which(OCT_info,origin, ptr->Coord); new_num = Zoltan_Oct_convert_idx_from_map(OCT_info, dir, k); dir = Zoltan_Oct_get_child_dir(OCT_info, dir, new_num); j += new_num; Zoltan_Oct_child_bounds(min, max, origin, k, cmin, cmax); vector_set(min, cmin); vector_set(max, cmax); } /* inform message which processor to send to */ npids[n] = array[j].npid; RootList = array[j].list; while((RootOct = RL_nextRootOctant(&RootList))) { Zoltan_Oct_bounds(RootOct,rmin,rmax); if (Zoltan_Oct_in_box_closure(OCT_info, ptr->Coord ,rmin, rmax)) { npids[n] = RootOct->npid; break; } } if((npids[n] != -1) && (npids[n] != zz->Proc)) { Zoltan_Oct_copy_info(zz, ptr, &(regions[n++])); } else { Zoltan_Oct_insert_orphan(zz, *ptr); } nreg++; /* increment region counter */ ptr = ptr->next; /* look at next region */ } /* * if regions looked at != number of regions in region list, * then there is an error */ if (nreg!=nregions) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "regions found != to expected number of regions"); return ZOLTAN_FATAL; } regions2 = (Region *) ZOLTAN_MALLOC(n * sizeof(Region)); gids2 = ZOLTAN_MALLOC_GID_ARRAY(zz, n); lids2 = ZOLTAN_MALLOC_LID_ARRAY(zz, n); npids2 = (int *) ZOLTAN_MALLOC(n * sizeof(int)); for(i=0; i<n; i++) { npids2[i] = npids[i]; vector_set(regions2[i].Coord, regions[i]->Coord); regions2[i].Weight = regions[i]->Weight; regions2[i].Global_ID = &(gids2[i*num_gid_entries]); regions2[i].Local_ID = (num_lid_entries ? &(lids2[i*num_lid_entries]) : NULL); ZOLTAN_SET_GID(zz, &(gids2[i*num_gid_entries]), regions[i]->Global_ID); ZOLTAN_SET_LID(zz, &(lids2[i*num_lid_entries]), regions[i]->Local_ID); regions2[i].Proc = regions[i]->Proc; regions2[i].attached = 0; } *c1 = n; /* migrate the orphan regions according to the message array */ Zoltan_Oct_migreg_migrate_regions(zz, regions2, gids2, lids2, npids2, n, c2); for (i=0; i < n; i++) { ZOLTAN_FREE(&(regions[i]->Global_ID)); ZOLTAN_FREE(&(regions[i]->Local_ID)); ZOLTAN_FREE(&(regions[i])); } ZOLTAN_FREE(®ions); ZOLTAN_FREE(&npids); ZOLTAN_FREE(®ions2); ZOLTAN_FREE(&gids2); ZOLTAN_FREE(&lids2); ZOLTAN_FREE(&npids2); return ierr; }