/*
* void Zoltan_Oct_fix_tags()
*
* fixes the import tags so that region tags that were previously
* exported aren't counted when imported back.
*/
int Zoltan_Oct_fix_tags(ZZ *zz, ZOLTAN_ID_PTR *import_global_ids,
ZOLTAN_ID_PTR *import_local_ids, int **import_procs,
int **import_to_part, int nrectags,
pRegion import_regs) {
char *yo = "Zoltan_Oct_fix_tags";
int i; /* index counter */
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
/* allocate memory */
if (!Zoltan_Special_Malloc(zz,(void **)import_global_ids,nrectags,
ZOLTAN_SPECIAL_MALLOC_GID)) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)import_local_ids,nrectags,
ZOLTAN_SPECIAL_MALLOC_LID)) {
Zoltan_Special_Free(zz,(void **)import_global_ids,
ZOLTAN_SPECIAL_MALLOC_GID);
ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)import_procs,nrectags,
ZOLTAN_SPECIAL_MALLOC_INT)) {
Zoltan_Special_Free(zz,(void **)import_global_ids,
ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void **)import_local_ids,
ZOLTAN_SPECIAL_MALLOC_LID);
ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)import_to_part,nrectags,
ZOLTAN_SPECIAL_MALLOC_INT)) {
Zoltan_Special_Free(zz,(void **)import_global_ids,
ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void **)import_local_ids,
ZOLTAN_SPECIAL_MALLOC_LID);
Zoltan_Special_Free(zz,(void **)import_procs,ZOLTAN_SPECIAL_MALLOC_INT);
ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
/* for each region imported, look at its originating processor */
for(i=0; i<nrectags; i++) {
ZOLTAN_SET_GID(zz, &((*import_global_ids)[i*num_gid_entries]),
import_regs[i].Global_ID);
ZOLTAN_SET_LID(zz, &((*import_local_ids)[i*num_lid_entries]),
import_regs[i].Local_ID);
(*import_procs)[i] = import_regs[i].newProc;
/*(*import_to_part)[i] = zz->Proc;*/
(*import_to_part)[i] = import_regs[i].newProc;
}
return ierr;
}
int Zoltan_PHG(
ZZ *zz, /* The Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work assigned
to each partition. */
int *num_imp, /* not computed */
ZOLTAN_ID_PTR *imp_gids, /* not computed */
ZOLTAN_ID_PTR *imp_lids, /* not computed */
int **imp_procs, /* not computed */
int **imp_to_part, /* not computed */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part ) /* list of partitions to which exported objs
are assigned. */
{
char *yo = "Zoltan_PHG";
ZHG *zoltan_hg = NULL;
PHGPartParams hgp; /* Hypergraph parameters. */
HGraph *hg = NULL; /* Hypergraph itself */
Partition parts = NULL; /* Partition assignments in
2D distribution. */
int err = ZOLTAN_OK, p=0;
struct phg_timer_indices *timer = NULL;
int do_timing = 0;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Initialization of return arguments. */
*num_imp = *num_exp = -1;
*imp_gids = *exp_gids = NULL;
*imp_lids = *exp_lids = NULL;
*imp_procs = *exp_procs = NULL;
/* Initialize HG parameters. */
err = Zoltan_PHG_Initialize_Params(zz, part_sizes, &hgp);
if (err != ZOLTAN_OK)
goto End;
if (hgp.use_timers) {
if (!zz->LB.Data_Structure) {
zz->LB.Data_Structure = (struct phg_timer_indices *)
ZOLTAN_MALLOC(sizeof(struct phg_timer_indices));
initialize_timer_indices((struct phg_timer_indices *)zz->LB.Data_Structure);
}
timer = zz->LB.Data_Structure;
if (timer->all < 0)
timer->all = Zoltan_Timer_Init(zz->ZTime, 1, "Zoltan_PHG");
}
if (hgp.use_timers > 1) {
do_timing = 1;
if (timer->build < 0)
timer->build = Zoltan_Timer_Init(zz->ZTime, 1, "Build");
if (timer->setupvmap < 0)
timer->setupvmap = Zoltan_Timer_Init(zz->ZTime, 0, "Vmaps");
}
if (hgp.use_timers)
ZOLTAN_TIMER_START(zz->ZTime, timer->all, zz->Communicator);
if (do_timing)
ZOLTAN_TIMER_START(zz->ZTime, timer->build, zz->Communicator);
/* build initial Zoltan hypergraph from callback functions. */
err = Zoltan_PHG_Build_Hypergraph (zz, &zoltan_hg, &parts, &hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error building hypergraph.");
goto End;
}
if (zoltan_hg->GnObj == 0){
/* degenerate case - no objects to partition */
hgp.final_output = 0;
goto End;
}
hg = &zoltan_hg->HG;
p = zz->LB.Num_Global_Parts;
zoltan_hg->HG.redl = MAX(hgp.redl, p); /* redl needs to be dynamic */
/* RTHRTH -- redl may need to be scaled by number of procs */
/* EBEB -- at least make sure redl > #procs */
if (hgp.UseFixedVtx)
hg->bisec_split = 1; /* this will be used only #parts=2
otherwise rdivide will set to appropriate
value */
if (hgp.UsePrefPart || hgp.UseFixedVtx) { /* allocate memory for pref_part */
if (hg->nVtx &&
!(hg->pref_part = (int*) ZOLTAN_MALLOC (sizeof(int) * hg->nVtx))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error building hypergraph.");
goto End;
}
}
if (hgp.UsePrefPart) /* copy input parts as pref_part;
UVCUVC: TODO: this code (and alloc of pref_part)
should go to Build_Hypergraph later */
memcpy(hg->pref_part, parts, sizeof(int) * hg->nVtx);
if (hgp.UseFixedVtx) {
int i;
for (i=0; i<hg->nVtx; ++i)
if (hg->fixed_part[i]>=0)
hg->pref_part[i] = hg->fixed_part[i];
else if (!hgp.UsePrefPart)
hg->pref_part[i] = -1;
}
hgp.UsePrefPart |= hgp.UseFixedVtx;
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->build, zz->Communicator);
/*
UVCUVC DEBUG PRINT
uprintf(hg->comm, "Zoltan_PHG kway=%d #parts=%d\n", hgp.kway, zz->LB.Num_Global_Parts);
*/
if (!strcasecmp(hgp.hgraph_pkg, "PARKWAY")){
if (do_timing) {
if (timer->parkway < 0)
timer->parkway = Zoltan_Timer_Init(zz->ZTime, 0, "PHG_ParKway");
ZOLTAN_TIMER_START(zz->ZTime, timer->parkway, zz->Communicator);
}
err = Zoltan_PHG_ParKway(zz, hg, p,
parts, &hgp);
if (err != ZOLTAN_OK)
goto End;
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->parkway, zz->Communicator);
} else if (!strcasecmp(hgp.hgraph_pkg, "PATOH")){
if (hgp.use_timers > 1) {
if (timer->patoh < 0)
timer->patoh = Zoltan_Timer_Init(zz->ZTime, 0, "HG_PaToH");
ZOLTAN_TIMER_START(zz->ZTime, timer->patoh, zz->Communicator);
}
err = Zoltan_PHG_PaToH(zz, hg, p,
parts, &hgp);
if (err != ZOLTAN_OK)
goto End;
if (hgp.use_timers > 1)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->patoh, zz->Communicator);
}
else { /* it must be PHG */
/* UVC: if it is bisection anyways; no need to create vmap etc;
rdivide is going to call Zoltan_PHG_Partition anyways... */
if (hgp.globalcomm.Communicator != MPI_COMM_NULL) {
/* This processor is part of the 2D data distribution; it should
participate in partitioning. */
if (hgp.kway || zz->LB.Num_Global_Parts == 2) {
/* call main V cycle routine */
err = Zoltan_PHG_Partition(zz, hg, p,
hgp.part_sizes, parts, &hgp);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error partitioning hypergraph.");
goto End;
}
}
else {
int i;
if (do_timing)
ZOLTAN_TIMER_START(zz->ZTime, timer->setupvmap, zz->Communicator);
/* vmap associates original vertices to sub hypergraphs */
if (hg->nVtx &&
!(hg->vmap = (int*) ZOLTAN_MALLOC(hg->nVtx*sizeof (int)))) {
err = ZOLTAN_MEMERR;
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
goto End;
}
for (i = 0; i < hg->nVtx; ++i)
hg->vmap[i] = i;
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->setupvmap, zz->Communicator);
/* partition hypergraph */
err = Zoltan_PHG_rdivide (0, p-1, parts, zz, hg, &hgp, 0);
if (hgp.output_level >= PHG_DEBUG_LIST)
uprintf(hg->comm, "FINAL %3d |V|=%6d |E|=%6d #pins=%6d %s/%s/%s/%s p=%d "
"bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins,
hgp.convert_str, hgp.redm_str,
hgp.coarsepartition_str, hgp.refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, hgp.part_sizes, 0,
p, parts),
Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, p, &err));
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error partitioning hypergraph.");
goto End;
}
ZOLTAN_FREE (&hg->vmap);
}
#ifdef CHECK_LEFTALONE_VERTICES
findAndSaveLeftAloneVertices(zz, hg, p, parts, &hgp);
#endif
}
}
if (!strcasecmp(hgp.hgraph_method, "REPARTITION")) {
Zoltan_PHG_Remove_Repart_Data(zz, zoltan_hg, hg, &hgp);
}
/* UVC DEBUG PRINT
if (!strcasecmp(hgp->hgraph_method, "REFINE")){
uprintf(hg->comm,
"UVC ATTHEEND |V|=%6d |E|=%6d #pins=%6d p=%d bal=%.2f cutl=%.2f\n",
hg->nVtx, hg->nEdge, hg->nPins, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, p, parts),
Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, p, &err));
detailed_balance_info(zz, hg, part_sizes, p, parts);
}
*/
/* Initialize these timers here so their output is near end of printout */
if (do_timing)
if (timer->retlist < 0)
timer->retlist = Zoltan_Timer_Init(zz->ZTime, 1, "Return_Lists");
if (hgp.use_timers)
if (timer->finaloutput < 0)
timer->finaloutput = Zoltan_Timer_Init(zz->ZTime, 1, "Final_Output");
if (do_timing)
ZOLTAN_TIMER_START(zz->ZTime, timer->retlist, zz->Communicator);
/* Build Zoltan's Output_Parts, mapped from 2D distribution
to input distribution. */
Zoltan_PHG_Output_Parts(zz, zoltan_hg, parts);
/* Build Zoltan's return arguments. */
Zoltan_PHG_Return_Lists(zz, zoltan_hg, num_exp, exp_gids,
exp_lids, exp_procs, exp_to_part);
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->retlist, zz->Communicator);
End:
if (err == ZOLTAN_MEMERR)
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Memory error.")
else if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Error partitioning hypergraph.")
/* KDDKDD The following code prints a final quality result even when
* KDDKDD phg_output_level is zero. It is useful for our tests and
* KDDKDD data collection. */
if ((err == ZOLTAN_OK) && hgp.final_output) {
static int nRuns=0;
static double balsum = 0.0, cutlsum = 0.0, cutnsum = 0.0, movesum = 0.0, repartsum = 0.0;
static double balmax = 0.0, cutlmax = 0.0, cutnmax = 0.0, movemax = 0.0, repartmax = 0.0;
static double balmin = 1e100, cutlmin = 1e100, cutnmin = 1e100, movemin = 1e100, repartmin = 1e100;
double bal = 0.;
double cutl = 0.; /* Connnectivity cuts: sum_over_edges((npart-1)*ewgt) */
double cutn = 0.; /* Net cuts: sum_over_edges((nparts>1)*ewgt) */
double rlocal[2]; /* local cut stats for removed edges */
double rglobal[2]; /* global cut stats for removed edges */
int gnremove, i;
double move=0.0, gmove; /* local and global migration costs */
double repart=0.0; /* total repartitioning cost: comcost x multiplier + migration_cost */
if (hgp.use_timers) {
/* Do not include final output time in partitioning time */
ZOLTAN_TIMER_STOP(zz->ZTime, timer->all, zz->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->finaloutput, zz->Communicator);
}
if (hgp.globalcomm.Communicator != MPI_COMM_NULL) {
/* Processor participated in partitioning */
bal = Zoltan_PHG_Compute_Balance(zz, hg, hgp.part_sizes, 0,
zz->LB.Num_Global_Parts, parts);
cutl= Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts,
zz->LB.Num_Global_Parts, &err);
cutn = Zoltan_PHG_Compute_NetCut(hg->comm, hg, parts,
zz->LB.Num_Global_Parts);
for (i = 0; i < zoltan_hg->nObj; ++i) {
/* uprintf(hg->comm, " obj[%d] = %d in=%d out=%d\n", i, zoltan_hg->AppObjSizes[i], zoltan_hg->Input_Parts[i], zoltan_hg->Output_Parts[i]); */
if (zoltan_hg->Input_Parts[i] != zoltan_hg->Output_Parts[i])
move += (double) ((zoltan_hg->AppObjSizes) ? zoltan_hg->AppObjSizes[i] : 1.0);
}
}
if (!err) {
/* Add in cut contributions from removed edges */
MPI_Allreduce(&(zoltan_hg->nRemove), &gnremove, 1, MPI_INT, MPI_SUM,
zz->Communicator);
if (gnremove) {
err = Zoltan_PHG_Removed_Cuts(zz, zoltan_hg, rlocal);
MPI_Allreduce(rlocal, rglobal, 2, MPI_DOUBLE,MPI_SUM,zz->Communicator);
cutl += rglobal[0];
cutn += rglobal[1];
}
MPI_Allreduce(&move, &gmove, 1, MPI_DOUBLE, MPI_SUM, zz->Communicator);
repart = cutl*hgp.RepartMultiplier + gmove;
repartsum += repart;
if (repart > repartmax) repartmax = repart;
if (repart < repartmin) repartmin = repart;
movesum += gmove;
if (gmove > movemax) movemax = gmove;
if (gmove < movemin) movemin = gmove;
cutlsum += cutl;
if (cutl > cutlmax) cutlmax = cutl;
if (cutl < cutlmin) cutlmin = cutl;
cutnsum += cutn;
if (cutn > cutnmax) cutnmax = cutn;
if (cutn < cutnmin) cutnmin = cutn;
balsum += bal;
if (bal > balmax) balmax = bal;
if (bal < balmin) balmin = bal;
nRuns++;
if (zz->Proc == 0) {
uprintf(hg->comm,
"STATS Runs %d bal CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, bal, balmax, balmin, balsum/nRuns);
uprintf(hg->comm,
"STATS Runs %d cutl CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, cutl, cutlmax, cutlmin, cutlsum/nRuns);
uprintf(hg->comm,
"STATS Runs %d cutn CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, cutn, cutnmax, cutnmin, cutnsum/nRuns);
uprintf(hg->comm,
"STATS Runs %d %s CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, (zoltan_hg->showMoveVol) ? "moveVol" : "moveCnt", gmove, movemax, movemin, movesum/nRuns);
if (zoltan_hg->showMoveVol)
uprintf(hg->comm,
"STATS Runs %d repart CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, repart, repartmax, repartmin, repartsum/nRuns);
}
}
if (hgp.use_timers) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->finaloutput, zz->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->all, zz->Communicator);
}
}
/* KDDKDD End of printing section. */
ZOLTAN_FREE(&parts);
if (zoltan_hg != NULL) {
Zoltan_PHG_Free_Hypergraph_Data(zoltan_hg);
ZOLTAN_FREE (&zoltan_hg);
}
if (hgp.use_timers) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->all, zz->Communicator);
if (hgp.globalcomm.Communicator != MPI_COMM_NULL)
Zoltan_Timer_PrintAll(zz->ZTime, 0, hgp.globalcomm.Communicator, stdout);
}
if (hgp.globalcomm.row_comm != MPI_COMM_NULL)
MPI_Comm_free(&(hgp.globalcomm.row_comm));
if (hgp.globalcomm.col_comm != MPI_COMM_NULL)
MPI_Comm_free(&(hgp.globalcomm.col_comm));
if (hgp.globalcomm.Communicator != MPI_COMM_NULL)
MPI_Comm_free(&(hgp.globalcomm.Communicator));
/* Free part_sizes if created new due to ADD_OBJ_WEIGHT */
if (hgp.part_sizes != part_sizes)
ZOLTAN_FREE(&hgp.part_sizes);
ZOLTAN_TRACE_EXIT(zz, yo);
return err;
}
int Zoltan_ParMetis_Order(
ZZ *zz, /* Zoltan structure */
int num_obj, /* Number of (local) objects to order. */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */
int *iperm,
ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */
)
{
static char *yo = "Zoltan_ParMetis_Order";
int i, n, ierr;
ZOLTAN_Output_Order ord;
ZOLTAN_Third_Graph gr;
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't want to risk letting external
packages changing our communicator */
#endif
indextype numflag = 0;
int timer_p = 0;
int get_times = 0;
int use_timers = 0;
double times[5];
ZOLTAN_ID_PTR l_gids = NULL;
ZOLTAN_ID_PTR l_lids = NULL;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
ZOLTAN_TRACE_ENTER(zz, yo);
#ifdef ZOLTAN_PARMETIS
#if TPL_USE_DATATYPE != TPL_METIS_DATATYPES
#ifdef TPL_FLOAT_WEIGHT
i = 1;
#else
i = 0;
#endif
if ((sizeof(indextype) != sizeof(idxtype)) ||
(sizeof(weighttype) != sizeof(idxtype)) || i){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,
"Not supported: Multiple 3rd party libraries with incompatible "
"data types.");
return ZOLTAN_FATAL;
}
#endif
#endif
memset(&gr, 0, sizeof(ZOLTAN_Third_Graph));
memset(&ord, 0, sizeof(ZOLTAN_Output_Order));
memset(times, 0, sizeof(times));
ord.order_opt = order_opt;
if (!order_opt){
/* If for some reason order_opt is NULL, allocate a new ZOOS here. */
/* This should really never happen. */
order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
strcpy(order_opt->method,"PARMETIS");
}
ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord);
/* ParMetis only computes the rank vector */
order_opt->return_args = RETURN_RANK;
/* Check that num_obj equals the number of objects on this proc. */
/* This constraint may be removed in the future. */
n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){
/* Return error code */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
return(ZOLTAN_FATAL);
}
if (n != num_obj){
/* Currently this is a fatal error. */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the "
"number of objects.");
return(ZOLTAN_FATAL);
}
/* Do not use weights for ordering */
gr.obj_wgt_dim = -1;
gr.edge_wgt_dim = -1;
gr.num_obj = num_obj;
/* Check what ordering type is requested */
if (order_opt){
SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */
#ifdef ZOLTAN_PARMETIS
if ((strcmp(order_opt->method, "METIS") == 0))
#endif /* ZOLTAN_PARMETIS */
SET_LOCAL_GRAPH(&gr.graph_type);
}
gr.get_data = 1;
if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: "
"set ParMetis instead.");
return(ZOLTAN_FATAL);
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
return (ierr);
}
/* Allocate space for separator sizes */
if (IS_GLOBAL_GRAPH(gr.graph_type)) {
if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) {
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype));
if (ord.sep_sizes == NULL) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */
}
/* Allocate space for direct perm */
ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.rank){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
if (IS_LOCAL_GRAPH(gr.graph_type)){
/* Allocate space for inverse perm */
ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.iperm){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
}
else
ord.iperm = NULL;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_PARMETIS
if (IS_GLOBAL_GRAPH(gr.graph_type)){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library");
ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy,
&numflag, options, ord.rank, ord.sep_sizes, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else
#endif /* ZOLTAN_PARMETIS */
#if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS)
if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */
indextype numobj = gr.num_obj;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library");
order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0; /* Use default options for METIS. */
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options,
ord.iperm, ord.rank);
#else
METIS_SetDefaultOptions(options);
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, NULL, options,
ord.iperm, ord.rank); /* NULL is vwgt -- new interface in v4 */
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */
int numbloc;
int start;
int leaf;
int *converttab;
int levelmax;
levelmax = mylog2(zz->Num_Proc) + 1;
converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int));
memset(converttab, 0, zz->Num_Proc*2*sizeof(int));
/* Determine the first node in each separator, store it in zz->TPL_Order.start */
for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) {
int father;
father = zz->Num_Proc + numbloc;
converttab[start] = 2*numbloc;
zz->TPL_Order.leaves[leaf++]=start;
zz->TPL_Order.ancestor[start] = start + 2;
converttab[start+1] = 2*numbloc+1;
zz->TPL_Order.leaves[leaf++]=start+1;
zz->TPL_Order.ancestor[start+1] = start + 2;
start+=2;
do {
converttab[start] = father;
if (father %2 == 0) {
int nextoffset;
int level;
level = mylog2(2*zz->Num_Proc - 1 - father);
nextoffset = (1<<(levelmax-level));
zz->TPL_Order.ancestor[start] = start+nextoffset;
start++;
break;
}
else {
zz->TPL_Order.ancestor[start] = start+1;
start++;
father = zz->Num_Proc + father/2;
}
} while (father < 2*zz->Num_Proc - 1);
}
zz->TPL_Order.start[0] = 0;
zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1;
for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) {
int oldblock=converttab[numbloc-1];
zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock];
}
ZOLTAN_FREE(&converttab);
ZOLTAN_FREE(&ord.sep_sizes);
zz->TPL_Order.leaves[zz->Num_Proc] = -1;
zz->TPL_Order.nbr_leaves = zz->Num_Proc;
zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1;
}
else { /* No tree */
zz->TPL_Order.nbr_blocks = 0;
zz->TPL_Order.start = NULL;
zz->TPL_Order.ancestor = NULL;
zz->TPL_Order.leaves = NULL;
}
/* Correct because no redistribution */
memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE));
memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE));
ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL);
ZOLTAN_FREE(&l_gids);
ZOLTAN_FREE(&l_lids);
/* Get a time here */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){
memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i];
}
}
if ((ord.iperm != NULL) && (iperm != NULL)){
if (sizeof(indextype) == sizeof(int)){
memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
iperm[i] = (int)ord.iperm[i];
}
}
}
if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm);
ZOLTAN_FREE(&ord.rank);
/* Free all other "graph" stuff */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
int
Zoltan_PHG_2ways_hyperedge_partition (
ZZ *zz, /* Input : Zoltan data structure */
HGraph *hg,
Partition parts,
Zoltan_PHG_Tree *tree,
struct Zoltan_DD_Struct * gnoToGID,
struct Zoltan_DD_Struct **dd,
int *numParts,
int **sizeParts
)
{
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_PHG_2ways_hyperedge_partition";
int nEdge, hEdge;
int *interval;
int *partnumber = NULL;
int tree_size;
ZOLTAN_ID_TYPE *rowpart =NULL; /* ZOLTAN_ID_TYPE because it's used in Zoltan_DD_* */
ZOLTAN_GNO_TYPE *rowGNO = NULL;
ZOLTAN_ID_PTR rowGID=NULL;
int index;
int offset;
ZOLTAN_TRACE_ENTER(zz, yo);
nEdge = hg->nEdge;
fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge);
interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int));
if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR;
tree_size = get_tree_size(tree) + 1;
for (index = 0 ; index < nEdge ; ++index){
SET_MIN_NODE(interval, index, tree_size);
SET_MAX_NODE(interval, index, -1);
}
/* Update interval with the local knowledge */
/* XXX: I loop on the hyperedges, as I think it's more cache friendly
* and it allows me to discoupled the computation if a non blocking MPI_Reduce is
* available
*/
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){
int part;
int max = -1; /* Trick : we use the initialized values */
int min = tree_size + 1;
for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) {
part = parts[hg->hvertex[index]];
max = MAX(max, part);
min = MIN(min, part);
}
SET_MIN_NODE(interval, hEdge, min);
SET_MAX_NODE(interval, hEdge, max);
}
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm);
/* Now I have to compute the partition of hyperedges according to the "interval"
* and the tree */
/* First, compute the partition number corresponding to the nodes in the tree */
partnumber = compute_part_number(tree);
if (partnumber == NULL) {
ierr = ZOLTAN_FATAL;
goto End;
}
(*numParts) = get_tree_size(tree);
rowpart = (ZOLTAN_ID_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_ID_TYPE));
if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR;
rowGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_GNO_TYPE));
if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR;
(*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int));
if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR;
offset = hg->dist_y[hg->comm->myProc_y];
/* Then we search we is the hyperedge in the tree */
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
int node;
node = find_interval_in_tree(tree, interval+2*hEdge);
rowpart[hEdge] = partnumber[node];
(*sizeParts)[rowpart[hEdge]] ++;
rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge);
#if 0
fprintf (stderr, "%zd : " ZOLTAN_ID_SPEC " (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]);
#endif
}
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
/* Compute number of elements per parts */
/* TODO: support processor which are not part of the distribution */
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm);
/* Export results to data directory */
/* First, get the GIDs of our edges */
rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge);
if (nEdge && rowGID == NULL) MEMORY_ERROR;
ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL,
nEdge, NULL);
ZOLTAN_FREE(&rowGNO);
ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, rowpart, NULL, NULL, nEdge);
#ifdef CEDRIC_PRINT
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]);
}
#endif
End:
ZOLTAN_FREE(&rowGID);
ZOLTAN_FREE(&rowGNO);
ZOLTAN_FREE(&rowpart);
if (partnumber != NULL)
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/*
* void lb_oct_init();
*
* initialize the calls needed to start the octree load balancing rounties
*/
static int lb_oct_init(
ZZ *zz, /* The Zoltan structure with info for
the OCTPART balancer. */
int *num_export, /* Number of non-local objs assigned to this
processor in the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition. */
int **export_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition. */
int **export_to_part, /* Returned value: array of partitions to
which objects are imported.
KDDKDD Assume #parts==#procs. */
int oct_dim, /* Dimension of method (2D or 3D) */
int oct_method, /* Flag specifying curve to be used. */
int oct_maxoctregions, /* max # of objects in leaves of octree. */
int oct_minoctregions, /* min # of objects in leaves of octree. */
int oct_output_level, /* Flag specifying amount of output. */
int oct_wgtflag, /* Flag specifying use of object weights. */
float *part_sizes /* Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
)
{
char *yo = "lb_oct_init";
OCT_Global_Info *OCT_info;
int nsentags; /* number of tags being sent */
pRegion export_regs; /* */
int nrectags; /* number of tags received */
int kk;
double time1,time2; /* timers */
double timestart,timestop; /* timers */
double timers[4]; /* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
int counters[6]; /* diagnostic counts
0 = # of median iterations
1 = # of objects sent
2 = # of objects received
3 = most objects this proc ever owns
*/
float c[4];
int createpartree = 0;
/*int num_gid_entries = zz->Num_GID;*/
/*int num_lid_entries = zz->Num_LID;*/
ZOLTAN_TRACE_ENTER(zz, yo);
MPI_Barrier(zz->Communicator);
timestart = MPI_Wtime();
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = 0;
counters[4] = 0;
counters[5] = 0;
c[0] = 0;
c[1] = 0;
c[2] = 0;
c[3] = 0;
timers[1] = 0.0;
timers[2] = 0.0;
timers[3] = 0.0;
nsentags = nrectags = 0;
if(zz->LB.Data_Structure == NULL) {
OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim);
Zoltan_Oct_set_method(OCT_info, oct_method);
Zoltan_Oct_set_maxregions(oct_maxoctregions);
Zoltan_Oct_set_minregions(oct_minoctregions);
createpartree = 1;
}
else {
OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
}
/* create the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data");
Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1],
&counters[2], &counters[3], &c[0],
createpartree);
time2 = MPI_Wtime();
timers[0] = time2 - time1; /* time took to create octree */
/* Zoltan_Oct_POct_printResults(OCT_info); */
/* partition the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition");
/* old call to dfs_paritition: */
#if 0
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]);
#else
/***************************
if(zz->Proc == 0) {
int debug_i;
for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) {
fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]);
}
}
****************************/
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes);
#endif
time2 = MPI_Wtime();
timers[1] = time2 - time1; /* time took to partition octree */
if (oct_output_level > 2) {
Zoltan_Oct_Plots(zz);
}
/* set up tags for migrations */
time1 = MPI_Wtime();
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate");
Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags,
&c[2], &c[3], &counters[3], &counters[5]);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags");
if (zz->LB.Return_Lists) {
*num_export = nrectags;
if (nrectags > 0)
Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids,
export_procs, export_to_part, nrectags,
export_regs);
}
time2 = MPI_Wtime();
timers[2] = time2 - time1; /* time took to setup migration */
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
/* count the number of objects on this processor */
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
counters[4] = nsentags;
MPI_Barrier(zz->Communicator);
timestop = MPI_Wtime();
if (oct_output_level > 0) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats");
Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c,
oct_output_level);
}
for (kk = 0; kk < nrectags; kk++) {
ZOLTAN_FREE(&(export_regs[kk].Global_ID));
ZOLTAN_FREE(&(export_regs[kk].Local_ID));
}
ZOLTAN_FREE(&export_regs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear");
Zoltan_Oct_global_clear(OCT_info);
/* KDDKDD Don't understand how re-used octree will work, especially without
* KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything;
* KDDKDD we can move back to saving some of the tree later.
*/
Zoltan_Oct_Free_Structure(zz);
/* KDDKDD END */
/* Temporary return value until error codes are fully implemented. */
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
static int Zoltan_LB(
ZZ *zz,
int include_parts, /* Flag indicating whether to generate
partition informtion;
0 if called by Zoltan_LB_Balance,
1 if called by Zoltan_LB_Partition. */
int *changes, /* Set to zero or one depending on if
Zoltan determines a new
decomposition or not:
zero - No changes to the decomposition
were made by the load-balancing
algorithm; migration is not needed.
one - A new decomposition is suggested
by the load-balancer; migration is
needed to establish the new
decomposition. */
int *num_gid_entries, /* The number of array entries in a global ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Global_ID_Entries. */
int *num_lid_entries, /* The number of array entries in a local ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Local_ID_Entries. */
int *num_import_objs, /* The number of non-local objects in the
processor's new decomposition. */
ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
int **import_procs, /* Array of processor IDs for processors
currently owning non-local objects (i.e.,
objs to be imported) in this processor's
new decomposition. */
int **import_to_part, /* Partition to which the objects should be
imported. */
int *num_export_objs, /* The number of local objects that need to
be exported from the processor to establish
the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
int **export_procs, /* Array of destination processor IDs for
objects that need to be exported
to establish the new decomposition. */
int **export_to_part /* Partition to which objects should be
exported. */
)
{
/*
* Main load-balancing routine.
* Input: a Zoltan structure with appropriate function pointers set.
* Output:
* changes
* num_import_objs
* import_global_ids
* import_local_ids
* import_procs
* import_to_part
* num_export_objs
* export_global_ids
* export_local_ids
* export_procs
* export_to_part
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_LB";
int gmax; /* Maximum number of imported/exported objects
over all processors. */
int error = ZOLTAN_OK; /* Error code */
double start_time, end_time;
double lb_time[2] = {0.0,0.0};
char msg[256];
int comm[3],gcomm[3];
float *part_sizes = NULL, *fdummy = NULL;
int wgt_dim, part_dim;
int all_num_obj, i, ts, idIdx;
struct Hash_Node **ht;
int *export_all_procs, *export_all_to_part, *parts=NULL;
ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL;
ZOLTAN_ID_PTR gid;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_DRUM
/* initialize DRUM if needed */
Zoltan_Drum_Create_Model(zz);
/* stop DRUM monitors */
Zoltan_Drum_Stop_Monitors(zz);
#endif
/*
* Compute Max number of array entries per ID over all processors.
* Compute Max number of return arguments for Zoltan_LB_Balance.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
comm[2] = zz->LB.Return_Lists;
MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
zz->LB.Return_Lists = gcomm[2];
/* assume no changes */
*changes = 0;
*num_import_objs = *num_export_objs = 0;
*import_global_ids = NULL;
*import_local_ids = NULL;
*import_procs = NULL;
*import_to_part = NULL;
*export_global_ids = NULL;
*export_local_ids = NULL;
*export_procs = NULL;
*export_to_part = NULL;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz))
goto End;
if (zz->LB.Method == NONE) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s Balancing method selected == NONE; no balancing performed\n",
yo);
error = ZOLTAN_WARN;
goto End;
}
/*
* Sync the random number generator across processors.
*/
Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator);
/*
* Construct the heterogenous machine description.
*/
error = Zoltan_Build_Machine_Desc(zz);
if (error == ZOLTAN_FATAL)
goto End;
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/* Since generating a new partition, need to free old mapping vector */
zz->LB.OldRemap = zz->LB.Remap;
zz->LB.Remap = NULL;
error = Zoltan_LB_Build_PartDist(zz);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN)
goto End;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
int i, np, fp;
for (i = 0; i < zz->Num_Proc; i++) {
Zoltan_LB_Proc_To_Part(zz, i, &np, &fp);
printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n",
zz->Proc, i, np, fp);
}
}
/*
* Generate partitions sizes.
*/
#ifdef ZOLTAN_DRUM
/* set partition sizes computed by DRUM, if requested */
Zoltan_Drum_Set_Part_Sizes(zz);
#endif
wgt_dim = zz->Obj_Weight_Dim;
part_dim = ((wgt_dim > 0) ? wgt_dim : 1);
part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim
* zz->LB.Num_Global_Parts);
if (part_sizes == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
/* Get partition sizes. */
Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim,
part_sizes);
/*
* Call the actual load-balancing function.
*/
error = zz->LB.LB_Fn(zz, part_sizes,
num_import_objs, import_global_ids, import_local_ids,
import_procs, import_to_part,
num_export_objs, export_global_ids, export_local_ids,
export_procs, export_to_part);
ZOLTAN_FREE(&part_sizes);
if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){
sprintf(msg, "Partitioning routine returned code %d.", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
else if (error){
if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) {
sprintf(msg, "Partitioning routine returned code %d.", error);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning");
#ifdef ZOLTAN_DRUM
/* restart DRUM monitors -- should happen later but there are a lot
of ways out of Zoltan_LB and we want to make sure they do start */
Zoltan_Drum_Start_Monitors(zz);
#endif
if (*num_import_objs >= 0)
MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
else /* use export data */
MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
if (gmax == 0) {
/*
* Decomposition was not changed by the load balancing; no migration
* is needed.
*/
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s No changes to the decomposition due to partitioning; "
"no migration is needed.\n", yo);
/*
* Reset num_import_objs and num_export_objs; don't want to return
* -1 for the arrays that weren't returned by ZOLTAN_LB_FN.
*/
*num_import_objs = *num_export_objs = 0;
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){
/*
* This parameter setting requires that all local objects
* and their assignments appear in the export list.
*/
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
goto End;
}
/*
* Check whether we know the import data, export data, or both.
*
* If we were given the import data,
* we know what the new decomposition should look like on the
* processor, but we don't know which of our local objects we have
* to export to other processors to establish the new decomposition.
* Reverse the argument if we were given the export data.
*
* Unless we were given both maps, compute the inverse map.
*/
if (*num_import_objs >= 0){
if (*num_export_objs >= 0) {
/* Both maps already available; nothing to do. */;
if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) {
/* This condition should never happen!! */
/* Methods should not return arrays if no lists are requested. */
*num_import_objs = *num_export_objs = -1;
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
ZOLTAN_PRINT_WARN(zz->Proc, yo,
"Method returned lists, but no lists requested.");
}
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Export lists are requested; compute export map */
error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids,
*import_local_ids, *import_procs,
*import_to_part,
num_export_objs, export_global_ids,
export_local_ids, export_procs,
export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Method returned import lists, but only export lists were desired. */
/* Import lists not needed; free them. */
*num_import_objs = -1;
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
}
}
}
else { /* (*num_import_objs < 0) */
if (*num_export_objs >= 0) {
/* Only export lists have been returned. */
if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Compute import map */
error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids,
*export_local_ids, *export_procs,
*export_to_part,
num_import_objs, import_global_ids,
import_local_ids, import_procs,
import_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Method returned export lists, but only import lists are desired. */
/* Export lists not needed; free them. */
*num_export_objs = -1;
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
}
}
}
else { /* *num_export_objs < 0 && *num_import_objs < 0) */
if (zz->LB.Return_Lists) {
/* No map at all available */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned "
"neither import nor export data.");
error = ZOLTAN_WARN;
goto End;
}
}
}
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/*
* Normally, Zoltan_LB returns in the export lists all local
* objects that are moving off processor, or that are assigned
* to a partition on the local processor that is not the
* default partition. This setting of Return_Lists requests
* that all local objects be included in the export list.
*/
if (*num_export_objs == 0){
/* all local objects are remaining on processor */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End;
}
else{
all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error);
if (*num_export_objs < all_num_obj){
/* Create a lookup table for exported IDs */
if (*num_export_objs > 16){ /* could be 0, maybe only importing */
ts = (*num_export_objs) / 4; /* what's a good table size? */
}
else{
ts = *num_export_objs;
}
ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts);
/* Create a list of all gids, lids and partitions */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj,
&all_global_ids, &all_local_ids,
wgt_dim, &fdummy, &parts);
if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){
ZOLTAN_FREE(&fdummy);
if ((Zoltan_Special_Malloc(zz, (void **)&export_all_procs,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) ||
(Zoltan_Special_Malloc(zz, (void **)&export_all_to_part,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){
error = ZOLTAN_MEMERR;
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){
sprintf(msg, "Error building complete export list; "
"%d returned by Zoltan_Get_Obj_List\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
gid = all_global_ids;
for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){
idIdx = search_hash_table(zz, gid, ht, ts);
if (idIdx >= 0){
export_all_procs[i] = (*export_procs)[idIdx];
export_all_to_part[i] = (*export_to_part)[idIdx];
}
else{
export_all_procs[i] = zz->Proc;
export_all_to_part[i] = parts[i];
}
}
free_hash_table(ht, ts);
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
Zoltan_Special_Free(zz, (void **)&parts,
ZOLTAN_SPECIAL_MALLOC_INT);
*export_global_ids = all_global_ids;
*export_local_ids = all_local_ids;
*export_procs = export_all_procs;
*export_to_part = export_all_to_part;
*num_export_objs = all_num_obj;
}
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments");
end_time = Zoltan_Time(zz->Timer);
lb_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc);
for (i = 0; i < *num_import_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID]));
printf(" To partition: %4d",
(*import_to_part != NULL ? (*import_to_part)[i]
: zz->Proc));
printf(" From processor: %4d\n", (*import_procs)[i]);
}
printf("\n");
printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc);
for (i = 0; i < *num_export_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID]));
printf(" To partition: %4d",
(*export_to_part != NULL ? (*export_to_part)[i]
: (*export_procs)[i]));
printf(" To processor: %4d\n", (*export_procs)[i]);
}
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/*
* If the Help_Migrate flag is set, perform migration for the application.
*/
if (zz->Migrate.Auto_Migrate) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration");
start_time = Zoltan_Time(zz->Timer);
error = Zoltan_Migrate(zz,
*num_import_objs, *import_global_ids,
*import_local_ids, *import_procs, *import_to_part,
*num_export_objs, *export_global_ids,
*export_local_ids, *export_procs, *export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error in auto-migration; %d returned from "
"Zoltan_Help_Migrate\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
end_time = Zoltan_Time(zz->Timer);
lb_time[1] = end_time - start_time;
ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration");
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0],
"ZOLTAN Partition: ");
if (zz->Migrate.Auto_Migrate)
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1],
"ZOLTAN Migrate: ");
}
*changes = 1;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (error);
}
/*
* Each processor:
* owns a set of pins (nonzeros)
* may provide some edge weights
*
* We assume that no two processes will supply the same pin.
* But more than one process may supply pins for the same edge.
*/
static int
matrix_get_edges(ZZ *zz, Zoltan_matrix *matrix, ZOLTAN_ID_PTR *yGID, ZOLTAN_ID_PTR *pinID, int nX,
ZOLTAN_ID_PTR *xGID, ZOLTAN_ID_PTR *xLID, ZOLTAN_GNO_TYPE **xGNO, float **xwgt, int use_full_dd)
{
static char *yo = "matrix_get_edges";
int ierr = ZOLTAN_OK;
int hypergraph_callbacks = 0, graph_callbacks = 0;
int *nbors_proc = NULL; /* Pointers are global for the function to ensure proper free */
int *edgeSize = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Get_HG_Size_CS && zz->Get_HG_CS) {
hypergraph_callbacks = 1;
}
if ((zz->Get_Num_Edges != NULL || zz->Get_Num_Edges_Multi != NULL) &&
(zz->Get_Edge_List != NULL || zz->Get_Edge_List_Multi != NULL)) {
graph_callbacks = 1;
}
/* TEMPORARY FIX */
if (!graph_callbacks){
fprintf(stderr,"Bug #5470: matrix_get_edges fails for hypergraph queries\n");
return ZOLTAN_FATAL;
}
hypergraph_callbacks=0;
/* TEMPORARY FIX */
if (graph_callbacks && hypergraph_callbacks){
/* if (hgraph_model == GRAPH) */
/* hypergraph_callbacks = 0; */
graph_callbacks = 1; /* I prefer graph (allow to do "inplace") ! */
}
if (hypergraph_callbacks) {
matrix->redist = 1;
if (use_full_dd || ((ZOLTAN_ID_PTR) *xGNO != *xGID))
ZOLTAN_FREE(xGID);
else
*xGID = NULL;
ZOLTAN_FREE(xLID);
ZOLTAN_FREE(xGNO);
ierr = Zoltan_Hypergraph_Queries(zz, &matrix->nY,
&matrix->nPins, yGID, &matrix->ystart,
pinID);
CHECK_IERR;
matrix->yend = matrix->ystart + 1;
}
else if (graph_callbacks) {
int max_edges = 0;
int vertex;
int numGID, numLID;
matrix->opts.enforceSquare = 1;
matrix->nY = nX; /* It is square ! */
matrix->yGNO = *xGNO;
*yGID = NULL;
matrix->ywgtdim = zz->Obj_Weight_Dim;
*xwgt = NULL;
numGID = zz->Num_GID;
numLID = zz->Num_LID;
/* TODO : support local graphs */
/* TODO : support weights ! */
/* Get edge data */
Zoltan_Get_Num_Edges_Per_Obj(zz, matrix->nY, *xGID, *xLID,
&edgeSize, &max_edges, &matrix->nPins);
(*pinID) = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nPins);
nbors_proc = (int *)ZOLTAN_MALLOC(matrix->nPins * sizeof(int));
if (matrix->nPins && ((*pinID) == NULL || nbors_proc == NULL))
MEMORY_ERROR;
matrix->pinwgt = (float*)ZOLTAN_MALLOC(matrix->nPins*matrix->pinwgtdim*sizeof(float));
if (matrix->nPins && matrix->pinwgtdim && matrix->pinwgt == NULL)
MEMORY_ERROR;
if (zz->Get_Edge_List_Multi) {
zz->Get_Edge_List_Multi(zz->Get_Edge_List_Multi_Data,
numGID, numLID,
matrix->nY, *xGID, *xLID,
edgeSize,
(*pinID), nbors_proc, matrix->pinwgtdim,
matrix->pinwgt, &ierr);
}
else {
int edge;
for (vertex = 0, edge = 0 ; vertex < matrix->nY ; ++vertex) {
zz->Get_Edge_List(zz->Get_Edge_List_Data, numGID, numLID,
(*xGID)+vertex*numGID, (*xLID)+vertex*numLID,
(*pinID)+edge*numGID, nbors_proc+edge, matrix->pinwgtdim,
matrix->pinwgt+edge*matrix->pinwgtdim, &ierr);
edge += edgeSize[vertex];
}
}
CHECK_IERR;
/* Not Useful anymore */
ZOLTAN_FREE(xLID);
if (use_full_dd || ((ZOLTAN_ID_PTR) *xGNO != *xGID)) {
ZOLTAN_FREE(xGID);
*xGNO = NULL;
}
else {
*xGID = NULL;
*xGNO = NULL;
}
ZOLTAN_FREE(&nbors_proc);
/* Now construct CSR indexing */
matrix->ystart = (int*) ZOLTAN_MALLOC((matrix->nY+1)*sizeof(int));
if (matrix->ystart == NULL)
MEMORY_ERROR;
matrix->ystart[0] = 0;
matrix->yend = matrix->ystart + 1;
for (vertex = 0 ; vertex < matrix->nY ; ++vertex)
matrix->ystart[vertex+1] = matrix->ystart[vertex] + edgeSize[vertex];
}
else {
FATAL_ERROR ("You have to define Hypergraph or Graph queries");
}
if (matrix->opts.enforceSquare) {
matrix->globalY = matrix->globalX;
matrix->ddY = matrix->ddX;
matrix->ywgtdim = zz->Obj_Weight_Dim;
}
End:
ZOLTAN_FREE(&edgeSize);
ZOLTAN_FREE(&nbors_proc);
ZOLTAN_FREE(xLID);
ZOLTAN_FREE(xGID);
ZOLTAN_FREE(xGNO);
ZOLTAN_FREE(xwgt);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
void Zoltan_HG_Print(
ZZ *zz,
HGraph *hg,
Partition parts,
FILE *fp,
char *str
)
{
/* Routine to print hypergraph weights and edges. Assumes serial execution;
* put inside Zoltan_Print_Sync_Start/Zoltan_Print_Sync_End for parallel
* programs.
*/
int i, j;
int num_vwgt;
int num_ewgt;
float *sum;
char *yo = "Zoltan_HG_Print";
if (hg == NULL)
return;
ZOLTAN_TRACE_ENTER(zz, yo);
num_vwgt = hg->VtxWeightDim;
num_ewgt = hg->EdgeWeightDim;
sum = (float *) ZOLTAN_MALLOC(MAX(num_vwgt, num_ewgt) * sizeof(float));
fprintf(fp, "%s nVtx=%d nEdge=%d nPins=%d vWgt=%d eWgt=%d\n",
str, hg->nVtx, hg->nEdge, hg->nPins,
hg->VtxWeightDim, hg->EdgeWeightDim);
/* Print Vertex Info */
fprintf(fp, "%s Vertices: (edges)\n", str);
for (i = 0; i < hg->nVtx; i++) {
fprintf(fp, "%d (" ZOLTAN_GNO_SPEC ") in part %d: ",
i, VTX_LNO_TO_GNO(hg, i), (parts ? parts[i] : -1));
fprintf(fp, "(");
for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++)
fprintf(fp, "%d ", hg->vedge[j]);
fprintf(fp, ")\n");
}
if (hg->vwgt != NULL) {
for (j = 0; j < num_vwgt; j++) sum[j] = 0;
fprintf(fp, "%s Vertices: [weights])\n", str);
for (i = 0; i < hg->nVtx; i++) {
fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "): [", i, VTX_LNO_TO_GNO(hg, i));
for (j = 0; j < num_vwgt; j++) {
fprintf(fp, "%f ", hg->vwgt[i*num_vwgt + j]);
sum[j] += hg->vwgt[i*num_vwgt + j];
}
fprintf(fp, "])\n");
}
fprintf(fp, "Total vertex weight = [");
for (j = 0; j < num_vwgt; j++) fprintf(fp, "%f ", sum[j]);
fprintf(fp, "]\n");
}
/* Print Hyperedge Info */
fprintf(fp, "%s Hyperedges: (vertices)\n", str);
for (i = 0; i < hg->nEdge; i++) {
fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "): ", i, EDGE_LNO_TO_GNO(hg, i));
fprintf(fp, "(");
for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++)
fprintf(fp, "%d ", hg->hvertex[j]);
fprintf(fp, ")\n");
}
if (hg->ewgt != NULL) {
for (j = 0; j < num_ewgt; j++) sum[j] = 0;
fprintf(fp, "%s Hyperedge Weights: [weights]\n", str);
for (i = 0; i < hg->nEdge; i++) {
fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "): ", i, EDGE_LNO_TO_GNO(hg, i));
fprintf(fp, "[");
for (j = 0; j < num_ewgt; j++) {
fprintf(fp, "%f ", hg->ewgt[i*num_ewgt + j]);
sum[j] += hg->ewgt[i*num_ewgt + j];
}
fprintf(fp, "])\n");
}
fprintf(fp, "Total hyperedge weight = [");
for (j = 0; j < num_ewgt; j++) fprintf(fp, "%f ", sum[j]);
fprintf(fp, "]\n");
}
ZOLTAN_FREE(&sum);
ZOLTAN_TRACE_EXIT(zz, yo);
}
int Zoltan_HG_Info (
ZZ *zz,
HGraph *hg
)
{
int i, dd, size, size_min, size_max, count;
float wgt_min, wgt_max, wgt_tot;
double mean, var, temp;
char *yo = "Zoltan_HG_Info";
ZOLTAN_TRACE_ENTER(zz, yo);
printf("--------- HGraph Information (min/ave/max/tot) ------------------\n");
printf("INFO MAY BE WRONG FOR 2D DATA DISTRIBUTION. KDDKDD \n");
printf("info:%d |V|=%d |E|=%d |P|=%d \n", hg->info, hg->nVtx, hg->nEdge,
hg->nPins);
/* print weights */
if (hg->nVtx && hg->vwgt) {
for (dd = 0; dd < hg->VtxWeightDim; dd++) {
wgt_tot = 0.0;
wgt_min = FLT_MAX;
wgt_max = FLT_MIN;
for (i = 0; i < hg->nVtx; i++) {
wgt_tot += hg->vwgt[i*hg->VtxWeightDim+dd];
wgt_min = MIN(wgt_min, hg->vwgt[i*hg->VtxWeightDim+dd]);
wgt_max = MAX(wgt_max, hg->vwgt[i*hg->VtxWeightDim+dd]);
}
printf("Vertex weights : %9.2f %9.2f %9.2f %12.2f\n", wgt_min,
wgt_tot/hg->nVtx, wgt_max, wgt_tot);
mean = var = 0.0;
if (hg->nVtx > 1) {
mean = wgt_tot / hg->nVtx;
for (i = 0; i < hg->nVtx; i++) {
temp = hg->vwgt[i*hg->VtxWeightDim+dd] - mean;
var += (temp*temp);
}
var = sqrt(var/(hg->nVtx-1));
printf ("Vertex Stats: stdev %.2f, Coef of Var %.2f\n", var, var/mean);
}
count=0;
temp=0.0;
for (i = 0; i < hg->nVtx; i++)
if (hg->vwgt[i*hg->VtxWeightDim+dd] > (mean + 3.0 * var)) {
count++;
temp += hg->vwgt[i*hg->VtxWeightDim+dd];
}
printf ("Vertex >3sigma: count %d, rel count x10000 %.1f, "
"rel weight x100 %.1f\n",
count, (float)10000*count/(float)hg->nVtx, 100.0*temp/wgt_tot);
}
}
if (hg->nEdge && hg->ewgt) {
wgt_tot = 0.0;
wgt_min = FLT_MAX;
wgt_max = FLT_MIN;
for (i = 0; i < hg->nEdge; i++) {
wgt_tot += hg->ewgt[i];
wgt_min = MIN(wgt_min, hg->ewgt[i]);
wgt_max = MAX(wgt_max, hg->ewgt[i]);
}
printf("HEdge weights : %9.2f %9.2f %9.2f %12.2f\n", wgt_min,
wgt_tot/hg->nEdge, wgt_max, wgt_tot);
if (hg->nEdge > 1) {
var = 0.0;
mean = wgt_tot / hg->nEdge;
for (i = 0; i < hg->nEdge; i++) {
temp = hg->ewgt[i] - mean;
var += (temp*temp);
}
var = sqrt(var/(hg->nEdge-1));
printf ("HEdge Stats: STDV %.2f, Coef of Var %.2f\n", var, var/mean);
}
}
/* print sizes */
if (hg->nPins && hg->hindex) {
size_min = INT_MAX;
size_max = INT_MIN;
for (i = 0; i < hg->nEdge; i++) {
size = hg->hindex[i+1] - hg->hindex[i];
size_min = MIN(size_min, size);
size_max = MAX(size_max, size);
}
printf("Edge sizes : %6d %9.2f %6d %9d\n", size_min,
(float)hg->nPins / hg->nEdge, size_max, hg->nPins);
if (hg->nEdge > 1) {
var = 0.0;
mean = (float)hg->nPins / hg->nEdge;
for (i = 0; i < hg->nEdge; i++) {
temp = (float)(hg->hindex[i+1]-hg->hindex[i]) - mean;
var += (temp*temp);
}
var = sqrt(var/(hg->nEdge-1));
printf ("Edge Stats: stdev %.2f, Coef of Var %.2f\n", var, var/mean);
}
}
if (hg->nPins && hg->vindex) {
size_min = INT_MAX;
size_max = INT_MIN;
for (i = 0; i < hg->nVtx; i++) {
size = hg->vindex[i+1] - hg->vindex[i];
size_min = MIN(size_min, size);
size_max = MAX(size_max, size);
}
printf("Vertex sizes : %6d %9.2f %6d %9d\n", size_min,
(float)hg->nPins / hg->nVtx, size_max, hg->nPins);
if (hg->nVtx > 1) {
var = 0.0;
mean = (float)hg->nPins / hg->nVtx;
for (i = 0; i < hg->nVtx; i++) {
temp = (float)(hg->vindex[i+1]-hg->vindex[i]) - mean;
var += (temp*temp);
}
var = sqrt(var/(hg->nVtx-1));
printf ("Vertex Stats: stdev %.2f, Coef of Var %.2f\n", var, var/mean);
}
}
printf("-----------------------------------------------------------------\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_OK;
}
int Zoltan_HG_Create_Mirror (
ZZ *zz,
HGraph *hg
)
{
int inlength, outlength; /* input/output array lengths */
int *index; /* pointers to input information */
int *outindex;
int *data, *outdata;
char *yo = "Zoltan_HG_Create_Mirror";
ZOLTAN_TRACE_ENTER(zz, yo);
/* determine which data to "mirror" and set corresponding data pointers. */
if (hg && (hg->nEdge == 0 || hg->hindex) && (hg->nPins == 0 || hg->hvertex)
&& !hg->vindex && !hg->vedge) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Have hindex; building vindex.");
inlength = hg->nEdge;
outlength = hg->nVtx;
index = hg->hindex;
data = hg->hvertex;
outindex = hg->vindex = (int*) ZOLTAN_MALLOC((hg->nVtx+1) * sizeof(int));
outdata = hg->vedge = (int*) ZOLTAN_MALLOC (hg->nPins * sizeof(int));
if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) {
Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->vindex, &hg->vedge);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
else if (hg && (hg->nVtx == 0 || hg->vindex) && (hg->nPins == 0 || hg->vedge)
&& !hg->hindex && !hg->hvertex) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Have vindex; building hindex.");
inlength = hg->nVtx;
outlength = hg->nEdge;
index = hg->vindex;
data = hg->vedge;
outindex = hg->hindex = (int*) ZOLTAN_MALLOC((hg->nEdge+1) * sizeof(int));
outdata = hg->hvertex = (int*) ZOLTAN_MALLOC(hg->nPins * sizeof(int));
if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) {
Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->hindex, &hg->hvertex);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
else {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input error.");
return ZOLTAN_FATAL; /* unable to proceed */
}
Zoltan_HG_Mirror(inlength, index, data,
outlength, outindex, outdata);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_OK;
}
int Zoltan_HG_Check (
ZZ *zz,
HGraph *hg
)
{
int i;
int iedge; /* iedge denotes an hyperedge */
int j; /* j is the index of a vertex */
int k; /* k is an index of hyperedge */
int *check;
char str[256];
int err = ZOLTAN_OK;
char *yo = "Zoltan_HG_Check";
ZOLTAN_TRACE_ENTER(zz, yo);
if ((hg->nEdge && !hg->hindex) ||
(hg->nVtx && !hg->vindex) ||
(hg->nPins && (!hg->vedge || !hg->hvertex))) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "NULL arrays found");
err = ZOLTAN_WARN;
goto End;
}
if (hg->nPins) {
check = (int*) ZOLTAN_MALLOC(((hg->nEdge > hg->nVtx) ? hg->nEdge : hg->nVtx)
* sizeof(int));
if (check == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unable to allocate memory.");
err = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < hg->nEdge; i++)
check[i] = -1;
for (i = 0; i < hg->nVtx; i++)
for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++)
if (check[hg->vedge[j]] < i)
check[hg->vedge[j]] = i;
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found multiple hedge for same vertex.");
err = ZOLTAN_WARN;
}
for (i = 0; i < hg->nVtx; i++)
check[i] = -1;
for (i = 0; i < hg->nEdge; i++)
for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++)
if (check[hg->hvertex[j]] < i)
check[hg->hvertex[j]] = i;
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found multiple vertex for same hedge.");
err = ZOLTAN_WARN;
}
ZOLTAN_FREE (&check);
}
for (i = 0; i < hg->VtxWeightDim * hg->nVtx; i++)
if (hg->vwgt[i] < 0.0) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found negative vertex weight.");
err = ZOLTAN_WARN;
}
for (i = 0; i < hg->EdgeWeightDim * hg->nEdge; i++)
if (hg->ewgt[i] < 0.0) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found negative edge weight.");
err = ZOLTAN_WARN;
}
if (hg->comm && hg->comm->nProc_x == 1) {
/* In 2D distribution, check makes sense only if proc has entire hedge */
for (i = 0; i < hg->nEdge; i++)
if ((hg->hindex[i+1] == hg->hindex[i])) {
sprintf (str, "Found hyperedge with no vertices: "
"edge %d has %d vtxs\n", i, (hg->hindex[i+1] - hg->hindex[i]));
ZOLTAN_PRINT_WARN(zz->Proc, yo, str);
err = ZOLTAN_WARN;
}
}
for (i = 0; i < hg->nEdge; i++)
for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++)
if (hg->hvertex[j] < 0 || hg->hvertex[j] >= hg->nVtx) {
ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found vertex out of range in hvertex.");
err = ZOLTAN_WARN;
}
for (i = 0; i < hg->nVtx; i++)
for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++)
if (hg->vedge[j] < 0 || hg->vedge[j] >= hg->nEdge) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found edge out of range in vedge.");
err = ZOLTAN_WARN;
}
/* starting from (hindex,hvertex), for each edge determine each associated
* vertex. Then for each vertex lookup associated edges using (vindex, vedge)
*/
for (iedge = 0; iedge < hg->nEdge; iedge++)
for (j = hg->hindex[iedge]; j < hg->hindex[iedge+1]; j++) {
/* for each hyperedge get index to vertices */
for (k=hg->vindex[hg->hvertex[j]]; k<hg->vindex[hg->hvertex[j]+1]; k++)
/* for each vertex of current hyperedge get index to hyperedges */
if (hg->vedge[k] == iedge) /* does it match with original edge? */
break;
if (k == hg->vindex[hg->hvertex[j]+1]) {
/* if no match was found then failure, else keep on */
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Inconsistent hvertex/vedge");
err = ZOLTAN_WARN;
break;
}
}
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return err;
}
int Zoltan_Get_Coordinates(
ZZ *zz,
int num_obj, /* Input: number of objects */
ZOLTAN_ID_PTR global_ids, /* Input: global IDs of objects */
ZOLTAN_ID_PTR local_ids, /* Input: local IDs of objects; may be NULL. */
int *num_dim, /* Output: dimension of coordinates */
double **coords /* Output: array of coordinates; malloc'ed by
fn if NULL upon input. */
)
{
char *yo = "Zoltan_Get_Coordinates";
int i,j,rc;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
int alloced_coords = 0;
ZOLTAN_ID_PTR lid; /* Temporary pointers to local IDs; used to pass
NULL to query functions when NUM_LID_ENTRIES == 0. */
double dist[3];
double im[3][3];
double deg_ratio;
double x;
int order[3];
int reduce_dimensions, d, axis_aligned;
int target_dim;
int ierr = ZOLTAN_OK;
char msg[256];
ZZ_Transform *tran;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Error check -- make sure needed callback functions are registered. */
if (zz->Get_Num_Geom == NULL ||
(zz->Get_Geom == NULL && zz->Get_Geom_Multi == NULL)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_GEOM_FN and "
"either ZOLTAN_GEOM_MULTI_FN or ZOLTAN_GEOM_FN");
goto End;
}
/* Get problem dimension. */
*num_dim = zz->Get_Num_Geom(zz->Get_Num_Geom_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_NUM_GEOM_FN");
goto End;
}
if (*num_dim < 0 || *num_dim > 3) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Invalid dimension returned from ZOLTAN_NUM_GEOM_FN");
goto End;
}
/* Get coordinates for object; allocate memory if not already provided. */
if (*num_dim > 0 && num_obj > 0) {
if (*coords == NULL) {
alloced_coords = 1;
*coords = (double *) ZOLTAN_MALLOC(num_obj * (*num_dim) * sizeof(double));
if (*coords == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error");
goto End;
}
}
if (zz->Get_Geom_Multi != NULL) {
zz->Get_Geom_Multi(zz->Get_Geom_Multi_Data, zz->Num_GID, zz->Num_LID,
num_obj, global_ids, local_ids, *num_dim, *coords,
&ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_GEOM_MULTI_FN");
goto End;
}
}
else {
for (i = 0; i < num_obj; i++) {
lid = (num_lid_entries ? &(local_ids[i*num_lid_entries]) : NULL);
zz->Get_Geom(zz->Get_Geom_Data, num_gid_entries, num_lid_entries,
global_ids + i*num_gid_entries, lid,
(*coords) + i*(*num_dim), &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_GEOM_FN");
goto End;
}
}
}
}
/*
* For RCB, RIB, and HSFC: if REDUCE_DIMENSIONS was selected, compute the
* center of mass and inertial matrix of the coordinates.
*
* For 3D problems: If the geometry is "flat", transform the points so the
* two primary directions lie along the X and Y coordinate axes and project
* to the Z=0 plane. If in addition the geometry is "skinny", project to
* the X axis. (This creates a 2D or 1D problem respectively.)
*
* For 2D problems: If the geometry is essentially a line, transform it's
* primary direction to the X axis and project to the X axis, yielding a
* 1D problem.
*
* Return these points to the partitioning algorithm, in effect partitioning
* in only the 2 (or 1) significant dimensions.
*/
if (((*num_dim == 3) || (*num_dim == 2)) &&
((zz->LB.Method==RCB) || (zz->LB.Method==RIB) || (zz->LB.Method==HSFC))){
Zoltan_Bind_Param(Reduce_Dim_Params, "KEEP_CUTS", (void *)&i);
Zoltan_Bind_Param(Reduce_Dim_Params, "REDUCE_DIMENSIONS",
(void *)&reduce_dimensions);
Zoltan_Bind_Param(Reduce_Dim_Params, "DEGENERATE_RATIO", (void *)°_ratio);
i = 0;
reduce_dimensions = 0;
deg_ratio = 10.0;
Zoltan_Assign_Param_Vals(zz->Params, Reduce_Dim_Params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
if (reduce_dimensions == 0){
goto End;
}
if (deg_ratio <= 1){
if (zz->Proc == 0){
ZOLTAN_PRINT_WARN(0, yo, "DEGENERATE_RATIO <= 1, setting it to 10.0");
}
deg_ratio = 10.0;
}
if (zz->LB.Method == RCB){
tran = &(((RCB_STRUCT *)(zz->LB.Data_Structure))->Tran);
}
else if (zz->LB.Method == RIB){
tran = &(((RIB_STRUCT *)(zz->LB.Data_Structure))->Tran);
}
else{
tran = &(((HSFC_Data*)(zz->LB.Data_Structure))->tran);
}
d = *num_dim;
if (tran->Target_Dim >= 0){
/*
* On a previous load balancing call, we determined whether
* or not the geometry was degenerate. If the geometry was
* determined to be not degenerate, then we assume it is still
* not degenerate, and we skip the degeneracy calculation.
*/
if (tran->Target_Dim > 0){
/*
* The geometry *was* degenerate. We test the extent
* of the geometry along the principal directions determined
* last time to determine if it is still degenerate with that
* orientation. If so, we transform the coordinates using the
* same transformation we used last time. If not, we do the
* entire degeneracy calculation again.
*/
if ((tran->Axis_Order[0] >= 0) &&
(tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){
axis_aligned = 1;
}
else{
axis_aligned = 0;
}
projected_distances(zz, *coords, num_obj, tran->CM,
tran->Evecs, dist, d, axis_aligned, tran->Axis_Order);
target_dim = get_target_dimension(dist, order, deg_ratio, d);
if (target_dim > 0){
transform_coordinates(*coords, num_obj, d, tran);
}
else{
/* Set's Target_Dim to -1, flag to recompute degeneracy */
Zoltan_Initialize_Transformation(tran);
}
}
}
if (tran->Target_Dim < 0){
tran->Target_Dim = 0;
/*
* Get the center of mass and inertial matrix of coordinates. Ignore
* vertex weights, we are only interested in geometry. Global operation.
*/
if (d == 2){
inertial_matrix2D(zz, *coords, num_obj, tran->CM, im);
}
else{
inertial_matrix3D(zz, *coords, num_obj, tran->CM, im);
}
/*
* The inertial matrix is a 3x3 or 2x2 real symmetric matrix. Get its
* three or two orthonormal eigenvectors. These usually indicate the
* orientation of the geometry.
*/
rc = eigenvectors(im, tran->Evecs, d);
if (rc){
if (zz->Proc == 0){
ZOLTAN_PRINT_WARN(0, yo, "REDUCE_DIMENSIONS calculation failed");
}
goto End;
}
/*
* Here we check to see if the eigenvectors are very close
* to the coordinate axes. If so, we can more quickly
* determine whether the geometry is degenerate, and also more
* quickly transform the geometry to the lower dimensional
* space.
*/
axis_aligned = 0;
for (i=0; i<d; i++){
tran->Axis_Order[i] = -1;
}
for (j=0; j<d; j++){
for (i=0; i<d; i++){
x = fabs(tran->Evecs[i][j]);
if (NEAR_ONE(x)){
tran->Axis_Order[j] = i; /* e'vector j is very close to i axis */
break;
}
}
if (tran->Axis_Order[j] < 0){
break;
}
}
if ((tran->Axis_Order[0] >= 0) &&
(tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){
axis_aligned = 1;
}
/*
* Calculate the extent of the geometry along the three lines defined
* by the direction of the eigenvectors through the center of mass.
*/
projected_distances(zz, *coords, num_obj, tran->CM, tran->Evecs, dist,
d, axis_aligned, tran->Axis_Order);
/*
* Decide whether these distances indicate the geometry is
* very flat in one or two directions.
*/
target_dim = get_target_dimension(dist, order, deg_ratio, d);
if (target_dim > 0){
/*
* Yes, geometry is degenerate
*/
if ((zz->Debug_Level > 0) && (zz->Proc == 0)){
if (d == 2){
sprintf(msg,
"Geometry (~%f x %f), exceeds %f to 1.0 ratio",
dist[order[0]], dist[order[1]], deg_ratio);
}
else{
sprintf(msg,
"Geometry (~%f x %f x %f), exceeds %f to 1.0 ratio",
dist[order[0]], dist[order[1]], dist[order[2]], deg_ratio);
}
ZOLTAN_PRINT_INFO(zz->Proc, yo, msg);
sprintf(msg, "We'll treat it as %d dimensional",target_dim);
ZOLTAN_PRINT_INFO(zz->Proc, yo, msg);
}
if (axis_aligned){
/*
** Create new geometry, transforming the primary direction
** to the X-axis, and the secondary to the Y-axis.
*/
tran->Permutation[0] = tran->Axis_Order[order[0]];
if (target_dim == 2){
tran->Permutation[1] = tran->Axis_Order[order[1]];
}
}
else{
/*
* Reorder the eigenvectors (they're the columns of evecs) from
* longest projected distance to shorted projected distance. Compute
* the transpose (the inverse) of the matrix. This will transform
* the geometry to align along the X-Y plane, or along the X axis.
*/
for (i=0; i< target_dim; i++){
tran->Transformation[i][2] = 0.0;
for (j=0; j<d; j++){
tran->Transformation[i][j] = tran->Evecs[j][order[i]];
}
}
for (i=target_dim; i< 3; i++){
for (j=0; j<3; j++){
tran->Transformation[i][j] = 0.0;
}
}
}
tran->Target_Dim = target_dim;
transform_coordinates(*coords, num_obj, d, tran);
} /* If geometry is very flat */
} /* If REDUCE_DIMENSIONS is true */
} /* If 2-D or 3-D rcb, rib or hsfc */
End:
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no coordinates returned.");
if (alloced_coords) ZOLTAN_FREE(coords);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
/* Point drop for refinement after above partitioning */
int Zoltan_HSFC_Point_Assign (
ZZ *zz,
double *x,
int *proc,
int *part)
{
double scaled[3];
double pt[3];
double fsfc;
Partition *p;
int i;
int dim;
HSFC_Data *d;
int err;
char *yo = "Zoltan_HSFC_Point_Assign";
ZOLTAN_TRACE_ENTER (zz, yo);
d = (HSFC_Data *) zz->LB.Data_Structure;
if (d == NULL)
ZOLTAN_HSFC_ERROR (ZOLTAN_FATAL,
"No Decomposition Data available; use KEEP_CUTS parameter.");
for (i=0; i<d->ndimension; i++){
pt[i] = x[i]; /* we don't want to change caller's "x" */
}
if (d->tran.Target_Dim > 0){ /* degenerate geometry */
dim = d->tran.Target_Dim;
Zoltan_Transform_Point(pt, d->tran.Transformation, d->tran.Permutation,
d->ndimension, dim, pt);
}
else{
dim = d->ndimension;
}
/* Calculate scaled coordinates, calculate HSFC coordinate */
for (i = 0; i < dim; i++)
{
scaled[i] = (pt[i] - d->bbox_lo[i]) / d->bbox_extent[i];
if (scaled[i] < HSFC_EPSILON) scaled[i] = HSFC_EPSILON;
if (scaled[i] > 1.0 - HSFC_EPSILON) scaled[i] = 1.0 - HSFC_EPSILON;
}
fsfc = d->fhsfc (zz, scaled); /* Note, this is a function call */
/* Find partition containing point and return its number */
p = (Partition *) bsearch (&fsfc, d->final_partition, zz->LB.Num_Global_Parts,
sizeof (Partition), Zoltan_HSFC_compare);
if (p == NULL)
ZOLTAN_HSFC_ERROR (ZOLTAN_FATAL, "programming error, shouldn't happen");
if (part != NULL) {
if (zz->LB.Remap)
*part = zz->LB.Remap[p->index];
else
*part = p->index;
}
if (proc != NULL) {
if (zz->LB.Remap)
*proc = Zoltan_LB_Part_To_Proc(zz, zz->LB.Remap[p->index], NULL);
else
*proc = Zoltan_LB_Part_To_Proc(zz, p->index, NULL);
}
err = ZOLTAN_OK;
End:
ZOLTAN_TRACE_EXIT (zz, yo);
return err;
}
int Zoltan_LocalHSFC_Order(
ZZ *zz, /* Zoltan structure */
int num_obj, /* Number of (local) objects to order. */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm,
ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */
)
{
static char *yo = "Zoltan_LocalHSFC_Order";
int n, ierr=ZOLTAN_OK;
double (*fhsfc)(ZZ*, double*); /* space filling curve function */
int wgt_dim=0;
float *obj_wgts=0;
int *parts=0;
int numGeomDims=0;
double *geomArray=0;
/* Variables for bounding box */
double *minValInDim;
double *maxValInDim;
double *widthDim;
double *hsfcKey=0;
int *coordIndx=0;
/* Counters */
int objNum;
int dimNum;
int offset=0;
int myrank;
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
ZOLTAN_TRACE_ENTER(zz, yo);
/******************************************************************/
/* If for some reason order_opt is NULL, allocate a new ZOOS here. */
/* This should really never happen. */
/******************************************************************/
if (!order_opt)
{
order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
strcpy(order_opt->method,"LOCAL_HSFC");
}
/******************************************************************/
/* local HSFC only computes the rank vector */
order_opt->return_args = RETURN_RANK;
/******************************************************************/
/* Check that num_obj equals the number of objects on this proc. */
/* This constraint may be removed in the future. */
/******************************************************************/
n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN))
{
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
return(ZOLTAN_FATAL);
}
if (n != num_obj)
{
/* Currently this is a fatal error. */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the number of objects.");
return(ZOLTAN_FATAL);
}
/******************************************************************/
/******************************************************************/
/* Get lists of objects */
/******************************************************************/
ierr = Zoltan_Get_Obj_List(zz, &n, &gids, &lids, wgt_dim, &obj_wgts, &parts);
if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN))
{
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Obj_List returned error.");
return(ZOLTAN_FATAL);
}
/******************************************************************/
/******************************************************************/
/* Get geometry for objects*/
/******************************************************************/
ierr = Zoltan_Get_Coordinates(zz, n, gids, lids, &numGeomDims,
&geomArray);
if (ierr != 0)
{
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Zoltan_Get_Coordinates returned error.");
return(ZOLTAN_FATAL);
}
/******************************************************************/
/******************************************************************/
/* Place coords in bounding box */
/******************************************************************/
minValInDim = (double *) malloc(numGeomDims * sizeof (double));
maxValInDim = (double *) malloc(numGeomDims * sizeof (double));
widthDim = (double *) malloc(numGeomDims * sizeof (double));
for(dimNum=0; dimNum<numGeomDims; dimNum++)
{
minValInDim[dimNum] = HUGE_VAL;
maxValInDim[dimNum] = -HUGE_VAL;
}
/*************************************************************/
/* Determine min, max, and width for each dimension */
/*************************************************************/
for (objNum=0; objNum<n; objNum++)
{
for(dimNum=0; dimNum<numGeomDims; dimNum++)
{
if (geomArray[objNum * numGeomDims + dimNum] < minValInDim[dimNum])
{
minValInDim[dimNum] = geomArray[objNum * numGeomDims + dimNum];
}
if (geomArray[objNum * numGeomDims + dimNum] > maxValInDim[dimNum])
{
maxValInDim[dimNum] = geomArray[objNum * numGeomDims + dimNum];
}
}
}
for(dimNum=0; dimNum<numGeomDims; dimNum++)
{
widthDim[dimNum] = maxValInDim[dimNum] - minValInDim[dimNum];
}
/*************************************************************/
/*************************************************************/
/* Rescale values to fit in bounding box */
/*************************************************************/
for (objNum=0; objNum<n; objNum++)
{
for(dimNum=0; dimNum<numGeomDims; dimNum++)
{
geomArray[objNum * numGeomDims + dimNum] -= minValInDim[dimNum];
geomArray[objNum * numGeomDims + dimNum] /= widthDim[dimNum];
}
}
/*************************************************************/
free(minValInDim); minValInDim=0;
free(maxValInDim); maxValInDim=0;
free(widthDim); widthDim=0;
/******************************************************************/
/******************************************************************/
/* Specify which HSFC function to use (based on dim) */
/******************************************************************/
if (numGeomDims==1)
{
fhsfc = Zoltan_HSFC_InvHilbert1d;
}
else if (numGeomDims==2)
{
fhsfc = Zoltan_HSFC_InvHilbert2d;
}
else if (numGeomDims==3)
{
fhsfc = Zoltan_HSFC_InvHilbert3d;
}
else /* this error should have been previously caught */
{
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Geometry should be of dimension 1, 2, or 3.");
return(ZOLTAN_FATAL);
}
/******************************************************************/
/******************************************************************/
/* Generate hsfc keys and indices to be sorted */
/******************************************************************/
hsfcKey = (double *) malloc(n * sizeof (double));
coordIndx = (int *) malloc(n *sizeof(int));
for (objNum=0; objNum<n; objNum++)
{
hsfcKey[objNum] = fhsfc(zz, &(geomArray[objNum * numGeomDims]) );
coordIndx[objNum] = objNum;
}
/******************************************************************/
/******************************************************************/
/* Sort indices based on keys */
/******************************************************************/
Zoltan_quicksort_pointer_dec_double (coordIndx, hsfcKey, 0, n-1);
/******************************************************************/
/******************************************************************/
/* get ranks */
/******************************************************************/
/******************************************************/
/* Determine offsets */
/******************************************************/
MPI_Scan(&n, &offset, 1, MPI_INT, MPI_SUM, zz->Communicator);
offset -= n; /* MPI_Scan is inclusive, so subtract off local size */
/******************************************************/
for(objNum=0; objNum<n; objNum++)
{
/*MMW temporary hack to make Cedric's interface give me want I need */
/*rank[coordIndx[objNum]] = objNum + offset; */
rank[objNum] = coordIndx[objNum] + offset;
}
/******************************************************************/
/* iperm is to be deprecated so not calculated*/
free(hsfcKey);
free(coordIndx);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
int Zoltan_Preprocess_Graph(
ZZ *zz, /* Zoltan structure */
ZOLTAN_ID_PTR *global_ids,
ZOLTAN_ID_PTR *local_ids,
ZOLTAN_Third_Graph *gr, /* Graph for third part libs */
ZOLTAN_Third_Geom *geo,
ZOLTAN_Third_Part *prt,
ZOLTAN_Third_Vsize *vsp
)
{
static char *yo = "Zoltan_Preprocess_Graph";
int ierr;
float *float_vwgt, *float_ewgts;
char msg[256];
char add_obj_weight[MAX_PARAM_STRING_LEN+1];
ZOLTAN_TRACE_ENTER(zz, yo);
/* Initialize all local pointers to NULL. This is necessary
* because we free all non-NULL pointers upon errors.
*/
gr->vtxdist = gr->xadj = gr->adjncy = NULL;
gr->vwgt = gr->ewgts = NULL;
float_vwgt = float_ewgts = NULL;
if (gr->obj_wgt_dim >= 0) {
/* Check weight dimensions */
if (zz->Obj_Weight_Dim<0){
sprintf(msg, "Object weight dimension is %d, "
"but should be >= 0. Using Obj_Weight_Dim = 0.",
zz->Obj_Weight_Dim);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
gr->obj_wgt_dim = 0;
}
else {
gr->obj_wgt_dim = zz->Obj_Weight_Dim;
}
}
else
gr->obj_wgt_dim = 0;
if (gr->edge_wgt_dim >= 0) {
if (zz->Edge_Weight_Dim<0){
sprintf(msg, "Edge weight dimension is %d, "
"but should be >= 0. Using Edge_Weight_Dim = 0.",
zz->Edge_Weight_Dim);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
gr->edge_wgt_dim = 0;
}
else if (zz->Edge_Weight_Dim>1){
ZOLTAN_PRINT_WARN(zz->Proc, yo, "This method does not support "
"multidimensional edge weights. Using Edge_Weight_Dim = 1.");
gr->edge_wgt_dim = 1;
}
else {
gr->edge_wgt_dim = zz->Edge_Weight_Dim;
}
}
else
gr->edge_wgt_dim = 0;
if (gr->graph_type >= 0)
/* Default graph type is GLOBAL. */
gr->graph_type = GLOBAL_GRAPH;
else
gr->graph_type = - gr->graph_type;
/* Get parameter options shared by ParMetis and Jostle */
gr->check_graph = 1; /* default */
gr->scatter = 1; /* default */
gr->final_output = 0;
strcpy(add_obj_weight, "NONE"); /* default */
Zoltan_Bind_Param(Graph_params, "CHECK_GRAPH", (void *) &gr->check_graph);
Zoltan_Bind_Param(Graph_params, "SCATTER_GRAPH", (void *) &gr->scatter);
Zoltan_Bind_Param(Graph_params, "FINAL_OUTPUT", (void *) &gr->final_output);
Zoltan_Bind_Param(Graph_params, "ADD_OBJ_WEIGHT", (void *) add_obj_weight);
Zoltan_Assign_Param_Vals(zz->Params, Graph_params, zz->Debug_Level, zz->Proc,
zz->Debug_Proc);
/* If reorder is true, we already have the id lists. Ignore weights. */
if ((*global_ids == NULL) || (!gr->id_known)){
int * input_part;
ierr = Zoltan_Get_Obj_List(zz, &gr->num_obj, global_ids, local_ids,
gr->obj_wgt_dim, &float_vwgt, &input_part);
if (prt) {
prt->input_part = input_part;
}
else { /* Ordering, dont need part */
ZOLTAN_FREE(&input_part);
}
if (ierr){
/* Return error */
ZOLTAN_PARMETIS_ERROR(ierr, "Get_Obj_List returned error.");
}
}
/* Build Graph for third party library data structures, or just get vtxdist. */
ierr = Zoltan_Build_Graph(zz, gr->graph_type, gr->check_graph, gr->num_obj,
*global_ids, *local_ids, gr->obj_wgt_dim, gr->edge_wgt_dim,
&gr->vtxdist, &gr->xadj, &gr->adjncy, &float_ewgts, &gr->adjproc);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
ZOLTAN_PARMETIS_ERROR(ierr, "Zoltan_Build_Graph returned error.");
}
if (prt) {
prt->part_sizes = prt->input_part_sizes;
if (gr->num_obj >0) {
prt->part = (indextype *)ZOLTAN_MALLOC((gr->num_obj+1) * sizeof(indextype));
if (!prt->part){
/* Not enough memory */
ZOLTAN_PARMETIS_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
memcpy (prt->part, prt->input_part, (gr->num_obj) * sizeof(indextype));
}
else {
prt->input_part = prt->part = NULL;
}
}
/* Convert from float. */
/* Get vertex weights if needed */
if (gr->obj_wgt_dim){
ierr = Zoltan_Preprocess_Scale_Weights (gr, float_vwgt, &gr->vwgt,
gr->num_obj, gr->obj_wgt_dim, 1, zz,
"vertex", gr->vtxdist[zz->Proc]);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
/* Return error code */
ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights.");
}
ZOLTAN_FREE(&float_vwgt);
}
if (strcasecmp(add_obj_weight, "NONE")){
if (Zoltan_Preprocess_Add_Weight(zz, gr, prt, add_obj_weight) != ZOLTAN_OK)
ZOLTAN_PARMETIS_ERROR(ierr, "Error in adding vertex weights.");
}
/* Get edge weights if needed */
if (gr->get_data)
gr->num_edges = gr->xadj[gr->num_obj];
else {
gr->num_edges = 0;
gr->edge_wgt_dim = 0;
}
if (gr->edge_wgt_dim){
ierr = Zoltan_Preprocess_Scale_Weights (gr, float_ewgts, &gr->ewgts,
gr->num_edges, gr->edge_wgt_dim, 1, zz,
"edge", 0);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
/* Return error code */
ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights.");
}
if (!gr->final_output)
ZOLTAN_FREE(&float_ewgts);
else
gr->float_ewgts = float_ewgts;
}
else
ZOLTAN_FREE(&float_ewgts);
if (geo){
ierr = Zoltan_Preprocess_Extract_Geom (zz, global_ids, local_ids, gr, geo);
if (ierr) {
ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
"Error returned from Zoltan_Preprocess_Extract_Geom");
}
}
if (vsp) {
ierr = Zoltan_Preprocess_Extract_Vsize (zz, global_ids, local_ids, gr, vsp);
if (ierr) {
ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
"Error returned from Zoltan_Preprocess_Extract_Vsize");
}
}
/* Scatter graph?
* If data distribution is highly imbalanced, it is better to
* redistribute the graph data structure before calling ParMetis.
* After partitioning, the results must be mapped back.
*/
if (gr->scatter < gr->scatter_min) gr->scatter = gr->scatter_min;
if (gr->scatter>0) {
ierr = Zoltan_Preprocess_Scatter_Graph (zz, gr, prt, geo, vsp);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
"Error returned from Zoltan_Preprocess_Scatter_Graph");
}
}
/* Verify that graph is correct */
if (gr->get_data){
int flag;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
flag = 2; /* Maximum output level */
else
flag = 1; /* Medium output level */
ierr = Zoltan_Verify_Graph(zz->Communicator, gr->vtxdist, gr->xadj, gr->adjncy, gr->vwgt,
gr->ewgts, gr->obj_wgt_dim, gr->edge_wgt_dim, gr->graph_type, gr->check_graph, flag);
}
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Migrate(
ZZ *zz, /* Zoltan structure. */
int num_import, /* Number of non-local objects assigned to the
processor in the new decomposition. */
ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects
assigned to this processor in the new
decomposition; this field can be NULL if
the application doesn't provide import IDs.*/
ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects
assigned to the processor in the new
decomposition; this field can be NULL if the
application does not provide import IDs. */
int *import_procs, /* Array of processor IDs of processors owning
the non-local objects that are assigned to
this processor in the new decomposition; this
field can be NULL if the application does
not provide import IDs. */
int *import_to_part, /* Array of partition numbers to which imported
objects should be assigned. */
int num_export, /* Number of objs to be exported
to other processors to establish the new
decomposition. */
ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of
objects to be exported to other processors
to establish the new decomposition. */
ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of
objects to be exported to other processors
to establish the new decomposition. */
int *export_procs, /* Array of processor IDs
to which objects will be exported
to establish the new decomposition. */
int *export_to_part /* Array of partition numbers to which exported
objects should be assigned. */
)
{
/*
* Routine to help perform migration. If migration pre-processing routine
* (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn.
* It then calls a function to obtain the size of the migrating objects
* (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified
* object packing routine (ZOLTAN_PACK_OBJ_FN) for each object
* to be exported. It develops the needed communication map to move the
* objects to other processors. It performs the communication according
* to the map, and then calls an application-specified object unpacking
* routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported.
*/
char *yo = "Zoltan_Migrate";
int num_gid_entries, num_lid_entries; /* lengths of global & local ids */
int *sizes = NULL; /* sizes (in bytes) of the object data for export. */
int id_size; /* size (in bytes) of ZOLTAN_GID + padding for
alignment */
int tag_size; /* size (in bytes) of ZOLTAN_GID + one int
(for message size) */
char *export_buf = NULL; /* buffer for packing export data. */
char *import_buf = NULL; /* buffer for receiving imported data. */
char *tmp; /* temporary pointer into buffers. */
int i; /* loop counter. */
int tmp_size; /* size of a single object's data. */
int *idx = NULL; /* index used for multi-fn packs and unpacks. */
int idx_cnt = 0; /* index counter for idx array. */
ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm
buf */
ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass
NULL to query functions when NUM_LID_ENTRIES=0. */
ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */
ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */
int msgtag, msgtag2; /* Tags for communication routines */
int total_send_size; /* Total size of outcoming message (in #items) */
int total_recv_size; /* Total size of incoming message (in #items) */
int aligned_int; /* size of an int padded for alignment */
int dest; /* temporary destination partition. */
int include_parts = 0; /* flag indicating whether partition info is
provided */
int ierr = ZOLTAN_OK;
int actual_num_exp = 0;
int actual_exp_allocated = 0;
ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */
int *actual_exp_procs = NULL; /* are changing partition but not */
int *actual_exp_to_part = NULL; /* processor may not be included. */
int actual_num_imp = 0;
int actual_imp_allocated = 0;
ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */
int *actual_imp_procs = NULL; /* are changing partition but not */
int *actual_imp_to_part = NULL; /* processor may not be included. */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
goto End;
}
/*
* Check that all procs use the same id types.
*/
ierr = check_input(zz,
((num_export >= 0 && export_to_part) ||
(num_import >= 0 && import_to_part)),
&include_parts);
if (ierr != ZOLTAN_OK)
goto End;
num_gid_entries = zz->Num_GID;
num_lid_entries = zz->Num_LID;
/*
* Check that all necessary query functions are available.
*/
if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (num_export >= 0) {
/* Build the actual export arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_export, export_global_ids, export_local_ids,
export_procs, export_to_part,
&actual_num_exp, &actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part,
&actual_exp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on actual exports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs,
zz->Communicator, msgtag, &actual_num_imp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
}
else if (num_import >= 0) {
/* Build the actual import arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_import, import_global_ids, import_local_ids,
import_procs, import_to_part,
&actual_num_imp, &actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part,
&actual_imp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on imports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs,
zz->Communicator, msgtag, &actual_num_exp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
/* Compute actual export lists for packing objects */
if (actual_num_exp > 0) {
actual_exp_allocated = 1;
actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp);
actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp);
actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp);
if (include_parts)
actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp);
if (actual_exp_gids == NULL ||
(num_lid_entries && actual_exp_lids == NULL) ||
actual_exp_procs == NULL ||
(import_to_part != NULL && actual_exp_to_part == NULL)) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
ierr = ZOLTAN_MEMERR;
goto End;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) actual_exp_gids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) actual_exp_lids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part,
(int) sizeof(int), (char *) actual_exp_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
/* Create inverse plan (i.e., plan based on exports) so can set
* variable sizes.
* (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable
* but does not tell how large to allocate receive buffer.
*/
ierr = Zoltan_Comm_Invert_Plan(&imp_plan);
exp_plan = imp_plan;
imp_plan = NULL;
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Migrate.Pre_Migrate_PP != NULL) {
zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Pre_Migrate != NULL) {
zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_FN function.");
goto End;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing");
id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE));
/* Note that alignment is not strictly necessary
when ZOLTAN_ID_TYPE is int or unsigned int. */
aligned_int = Zoltan_Align(sizeof(int));
tag_size = id_size + aligned_int;
/*
* For each object, allow space for its global ID and its data plus
* one int (for the object data size).
* Zoltan will pack the global IDs; the application must pack the data
* through the pack routine. Zoltan needs the global IDs for unpacking,
* as the order of the data received during communication is not
* necessarily the same order as import_global_ids[].
* Zoltan also needs to communicate the sizes of the objects because
* only the sender knows the size of each object.
*/
if (actual_num_exp > 0) {
sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!sizes) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_Size_Multi != NULL) {
zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids, sizes, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE_MULTI function.");
goto End;
}
}
else {
for (i = 0; i < actual_num_exp; i++){
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]),
lid, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE function.");
goto End;
}
}
}
total_send_size = 0;
for (i = 0; i < actual_num_exp; i++) {
sizes[i] = Zoltan_Align(sizes[i]);
total_send_size += sizes[i] + tag_size;
}
export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char));
if (!export_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Pack_Obj_Multi != NULL) {
/* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */
idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Pack the objects for export.
*/
idx_cnt = 0;
tmp = export_buf;
for (i = 0; i < actual_num_exp; i++) {
/* Pack the object's global ID */
tmp_id = (ZOLTAN_ID_PTR) tmp;
ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries]));
tmp += id_size;
/* Pack the object's size */
*((int *)tmp) = sizes[i];
tmp += aligned_int;
/* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Pack_Obj_Multi != NULL) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n",
zz->Proc, yo);
}
zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids,
(actual_exp_to_part!=NULL ? actual_exp_to_part
: actual_exp_procs),
sizes, idx, export_buf, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ_MULTI function.");
goto End;
}
}
else {
tmp = export_buf + tag_size;
for (i = 0; i < actual_num_exp; i++) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries]));
printf("size = %d bytes\n", sizes[i]);
}
/* Pack the object's data */
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i]
: actual_exp_procs[i]);
zz->Pack_Obj(zz->Pack_Obj_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]), lid, dest,
sizes[i], tmp, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ function.");
goto End;
}
tmp += sizes[i] + tag_size;
}
}
ZOLTAN_FREE(&idx);
tmp_id = NULL;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects");
/* Modify sizes[] to contain message sizes, not object sizes */
for (i=0; i<actual_num_exp; i++) {
sizes[i] += tag_size;
}
msgtag--;
ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize.");
goto End;
}
if (actual_num_imp > 0) {
import_buf = (char *) ZOLTAN_MALLOC(total_recv_size);
if (!import_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Send the export data using the communication plan.
*/
msgtag2 = 32765;
ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
/*
* Free whatever memory we can.
*/
Zoltan_Comm_Destroy(&exp_plan);
ZOLTAN_FREE(&export_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication");
/*
* Perform application-specified processing before unpacking the data.
*/
if (zz->Migrate.Mid_Migrate_PP != NULL) {
zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Mid_Migrate != NULL) {
zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_FN function.");
goto End;
}
}
/*
* Unpack the object data.
*/
if (actual_num_imp > 0) {
if (zz->Unpack_Obj_Multi != NULL) {
/* Allocate and fill input arrays for Unpack_Obj_Multi. */
sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp);
idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
if (!sizes || !tmp_id || !idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
tmp = import_buf;
idx_cnt = 0;
for (i = 0; i < actual_num_imp; i++) {
/* Unpack the object's global ID */
ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp);
tmp += id_size;
/* Unpack the object's size */
sizes[i] = *((int *)tmp);
tmp += aligned_int;
/* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n",
zz->Proc,yo);
}
zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries,
actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr);
ZOLTAN_FREE(&import_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_FREE(&tmp_id);
ZOLTAN_FREE(&idx);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_MULTI_FN.");
goto End;
}
}
else {
tmp = import_buf;
for (i = 0; i < actual_num_imp; i++) {
tmp_size = *((int *)(tmp + id_size));
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp);
printf("size = %d bytes\n", tmp_size);
}
/* Unpack the object's data */
zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries,
(ZOLTAN_ID_PTR) tmp, tmp_size,
tmp + tag_size, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_FN.");
goto End;
}
tmp += (tmp_size + tag_size);
}
ZOLTAN_FREE(&import_buf);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects");
if (zz->Migrate.Post_Migrate_PP != NULL) {
zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Post_Migrate != NULL) {
zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_FN function.");
goto End;
}
}
End:
if (actual_exp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
}
if (actual_imp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part);
}
if (ierr < 0) {
if (exp_plan) Zoltan_Comm_Destroy(&exp_plan);
Zoltan_Multifree(__FILE__, __LINE__, 5,
&import_buf, &tmp_id, &sizes, &idx, &export_buf);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_LB_Balance(
ZZ *zz,
int *changes,
int *num_gid_entries,
int *num_lid_entries,
int *num_import_objs,
ZOLTAN_ID_PTR *import_global_ids,
ZOLTAN_ID_PTR *import_local_ids,
int **import_procs,
int *num_export_objs,
ZOLTAN_ID_PTR *export_global_ids,
ZOLTAN_ID_PTR *export_local_ids,
int **export_procs
)
{
/*
* Wrapper around Zoltan_LB for backward compatibility with
* previous Zoltan versions.
* Appropriate only when (# requested partitions == # processors), uniformly
* distributed.
* Arguments correspond directly with arguments of Zoltan_LB.
*/
char *yo = "Zoltan_LB_Balance";
int ierr = ZOLTAN_OK; /* Error code */
int *import_to_part = NULL; /* Array used as dummy arg in partitioning. */
int *export_to_part = NULL; /* Array used as dummy arg in partitioning. */
ZOLTAN_TRACE_ENTER(zz, yo);
/* Determine whether partition parameters were set. Report error if
* values are unreasonable. */
if ((zz->LB.Num_Global_Parts_Param != -1 &&
zz->LB.Num_Global_Parts_Param != zz->Num_Proc) ||
(zz->LB.Num_Local_Parts_Param != -1 &&
zz->LB.Num_Local_Parts_Param != 1)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Non-uniform distribution of partitions over processors is specified; "
"use Zoltan_LB_Partition.");
ierr = ZOLTAN_FATAL;
goto End;
}
ierr = Zoltan_LB(zz, 0, changes, num_gid_entries, num_lid_entries,
num_import_objs, import_global_ids, import_local_ids,
import_procs, &import_to_part,
num_export_objs, export_global_ids,
export_local_ids, export_procs, &export_to_part);
End:
/* Not returning import/export partition information; free it if allocated. */
if (import_to_part != NULL)
Zoltan_Special_Free(zz, (void **) &import_to_part,
ZOLTAN_SPECIAL_MALLOC_INT);
if (export_to_part != NULL)
Zoltan_Special_Free(zz, (void **) &export_to_part,
ZOLTAN_SPECIAL_MALLOC_INT);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
int Zoltan_Random(
ZZ *zz, /* The Zoltan structure. */
float *part_sizes, /* Input: Array of size zz->LB.Num_Global_Parts
* zz->Obj_Weight_Dim
containing the percentage of work to be
assigned to each partition. */
int *num_import, /* Return -1. Random uses only export lists. */
ZOLTAN_ID_PTR *import_global_ids, /* Not used. */
ZOLTAN_ID_PTR *import_local_ids, /* Not used. */
int **import_procs, /* Not used. */
int **import_to_part, /* Not used. */
int *num_export, /* Output: Number of objects to export. */
ZOLTAN_ID_PTR *export_global_ids, /* Output: GIDs to export. */
ZOLTAN_ID_PTR *export_local_ids, /* Output: LIDs to export. */
int **export_procs, /* Output: Processsors to export to. */
int **export_to_part /* Output: Partitions to export to. */
)
{
int ierr = ZOLTAN_OK;
int i, count, num_obj;
int max_export;
double rand_frac = 1.0; /* Default is to move all objects. */
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int *parts = NULL;
float *dummy = NULL;
static char *yo = "Zoltan_Random";
static int first_time = 1;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Synchronize the random number generator.
* This synchronization is needed only for sanity in our nightly testing.
* If some other operation (eg., Zoltan_LB_Eval) changes the status of
* the random number generator, the answers here will change. They won't
* be wrong, but they will be different from our accepted answers.
*/
if (first_time) {
Zoltan_Srand(zz->Seed, NULL);
Zoltan_Rand(NULL);
first_time=0;
}
/* No import lists computed. */
*num_import = -1;
/* Get parameter values. */
Zoltan_Bind_Param(Random_params, "RANDOM_MOVE_FRACTION", (void *) &rand_frac);
Zoltan_Assign_Param_Vals(zz->Params, Random_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
/* Get list of local objects. */
ierr = Zoltan_Get_Obj_List(zz, &num_obj, &global_ids, &local_ids, 0,
&dummy, &parts);
/* Bound number of objects to export. */
max_export = 1.5*rand_frac*num_obj;
/* Allocate export lists. */
*export_global_ids = *export_local_ids = NULL;
*export_procs = *export_to_part = NULL;
if (max_export > 0) {
if (!Zoltan_Special_Malloc(zz, (void **)export_global_ids, max_export,
ZOLTAN_SPECIAL_MALLOC_GID)
|| !Zoltan_Special_Malloc(zz, (void **)export_local_ids, max_export,
ZOLTAN_SPECIAL_MALLOC_LID)
|| !Zoltan_Special_Malloc(zz, (void **)export_procs, max_export,
ZOLTAN_SPECIAL_MALLOC_INT)
|| !Zoltan_Special_Malloc(zz, (void **)export_to_part, max_export,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/* Randomly assign ids to procs. */
count=0;
for (i=0; i<num_obj; i++){
/* Randomly select some objects to move (export) */
if ((count<max_export) && (Zoltan_Rand(NULL)<rand_frac*ZOLTAN_RAND_MAX)){
/* export_global_ids[count] = global_ids[i]; */
ZOLTAN_SET_GID(zz, &((*export_global_ids)[count*zz->Num_GID]),
&global_ids[i*zz->Num_GID]);
if (local_ids)
/* export_local_ids[count] = local_ids[i]; */
ZOLTAN_SET_LID(zz, &((*export_local_ids)[count*zz->Num_LID]),
&local_ids[i*zz->Num_LID]);
/* Randomly pick new partition number. */
(*export_to_part)[count] = Zoltan_Rand_InRange(NULL, zz->LB.Num_Global_Parts);
/* Processor number is derived from partition number. */
(*export_procs)[count] = Zoltan_LB_Part_To_Proc(zz,
(*export_to_part)[count], &global_ids[i*zz->Num_GID]);
/* printf("Debug: Export gid %u to part %d and proc %d.\n", (*export_global_ids)[count], (*export_to_part)[count], (*export_procs)[count]); */
++count;
}
}
(*num_export) = count;
End:
/* Free local memory, but not export lists. */
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_FREE(&parts);
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
/* Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
ZZ *zz, /* Zoltan data structure */
HGraph *hg, /* Input hypergraph to be partitioned */
int p, /* Input: number partitions to be generated */
float *part_sizes, /* Input: array of length p containing percentages
of work to be assigned to each partition */
Partition parts, /* Input: initial partition #s; aligned with vtx
arrays.
Output: computed partition #s */
PHGPartParams *hgp) /* Input: parameters for hgraph partitioning. */
{
PHGComm *hgc = hg->comm;
VCycle *vcycle=NULL, *del=NULL;
int i, err = ZOLTAN_OK, middle;
ZOLTAN_GNO_TYPE origVpincnt; /* for processor reduction test */
ZOLTAN_GNO_TYPE prevVcnt = 2*hg->dist_x[hgc->nProc_x]; /* initialized so that the */
ZOLTAN_GNO_TYPE prevVedgecnt = 2*hg->dist_y[hgc->nProc_y]; /* while loop will be entered
before any coarsening */
ZOLTAN_GNO_TYPE tot_nPins, local_nPins;
MPI_Datatype zoltan_gno_mpi_type;
char *yo = "Zoltan_PHG_Partition";
int do_timing = (hgp->use_timers > 1);
int fine_timing = (hgp->use_timers > 2);
int vcycle_timing = (hgp->use_timers > 4 && hgp->ProRedL == 0);
short refine = 0;
struct phg_timer_indices *timer = Zoltan_PHG_LB_Data_timers(zz);
int reset_geometric_matching = 0;
char reset_geometric_string[4];
ZOLTAN_TRACE_ENTER(zz, yo);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
if (do_timing) {
if (timer->vcycle < 0)
timer->vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
if (timer->procred < 0)
timer->procred = Zoltan_Timer_Init(zz->ZTime, 0, "Processor Reduction");
if (timer->match < 0)
timer->match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
if (timer->coarse < 0)
timer->coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
if (timer->coarsepart < 0)
timer->coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
"Coarse_Partition");
if (timer->refine < 0)
timer->refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
if (timer->project < 0)
timer->project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
MPI_Allreduce(&local_nPins,&tot_nPins,1,zoltan_gno_mpi_type,MPI_SUM,hgc->Communicator);
origVpincnt = tot_nPins;
if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
/* For geometric coarsening, hgp->matching pointer and string are reset
* after geometric_levels of coarsening. Will need to reset them after
* this vcycle is completed. Capture that fact now! */
if (!strcasecmp(hgp->redm_str, "rcb") || !strcasecmp(hgp->redm_str, "rib")) {
reset_geometric_matching = 1;
strcpy(reset_geometric_string, hgp->redm_str);
}
/****** Coarsening ******/
#define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */
while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > (ZOLTAN_GNO_TYPE)hg->redl)
&& ((hg->dist_x[hgc->nProc_x] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5)) /* prevVcnt initialized to 2*hg->dist_x[hgc->nProc_x] */
|| (hg->dist_y[hgc->nProc_y] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5))) /* prevVedgecnt initialized to 2*hg->dist_y[hgc->nProc_y] */
&& hg->dist_y[hgc->nProc_y] && hgp->matching) {
ZOLTAN_GNO_TYPE *match = NULL;
VCycle *coarser=NULL, *redistributed=NULL;
prevVcnt = hg->dist_x[hgc->nProc_x];
prevVedgecnt = hg->dist_y[hgc->nProc_y];
#ifdef _DEBUG
/* UVC: load balance stats */
Zoltan_PHG_LoadBalStat(zz, hg);
#endif
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc,
"START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->match, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_match < 0) {
char str[80];
sprintf(str, "VC Matching %d", hg->info);
vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
}
/* Allocate and initialize Matching Array */
if (hg->nVtx && !(match = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC (hg->nVtx*sizeof(ZOLTAN_GNO_TYPE)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
match[i] = i;
/* Calculate matching (packing or grouping) */
err = Zoltan_PHG_Matching (zz, hg, match, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_FREE (&match);
goto End;
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->match, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->coarse, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_coarse < 0) {
char str[80];
sprintf(str, "VC Coarsening %d", hg->info);
vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
}
if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
ZOLTAN_FREE (&match);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
goto End;
}
/* Construct coarse hypergraph and LevelMap */
err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
&vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData,
&vcycle->comm_plan, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarse, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
ZOLTAN_FREE (&match);
if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
goto End;
vcycle = coarser;
hg = vcycle->hg;
if (hgc->nProc > 1 && hgp->ProRedL > 0) {
local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
MPI_Allreduce(&local_nPins, &tot_nPins, 1, zoltan_gno_mpi_type, MPI_SUM,
hgc->Communicator);
if (tot_nPins < (ZOLTAN_GNO_TYPE)(hgp->ProRedL * origVpincnt + 0.5)) {
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->procred, hgc->Communicator);
}
/* redistribute to half the processors */
origVpincnt = tot_nPins; /* update for processor reduction test */
if(hg->nVtx&&!(hg->vmap=(int*)ZOLTAN_MALLOC(hg->nVtx*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: hg->vmap");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
hg->vmap[i] = i;
middle = (int)((float) (hgc->nProc-1) * hgp->ProRedL);
if (hgp->nProc_x_req!=1&&hgp->nProc_y_req!=1) { /* Want 2D decomp */
if ((middle+1) > SMALL_PRIME && Zoltan_PHG_isPrime(middle+1))
--middle; /* if it was prime just use one less #procs (since
it should be bigger than SMALL_PRIME it is safe to
decrement) */
}
if (!(hgc = (PHGComm*) ZOLTAN_MALLOC (sizeof(PHGComm)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: PHGComm");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!(redistributed=newVCycle(zz,NULL,NULL,vcycle,vcycle_timing))) {
ZOLTAN_FREE (&hgc);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "redistributed is NULL.");
goto End;
}
Zoltan_PHG_Redistribute(zz,hgp,hg,0,middle,hgc, redistributed->hg,
&vcycle->vlno,&vcycle->vdest);
if (hgp->UseFixedVtx || hgp->UsePrefPart)
redistributed->hg->bisec_split = hg->bisec_split;
if ((err=allocVCycle(redistributed))!= ZOLTAN_OK)
goto End;
vcycle = redistributed;
if (hgc->myProc < 0)
/* I'm not in the redistributed part so I should go to uncoarsening
refinement and wait */ {
if (fine_timing) {
if (timer->cpgather < 0)
timer->cpgather = Zoltan_Timer_Init(zz->ZTime, 1, "CP Gather");
if (timer->cprefine < 0)
timer->cprefine =Zoltan_Timer_Init(zz->ZTime, 0, "CP Refine");
if (timer->cpart < 0)
timer->cpart = Zoltan_Timer_Init(zz->ZTime, 0, "CP Part");
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
goto Refine;
}
hg = vcycle->hg;
hg->redl = hgp->redl; /* not set with hg creation */
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
}
}
}
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
/* free array that may have been allocated in matching */
if (hgp->vtx_scal) {
hgp->vtx_scal_size = 0;
ZOLTAN_FREE(&(hgp->vtx_scal));
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->coarsepart, hgc->Communicator);
}
/****** Coarse Partitioning ******/
err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarsepart, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
Refine:
del = vcycle;
refine = 1;
/****** Uncoarsening/Refinement ******/
while (vcycle) {
VCycle *finer = vcycle->finer;
hg = vcycle->hg;
if (refine && hgc->myProc >= 0) {
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->refine, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_refine < 0) {
char str[80];
sprintf(str, "VC Refinement %d", hg->info);
vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
}
err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->refine, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
if (hgp->output_level >= PHG_DEBUG_LIST)
uprintf(hgc,
"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, 0, p,
vcycle->Part),
Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
"partitioned plot");
}
if (finer) {
int *rbuffer;
/* Project coarse partition to fine partition */
if (finer->comm_plan) {
refine = 1;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->project, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_project < 0) {
char str[80];
sprintf(str, "VC Project Up %d", hg->info);
vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
}
/* easy to assign partitions to internal matches */
for (i = 0; i < finer->hg->nVtx; i++)
if (finer->LevelMap[i] >= 0) /* if considers only the local vertices */
finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
/* now that the course partition assignments have been propagated */
/* upward to the finer level for the local vertices, we need to */
/* fill the LevelData (matched pairs of a local vertex with a */
/* off processor vertex) with the partition assignment of the */
/* local vertex - can be done totally in the finer level! */
for (i = 0; i < finer->LevelCnt; i++) {
++i; /* skip over off processor lno */
finer->LevelData[i] = finer->Part[finer->LevelData[i]];
}
/* allocate rec buffer to exchange LevelData information */
rbuffer = NULL;
if (finer->LevelSndCnt > 0) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
/* get partition assignments from owners of externally matched vtxs */
Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1,
(char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);
/* process data to assign partitions to expernal matches */
for (i = 0; i < 2 * finer->LevelSndCnt;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->project, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
} else {
int *sendbuf = NULL, size;
refine = 0;
/* ints local and partition numbers */
if (finer->vlno) {
sendbuf = (int*) ZOLTAN_MALLOC (2 * hg->nVtx * sizeof(int));
if (!sendbuf) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; ++i) {
sendbuf[2 * i] = finer->vlno[i]; /* assign local numbers */
sendbuf[2 * i + 1] = vcycle->Part[i];/* assign partition numbers */
}
}
ZOLTAN_FREE (&hgc);
hgc = finer->hg->comm; /* updating hgc is required when the processors
change */
/* Create comm plan to unredistributed processors */
err = Zoltan_Comm_Create(&finer->comm_plan, finer->vlno ? hg->nVtx : 0,
finer->vdest, hgc->Communicator, COMM_TAG+2,
&size);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(hgc->myProc, yo, "Zoltan_Comm_Create failed.");
goto End;
}
/* allocate rec buffer to exchange sendbuf information */
rbuffer = NULL;
if (finer->hg->nVtx) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->hg->nVtx * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
/* Use plan to send partitions to the unredistributed processors */
Zoltan_Comm_Do(finer->comm_plan, COMM_TAG+3, (char *) sendbuf,
2*sizeof(int), (char *) rbuffer);
MPI_Bcast(rbuffer, 2*finer->hg->nVtx, MPI_INT, 0, hgc->col_comm);
/* process data to assign partitions to unredistributed processors */
for (i = 0; i < 2 * finer->hg->nVtx;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
if (finer->vlno)
ZOLTAN_FREE (&sendbuf);
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
}
}
vcycle = finer;
} /* while (vcycle) */
End:
vcycle = del;
while (vcycle) {
if (vcycle_timing) {
Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
Zoltan_Timer_Destroy(&vcycle->timer);
}
if (vcycle->finer) { /* cleanup by level */
Zoltan_HG_HGraph_Free (vcycle->hg);
if (vcycle->LevelData)
Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part,
&vcycle->LevelMap, &vcycle->LevelData, &vcycle->hg);
else if (vcycle->vlno)
Zoltan_Multifree (__FILE__, __LINE__, 5, &vcycle->Part, &vcycle->vdest,
&vcycle->vlno, &vcycle->LevelMap, &vcycle->hg);
else
Zoltan_Multifree (__FILE__, __LINE__, 3, &vcycle->Part,
&vcycle->LevelMap, &vcycle->hg);
}
else /* cleanup top level */
Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
&vcycle->LevelData);
del = vcycle;
vcycle = vcycle->finer;
ZOLTAN_FREE(&del);
}
if (reset_geometric_matching) {
strcpy(hgp->redm_str, reset_geometric_string);
Zoltan_PHG_Set_Matching_Fn(hgp);
}
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TRACE_EXIT(zz, yo) ;
return err;
}
int Zoltan_Oct_migreg_migrate_orphans(ZZ *zz, pRegion RegionList, int nregions,
int level, Map *array, int *c1, int *c2) {
int i, j, k; /* index counters */
pRegion ptr; /* region in the mesh */
COORD origin; /* centroid coordinate information */
pRegion *regions = NULL; /* an array of regions */
int *npids = NULL;
Region *regions2 = NULL; /* an array of regions */
int *npids2 = NULL;
int nreg; /* number of regions */
COORD min, /* minimum bounds of an octant */
max; /* maximum bounds of an octant */
COORD cmin, /* minimum bounds of a child octant */
cmax; /* maximum bounds of a child octant */
COORD rmin, /* minimum bounds of a remote octant */
rmax; /* maximum bounds of a remote octant */
int new_num;
int n;
int dir = 0;
pRList RootList;
pOctant RootOct;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
char *yo = "Zoltan_Oct_migreg_migrate_orphans_static";
int ierr = ZOLTAN_OK;
ZOLTAN_ID_PTR gids2, lids2;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
if(nregions > 0) {
/* create the array of messages to be sent to other processors */
/* Array = (Message *) ZOLTAN_MALLOC(nregions * sizeof(Message)); */
if((regions = (pRegion *) ZOLTAN_MALLOC(nregions * sizeof(pRegion))) == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((npids = (int *) ZOLTAN_MALLOC(nregions * sizeof(int))) == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(®ions);
return ZOLTAN_MEMERR;
}
}
ptr = RegionList;
n = nreg = 0;
while((ptr != NULL) && (nregions > 0)) {
if(ptr->attached == 1) {
/* if region already attached to an octant, then skip to next region */
ptr = ptr->next;
continue;
}
/* region not attached, have to find which processor to send to */
j=0;
dir = 0;
vector_set(min, OCT_info->OCT_gmin);
vector_set(max, OCT_info->OCT_gmax);
/*
* for each level of refinement, find which child region belongs to.
* translate which child to which entry in map array.
*/
for(i=0; i<level; i++) {
Zoltan_Oct_bounds_to_origin(min, max, origin);
if(OCT_info->OCT_dimension == 2)
j = j * 4;
else
j = j * 8;
k = Zoltan_Oct_child_which(OCT_info,origin, ptr->Coord);
new_num = Zoltan_Oct_convert_idx_from_map(OCT_info, dir, k);
dir = Zoltan_Oct_get_child_dir(OCT_info, dir, new_num);
j += new_num;
Zoltan_Oct_child_bounds(min, max, origin, k, cmin, cmax);
vector_set(min, cmin);
vector_set(max, cmax);
}
/* inform message which processor to send to */
npids[n] = array[j].npid;
RootList = array[j].list;
while((RootOct = RL_nextRootOctant(&RootList))) {
Zoltan_Oct_bounds(RootOct,rmin,rmax);
if (Zoltan_Oct_in_box_closure(OCT_info, ptr->Coord ,rmin, rmax)) {
npids[n] = RootOct->npid;
break;
}
}
if((npids[n] != -1) && (npids[n] != zz->Proc)) {
Zoltan_Oct_copy_info(zz, ptr, &(regions[n++]));
}
else {
Zoltan_Oct_insert_orphan(zz, *ptr);
}
nreg++; /* increment region counter */
ptr = ptr->next; /* look at next region */
}
/*
* if regions looked at != number of regions in region list,
* then there is an error
*/
if (nreg!=nregions) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "regions found != to expected number of regions");
return ZOLTAN_FATAL;
}
regions2 = (Region *) ZOLTAN_MALLOC(n * sizeof(Region));
gids2 = ZOLTAN_MALLOC_GID_ARRAY(zz, n);
lids2 = ZOLTAN_MALLOC_LID_ARRAY(zz, n);
npids2 = (int *) ZOLTAN_MALLOC(n * sizeof(int));
for(i=0; i<n; i++) {
npids2[i] = npids[i];
vector_set(regions2[i].Coord, regions[i]->Coord);
regions2[i].Weight = regions[i]->Weight;
regions2[i].Global_ID = &(gids2[i*num_gid_entries]);
regions2[i].Local_ID = (num_lid_entries
? &(lids2[i*num_lid_entries])
: NULL);
ZOLTAN_SET_GID(zz, &(gids2[i*num_gid_entries]), regions[i]->Global_ID);
ZOLTAN_SET_LID(zz, &(lids2[i*num_lid_entries]), regions[i]->Local_ID);
regions2[i].Proc = regions[i]->Proc;
regions2[i].attached = 0;
}
*c1 = n;
/* migrate the orphan regions according to the message array */
Zoltan_Oct_migreg_migrate_regions(zz, regions2, gids2, lids2, npids2, n, c2);
for (i=0; i < n; i++) {
ZOLTAN_FREE(&(regions[i]->Global_ID));
ZOLTAN_FREE(&(regions[i]->Local_ID));
ZOLTAN_FREE(&(regions[i]));
}
ZOLTAN_FREE(®ions);
ZOLTAN_FREE(&npids);
ZOLTAN_FREE(®ions2);
ZOLTAN_FREE(&gids2);
ZOLTAN_FREE(&lids2);
ZOLTAN_FREE(&npids2);
return ierr;
}
int
Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix,
int request_GNOs, /* Input: Flag indicating calling code
needs translation of extra GIDs
to GNOs; partial 2D coloring
needs this feature. */
int num_requested, /* Input: Local # of GIDs needing
translation to GNOs. */
ZOLTAN_ID_PTR requested_GIDs, /* Input: Calling code requests the
GNOs for these GIDs */
ZOLTAN_GNO_TYPE *requested_GNOs /* Output: Return GNOs of
the requested GIDs. */
)
{
static char *yo = "Zoltan_Matrix_Build";
int ierr = ZOLTAN_OK;
int nX;
ZOLTAN_GNO_TYPE tmp;
ZOLTAN_GNO_TYPE *xGNO = NULL;
ZOLTAN_ID_PTR xLID=NULL;
ZOLTAN_ID_PTR xGID=NULL;
ZOLTAN_ID_PTR yGID=NULL;
ZOLTAN_ID_PTR pinID=NULL;
float *xwgt = NULL;
int * Input_Parts=NULL;
struct Zoltan_DD_Struct *dd = NULL;
int *proclist = NULL;
int *xpid = NULL;
int i;
int gno_size_for_dd;
MPI_Datatype zoltan_gno_mpi_type;
int use_full_dd = (opt->speed == MATRIX_FULL_DD);
int fast_build_base = opt->fast_build_base;
matrix->opts.speed = opt->speed;
matrix->opts.fast_build_base = opt->fast_build_base;
ZOLTAN_TRACE_ENTER(zz, yo);
if (num_requested && (!requested_GIDs || !requested_GNOs)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error in requested input; needed arrays are NULL.\n");
}
/* ZOLTAN_GNO_TYPE is >= ZOLTAN_ID_TYPE */
gno_size_for_dd = sizeof(ZOLTAN_GNO_TYPE) / sizeof(ZOLTAN_ID_TYPE);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */
memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options));
/**************************************************/
/* Obtain vertex information from the application */
/**************************************************/
ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID,
zz->Obj_Weight_Dim, &xwgt,
&Input_Parts);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data");
goto End;
}
ZOLTAN_FREE(&Input_Parts);
ZOLTAN_FREE(&xwgt);
/*******************************************************************/
/* Assign vertex consecutive numbers (gnos) */
/*******************************************************************/
if (use_full_dd) {
/* Zoltan computes a translation */
/* Have to use Data Directory if request_GNOs is true. */
if (nX) {
xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE));
if (xGNO == NULL)
MEMORY_ERROR;
}
ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices");
goto End;
}
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, gno_size_for_dd, 0, nX, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL, NULL, nX);
if (request_GNOs) {
Zoltan_DD_Find(dd, requested_GIDs, (ZOLTAN_ID_PTR) requested_GNOs,
NULL, NULL, num_requested, NULL);
}
}
else { /* We don't want to use the DD */
/*
* KDDKDD 2/10/11 This code cannot work when NUM_GID_ENTRIES>1.
* KDDKDD 2/10/11 The assumption is that, if a user sets the
* KDDKDD 2/10/11 appropriate parameter to enable this code, the user
* KDDKDD 2/10/11 knows that his GIDs are compatible with integers.
*/
if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){
xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE));
if (nX && xGNO == NULL)
MEMORY_ERROR;
for (i=0; i < nX; i++)
xGNO[i] = (ZOLTAN_GNO_TYPE)xGID[i] - fast_build_base;
}
else {
xGNO = (ZOLTAN_GNO_TYPE *)xGID;
if (fast_build_base)
for (i = 0; i < nX; i++)
xGNO[i] -= fast_build_base;
}
for (i = 0; i < num_requested; i++)
requested_GNOs[i] = (ZOLTAN_GNO_TYPE)requested_GIDs[i]
- fast_build_base;
tmp = (ZOLTAN_GNO_TYPE)nX;
MPI_Allreduce(&tmp, &matrix->globalX, 1, zoltan_gno_mpi_type, MPI_SUM, zz->Communicator);
}
/* I store : xGNO, xGID, xpid, */
ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, gno_size_for_dd, zz->Num_GID,
sizeof(int), matrix->globalX/zz->Num_Proc, 0);
CHECK_IERR;
/* Hope a linear assignment will help a little */
if (matrix->globalX/zz->Num_Proc)
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
/* Associate all the data with our xGNO */
xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int));
if (nX >0 && xpid == NULL) MEMORY_ERROR;
for (i = 0 ; i < nX ; ++i)
xpid[i] = zz->Proc;
Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (char *)xpid, NULL, nX);
ZOLTAN_FREE(&xpid);
if (matrix->opts.pinwgt)
matrix->pinwgtdim = zz->Edge_Weight_Dim;
else
matrix->pinwgtdim = 0;
ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt, use_full_dd);
CHECK_IERR;
matrix->nY_ori = matrix->nY;
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){
goto End;
}
if (matrix->opts.enforceSquare && matrix->redist) {
/* Convert yGID to yGNO using the same translation as x */
/* Needed for graph : rowID = colID */
/* y and x may have different distributions */
matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nY * sizeof(ZOLTAN_GNO_TYPE));
if (matrix->nY && matrix->yGNO == NULL) {
ZOLTAN_FREE(&pinID);
MEMORY_ERROR;
}
ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL,
matrix->nY, NULL);
if (ierr != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n");
ierr = ZOLTAN_FATAL;
ZOLTAN_FREE(&pinID);
goto End;
}
}
if (matrix->opts.local) { /* keep only local edges */
proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int));
if (matrix->nPins && proclist == NULL) {
ZOLTAN_FREE(&pinID);
MEMORY_ERROR;
}
}
else
proclist = NULL;
/* Convert pinID to pinGNO using the same translation as x */
if (use_full_dd) {
matrix->pinGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nPins* sizeof(ZOLTAN_GNO_TYPE));
if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) {
ZOLTAN_FREE(&pinID);
MEMORY_ERROR;
}
ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL,
matrix->nPins, proclist);
if (ierr != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n");
ierr = ZOLTAN_FATAL;
goto End;
}
ZOLTAN_FREE(&pinID);
Zoltan_DD_Destroy(&dd);
dd = NULL;
}
else {
if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){
matrix->pinGNO = (ZOLTAN_GNO_TYPE *)ZOLTAN_MALLOC(matrix->nPins * sizeof(ZOLTAN_GNO_TYPE));
if (matrix->nPins && !matrix->pinGNO){
ZOLTAN_FREE(&pinID);
MEMORY_ERROR;
}
for (i=0; i < matrix->nPins; i++)
matrix->pinGNO[i] = (ZOLTAN_GNO_TYPE)pinID[i] - fast_build_base;
ZOLTAN_FREE(&pinID);
}
else{
matrix->pinGNO = (ZOLTAN_GNO_TYPE *) pinID;
if (fast_build_base)
for (i=0; i < matrix->nPins; i++)
matrix->pinGNO[i] -= fast_build_base;
pinID = NULL;
}
}
/* if (matrix->opts.local) { /\* keep only local edges *\/ */
/* int *nnz_list; /\* nnz offset to delete *\/ */
/* int nnz; /\* number of nnz to delete *\/ */
/* int i; */
/* nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */
/* if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */
/* for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */
/* if (proclist[i] == zz->Proc) continue; */
/* nnz_list[nnz++] = i; */
/* } */
/* ZOLTAN_FREE(&proclist); */
/* Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */
/* } */
if (!matrix->opts.enforceSquare) {
/* Hyperedges name translation is different from the one of vertices */
matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_CALLOC(matrix->nY, sizeof(ZOLTAN_GNO_TYPE));
if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR;
/* int nGlobalEdges = 0; */
ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY,
matrix->opts.randomize, &matrix->globalY);
CHECK_IERR;
/* /\**************************************************************************************** */
/* * If it is desired to remove dense edges, divide the list of edges into */
/* * two lists. The ZHG structure will contain the removed edges (if final_output is true), */
/* * and the kept edges will be returned. */
/* ****************************************************************************************\/ */
/* totalNumEdges = zhg->globalHedges; */
/* ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */
/* &nLocalEdges, &nGlobalEdges, &nPins, */
/* &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */
/* if (nGlobalEdges < totalNumEdges){ */
/* /\* re-assign edge global numbers if any edges were removed *\/ */
/* ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */
/* randomizeInitDist, &totalNumEdges); */
/* if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */
/* goto End; */
/* } */
/* } */
/* We have to define ddY : yGNO, yGID, ywgt */
ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, gno_size_for_dd, zz->Num_GID,
0, matrix->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
if (matrix->globalY/zz->Num_Proc)
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc);
/* Associate all the data with our yGNO */
Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY);
}
End:
ZOLTAN_FREE(&xpid);
ZOLTAN_FREE(&xLID);
ZOLTAN_FREE(&xGNO);
ZOLTAN_FREE(&xGID);
ZOLTAN_FREE(&xwgt);
ZOLTAN_FREE(&Input_Parts);
ZOLTAN_FREE(&proclist);
if (dd != NULL)
Zoltan_DD_Destroy(&dd);
/* Already stored in the DD */
ZOLTAN_FREE(&yGID);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions,
ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
int *npids, int nregions, int *c2)
{
char *yo = "Zoltan_Oct_migreg_migrate_regions";
int i; /* index counter */
int ierr = ZOLTAN_OK;
int n_import;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
Region *import_objs = NULL; /* Array of import objects used to request
the objs from other processors. */
ZOLTAN_ID_PTR import_gids = NULL; /* Array of global IDs of import_objs. */
ZOLTAN_ID_PTR import_lids = NULL; /* Array of local IDs of import_objs. */
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator,
MIGMIGREGCommCreate, &n_import);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
*c2 = n_import;
if (n_import > 0) {
import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region));
import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import);
import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import);
if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region),
(char *) import_objs);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids,
sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
if (num_lid_entries > 0) {
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids,
sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
}
for (i=0; i<n_import; i++) {
import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]);
import_objs[i].Local_ID = (num_lid_entries
? &(import_lids[i*num_lid_entries])
: NULL);
Zoltan_Oct_insert_orphan(zz, import_objs[i]);
}
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
return ierr;
}
static int _Zoltan_Get_Obj_List(
ZZ *zz,
int *num_obj,
ZOLTAN_ID_PTR *global_ids,
ZOLTAN_ID_PTR *local_ids,
int wdim,
float **objwgts,
int **parts,
int special_malloc
)
{
char *yo = "_Zoltan_Get_Obj_List";
int i, n;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
int alloced_gids = 0, alloced_lids = 0;
int gid_off, lid_off;
ZOLTAN_ID_PTR lid, next_lid; /* Temporary pointers to local IDs; used to pass
NULL to query functions when
NUM_LID_ENTRIES == 0. */
float *next_objwgt; /* Temporarry pointer to an object weight; used
to pass NULL to query functions when wdim=0 */
int ierr = ZOLTAN_OK;
ZOLTAN_TRACE_ENTER(zz, yo);
*num_obj = 0;
*objwgts = NULL;
if (zz->Get_Num_Obj != NULL) {
*num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
goto End;
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (*num_obj > 0) {
/*
* Test global_ids and local_ids for NULL.
* Should be NULL for doing partitioning.
* Should not be NULL if doing ordering.
*/
if (special_malloc){
if (*global_ids == NULL) {
Zoltan_Special_Malloc(zz, (void **)global_ids,
*num_obj, ZOLTAN_SPECIAL_MALLOC_GID);
alloced_gids = 1;
}
if (*local_ids == NULL) {
Zoltan_Special_Malloc(zz, (void **)local_ids,
*num_obj, ZOLTAN_SPECIAL_MALLOC_LID);
alloced_lids = 1;
}
Zoltan_Special_Malloc(zz, (void **)parts,
*num_obj, ZOLTAN_SPECIAL_MALLOC_INT);
}
else{
if (*global_ids == NULL) {
*global_ids = ZOLTAN_MALLOC_GID_ARRAY(zz, *num_obj);
alloced_gids = 1;
}
if (*local_ids == NULL) {
*local_ids = ZOLTAN_MALLOC_LID_ARRAY(zz, *num_obj);
alloced_lids = 1;
}
*parts = (int *) ZOLTAN_MALLOC(*num_obj * sizeof(int));
}
if (wdim > 0)
*objwgts = (float*) ZOLTAN_MALLOC (sizeof(float) * *num_obj * wdim);
if ((*global_ids == NULL) || (num_lid_entries > 0 && *local_ids == NULL) ||
(*parts == NULL) ||
(wdim > 0 && *objwgts == NULL)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory Error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_List != NULL){
/* Get object list directly */
zz->Get_Obj_List(zz->Get_Obj_List_Data,
num_gid_entries, num_lid_entries,
*global_ids, *local_ids,
wdim, *objwgts, &ierr);
}
else if ((zz->Get_First_Obj != NULL) && (zz->Get_Next_Obj != NULL)){
/* Use iterator functions to loop through object list */
if (zz->Get_First_Obj(zz->Get_First_Obj_Data,
num_gid_entries, num_lid_entries,
*global_ids, *local_ids,
wdim, *objwgts, &ierr)){
n = *num_obj;
i = 0;
while (!ierr && (i<n-1)){
gid_off = i * num_gid_entries;
lid_off = i * num_lid_entries;
lid = (num_lid_entries ? &((*local_ids)[lid_off]) : NULL);
next_lid = (num_lid_entries ? &((*local_ids)[lid_off+num_lid_entries])
: NULL);
next_objwgt = (wdim ? (*objwgts) + (i+1)*wdim : NULL);
zz->Get_Next_Obj(zz->Get_Next_Obj_Data,
num_gid_entries, num_lid_entries,
&((*global_ids)[gid_off]), lid,
&((*global_ids)[gid_off+num_gid_entries]),
next_lid,
wdim, next_objwgt, &ierr);
i++;
}
}
}
else { /* No way to get objects */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_OBJ_LIST_FN or "
"ZOLTAN_FIRST_OBJ_FN/ZOLTAN_NEXT_OBJ_FN.");
ierr = ZOLTAN_FATAL;
goto End;
}
/* Get partition information for objects. */
/* Call user-callback if provided; otherwise, all parts == zz->Proc */
if (zz->Get_Part == NULL && zz->Get_Part_Multi == NULL) {
for (i = 0; i < *num_obj; i++)
(*parts)[i] = zz->Proc;
}
else if (zz->Get_Part_Multi) {
zz->Get_Part_Multi(zz->Get_Part_Multi_Data,
num_gid_entries, num_lid_entries, *num_obj,
*global_ids, *local_ids, *parts, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_PART_MULTI_FN");
goto End;
}
}
else {
for (i = 0; i < *num_obj; i++) {
lid = (num_lid_entries ? &((*local_ids)[i*num_lid_entries]) : NULL);
(*parts)[i] = zz->Get_Part(zz->Get_Part_Data,
num_gid_entries, num_lid_entries,
&((*global_ids)[i*num_gid_entries]), lid, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_PART_FN");
goto End;
}
}
}
}
End:
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no lists returned.");
if (special_malloc){
if (alloced_gids)
Zoltan_Special_Free(zz, (void **)global_ids, ZOLTAN_SPECIAL_MALLOC_GID);
if (alloced_lids)
Zoltan_Special_Free(zz, (void **)local_ids, ZOLTAN_SPECIAL_MALLOC_LID);
Zoltan_Special_Free(zz, (void **)parts, ZOLTAN_SPECIAL_MALLOC_INT);
}
else{
if (alloced_gids) ZOLTAN_FREE(global_ids);
if (alloced_lids) ZOLTAN_FREE(local_ids);
ZOLTAN_FREE(parts);
}
ZOLTAN_FREE(objwgts);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
int Zoltan_Invert_Lists(
ZZ *zz, /* Zoltan structure. */
int num_in, /* Number of objects in the input lists. */
ZOLTAN_ID_PTR in_global_ids, /* Array of input global IDs. */
ZOLTAN_ID_PTR in_local_ids, /* Array of input local IDs. */
int *in_procs, /* Array of processor IDs of processors owning
the input objects. */
int *in_to_part, /* Optional: Array of partition numbers to
which input objects should be assigned. */
int *num_out, /* Returned value: Number of output objs. */
ZOLTAN_ID_PTR *out_global_ids,/* Returned value: Array of global IDs of
output objects. */
ZOLTAN_ID_PTR *out_local_ids, /* Returned value: Array of local IDs of
output objects. */
int **out_procs, /* Returned value: Array of processor IDs
to which output objects are assigned. */
int **out_to_part /* Optional: Returned value: Array of
partition numbers to which output
objects should be assigned. */
)
{
/*
* Routine to compute the inverse map. Can be used in two ways:
* 1. Given, for each processor, a list of objects to be received by the
* processor, compute the list of objects that the processor needs to send
* to other processors to satisfy their needs.
* 2. Given, for each processor, a list of objects to be sent to other
* processors, compute the list of objects that the processor needs to receive
* to satisfy its needs.
*/
char *yo = "Zoltan_Invert_Lists";
char msg[256];
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned communication routines */
int msgtag, msgtag2; /* Message tags for communication routines */
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
int include_parts; /* Flag indicating whether to compute
inverse list for partitions. */
int ierr, ret_ierr = ZOLTAN_OK;
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Check that all procs use the same id types.
*/
ierr = check_invert_input(zz, num_in, in_procs, in_to_part,
&num_gid_entries, &num_lid_entries, &include_parts);
if (ierr != ZOLTAN_OK) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
/* Initialize returned arrays. */
*out_global_ids = NULL;
*out_local_ids = NULL;
*out_procs = NULL;
if (include_parts) *out_to_part = NULL;
/*
* Compute communication map and num_out, the number of objs this
* processor has to out to establish the new decomposition.
*/
msgtag = 32767;
ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator,
msgtag, num_out);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Create.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
goto End;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create");
/*
* Allocate space for the object tags that need to be outed. Communicate
* to get the list of objects to be outed.
*/
if (*num_out > 0) {
if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out,
ZOLTAN_SPECIAL_MALLOC_GID)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out,
ZOLTAN_SPECIAL_MALLOC_LID)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (include_parts) {
if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
}
}
/*
* Use the communication plan to send global IDs, local IDs, and processor
* numbers. Do in separate communications to avoid a memory copy and to
* simplify implementation when a data type is added to the comm. package
* (to support heterogeneous computing).
*/
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) *out_global_ids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) *out_local_ids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
}
Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, *out_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part,
(int) sizeof(int), (char *) *out_to_part);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do");
End:
Zoltan_Comm_Destroy(&comm_plan);
if (ret_ierr == ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT);
if (include_parts)
Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ret_ierr);
}
/*
* void Zoltan_Oct_gen_tree_from_input_data()
*
* This function will create a root node of on each processor which will
* then be used to create an octree with regions associated with it. The
* tree will then be balanced and the output used to balance "mesh regions"
* on several processors.
*/
static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag,
int *c1, int *c2, int *c3,
float *c0, int createpartree)
{
char *yo = "Zoltan_Oct_gen_tree_from_input_data";
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
COORD min, /* min coord bounds of objects */
max; /* max coord bounds of objects */
int num_extra; /* number of orphaned objects */
int num_objs; /* total number of local objects */
pRegion ptr, /* pointer to iterate trough region list */
ptr1; /* pointer to iterate trough region list */
pOctant root; /* root of the partition tree */
int i; /* index counter */
int count, /* count for leaf nodes in partition tree */
proc, /* proc leaf node of parition tree belongs to */
extra, /* extra leaf node flag, if not evenly divisible */
remainder; /* remainder of node, or processors to fill */
pOctant cursor, /* cursor to iterate through octant list */
cursor2, /* another cursor to iterate through octant list */
parent; /* parent of an octant */
int level, /* number of levels of refinement */
n, /* index counter */
part; /* partition counter */
Map *array; /* map of which processors own which octants */
int hold; /* used for calculating partition divisions */
int ierr = 0;
#ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX};
COORD global_min, global_max;
#endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */
int nroots = 0;
/*test*/
/* COORD gmin,gmax; */
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* If there are no objects on this processor, do not create a root octant.
* The partitioner will probably assign objects to this processor
*/
if(zz->Get_Num_Obj == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register ZOLTAN_NUM_OBJ_FN function");
abort();
}
*c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr) {
fprintf(stderr, "OCT [%d] %s: Error returned from user defined "
"Get_Num_Obj function.\n", zz->Proc, yo);
exit (-1);
}
ptr1 = NULL;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds");
/* Need A Function To Get The Bounds Of The Local Objects */
Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0);
#ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
/* For now, don't want to add the new query function to Zoltan. */
/* Zoltan_Oct_get_bounds appears to compute the global min and max from */
/* the object input. */
vector_set(OCT_info->OCT_gmin, min);
vector_set(OCT_info->OCT_gmax, max);
#else
/*test*/
/*getMaxBounds(&gmin, &gmax);*/
if(zz->Get_Bounds_Geom == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register Get_Bounds_Geom function");
abort();
}
zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr);
MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3,
MPI_DOUBLE, MPI_MIN, zz->Communicator);
MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3,
MPI_DOUBLE, MPI_MAX, zz->Communicator);
vector_set(OCT_info->OCT_gmin, global_min);
vector_set(OCT_info->OCT_gmax, global_max);
#endif
/*
* the following code segment was added to create a pseudo global octree
* needed for the partitioner. The basic idea is to regroup all the
* regions into something close to an octree partitioning and build the
* tree from that.
* NOTE: This way of doing things is very costly, especially when calling
* this for the first time on a mesh not partitioned in an octree style
* partitioning.
*/
level = 0; /* initialize level count */
/*
* if more than 1 processor, need to find what level of refinement needed
* to initially partition bounding box among the processors
*/
if(zz->Num_Proc > 1) {
n = zz->Num_Proc;
if(OCT_info->OCT_dimension == 2)
hold = 4;
else
hold = 8;
remainder = hold;
for(; remainder > 0; level++) {
int pr = (int)POW(hold, level);
remainder = n - pr;
}
level--;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree");
if(createpartree) {
/* create the global root octant */
root = Zoltan_Oct_POct_new(OCT_info);
Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax);
/* Zoltan_Oct_setOrientation(root, 0); */
/* subdivide to as many levels as calculated */
for(i=0; i<level; i++) {
cursor = root;
while(cursor != NULL) {
if(Zoltan_Oct_isTerminal(cursor)) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
Zoltan_Oct_terminal_refine(zz, cursor, 0);
cursor = cursor2;
}
else
cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
}
}
#if 0
if(zz->Proc == 0)
for(i=0; i<8; i++)
if(Zoltan_Oct_child(root, i) == NULL)
fprintf(stderr,"NULL child pointer\n");
else
fprintf(stderr, "child %d exists\n", i);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array");
/* this part creates the map array */
if(OCT_info->OCT_dimension == 2) {
hold = (int)POW(4, level); /* ignoring the z+ octants */
if(hold == 0)
hold = 1;
}
else
hold = (int)POW(8, level);
part = hold / zz->Num_Proc; /* how many octants per partition */
remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */
extra = zz->Num_Proc - remainder;/* where to start adding extra octants */
array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */
if(array == NULL) {
fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n",
zz->Proc);
abort();
}
/* initialize variables */
proc = 0;
count = 0;
i = 0;
cursor = root;
while(cursor != NULL) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) {
if(proc == extra) {
part++;
extra = -1;
}
if(count != part) {
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
count++;
}
else {
count = 1;
proc++;
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
}
if(proc == zz->Proc) {
array[i].npid = -1;
/* KDDKDD Added RL_freeList below. The
* KDDKDD implementation from RPI leaked memory because the
* KDDKDD test cases for setting array[i].list were not mutually
* KDDKDD exclusive. Freeing the list produces the result we got
* KDDKDD before, without the memory leak.
*/
/* LGG -- it seems to me that this array[i].list assignment is
* not really necessary. It looks as though it has already been
* assigned with the same information from the prev if-else
* commented out RL_freeList(), and RL_initRootList()
*/
/*RL_freeList(&(array[i].list));*/
/* KDDKDD End addition */
/*array[i].list = RL_initRootList();*/
parent = Zoltan_Oct_parent(cursor);
if(parent != NULL)
Zoltan_Oct_setchild(parent, cursor->which, NULL);
/* octant into local root list */
Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1);
Zoltan_Oct_setMapIdx(cursor, i);
nroots++;
/* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc);
octant into local root list */
}
i++;
}
cursor = cursor2;
}
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RootOct = RL_nextRootOctant(&RootList);
if(RootOct != root) {
/* KDDKDDFREE changed root to &root to allow root to be reset to NULL */
Zoltan_Oct_POct_delTree(OCT_info,&root);
}
OCT_info->map = array;
OCT_info->mapsize = hold;
}
/*
* attach the regions to the root... Zoltan_Oct_fix will create the octree
* starting with the root and subdividing as needed
*/
num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans");
Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map,
c1, c2);
/* ZOLTAN_FREE(&array); */
while(ptr1 != NULL) {
ptr = ptr1->next;
ZOLTAN_FREE(&(ptr1->Global_ID));
ZOLTAN_FREE(&(ptr1->Local_ID));
ZOLTAN_FREE(&ptr1);
ptr1 = ptr;
}
ZOLTAN_TRACE_EXIT(zz, yo);
}
/* if !copy, inmat is not usable after this call */
int
Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */
Zoltan_matrix_2d *outmat, int copy)
{
static char *yo = "Zoltan_Matrix2d_Distribute";
int ierr = ZOLTAN_OK;
int nProc_x, nProc_y;
int myProc_x, myProc_y;
int i, j, cnt;
int *proclist = NULL;
Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL;
ZOLTAN_GNO_TYPE *perm_y = NULL;
float *wgtarray = NULL;
float *tmpwgtarray = NULL;
int msg_tag = 1021982;
ZOLTAN_COMM_OBJ *plan;
MPI_Comm communicator = MPI_COMM_NULL;
int nProc;
ZOLTAN_GNO_TYPE *yGNO = NULL;
ZOLTAN_GNO_TYPE *pinGNO = NULL;
ZOLTAN_GNO_TYPE tmp_gno;
void *partdata = NULL;
MPI_Datatype zoltan_gno_mpi_type;
ZOLTAN_TRACE_ENTER(zz, yo);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix));
if(copy) {
/* TODO: We need to copy the arrays also */
Zoltan_Matrix_Reset (&outmat->mtx);
/* Copy also directories */
outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX);
if (inmat.ddY == inmat.ddX)
outmat->mtx.ddY = outmat->mtx.ddX;
else
outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY);
}
communicator = outmat->comm->Communicator;
nProc = outmat->comm->nProc;
nProc_x = outmat->comm->nProc_x;
nProc_y = outmat->comm->nProc_y;
myProc_x = outmat->comm->myProc_x;
myProc_y = outmat->comm->myProc_y;
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_Duplicates");
ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx);
/* KDDKDDKDD FIX INDENTATION OF THIS BLOCK */
if (inmat.opts.speed != MATRIX_NO_REDIST) {
if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) {
/* I need to know the original distribution */
if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */
outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear;
}
else {
int *cmember = NULL;
cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int));
if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR;
Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL,
outmat->mtx.nY, NULL);
KDDKDDKDD(zz->Proc, " Zoltan_Distribute_Partition_Register");
partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO,
cmember, zz->Num_Proc, zz->Num_Proc);
ZOLTAN_FREE(&cmember);
Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata);
}
}
/*
* Build comm plan for sending non-zeros to their target processors in
* 2D data distribution.
*/
/* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */
proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int));
sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc));
if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR;
wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float));
if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR;
yGNO = outmat->mtx.yGNO;
pinGNO = outmat->mtx.pinGNO;
KDDKDDKDD(zz->Proc, " CommPlan Hash");
cnt = 0;
for (i = 0; i < outmat->mtx.nY; i++) {
ZOLTAN_GNO_TYPE edge_gno=-1;
/* processor row for the edge */
edge_gno = yGNO[i];
for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) {
ZOLTAN_GNO_TYPE vtx_gno=-1;
/* processor column for the vertex */
vtx_gno = pinGNO[j];
proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData,
&sendbuf[cnt].part_y);
if (proclist[cnt] < 0) /* Discard this nnz */
continue;
sendbuf[cnt].GNO[0] = edge_gno;
sendbuf[cnt].GNO[1] = vtx_gno;
memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim,
outmat->mtx.pinwgtdim*sizeof(float));
cnt++;
}
if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) {
proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData,
&sendbuf[cnt].part_y);
if (proclist[cnt] < 0) /* Discard this nnz */
continue;
sendbuf[cnt].GNO[0] = edge_gno;
sendbuf[cnt].GNO[1] = -1;
memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float));
cnt++;
}
}
if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin)
Zoltan_Distribute_Partition_Free(&outmat->hashDistData);
if (outmat->mtx.yend != outmat->mtx.ystart + 1)
ZOLTAN_FREE(&outmat->mtx.yend);
outmat->mtx.yend = NULL;
ZOLTAN_FREE(&outmat->mtx.ystart);
ZOLTAN_FREE(&outmat->mtx.yGNO);
ZOLTAN_FREE(&outmat->mtx.pinGNO);
ZOLTAN_FREE(&outmat->mtx.pinwgt);
ZOLTAN_FREE(&outmat->mtx.yGID);
/*
* Send pins to their target processors.
* They become non-zeros in the 2D data distribution.
*/
KDDKDDKDD(zz->Proc, " CommPlan Create");
msg_tag--;
ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins);
ZOLTAN_FREE(&proclist);
nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc));
if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR;
msg_tag--;
Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros);
ZOLTAN_FREE(&sendbuf);
if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */
tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float));
if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR;
msg_tag--;
Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float),
(char *) tmpwgtarray);
ZOLTAN_FREE(&wgtarray);
}
Zoltan_Comm_Destroy(&plan);
/* Unpack the non-zeros received. */
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_DupArcs");
/* TODO: do take care about singletons */
Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray,
&outmat->mtx);
}
/* Now we just have to change numbering */
outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE));
outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE));
if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR;
/* FIXME: Work only in 1D */
tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY;
MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator);
for (i = 1 ; i <= nProc_y ; i ++) {
outmat->dist_y[i] += outmat->dist_y[i-1];
}
outmat->dist_x[1] = outmat->mtx.globalX;
perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE));
if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR;
for (i = 0 ; i < outmat->mtx.nY ; ++i){
perm_y[i] = i + outmat->dist_y[myProc_y];
}
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute");
Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y);
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute done");
End:
ZOLTAN_FREE(&perm_y);
ZOLTAN_FREE(&proclist);
ZOLTAN_FREE(&sendbuf);
ZOLTAN_FREE(&nonzeros);
ZOLTAN_FREE(&tmpwgtarray);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
realtype itr = 0.0;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
/* change our zz struct. */
#endif
indextype i;
realtype *imb_tols;
indextype ncon;
indextype edgecut;
indextype wgtflag;
indextype numflag = 0;
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
ZOLTAN_TRACE_ENTER(zz, yo);
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
if (sizeof(realtype) != sizeof(float)) {
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));
for (i = 0; i < tmp; i++)
prt.input_part_sizes[i] = (realtype) part_sizes[i];
/* KDD 2/2014: removed re-scaling part sizes so they sum to one.
* part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
* plus, the code here was wrong for multiple object weights.
* similar scaling code did not exist in the Scotch interface.
*/
prt.part_sizes = prt.input_part_sizes;
}
else
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
indextype j;
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
/* Now we can call ParMetis */
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
/* Use default options for METIS */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0;
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
#else
METIS_SetDefaultOptions(options);
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
prt.part_sizes, imb_tols, options,
&edgecut, prt.part);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) {
/* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int num_gid_entries, /* # of entries for a global id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm /* iperm[rank[i]]=i, only for sequential ordering */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
int * vtxdist = NULL;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL, *local_iperm=NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value");
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.rank = rank;
zz->Order.iperm = iperm;
zz->Order.gids = gids;
zz->Order.lids = lids;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
#ifdef HAVE_MPI
strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN);
#else
strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN);
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
zz->Order.start_index = opt.start_index;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && rank)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
/* strcpy(opt.order_type, "GLOBAL"); */
}
#endif /* ZOLTAN_SCOTCH */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
if (!strcmp(opt.order_type, "GLOBAL"))
strcpy (opt.order_type, "DIST");
if (!strcmp(opt.order_type, "LOCAL"))
strcpy (opt.order_type, "SERIAL");
strcpy(zz->Order.order_type, opt.order_type);
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 3,
&local_gids, &local_rank, &local_iperm);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
if ((!(opt.return_args & RETURN_RANK) && (rank != NULL))
|| (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) {
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr) {
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) {
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) {
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* I store : GNO, rank, iperm */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
ZOLTAN_FREE(&local_iperm);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
int Zoltan_Scatter_Graph(
indextype **vtxdist,
indextype **xadj,
indextype **adjncy,
weighttype **vwgt,
indextype **vsize,
weighttype **adjwgt,
realtype **xyz,
int ndims, /* # dimensions of xyz geometry data */
int vwgt_dim,
ZZ *zz,
ZOLTAN_COMM_OBJ **plan
)
{
static char *yo = "Zoltan_Scatter_Graph";
char msg[256];
indextype *old_vtxdist, *old_adjncy;
indextype *old_xadj;
indextype *old_vsize;
weighttype *old_vwgt, *old_adjwgt;
realtype *old_xyz;
int *ptr, *proclist = NULL, *proclist2 = NULL;
int i, j, num_obj, old_num_obj, num_edges, nrecv;
int use_graph; /* do we use graph data, or only the geometry? */
int use_vsize; /* do we use the vsize array? */
int ewgt_dim= zz->Edge_Weight_Dim;
ZOLTAN_COMM_OBJ *plan2;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Save pointers to "old" data distribution */
old_adjncy = NULL;
old_xadj = NULL;
old_vwgt = old_adjwgt = NULL;
old_vsize = NULL;
old_xyz = NULL;
old_vtxdist = *vtxdist;
if (xadj)
old_xadj = *xadj;
if (adjncy)
old_adjncy = *adjncy;
if (vwgt)
old_vwgt = *vwgt;
if (vsize)
old_vsize = *vsize;
if (adjwgt)
old_adjwgt = *adjwgt;
if (xyz)
old_xyz = *xyz;
old_num_obj = (int)(old_vtxdist[zz->Proc+1] - old_vtxdist[zz->Proc]);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Old number of objects = %d\n", zz->Proc, old_num_obj);
/* Compute new distribution, *vtxdist */
(*vtxdist) = (indextype *)ZOLTAN_MALLOC((zz->Num_Proc+1)* sizeof(indextype));
for (i=0; i<=zz->Num_Proc; i++){
(*vtxdist)[i] = (i*old_vtxdist[zz->Num_Proc])/zz->Num_Proc;
}
/* Check if any proc has graph data */
i = (old_xadj != NULL);
MPI_Allreduce(&i, &use_graph, 1, MPI_INT, MPI_LOR, zz->Communicator);
j = (old_vsize != NULL);
MPI_Allreduce(&j, &use_vsize, 1, MPI_INT, MPI_LOR, zz->Communicator);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: use_graph = %1d, use_vsize = %1d\n", zz->Proc,
use_graph, use_vsize);
/* Reset all data pointers to NULL for now */
*xadj = NULL;
*adjncy = NULL;
*vwgt = *adjwgt = NULL;
*xyz = NULL;
if (use_vsize) *vsize = NULL;
/* Convert the xdj array so that it contains the degree of each vertex */
if (use_graph){
for (i=0; i<old_num_obj; i++){
old_xadj[i] = old_xadj[i+1] - old_xadj[i];
}
}
/* Allocate new space for vertex data */
num_obj = (int)((*vtxdist)[zz->Proc+1] - (*vtxdist)[zz->Proc]);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: New number of objects = %d\n", zz->Proc, num_obj);
if (use_graph)
*xadj = (indextype *) ZOLTAN_MALLOC((num_obj+1)*sizeof(indextype));
if (vwgt_dim)
*vwgt = (weighttype *) ZOLTAN_MALLOC(vwgt_dim*num_obj*sizeof(weighttype));
if (use_vsize)
*vsize = (indextype *) ZOLTAN_MALLOC(num_obj*sizeof(indextype));
if (ndims)
*xyz = (realtype *) ZOLTAN_MALLOC(ndims*num_obj*sizeof(realtype));
if (old_num_obj > 0) {
/* Set up the communication plan for the vertex data */
proclist = (int *) ZOLTAN_MALLOC(old_num_obj * sizeof(int));
/* Let j be the new owner of vertex old_vtxdist[zz->Proc]+i */
j = 0;
while (old_vtxdist[zz->Proc] >= (*vtxdist)[j+1]) j++;
for (i=0; i<old_num_obj; i++){
if (old_vtxdist[zz->Proc]+i >= (*vtxdist)[j+1]) j++;
proclist[i] = j;
}
}
Zoltan_Comm_Create(plan, old_num_obj, proclist, zz->Communicator, TAG1, &nrecv);
if (nrecv != num_obj){
sprintf(msg,"Proc %d received %d object but expected %d.",
zz->Proc, nrecv, num_obj);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
/* Free data */
ZOLTAN_FREE(&proclist);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_FATAL;
}
/* Do the communication. To save memory, we do not pack all the data into
* a buffer, but send directly from the old arrays to the new arrays.
* We use the vertex communication plan for all the vertex-based arrays
* and the edge communication plan for all the edge-based arrays.
*/
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Starting vertex-based communication.\n", zz->Proc);
if (use_graph){
Zoltan_Comm_Do( *plan, TAG2, (char *) old_xadj, sizeof(indextype), (char *) *xadj);
}
if (vwgt_dim){
Zoltan_Comm_Do( *plan, TAG3, (char *) old_vwgt, vwgt_dim*sizeof(weighttype), (char *) *vwgt);
}
if (use_vsize){
Zoltan_Comm_Do( *plan, TAG4, (char *) old_vsize, sizeof(indextype), (char *) *vsize);
}
if (ndims){
Zoltan_Comm_Do( *plan, TAG5, (char *) old_xyz, ndims*sizeof(realtype), (char *) *xyz);
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Finished vertex-based communication.\n", zz->Proc);
if (use_graph){
/* Rebuild xadj from degrees */
for (i=1; i<num_obj; i++)
(*xadj)[i] += (*xadj)[i-1];
for (i=num_obj; i>0; i--)
(*xadj)[i] = (*xadj)[i-1];
(*xadj)[0] = 0;
/* Allocate space for new edge data structures */
num_edges = (*xadj)[num_obj];
*adjncy = (indextype *) ZOLTAN_MALLOC(num_edges*sizeof(indextype));
if (ewgt_dim)
*adjwgt = (weighttype *) ZOLTAN_MALLOC(ewgt_dim*num_edges*sizeof(weighttype));
/* Set up the communication plan for the edge data. */
ptr = proclist2 = (int *) ZOLTAN_MALLOC(old_xadj[old_num_obj] * sizeof(int));
for (i=0; i<old_num_obj; i++)
for (j=0; j<old_xadj[i]; j++)
*ptr++ = proclist[i];
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] Debug: Allocated proclist of length " TPL_IDX_SPEC " for edges.\n",
zz->Proc, old_xadj[old_num_obj]);
}
Zoltan_Comm_Create(&plan2, (int)old_xadj[old_num_obj], proclist2, zz->Communicator,
TAG1, &nrecv);
if (nrecv != num_edges){
sprintf(msg,"Proc %d received %d edges but expected %d.",
zz->Proc, nrecv, num_edges);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
/* Free data */
ZOLTAN_FREE(&proclist);
ZOLTAN_FREE(&proclist2);
ZOLTAN_FREE(&old_vtxdist);
ZOLTAN_FREE(&old_xadj);
ZOLTAN_FREE(&old_adjncy);
ZOLTAN_FREE(&old_vwgt);
ZOLTAN_FREE(&old_vsize);
ZOLTAN_FREE(&old_adjwgt);
ZOLTAN_FREE(&old_xyz);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_FATAL;
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Starting edge-based communication.\n", zz->Proc);
/* Do the communication. */
Zoltan_Comm_Do( plan2, TAG2, (char *) old_adjncy, sizeof(indextype), (char *) *adjncy);
if (ewgt_dim){
Zoltan_Comm_Do( plan2, TAG3, (char *) old_adjwgt, ewgt_dim*sizeof(weighttype), (char *) *adjwgt);
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Finished edge-based communication.\n", zz->Proc);
/* Free the comm. plan for edge data */
Zoltan_Comm_Destroy(&plan2);
} /* end of use_graph */
/* Free data structures */
ZOLTAN_FREE(&proclist);
ZOLTAN_FREE(&proclist2);
ZOLTAN_FREE(&old_vtxdist);
ZOLTAN_FREE(&old_xadj);
ZOLTAN_FREE(&old_adjncy);
ZOLTAN_FREE(&old_vwgt);
ZOLTAN_FREE(&old_vsize);
ZOLTAN_FREE(&old_adjwgt);
ZOLTAN_FREE(&old_xyz);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_OK;
}
/*
* void Zoltan_Oct_migrate_objects()
*
* sets up the export_tags, and import_tags for the application that called
* this load balancing routine
*/
int Zoltan_Oct_migrate_objects(ZZ *zz, pOctant *octs, int *newpids, int nocts,
int *nsenregs, pRegion *import_regions,
int *nrecregs, float *c2, float *c3,
int *counter3, int *counter4)
{
int i; /* index counter */
int *tag_pids; /* array of which processors to send information */
int np_regs; /* number of regions previously imported */
Region *p_reg; /* previously imported regions */
ZOLTAN_ID_PTR p_gids; /* global IDs of previously imported regions */
ZOLTAN_ID_PTR p_lids; /* local IDs of previously imported regions */
ZOLTAN_ID_PTR exported_gids, exported_lids;
Region *exported_regions;
int max_objs;
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_Oct_migrate_objects";
p_reg = *import_regions = exported_regions = NULL;
exported_gids = p_gids = NULL;
exported_lids = p_lids = NULL;
/* tag all the regions to be exported */
ierr = tag_regions(zz, octs, newpids, nocts, &exported_regions,
&exported_gids, &exported_lids, nsenregs, &tag_pids,
&p_reg, &p_gids, &p_lids, &np_regs, c2, &max_objs);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/* get all the region tags that are being imported */
ierr = malloc_new_objects(zz, *nsenregs, exported_regions,
exported_gids, exported_lids, tag_pids, nrecregs,
import_regions, p_reg, p_gids, p_lids,
np_regs, c3);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
if(np_regs > 0){
ZOLTAN_FREE(&p_reg);
ZOLTAN_FREE(&p_gids);
ZOLTAN_FREE(&p_lids);
}
ZOLTAN_FREE(&exported_regions);
ZOLTAN_FREE(&exported_gids);
ZOLTAN_FREE(&exported_lids);
if(max_objs > (*counter3))
(*counter3) = max_objs;
i = (max_objs - (*nsenregs) + (*nrecregs) - np_regs);
if(i > (*counter3))
(*counter3) = i;
(*counter4) = (*nrecregs) - np_regs;
/* fprintf(stderr,"(%d) nrectags = %d\n", zz->Proc, (*nrecregs));
* for(i=0; i<(*nrecregs); i++)
* fprintf(stderr,"%d\n", (*import_regions)[i].Proc);
*/
ZOLTAN_FREE(&tag_pids);
return ierr;
}
