int Zoltan_Oct_migrate_octants(ZZ *zz, int *newpids, pOctant *octs, int nocts, int *nrecocts) {
int i,j = 0;
int nsends = 0;
int nreceives = 0;
int *despid = NULL;
OCTNEW_msg *snd_reply = NULL;
OCTNEW_msg *rcv_reply = NULL;
int ierr = ZOLTAN_OK;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
char *yo = "Zoltan_Oct_migrate_octants";
pOctant *newocts = NULL; /* New foreign octant pointers */
if((newocts = (pOctant *) ZOLTAN_MALLOC(sizeof(pOctant)*(nocts+10))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i=0; i<nocts; i++)
newocts[i]=NULL;
/* count number of sends */
nsends=0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc)
nsends++;
/* build message array */
/* if(nsends > 0) { */
if((snd_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nsends + 1) * sizeof(OCTNEW_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((despid = (int *) ZOLTAN_MALLOC((nsends+10) * sizeof(int))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
return ZOLTAN_MEMERR;
}
/* } */
/* else { */
/* snd_reply = NULL; */
/* despid = NULL; */
/* } */
j = 0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc) {
snd_reply[j].num = i;
despid[j++] = newpids[i];
}
/* send messages */
ierr = Zoltan_Comm_Create(&comm_plan, nsends, despid, zz->Communicator, MigOctCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
return (ierr);
}
if((rcv_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nreceives + 1) * sizeof(OCTNEW_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return ZOLTAN_MEMERR;
}
ierr = Zoltan_Comm_Do(comm_plan, MigOctCommDo, (char *) snd_reply,
sizeof(OCTNEW_msg), (char *) rcv_reply);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
/* Reply to malloc requests and Receive malloc replies */
for (i=0; i< nreceives; i++) {
rcv_reply[i].ptr = Zoltan_Oct_POct_new((OCT_Global_Info *) (zz->LB.Data_Structure));
}
;
ierr = Zoltan_Comm_Do_Reverse(comm_plan, MigOctCommReverse, (char *) rcv_reply,
sizeof(OCTNEW_msg), NULL, (char *) snd_reply);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
/* store remote pointers locally for future migration */
for (i=0; i<nsends; i++) {
newocts[snd_reply[i].num] = snd_reply[i].ptr;
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
/* set return value */
*nrecocts = nreceives;
ierr = Zoltan_Oct_Update_Connections(zz, octs, newpids, newocts, nocts);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
abort();
return (ierr);
}
ierr = Zoltan_Oct_Final_Migration(zz, octs,newpids,newocts,nocts, *nrecocts);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
abort();
return (ierr);
}
Zoltan_Oct_Update_Map(zz);
ZOLTAN_FREE(&newocts);
return ierr;
}
int Zoltan_ParMetis_Order(
ZZ *zz, /* Zoltan structure */
int num_obj, /* Number of (local) objects to order. */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */
int *iperm,
ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */
)
{
static char *yo = "Zoltan_ParMetis_Order";
int i, n, ierr;
ZOLTAN_Output_Order ord;
ZOLTAN_Third_Graph gr;
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't want to risk letting external
packages changing our communicator */
#endif
indextype numflag = 0;
int timer_p = 0;
int get_times = 0;
int use_timers = 0;
double times[5];
ZOLTAN_ID_PTR l_gids = NULL;
ZOLTAN_ID_PTR l_lids = NULL;
indextype options[MAX_OPTIONS];
char alg[MAX_PARAM_STRING_LEN+1];
ZOLTAN_TRACE_ENTER(zz, yo);
#ifdef ZOLTAN_PARMETIS
#if TPL_USE_DATATYPE != TPL_METIS_DATATYPES
#ifdef TPL_FLOAT_WEIGHT
i = 1;
#else
i = 0;
#endif
if ((sizeof(indextype) != sizeof(idxtype)) ||
(sizeof(weighttype) != sizeof(idxtype)) || i){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,
"Not supported: Multiple 3rd party libraries with incompatible "
"data types.");
return ZOLTAN_FATAL;
}
#endif
#endif
memset(&gr, 0, sizeof(ZOLTAN_Third_Graph));
memset(&ord, 0, sizeof(ZOLTAN_Output_Order));
memset(times, 0, sizeof(times));
ord.order_opt = order_opt;
if (!order_opt){
/* If for some reason order_opt is NULL, allocate a new ZOOS here. */
/* This should really never happen. */
order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
strcpy(order_opt->method,"PARMETIS");
}
ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord);
/* ParMetis only computes the rank vector */
order_opt->return_args = RETURN_RANK;
/* Check that num_obj equals the number of objects on this proc. */
/* This constraint may be removed in the future. */
n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){
/* Return error code */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
return(ZOLTAN_FATAL);
}
if (n != num_obj){
/* Currently this is a fatal error. */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the "
"number of objects.");
return(ZOLTAN_FATAL);
}
/* Do not use weights for ordering */
gr.obj_wgt_dim = -1;
gr.edge_wgt_dim = -1;
gr.num_obj = num_obj;
/* Check what ordering type is requested */
if (order_opt){
SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */
#ifdef ZOLTAN_PARMETIS
if ((strcmp(order_opt->method, "METIS") == 0))
#endif /* ZOLTAN_PARMETIS */
SET_LOCAL_GRAPH(&gr.graph_type);
}
gr.get_data = 1;
if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: "
"set ParMetis instead.");
return(ZOLTAN_FATAL);
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
return (ierr);
}
/* Allocate space for separator sizes */
if (IS_GLOBAL_GRAPH(gr.graph_type)) {
if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) {
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype));
if (ord.sep_sizes == NULL) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */
}
/* Allocate space for direct perm */
ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.rank){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
if (IS_LOCAL_GRAPH(gr.graph_type)){
/* Allocate space for inverse perm */
ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.iperm){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
}
else
ord.iperm = NULL;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_PARMETIS
if (IS_GLOBAL_GRAPH(gr.graph_type)){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library");
ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy,
&numflag, options, ord.rank, ord.sep_sizes, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else
#endif /* ZOLTAN_PARMETIS */
#if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS)
if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */
indextype numobj = gr.num_obj;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library");
options[0] = 0; /* Use default options for METIS. */
order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options,
ord.iperm, ord.rank);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */
int numbloc;
int start;
int leaf;
int *converttab;
int levelmax;
levelmax = mylog2(zz->Num_Proc) + 1;
converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int));
memset(converttab, 0, zz->Num_Proc*2*sizeof(int));
/* Determine the first node in each separator, store it in zz->TPL_Order.start */
for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) {
int father;
father = zz->Num_Proc + numbloc;
converttab[start] = 2*numbloc;
zz->TPL_Order.leaves[leaf++]=start;
zz->TPL_Order.ancestor[start] = start + 2;
converttab[start+1] = 2*numbloc+1;
zz->TPL_Order.leaves[leaf++]=start+1;
zz->TPL_Order.ancestor[start+1] = start + 2;
start+=2;
do {
converttab[start] = father;
if (father %2 == 0) {
int nextoffset;
int level;
level = mylog2(2*zz->Num_Proc - 1 - father);
nextoffset = (1<<(levelmax-level));
zz->TPL_Order.ancestor[start] = start+nextoffset;
start++;
break;
}
else {
zz->TPL_Order.ancestor[start] = start+1;
start++;
father = zz->Num_Proc + father/2;
}
} while (father < 2*zz->Num_Proc - 1);
}
zz->TPL_Order.start[0] = 0;
zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1;
for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) {
int oldblock=converttab[numbloc-1];
zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock];
}
ZOLTAN_FREE(&converttab);
ZOLTAN_FREE(&ord.sep_sizes);
zz->TPL_Order.leaves[zz->Num_Proc] = -1;
zz->TPL_Order.nbr_leaves = zz->Num_Proc;
zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1;
}
else { /* No tree */
zz->TPL_Order.nbr_blocks = 0;
zz->TPL_Order.start = NULL;
zz->TPL_Order.ancestor = NULL;
zz->TPL_Order.leaves = NULL;
}
/* Correct because no redistribution */
memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE));
memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE));
ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL);
ZOLTAN_FREE(&l_gids);
ZOLTAN_FREE(&l_lids);
/* Get a time here */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){
memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i];
}
}
if ((ord.iperm != NULL) && (iperm != NULL)){
if (sizeof(indextype) == sizeof(int)){
memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
iperm[i] = (int)ord.iperm[i];
}
}
}
if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm);
ZOLTAN_FREE(&ord.rank);
/* Free all other "graph" stuff */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
static int Zoltan_Oct_Final_Migration(
ZZ *zz,
pOctant *octs, /* octs[nocts] */
int *newpids, /* newpids[nocts] */
pOctant *newocts, /* newocts[nocts] */
int nocts, /* number of octants leaving this processor */
int nrecocts) /* number of octants received in this processor */
{
int i;
Migrate_msg *msnd = NULL, *mrcv = NULL;
int remotecount;
int nsends;
int nreceives;
int *despid = NULL;
int ierr = ZOLTAN_OK;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
char *yo = "Zoltan_Oct_Final_Migration";
OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure;
/* count number of sends */
nsends=0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc)
nsends++;
if((despid = (int *) ZOLTAN_MALLOC((nocts+10) * sizeof(int))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((msnd = (Migrate_msg *) ZOLTAN_MALLOC((nocts+10) * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
return ZOLTAN_MEMERR;
}
remotecount = 0;
for (i=0; i<nocts; i++) /* Send and free */
if (newpids[i]!=zz->Proc)
{
FILLMIGRATEMSG(octs[i], newocts[i], msnd[remotecount], zz->Proc); /* bug */
despid[remotecount++] = newpids[i];
Zoltan_Oct_clearRegions(octs[i]);
/* KDDKDDFREE Change oct to &oct to allow NULL from ZOLTAN_FREE
* KDDKDDFREE to propagate back. */
Zoltan_Oct_POct_free(OCT_info, &(octs[i]));
}
ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator,
MigFinCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
return (ierr);
}
if((mrcv = (Migrate_msg *) ZOLTAN_MALLOC((nreceives+10) * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
return ZOLTAN_MEMERR;
}
ierr = Zoltan_Comm_Do(comm_plan, MigFinCommDo, (char *) msnd,
sizeof(Migrate_msg), (char *) mrcv);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
ZOLTAN_FREE(&mrcv);
return (ierr);
}
for (i=0; i<nreceives; i++) { /* Receive new parocts */
/* Zoltan_Oct_setID(mrcv[i].ptr,mrcv[i].id); */
/* Zoltan_Oct_POct_setparent(OCT_info, mrcv[i].ptr, mrcv[i].parent, mrcv[i].ppid); */
SETOCTFROMMIGRATEMSG(OCT_info, mrcv[i]);
/* Zoltan_Oct_setchildnum(mrcv[i].ptr,mrcv[i].childnum); */
/* Zoltan_Oct_setchildren(mrcv[i].ptr, mrcv[i].children, mrcv[i].cpids); */
/* Zoltan_Oct_setbounds(mrcv[i].ptr, mrcv[i].min, mrcv[i].max); */
/* Zoltan_Oct_setDir(mrcv[i].ptr,mrcv[i].dir); */
/* Zoltan_Oct_setMapIdx(mrcv[i].ptr,mrcv[i].mapidx); */
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
ZOLTAN_FREE(&mrcv);
return (ierr);
}
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
ZOLTAN_FREE(&mrcv);
return ierr;
}
static int Zoltan_Oct_build_global_rootlist(ZZ *zz,Migrate_msg **ret_rmsg, int *size) {
int j, k = 0;
int *despid = NULL;
int nroots, nreceives;
pRList RootList; /* list of the local roots */
pOctant RootOct;
Migrate_msg *snd_rmsg = NULL;
Migrate_msg *rcv_rmsg = NULL;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
/*Map *array = OCT_info->map;*/
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_Oct_build_global_rootlist";
nroots = RL_numRootOctants(Zoltan_Oct_POct_localroots(OCT_info));
if (nroots > 0) { /* KDDKDD -- Added test to prevent departure before comm */
if((despid = (int *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(int))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((snd_rmsg = (Migrate_msg *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
return ZOLTAN_MEMERR;
}
k = 0;
for (j=0; j<zz->Num_Proc; j++) {
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
/* if(array[Zoltan_Oct_mapidx(RootOct)].npid > 0) { */
FILLMIGRATEMSG(RootOct, RootOct, snd_rmsg[k], zz->Proc);
despid[k] = j;
k++;
/* } */
}
}
} /* KDDKDD */
ierr = Zoltan_Comm_Create(&comm_plan, k, despid, zz->Communicator,
RootListCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
return (ierr);
}
if (nreceives > 0) {
if((rcv_rmsg = (Migrate_msg *) ZOLTAN_MALLOC(nreceives * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
return ZOLTAN_MEMERR;
}
}
ierr = Zoltan_Comm_Do(comm_plan, RootListCommDo, (char *) snd_rmsg, sizeof(Migrate_msg), (char *) rcv_rmsg);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
ZOLTAN_FREE(&rcv_rmsg);
return (ierr);
}
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
*ret_rmsg = rcv_rmsg;
*size = nreceives;
return ierr;
}
int Zoltan_RCB_Build_Structure(ZZ *zz, int *num_obj, int *max_obj, int wgtflag,
int use_ids)
{
/*
* Function to build the geometry-based data structures for
* Steve Plimpton's RCB implementation.
*/
char *yo = "Zoltan_RCB_Build_Structure";
RCB_STRUCT *rcb; /* Data structure for RCB. */
struct rcb_tree *treeptr;
int i, ierr = 0;
/*
* Allocate an RCB data structure for this Zoltan structure.
* If the previous data structure is still there, free the Dots and IDs first;
* the other fields can be reused.
*/
if (zz->LB.Data_Structure == NULL) {
rcb = (RCB_STRUCT *) ZOLTAN_MALLOC(sizeof(RCB_STRUCT));
if (rcb == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
return(ZOLTAN_MEMERR);
}
zz->LB.Data_Structure = (void *) rcb;
rcb->Tree_Ptr = NULL;
rcb->Box = NULL;
rcb->Global_IDs = NULL;
rcb->Local_IDs = NULL;
rcb->Dots = NULL;
Zoltan_Initialize_Transformation(&(rcb->Tran));
rcb->Tree_Ptr = (struct rcb_tree *)
ZOLTAN_MALLOC(zz->LB.Num_Global_Parts * sizeof(struct rcb_tree));
rcb->Box = (struct rcb_box *) ZOLTAN_MALLOC(sizeof(struct rcb_box));
if (rcb->Tree_Ptr == NULL || rcb->Box == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_RCB_Free_Structure(zz);
return(ZOLTAN_MEMERR);
}
/* initialize Tree_Ptr */
for (i = 0; i < zz->LB.Num_Global_Parts; i++) {
treeptr = &(rcb->Tree_Ptr[i]);
/* initialize dim to -1 to prevent use of cut */
treeptr->dim = -1;
treeptr->cut = 0.0;
treeptr->parent = treeptr->left_leaf = treeptr->right_leaf = 0;
}
}
else {
rcb = (RCB_STRUCT *) zz->LB.Data_Structure;
ZOLTAN_FREE(&(rcb->Global_IDs));
ZOLTAN_FREE(&(rcb->Local_IDs));
ZOLTAN_FREE(&(rcb->Dots));
}
ierr = Zoltan_RB_Build_Structure(zz, &(rcb->Global_IDs), &(rcb->Local_IDs),
&(rcb->Dots), num_obj, max_obj,
&(rcb->Num_Dim),
wgtflag, use_ids);
if (ierr) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_RB_Build_Structure.");
Zoltan_RCB_Free_Structure(zz);
return(ierr);
}
return(ZOLTAN_OK);
}
static int Zoltan_Oct_Update_Connections(
ZZ *zz,
pOctant *octs, /* octs[nocts] */
int *newpids, /* newpids[nocts] */
pOctant *newocts, /* newocts[nocts] */
int nocts) /* number of octants leaving this processor */
{
int i, j;
int nsends;
int nreceives;
pOctant parent;
pOctant child;
int ppid;
int cpid;
int childnum;
int *despid = NULL;
Update_msg umsg;
Update_msg *localumsg = NULL;
Update_msg *remoteumsg = NULL;
Update_msg *rcv_umsg = NULL;
int localcount;
int remotecount;
int ierr = ZOLTAN_OK;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
char *yo = "Zoltan_Oct_Update_Connections";
OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure;
localcount=0;
remotecount=0;
/* count number of sends */
nsends = 0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc)
nsends++;
if(nocts > 0) {
if((remoteumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((localumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
return ZOLTAN_MEMERR;
}
if((despid = (int *) ZOLTAN_MALLOC((nocts+1) * sizeof(int)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
return ZOLTAN_MEMERR;
}
}
else {
remoteumsg = NULL;
localumsg = NULL;
despid = NULL;
}
localcount = 0;
remotecount = 0;
for (i=0; i<nocts; i++) /* Send connection updates */
if (newpids[i]!=zz->Proc) {
parent = Zoltan_Oct_parent(octs[i]);
ppid = Zoltan_Oct_Ppid(octs[i]);
childnum = Zoltan_Oct_childnum(octs[i]);
if (parent) { /* Let parent of oct[i] know that it's moving */
if (ppid==zz->Proc) {
FILLUPDATEMSG(localumsg[localcount], parent, childnum, newocts[i], newpids[i]);
localcount++;
}
else {
FILLUPDATEMSG(remoteumsg[remotecount], parent, childnum, newocts[i], newpids[i]);
despid[remotecount++] = ppid;
}
}
for (j=0; j<8; j++) {
child = Zoltan_Oct_child(octs[i],j);
cpid = octs[i]->cpid[j];
/* Tell child of oct[i] that it is moving */
if (child) {
if (cpid==zz->Proc) {
/* NOTE: -1 signals PARENT */
FILLUPDATEMSG(localumsg[localcount], child, -1, newocts[i], newpids[i]);
localcount++;
}
else {
/* NOTE: -1 signals PARENT */
FILLUPDATEMSG(remoteumsg[remotecount], child, -1, newocts[i], newpids[i]);
despid[remotecount++] = cpid;
}
}
}
}
ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator,
MigUpdCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
return (ierr);
}
/* if(nreceives > 0) { */
if((rcv_umsg = (Update_msg *) ZOLTAN_MALLOC((nreceives +1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
return ZOLTAN_MEMERR;
}
ierr = Zoltan_Comm_Do(comm_plan, MigUpdCommDo, (char *) remoteumsg,
sizeof(Update_msg), (char *) rcv_umsg);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_umsg);
return (ierr);
}
/* } */
/* else { */
/* rcv_umsg = NULL; */
/* } */
/* update new octants */
for (i=0; i< (localcount+nreceives); i++) {
if (i<localcount)
umsg=localumsg[i];
else
umsg=rcv_umsg[i-localcount];
if (umsg.childnum>=0) {
Zoltan_Oct_setchild(umsg.oct,umsg.childnum,umsg.newptr);
Zoltan_Oct_setCpid(umsg.oct,umsg.childnum,umsg.newpid);
}
else {
if((Zoltan_Oct_data_newpid(umsg.oct) == OCT_info->OCT_localpid) ||
((Zoltan_Oct_data_newpid(umsg.oct) != OCT_info->OCT_localpid) && (umsg.newpid == OCT_info->OCT_localpid)))
Zoltan_Oct_POct_setparent(OCT_info, umsg.oct,umsg.newptr,umsg.newpid);
else {
umsg.oct->ppid = umsg.newpid;
umsg.oct->parent = umsg.newptr;
}
}
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_umsg);
return (ierr);
}
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&rcv_umsg);
ZOLTAN_FREE(&despid);
return ierr;
}
/* This function needs a distribution : rows then cols to work properly */
int
Zoltan_ZG_Build (ZZ* zz, ZG* graph, int local)
{
static char *yo = "Zoltan_ZG_Build";
int ierr = ZOLTAN_OK;
int diag;
int *diagarray=NULL;
Zoltan_matrix_options opt;
char symmetrization[MAX_PARAM_STRING_LEN+1];
char bipartite_type[MAX_PARAM_STRING_LEN+1];
char weigth_type[MAX_PARAM_STRING_LEN+1];
char matrix_build_type[MAX_PARAM_STRING_LEN+1];
int bipartite = 0;
#ifdef CC_TIMERS
double times[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
double gtimes[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
char *timenames[9]= {"", "setup", "matrix build", "diag", "symmetrize", "dist lin", "2D dist", "complete", "clean up"};
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
#endif /* CC_TIMERS */
ZOLTAN_TRACE_ENTER(zz, yo);
memset (graph, 0, sizeof(ZG));
/* Read graph build parameters */
Zoltan_Bind_Param(ZG_params, "GRAPH_SYMMETRIZE", (void *) &symmetrization);
Zoltan_Bind_Param(ZG_params, "GRAPH_SYM_WEIGHT", (void *) &weigth_type);
Zoltan_Bind_Param(ZG_params, "GRAPH_BIPARTITE_TYPE", (void *) &bipartite_type);
Zoltan_Bind_Param(ZG_params, "GRAPH_BUILD_TYPE", (void*) &matrix_build_type);
/* Set default values */
strncpy(symmetrization, "NONE", MAX_PARAM_STRING_LEN);
strncpy(bipartite_type, "OBJ", MAX_PARAM_STRING_LEN);
strncpy(weigth_type, "ADD", MAX_PARAM_STRING_LEN);
strncpy(matrix_build_type, "NORMAL", MAX_PARAM_STRING_LEN);
Zoltan_Assign_Param_Vals(zz->Params, ZG_params, zz->Debug_Level, zz->Proc,
zz->Debug_Proc);
Zoltan_Matrix2d_Init(&graph->mtx);
graph->mtx.comm = (PHGComm*)ZOLTAN_MALLOC (sizeof(PHGComm));
if (graph->mtx.comm == NULL) MEMORY_ERROR;
Zoltan_PHGComm_Init (graph->mtx.comm);
memset(&opt, 0, sizeof(Zoltan_matrix_options));
opt.enforceSquare = 1; /* We want a graph: square matrix */
if (!strcasecmp(weigth_type, "ADD"))
opt.pinwgtop = ADD_WEIGHT;
else if (!strcasecmp(weigth_type, "MAX"))
opt.pinwgtop = MAX_WEIGHT;
else if (!strcasecmp(weigth_type, "CMP"))
opt.pinwgtop = MAX_WEIGHT;
opt.pinwgt = 1;
opt.randomize = 0;
opt.local = local;
opt.keep_distribution = 1;
if (strcasecmp(symmetrization, "NONE")) {
opt.symmetrize = 1;
}
if (!strcasecmp(matrix_build_type, "FAST"))
opt.speed = MATRIX_FAST;
else if (!strcasecmp(matrix_build_type, "FAST_NO_DUP"))
opt.speed = MATRIX_NO_REDIST;
else
opt.speed = MATRIX_FULL_DD;
#ifdef CC_TIMERS
times[1] = Zoltan_Time(zz->Timer);
#endif
ierr = Zoltan_Matrix_Build(zz, &opt, &graph->mtx.mtx);
CHECK_IERR;
#ifdef CC_TIMERS
times[2] = Zoltan_Time(zz->Timer);
#endif
ierr = Zoltan_Matrix_Mark_Diag (zz, &graph->mtx.mtx, &diag, &diagarray);
CHECK_IERR;
if (diag) { /* Some Diagonal Terms have to be removed */
ierr = Zoltan_Matrix_Delete_nnz(zz, &graph->mtx.mtx, diag, diagarray);
ZOLTAN_FREE(&diagarray);
CHECK_IERR;
}
#ifdef CC_TIMERS
times[3] = Zoltan_Time(zz->Timer);
#endif
if (opt.symmetrize) {
if (!strcasecmp(symmetrization, "BIPARTITE"))
bipartite = 1;
ierr = Zoltan_Matrix_Sym(zz, &graph->mtx.mtx, bipartite);
CHECK_IERR;
}
#ifdef CC_TIMERS
times[4] = Zoltan_Time(zz->Timer);
#endif
ierr = Zoltan_Distribute_LinearY(zz, graph->mtx.comm);
CHECK_IERR;
#ifdef CC_TIMERS
times[5] = Zoltan_Time(zz->Timer);
MPI_Barrier(zz->Communicator);
#endif
ierr = Zoltan_Matrix2d_Distribute (zz, graph->mtx.mtx, &graph->mtx, 0);
CHECK_IERR;
#ifdef CC_TIMERS
times[6] = Zoltan_Time(zz->Timer);
#endif
ierr = Zoltan_Matrix_Complete(zz, &graph->mtx.mtx);
#ifdef CC_TIMERS
times[7] = Zoltan_Time(zz->Timer);
#endif
if (bipartite) {
int vertlno;
int limit;
int offset;
graph->bipartite = 1;
graph->fixed_vertices = graph->mtx.mtx.ybipart;
/* graph->fixed_vertices = (int*) ZOLTAN_MALLOC(graph->mtx.mtx.nY*sizeof(int)); */
/* if (graph->mtx.mtx.nY && graph->fixed_vertices == NULL) MEMORY_ERROR; */
/* limit = graph->mtx.mtx.offsetY; */
/* /\* What kind of vertices do we want to keep ? *\/ */
/* graph->fixObj = !strcasecmp(bipartite_type, "OBJ"); /\* Non-zero value means "objects" *\/ */
/* offset = graph->mtx.mtx.offsetY - graph->mtx.dist_y[graph->mtx.comm->myProc_y]; */
/* if (graph->fixObj) /\* What kind of vertices do we want to keep ? *\/ */
/* for (vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++ vertlno) */
/* graph->fixed_vertices[vertlno] = (vertlno < offset); */
/* else */
/* for (vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++ vertlno) */
/* graph->fixed_vertices[vertlno] = (vertlno >= offset); */
}
#ifdef CC_TIMERS
MPI_Barrier(zz->Communicator);
times[8] = Zoltan_Time(zz->Timer);
MPI_Reduce(times, gtimes, 9, MPI_DOUBLE, MPI_MAX, 0, zz->Communicator);
if (!zz->Proc) {
int i;
printf("Total Build Time in Proc-0: %.2lf Max: %.2lf\n", times[8]-times[0], gtimes[8]-times[0]);
for (i=1; i<9; ++i)
printf("%-13s in Proc-0: %8.2lf Max: %8.2lf\n", timenames[i], times[i]-times[i-1], gtimes[i]-gtimes[i-1]);
}
#endif
End:
ZOLTAN_FREE(&diagarray);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Order (
struct Zoltan_Struct *zz,
int num_gid_entries,
int num_obj,
ZOLTAN_ID_PTR gids,
ZOLTAN_ID_PTR permuted_global_ids
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids.
* num_gid_entries
* Output:
* permuted_global_ids
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
char msg[253];
sprintf(msg, "num_gid_entries=%d is not equal to parameter setting "
"NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
#ifdef HAVE_MPI
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "GLOBAL");
#else
strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "LOCAL");
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
/*
* Check that the user has allocated space for the return args.
*/
if (num_obj && !(gids && permuted_global_ids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "LOCAL_HSFC"))
{
Order_fn = Zoltan_LocalHSFC_Order;
strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PTSCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_SCOTCH */
#ifdef ZOLTAN_HUND
else if (!strcasecmp(opt.method, "HUND")) {
ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL);
goto End;
}
#endif /* ZOLTAN_HUND */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/* TODO : Ask why useful ! */
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/************************************
* Check for required query function
************************************/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
/* TODO allocate all this stuff with the graph */
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 2,
&local_gids, &local_rank);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* TODO: At this time, I consider rank == permuted_global_ids */
/* I store : GNO, rank, permuted GID */
/* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0);
/* Hope a linear assignment will help a little */
if (local_num_obj)
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", permuted_global_ids[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
#ifdef ZOLTAN_HUND
End:
#endif /*ZOLTAN_HUND*/
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Comm_Do_Post(
ZOLTAN_COMM_OBJ * plan, /* communication data structure */
int tag, /* message tag for communicating */
char *send_data, /* array of data I currently own */
int nbytes, /* multiplier for sizes */
char *recv_data) /* array of data I'll own after comm */
{
char *send_buff; /* space to buffer outgoing data */
int my_proc; /* processor ID */
unsigned int self_recv_address = 0;/* where in recv_data self info starts */
int self_num=0; /* where in send list my_proc appears */
int offset; /* offset into array I'm copying into */
int self_index = 0; /* send offset for data I'm keeping */
int out_of_mem; /* am I out of memory? */
int nblocks; /* number of procs who need my data */
int proc_index; /* loop counter over procs to send to */
int i, j, k, jj; /* loop counters */
static char *yo = "Zoltan_Comm_Do_Post";
/* Check input parameters */
if (!plan) {
MPI_Comm_rank(MPI_COMM_WORLD, &my_proc);
ZOLTAN_COMM_ERROR("Communication plan = NULL", yo, my_proc);
return ZOLTAN_FATAL;
}
/* If not point to point, currently we do synchroneous communications */
if (plan->maxed_recvs){
int status;
status = Zoltan_Comm_Do_AlltoAll(plan, send_data, nbytes, recv_data);
return (status);
}
MPI_Comm_rank(plan->comm, &my_proc);
if ((plan->nsends + plan->self_msg) && !send_data) {
int sum = 0;
if (plan->sizes_to) /* Not an error if all sizes_to == 0 */
for (i = 0; i < (plan->nsends + plan->self_msg); i++)
sum += plan->sizes_to[i];
if (!plan->sizes_to || (plan->sizes_to && sum)) {
ZOLTAN_COMM_ERROR("nsends not zero, but send_data = NULL",
yo, my_proc);
return ZOLTAN_FATAL;
}
}
if ((plan->nrecvs + plan->self_msg) && !recv_data) {
int sum = 0;
if (plan->sizes_from) /* Not an error if all sizes_from == 0 */
for (i = 0; i < (plan->nrecvs + plan->self_msg); i++)
sum += plan->sizes_from[i];
if (!plan->sizes_from || (plan->sizes_from && sum)) {
ZOLTAN_COMM_ERROR("nrecvs not zero, but recv_data = NULL",
yo, my_proc);
return ZOLTAN_FATAL;
}
}
if (nbytes < 0) {
ZOLTAN_COMM_ERROR("Scale factor nbytes is negative", yo, my_proc);
return ZOLTAN_FATAL;
}
/* Post irecvs */
out_of_mem = 0;
if (plan->indices_from == NULL) {
/* Data can go directly into user space. */
plan->recv_buff = recv_data;
}
else { /* Need to buffer receive to reorder */
plan->recv_buff = (char *) ZOLTAN_MALLOC(plan->total_recv_size * nbytes);
if (plan->recv_buff == NULL && plan->total_recv_size * nbytes != 0)
out_of_mem = 1;
}
if (!out_of_mem) {
if (plan->sizes == NULL) { /* All data the same size */
k = 0;
for (i = 0; i < plan->nrecvs + plan->self_msg; i++) {
if (plan->procs_from[i] != my_proc) {
MPI_Irecv((void *) & plan->recv_buff[plan->starts_from[i] * nbytes],
plan->lengths_from[i] * nbytes,
(MPI_Datatype) MPI_BYTE, plan->procs_from[i], tag,
plan->comm, &plan->request[k]);
k++;
}
else {
self_recv_address = plan->starts_from[i] * nbytes;
}
}
}
else { /* Data of varying sizes */
k = 0;
for (i = 0; i < plan->nrecvs + plan->self_msg; i++) {
if (plan->procs_from[i] != my_proc) {
if (plan->sizes_from[i])
MPI_Irecv((void *)
&plan->recv_buff[plan->starts_from_ptr[i] * nbytes],
plan->sizes_from[i] * nbytes,
(MPI_Datatype) MPI_BYTE, plan->procs_from[i],
tag, plan->comm, &plan->request[k]);
else
plan->request[k] = MPI_REQUEST_NULL;
k++;
}
else {
self_recv_address = plan->starts_from_ptr[i] * nbytes;
}
}
}
}
/* Do remaining allocation to check for any mem problems. */
if (plan->indices_to != NULL) { /* can't sent straight from input */
send_buff = (char *) ZOLTAN_MALLOC(plan->max_send_size * nbytes);
if (send_buff == 0 && plan->max_send_size * nbytes != 0)
out_of_mem = 1;
}
else
send_buff = NULL;
/* Barrier to ensure irecvs are posted before doing any sends. */
/* Simultaneously see if anyone out of memory */
MPI_Allreduce(&out_of_mem, &j, 1, MPI_INT, MPI_SUM, plan->comm);
if (j > 0) { /* Some proc is out of memory -> Punt */
ZOLTAN_FREE(&send_buff);
if (plan->indices_from != NULL)
ZOLTAN_FREE(&plan->recv_buff);
return (ZOLTAN_MEMERR);
}
/* Send out data */
/* Scan through procs_to list to start w/ higher numbered procs */
/* This should balance message traffic. */
nblocks = plan->nsends + plan->self_msg;
proc_index = 0;
while (proc_index < nblocks && plan->procs_to[proc_index] < my_proc)
proc_index++;
if (proc_index == nblocks)
proc_index = 0;
if (plan->sizes == NULL) { /* Data all of same size */
if (plan->indices_to == NULL) { /* data already blocked by processor. */
for (i = proc_index, j = 0; j < nblocks; j++) {
if (plan->procs_to[i] != my_proc) {
MPI_Rsend((void *) &send_data[plan->starts_to[i] * nbytes],
plan->lengths_to[i] * nbytes,
(MPI_Datatype) MPI_BYTE, plan->procs_to[i], tag,
plan->comm);
}
else
self_num = i;
if (++i == nblocks)
i = 0;
}
if (plan->self_msg) { /* Copy data to self. */
/* I use array+offset instead of &(array[offset]) because of
a bug with PGI v9 */
/* I use memmove because I'm not sure that the pointer are not
overlapped. */
memmove(
plan->recv_buff+self_recv_address,
send_data+(size_t)(plan->starts_to[self_num])*(size_t)nbytes,
(size_t) (plan->lengths_to[self_num]) * (size_t) nbytes);
}
}
else { /* Not blocked by processor. Need to buffer. */
for (i = proc_index, jj = 0; jj < nblocks; jj++) {
if (plan->procs_to[i] != my_proc) {
/* Need to pack message first. */
offset = 0;
j = plan->starts_to[i];
for (k = 0; k < plan->lengths_to[i]; k++) {
memcpy(&send_buff[offset],
&send_data[plan->indices_to[j++] * nbytes], nbytes);
offset += nbytes;
}
MPI_Rsend((void *) send_buff, plan->lengths_to[i] * nbytes,
(MPI_Datatype) MPI_BYTE, plan->procs_to[i], tag, plan->comm);
}
else {
self_num = i;
self_index = plan->starts_to[i];
}
if (++i == nblocks)
i = 0;
}
if (plan->self_msg) { /* Copy data to self. */
for (k = 0; k < plan->lengths_to[self_num]; k++) {
memcpy(&plan->recv_buff[self_recv_address],
&send_data[plan->indices_to[self_index++] * nbytes], nbytes);
self_recv_address += nbytes;
}
}
ZOLTAN_FREE(&send_buff);
}
}
else { /* Data of differing sizes */
if (plan->indices_to == NULL) { /* data already blocked by processor. */
for (i = proc_index, j = 0; j < nblocks; j++) {
if (plan->procs_to[i] != my_proc) {
if (plan->sizes_to[i]) {
MPI_Rsend((void *)
&send_data[plan->starts_to_ptr[i] * nbytes],
plan->sizes_to[i] * nbytes,
(MPI_Datatype) MPI_BYTE, plan->procs_to[i],
tag, plan->comm);
}
}
else
self_num = i;
if (++i == nblocks)
i = 0;
}
if (plan->self_msg) { /* Copy data to self. */
if (plan->sizes_to[self_num]) {
char* lrecv = &plan->recv_buff[self_recv_address];
char* lsend = &send_data[plan->starts_to_ptr[self_num]
* nbytes];
int sindex = plan->sizes_to[self_num], idx;
for (idx=0; idx<nbytes; idx++) {
memcpy(lrecv, lsend, sindex);
lrecv += sindex;
lsend += sindex;
}
}
}
}
else { /* Not blocked by processor. Need to buffer. */
for (i = proc_index, jj = 0; jj < nblocks; jj++) {
if (plan->procs_to[i] != my_proc) {
/* Need to pack message first. */
offset = 0;
j = plan->starts_to[i];
for (k = 0; k < plan->lengths_to[i]; k++) {
if (plan->sizes[plan->indices_to[j]]) {
memcpy(&send_buff[offset],
&send_data[plan->indices_to_ptr[j] * nbytes],
plan->sizes[plan->indices_to[j]] * nbytes);
offset += plan->sizes[plan->indices_to[j]] * nbytes;
}
j++;
}
if (plan->sizes_to[i]) {
MPI_Rsend((void *) send_buff,
plan->sizes_to[i] * nbytes,
(MPI_Datatype) MPI_BYTE, plan->procs_to[i],
tag, plan->comm);
}
}
else
self_num = i;
if (++i == nblocks)
i = 0;
}
if (plan->self_msg) { /* Copy data to self. */
if (plan->sizes_to[self_num]) {
j = plan->starts_to[self_num];
for (k = 0; k < plan->lengths_to[self_num]; k++) {
int kk = plan->indices_to_ptr[j];
char* lrecv = &plan->recv_buff[self_recv_address];
unsigned int send_idx = kk * nbytes;
char* lsend = &send_data[send_idx];
int sindex = plan->sizes[plan->indices_to[j]], idx;
for (idx=0; idx<nbytes; idx++) {
memcpy(lrecv, lsend, sindex);
lrecv += sindex;
lsend += sindex;
}
self_recv_address += plan->sizes[plan->indices_to[j]]
* nbytes;
j++;
}
}
}
ZOLTAN_FREE(&send_buff);
}
}
return (ZOLTAN_OK);
}
int
Zoltan_PHG_2ways_hyperedge_partition (
ZZ *zz, /* Input : Zoltan data structure */
HGraph *hg,
Partition parts,
Zoltan_PHG_Tree *tree,
struct Zoltan_DD_Struct * gnoToGID,
struct Zoltan_DD_Struct **dd,
int *numParts,
int **sizeParts
)
{
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_PHG_2ways_hyperedge_partition";
int nEdge, hEdge;
int *interval;
int *partnumber = NULL;
int tree_size;
ZOLTAN_ID_TYPE *rowpart =NULL; /* ZOLTAN_ID_TYPE because it's used in Zoltan_DD_* */
ZOLTAN_GNO_TYPE *rowGNO = NULL;
ZOLTAN_ID_PTR rowGID=NULL;
int index;
int offset;
ZOLTAN_TRACE_ENTER(zz, yo);
nEdge = hg->nEdge;
fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge);
interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int));
if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR;
tree_size = get_tree_size(tree) + 1;
for (index = 0 ; index < nEdge ; ++index){
SET_MIN_NODE(interval, index, tree_size);
SET_MAX_NODE(interval, index, -1);
}
/* Update interval with the local knowledge */
/* XXX: I loop on the hyperedges, as I think it's more cache friendly
* and it allows me to discoupled the computation if a non blocking MPI_Reduce is
* available
*/
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){
int part;
int max = -1; /* Trick : we use the initialized values */
int min = tree_size + 1;
for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) {
part = parts[hg->hvertex[index]];
max = MAX(max, part);
min = MIN(min, part);
}
SET_MIN_NODE(interval, hEdge, min);
SET_MAX_NODE(interval, hEdge, max);
}
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm);
/* Now I have to compute the partition of hyperedges according to the "interval"
* and the tree */
/* First, compute the partition number corresponding to the nodes in the tree */
partnumber = compute_part_number(tree);
if (partnumber == NULL) {
ierr = ZOLTAN_FATAL;
goto End;
}
(*numParts) = get_tree_size(tree);
rowpart = (ZOLTAN_ID_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_ID_TYPE));
if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR;
rowGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_GNO_TYPE));
if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR;
(*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int));
if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR;
offset = hg->dist_y[hg->comm->myProc_y];
/* Then we search we is the hyperedge in the tree */
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
int node;
node = find_interval_in_tree(tree, interval+2*hEdge);
rowpart[hEdge] = partnumber[node];
(*sizeParts)[rowpart[hEdge]] ++;
rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge);
#if 0
fprintf (stderr, "%zd : " ZOLTAN_ID_SPEC " (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]);
#endif
}
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
/* Compute number of elements per parts */
/* TODO: support processor which are not part of the distribution */
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm);
/* Export results to data directory */
/* First, get the GIDs of our edges */
rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge);
if (nEdge && rowGID == NULL) MEMORY_ERROR;
ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL,
nEdge, NULL);
ZOLTAN_FREE(&rowGNO);
ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, rowpart, NULL, NULL, nEdge);
#ifdef CEDRIC_PRINT
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]);
}
#endif
End:
ZOLTAN_FREE(&rowGID);
ZOLTAN_FREE(&rowGNO);
ZOLTAN_FREE(&rowpart);
if (partnumber != NULL)
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Comm_Do_AlltoAll(
ZOLTAN_COMM_OBJ * plan, /* communication data structure */
char *send_data, /* array of data I currently own */
int nbytes, /* multiplier for sizes */
char *recv_data) /* array of data I'll own after comm */
{
static char *yo = "Zoltan_Comm_Do_AlltoAll";
char *outbuf=NULL, *inbuf=NULL, *buf=NULL;
int *outbufCounts=NULL, *outbufOffsets=NULL;
int *inbufCounts=NULL, *inbufOffsets=NULL;
int nprocs, me, rc, i, j, k, p, sorted;
int nSendMsgs, nSendItems, nRecvMsgs, nRecvItems;
int length, offset, itemSize, outbufLen;
int sm = (plan->self_msg > 0) ? 1 : 0;
nSendMsgs = plan->nsends + sm;
nRecvMsgs = plan->nrecvs + sm;
for (i=0, nSendItems=0; i <nSendMsgs; i++){
nSendItems += plan->lengths_to[i];
}
for (i=0, nRecvItems=0; i <nRecvMsgs; i++){
nRecvItems += plan->lengths_from[i];
}
MPI_Comm_size(plan->comm, &nprocs);
MPI_Comm_rank(plan->comm, &me);
outbufCounts = (int *) ZOLTAN_CALLOC(nprocs , sizeof(int));
outbufOffsets = (int *) ZOLTAN_CALLOC(nprocs , sizeof(int));
inbufCounts = (int *) ZOLTAN_CALLOC(nprocs , sizeof(int));
inbufOffsets = (int *) ZOLTAN_CALLOC(nprocs , sizeof(int));
if (!outbufCounts || !outbufOffsets || !inbufCounts || !inbufOffsets){
ZOLTAN_COMM_ERROR("memory error", yo, me);
}
/* The *_to fields of the plan refer to the items in the send_data buffer,
* and how to pull out the correct items for each receiver. The
* *_from fields of the plan refer to the recv_data buffer. Items
* arrive in process rank order, and these fields tell us where to
* put them in the recv_data buffer.
*/
/* CREATE SEND BUFFER */
sorted = 0;
if (plan->indices_to == NULL){
sorted = 1;
for (i=1; i< nSendMsgs; i++){
if (plan->starts_to[i] < plan->starts_to[i-1]){
sorted = 0;
break;
}
}
}
if (plan->sizes_to){
/*
* Each message contains items for a process, and each item may be
* a different size.
*/
for (i=0, outbufLen=0; i < nSendMsgs; i++){
outbufLen += plan->sizes_to[i];
}
if (plan->indices_to){
/*
* items are not grouped by message
*/
buf = outbuf = (char *)ZOLTAN_MALLOC(outbufLen * nbytes);
if (outbufLen && nbytes && !outbuf){
ZOLTAN_COMM_ERROR("memory error", yo, me);
}
for (p=0, i=0, k=0; p < nprocs; p++){
length = 0;
if (i < nSendMsgs){
if (plan->procs_to[i] == p){ /* procs_to is sorted */
for (j=0; j < plan->lengths_to[i]; j++,k++){
itemSize = plan->sizes[plan->indices_to[k]] * nbytes;
offset = plan->indices_to_ptr[k] * nbytes;
memcpy(buf, send_data + offset, itemSize);
buf += itemSize;
length += itemSize;
}
i++;
}
}
outbufCounts[p] = length;
if (p){
outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1];
}
}
}
else{
/*
* items are stored contiguously for each message
*/
if (!sorted || (plan->nvals > nSendItems) ){
buf = outbuf = (char *)ZOLTAN_MALLOC(outbufLen * nbytes);
if (outbufLen && nbytes && !outbuf){
ZOLTAN_COMM_ERROR("memory error", yo, me);
}
}
else{
/* All items in send_data are being sent, and they are sorted
* in process rank order.
*/
outbuf = send_data;
}
for (p=0, i=0; p < nprocs; p++){
length = 0;
if (i < nSendMsgs){
if (plan->procs_to[i] == p){ /* procs_to is sorted */
length = plan->sizes_to[i] * nbytes;
offset = plan->starts_to_ptr[i] * nbytes;
if ((!sorted || (plan->nvals > nSendItems)) && length){
memcpy(buf, send_data + offset, length);
buf += length;
}
i++;
}
}
outbufCounts[p] = length;
if (p){
outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1];
}
}
}
}
else if (plan->indices_to){
/*
* item sizes are constant, however the items belonging in a given
* message may not be contiguous in send_data
*/
buf = outbuf = (char *)ZOLTAN_MALLOC(nSendItems * nbytes);
if (nSendMsgs && nbytes && !outbuf){
ZOLTAN_COMM_ERROR("memory error", yo, me);
}
for (p=0, i=0, k=0; p < nprocs; p++){
length = 0;
if (i < nSendMsgs){
if (plan->procs_to[i] == p){ /* procs_to is sorted */
for (j=0; j < plan->lengths_to[i]; j++,k++){
offset = plan->indices_to[k] * nbytes;
memcpy(buf, send_data + offset, nbytes);
buf += nbytes;
}
length = plan->lengths_to[i] * nbytes;
i++;
}
}
outbufCounts[p] = length;
if (p){
outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1];
}
}
}
else{
/* item sizes are constant, and items belonging to a
* given message are always stored contiguously in send_data
*/
if (!sorted || (plan->nvals > nSendItems)){
buf = outbuf = (char *)ZOLTAN_MALLOC(nSendItems * nbytes);
if (nSendItems && nbytes && !outbuf){
ZOLTAN_COMM_ERROR("memory error", yo, me);
}
}
else{
/* send_data is sorted by process, and we don't skip
* any of the data in the buffer, so we can use send_data
* in the alltoall call
*/
outbuf = send_data;
}
for (p=0,i=0; p < nprocs; p++){
length = 0;
if (i < nSendMsgs){
if (plan->procs_to[i] == p){ /* procs_to is sorted */
offset = plan->starts_to[i] * nbytes;
length = plan->lengths_to[i] * nbytes;
if ((!sorted || (plan->nvals > nSendItems)) && length){
memcpy(buf, send_data + offset, length);
buf += length;
}
i++;
}
}
outbufCounts[p] = length;
if (p){
outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1];
}
}
}
/* CREATE RECEIVE BUFFER */
sorted = 0;
if (plan->indices_from == NULL){
sorted = 1;
for (i=1; i< nRecvMsgs; i++){
if (plan->starts_from[i] < plan->starts_from[i-1]){
sorted = 0;
break;
}
}
}
if (sorted){
/* Caller already expects received data to be ordered by
* the sending process rank.
*/
inbuf = recv_data;
}
else{
inbuf = (char *)ZOLTAN_MALLOC(plan->total_recv_size * nbytes);
if (plan->total_recv_size && nbytes && !inbuf){
ZOLTAN_COMM_ERROR("memory error", yo, me);
}
}
for (p=0, i=0; p < nprocs; p++){
length = 0;
if (i < nRecvMsgs){
if (plan->procs_from[i] == p){
if (plan->sizes == NULL){
length = plan->lengths_from[i] * nbytes;
}
else{
length = plan->sizes_from[i] * nbytes;
}
i++;
}
}
inbufCounts[p] = length;
if (p){
inbufOffsets[p] = inbufOffsets[p-1] + inbufCounts[p-1];
}
}
/* EXCHANGE DATA */
rc = MPI_Alltoallv(outbuf, outbufCounts, outbufOffsets, MPI_BYTE,
inbuf, inbufCounts, inbufOffsets, MPI_BYTE,
plan->comm);
if (outbuf != send_data){
ZOLTAN_FREE(&outbuf);
}
ZOLTAN_FREE(&outbufCounts);
ZOLTAN_FREE(&outbufOffsets);
/* WRITE RECEIVED DATA INTO USER'S BUFFER WHERE IT'S EXPECTED */
if (!sorted){
buf = inbuf;
if (plan->sizes == NULL){
/* each item in each message is nbytes long */
if (plan->indices_from == NULL){
for (i=0; i < nRecvMsgs; i++){
offset = plan->starts_from[i] * nbytes;
length = plan->lengths_from[i] * nbytes;
memcpy(recv_data + offset, buf, length);
buf += length;
}
}
else{
for (i=0,k=0; i < nRecvMsgs; i++){
for (j=0; j < plan->lengths_from[i]; j++,k++){
offset = plan->indices_from[k] * nbytes;
memcpy(recv_data + offset, buf, nbytes);
buf += nbytes;
}
}
}
}
else{ /* (sizes!=NULL) && (indices_from!=NULL) not allowed by Zoltan_Comm_Resize */
/* items can be different sizes */
for (i=0; i < nRecvMsgs; i++){
offset = plan->starts_from_ptr[i] * nbytes;
length = plan->sizes_from[i] * nbytes;
memcpy(recv_data + offset, buf, length);
buf += length;
}
}
ZOLTAN_FREE(&inbuf);
}
ZOLTAN_FREE(&inbufCounts);
ZOLTAN_FREE(&inbufOffsets);
return ZOLTAN_OK;
}
int Zoltan_Comm_Do_Wait(
ZOLTAN_COMM_OBJ * plan, /* communication data structure */
int tag, /* message tag for communicating */
char *send_data, /* array of data I currently own */
int nbytes, /* multiplier for sizes */
char *recv_data) /* array of data I'll own after comm */
{
MPI_Status status; /* return from Waitany */
int my_proc; /* processor ID */
int self_num; /* where in send list my_proc appears */
int i, j, k, jj; /* loop counters */
/* If not point to point, currently we do synchroneous communications */
if (plan->maxed_recvs){
/* Do nothing */
return (ZOLTAN_OK);
}
MPI_Comm_rank(plan->comm, &my_proc);
/* Wait for messages to arrive & unpack them if necessary. */
/* Note: since request is in plan, could wait in later routine. */
if (plan->indices_from == NULL) { /* No copying required */
if (plan->nrecvs > 0) {
MPI_Waitall(plan->nrecvs, plan->request, plan->status);
}
}
else { /* Need to copy into recv_data. */
if (plan->self_msg) { /* Unpack own data before waiting */
for (self_num = 0; self_num < plan->nrecvs + plan->self_msg; self_num++)
if (plan->procs_from[self_num] == my_proc) break;
k = plan->starts_from[self_num];
if (!plan->sizes_from || plan->sizes_from[self_num]) {
for (j = plan->lengths_from[self_num]; j; j--) {
memcpy(&recv_data[plan->indices_from[k] * nbytes],
&plan->recv_buff[k * nbytes], nbytes);
k++;
}
}
}
else
self_num = plan->nrecvs;
for (jj = 0; jj < plan->nrecvs; jj++) {
MPI_Waitany(plan->nrecvs, plan->request, &i, &status);
if (i == MPI_UNDEFINED) break; /* No more receives */
if (i >= self_num) i++;
k = plan->starts_from[i];
for (j = plan->lengths_from[i]; j; j--) {
memcpy(&recv_data[plan->indices_from[k] * nbytes],
&plan->recv_buff[k * nbytes], nbytes);
k++;
}
}
ZOLTAN_FREE(&plan->recv_buff);
}
return (ZOLTAN_OK);
}
int Zoltan_LB_Get_Part_Sizes(ZZ *zz,
int num_global_parts, int part_dim, float *part_sizes)
{
/*
* Function to get the scaled partition sizes.
*
* Input:
* zz -- The Zoltan structure to which this method
* applies.
* num_global_parts -- Number of global partitions.
* (This usually equals lb->Num_Global_Parts)
* part_dim -- The number of object weights per partition.
* (This usually equals lb->Obj_Wgt_Dim.)
*
* Output:
* part_sizes -- Array of floats that gives the set partition
* sizes, scaled such that they sum to one.
*/
int i, j, nparts, fpart;
float *temp_part_sizes=NULL, *sum=NULL;
int error = ZOLTAN_OK;
char msg[128];
static char *yo = "Zoltan_LB_Get_Part_Sizes";
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: num_global_parts = %d\n", zz->Proc, num_global_parts);
/* Barrier to make sure all procs have finished Zoltan_LB_Set_Part_Sizes */
MPI_Barrier(zz->Communicator);
/* For convenience, if no weights are used, set part_dim to 1 */
if (part_dim==0) part_dim = 1;
if (part_sizes == NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument part_sizes is NULL.");
error = ZOLTAN_FATAL;
goto End;
}
/* Find max Part_Info_Len over all procs to see if they are all zero. */
MPI_Allreduce((void*) &(zz->LB.Part_Info_Len), (void*) &j,
1, MPI_INT, MPI_MAX, zz->Communicator);
if (j == 0){
/* Uniform partition sizes. */
zz->LB.Uniform_Parts = 1;
for (i = 0; i < num_global_parts*part_dim; i++)
part_sizes[i] = 1.0 / (float)num_global_parts;
}
else {
/* Get the partition sizes set by the user (application).
* Each processor puts its data in a part_dim * num_global_parts
* array. Then we gather all the data across processors.
* Out-of-range partition size data is ignored.
*/
zz->LB.Uniform_Parts = 0;
/* Pack LB.Part_Info into temp array */
temp_part_sizes = (float *)ZOLTAN_MALLOC(num_global_parts*part_dim
*sizeof(float));
sum = (float *)ZOLTAN_MALLOC(part_dim*sizeof(float));
if ((!temp_part_sizes) || (!sum)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < num_global_parts*part_dim; i++){
temp_part_sizes[i] = -1.0;
}
for (i = 0; i < zz->LB.Part_Info_Len; i++){
/* Only assemble partition sizes for partitions and weights
in the requested range. */
if (zz->LB.Part_Info[i].Idx < part_dim){
j = zz->LB.Part_Info[i].Part_id;
if (zz->LB.Part_Info[i].Global_num == 0) {
Zoltan_LB_Proc_To_Part(zz, zz->Proc, &nparts, &fpart);
j += fpart;
}
if (j >= num_global_parts){
sprintf(msg, "Partition number %d is >= num_global_parts %d.",
j, num_global_parts);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
error = ZOLTAN_WARN;
}
else
temp_part_sizes[j*part_dim + zz->LB.Part_Info[i].Idx]
= zz->LB.Part_Info[i].Size;
}
}
/* Reduce over all procs */
MPI_Allreduce((void*) temp_part_sizes, (void*) part_sizes,
num_global_parts*part_dim, MPI_FLOAT, MPI_MAX, zz->Communicator);
/* Check for errors. Scale the sizes so they sum to one for each weight. */
for (j = 0; j < part_dim; j++)
sum[j] = 0.0;
for (i = 0; i < num_global_parts; i++){
for (j = 0; j < part_dim; j++){
if (part_sizes[i*part_dim+j]<0)
part_sizes[i*part_dim+j] = 1.0; /* default value if not set */
sum[j] += part_sizes[i*part_dim+j];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
printf("[%1d] In %s: Partition size %1d (before scaling) = ",
zz->Proc, yo, i);
for (j = 0; j < part_dim; j++)
printf("%f, ", part_sizes[i*part_dim+j]);
printf("\n");
}
}
/* Check for sum[j] == 0 (error). */
for (j = 0; j < part_dim; j++) {
if (sum[j] == 0.0) {
sprintf(msg, "Sum of weights (component %1d) is zero.", j);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
error = ZOLTAN_FATAL;
goto End;
}
}
/* Normalize partition sizes */
for (i = 0; i < num_global_parts; i++)
for (j = 0; j < part_dim; j++)
part_sizes[i*part_dim+j] /= sum[j];
}
End:
if (temp_part_sizes) ZOLTAN_FREE(&temp_part_sizes);
if (sum) ZOLTAN_FREE(&sum);
ZOLTAN_TRACE_EXIT(zz, yo);
return error;
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
realtype itr = 0.0;
indextype options[MAX_OPTIONS];
char alg[MAX_PARAM_STRING_LEN+1];
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
/* change our zz struct. */
#endif
indextype i;
realtype *imb_tols;
indextype ncon;
indextype edgecut;
indextype wgtflag;
indextype numflag = 0;
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
ZOLTAN_TRACE_ENTER(zz, yo);
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
if (sizeof(realtype) != sizeof(float)) {
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
realtype sum = 0.;
prt.input_part_sizes = (realtype *)
ZOLTAN_MALLOC(tmp * sizeof(realtype));
for (i = 0; i < tmp; i++) {
prt.input_part_sizes[i] = (realtype) part_sizes[i];
sum += prt.input_part_sizes[i];
}
if (sum != (realtype) 1.0) {
/* rescale part sizes in case of roundoff in conversion;
* ParMETIS requires sum of part sizes to be 1.0
*/
for (i = 0; i < tmp; i++) {
prt.input_part_sizes[i] /= sum;
}
}
prt.part_sizes = prt.input_part_sizes;
}
else
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
indextype j;
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
/* Now we can call ParMetis */
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
/* Use default options for METIS */
options[0] = 0;
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
#else
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
prt.part_sizes, imb_tols, options,
&edgecut, prt.part);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) {
/* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Oct_migreg_migrate_orphans(ZZ *zz, pRegion RegionList, int nregions,
int level, OMap *array, int *c1, int *c2) {
int i, j, k; /* index counters */
pRegion ptr; /* region in the mesh */
COORD origin; /* centroid coordinate information */
pRegion *regions = NULL; /* an array of regions */
int *npids = NULL;
Region *regions2 = NULL; /* an array of regions */
int *npids2 = NULL;
int nreg; /* number of regions */
COORD min, /* minimum bounds of an octant */
max; /* maximum bounds of an octant */
COORD cmin, /* minimum bounds of a child octant */
cmax; /* maximum bounds of a child octant */
COORD rmin, /* minimum bounds of a remote octant */
rmax; /* maximum bounds of a remote octant */
int new_num;
int n;
int dir = 0;
pRList RootList;
pOctant RootOct;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
char *yo = "Zoltan_Oct_migreg_migrate_orphans_static";
int ierr = ZOLTAN_OK;
ZOLTAN_ID_PTR gids2, lids2;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
if(nregions > 0) {
/* create the array of messages to be sent to other processors */
/* Array = (Message *) ZOLTAN_MALLOC(nregions * sizeof(Message)); */
if((regions = (pRegion *) ZOLTAN_MALLOC(nregions * sizeof(pRegion))) == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((npids = (int *) ZOLTAN_MALLOC(nregions * sizeof(int))) == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(®ions);
return ZOLTAN_MEMERR;
}
}
ptr = RegionList;
n = nreg = 0;
while((ptr != NULL) && (nregions > 0)) {
if(ptr->attached == 1) {
/* if region already attached to an octant, then skip to next region */
ptr = ptr->next;
continue;
}
/* region not attached, have to find which processor to send to */
j=0;
dir = 0;
vector_set(min, OCT_info->OCT_gmin);
vector_set(max, OCT_info->OCT_gmax);
/*
* for each level of refinement, find which child region belongs to.
* translate which child to which entry in map array.
*/
for(i=0; i<level; i++) {
Zoltan_Oct_bounds_to_origin(min, max, origin);
if(OCT_info->OCT_dimension == 2)
j = j * 4;
else
j = j * 8;
k = Zoltan_Oct_child_which(OCT_info,origin, ptr->Coord);
new_num = Zoltan_Oct_convert_idx_from_map(OCT_info, dir, k);
dir = Zoltan_Oct_get_child_dir(OCT_info, dir, new_num);
j += new_num;
Zoltan_Oct_child_bounds(min, max, origin, k, cmin, cmax);
vector_set(min, cmin);
vector_set(max, cmax);
}
/* inform message which processor to send to */
npids[n] = array[j].npid;
RootList = array[j].list;
while((RootOct = RL_nextRootOctant(&RootList))) {
Zoltan_Oct_bounds(RootOct,rmin,rmax);
if (Zoltan_Oct_in_box_closure(OCT_info, ptr->Coord ,rmin, rmax)) {
npids[n] = RootOct->npid;
break;
}
}
if((npids[n] != -1) && (npids[n] != zz->Proc)) {
Zoltan_Oct_copy_info(zz, ptr, &(regions[n++]));
}
else {
Zoltan_Oct_insert_orphan(zz, *ptr);
}
nreg++; /* increment region counter */
ptr = ptr->next; /* look at next region */
}
/*
* if regions looked at != number of regions in region list,
* then there is an error
*/
if (nreg!=nregions) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "regions found != to expected number of regions");
return ZOLTAN_FATAL;
}
regions2 = (Region *) ZOLTAN_MALLOC(n * sizeof(Region));
gids2 = ZOLTAN_MALLOC_GID_ARRAY(zz, n);
lids2 = ZOLTAN_MALLOC_LID_ARRAY(zz, n);
npids2 = (int *) ZOLTAN_MALLOC(n * sizeof(int));
for(i=0; i<n; i++) {
npids2[i] = npids[i];
vector_set(regions2[i].Coord, regions[i]->Coord);
regions2[i].Weight = regions[i]->Weight;
regions2[i].Global_ID = &(gids2[i*num_gid_entries]);
regions2[i].Local_ID = (num_lid_entries
? &(lids2[i*num_lid_entries])
: NULL);
ZOLTAN_SET_GID(zz, &(gids2[i*num_gid_entries]), regions[i]->Global_ID);
ZOLTAN_SET_LID(zz, &(lids2[i*num_lid_entries]), regions[i]->Local_ID);
regions2[i].Proc = regions[i]->Proc;
regions2[i].attached = 0;
}
*c1 = n;
/* migrate the orphan regions according to the message array */
Zoltan_Oct_migreg_migrate_regions(zz, regions2, gids2, lids2, npids2, n, c2);
for (i=0; i < n; i++) {
ZOLTAN_FREE(&(regions[i]->Global_ID));
ZOLTAN_FREE(&(regions[i]->Local_ID));
ZOLTAN_FREE(&(regions[i]));
}
ZOLTAN_FREE(®ions);
ZOLTAN_FREE(&npids);
ZOLTAN_FREE(®ions2);
ZOLTAN_FREE(&gids2);
ZOLTAN_FREE(&lids2);
ZOLTAN_FREE(&npids2);
return ierr;
}
int Zoltan_Block(
ZZ *zz, /* The Zoltan structure. */
float *part_sizes, /* Input: Array of size zz->LB.Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_import, /* Return -1. We use only export lists. */
ZOLTAN_ID_PTR *import_global_ids, /* Not used. */
ZOLTAN_ID_PTR *import_local_ids, /* Not used. */
int **import_procs, /* Not used. */
int **import_to_part, /* Not used. */
int *num_export, /* Output: Number of objects to export. */
ZOLTAN_ID_PTR *export_global_ids, /* Output: GIDs to export. */
ZOLTAN_ID_PTR *export_local_ids, /* Output: LIDs to export. */
int **export_procs, /* Output: Processsors to export to. */
int **export_to_part /* Output: Partitions to export to. */
)
{
int ierr = ZOLTAN_OK;
int i, count, num_obj;
int wtflag;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int *parts = NULL;
int *newparts = NULL;
float *wgts = NULL;
static char *yo = "Zoltan_Block";
ZOLTAN_TRACE_ENTER(zz, yo);
/* No import lists computed. */
*num_import = -1;
*export_global_ids = *export_local_ids = NULL;
*export_procs = *export_to_part = NULL;
/* Get list of local objects. */
if (zz->Obj_Weight_Dim > 1) {
ierr = ZOLTAN_FATAL;
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"OBJ_WEIGHT_DIM > 1 not supported by LB_METHOD BLOCK.");
goto End;
}
wtflag = (zz->Obj_Weight_Dim>0 ? 1 : 0);
ierr = Zoltan_Get_Obj_List(zz, &num_obj, &global_ids, &local_ids, wtflag,
&wgts, &parts);
/* Compute the new partition numbers. */
newparts = (int *) ZOLTAN_MALLOC(num_obj * sizeof(int));
if (num_obj && (!newparts)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
block_part(zz, num_obj, wtflag, wgts, part_sizes, newparts);
/* Check how many partition numbers changed. */
count=0;
for (i=0; i<num_obj; i++){
if (newparts[i] != parts[i])
++count;
}
(*num_export) = count;
/* Allocate export lists. */
if ((*num_export) > 0) {
if (!Zoltan_Special_Malloc(zz, (void **)export_global_ids, (*num_export),
ZOLTAN_SPECIAL_MALLOC_GID)
|| !Zoltan_Special_Malloc(zz, (void **)export_local_ids, (*num_export),
ZOLTAN_SPECIAL_MALLOC_LID)
|| !Zoltan_Special_Malloc(zz, (void **)export_procs, (*num_export),
ZOLTAN_SPECIAL_MALLOC_INT)
|| !Zoltan_Special_Malloc(zz, (void **)export_to_part, (*num_export),
ZOLTAN_SPECIAL_MALLOC_INT)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/* Loop over objects and fill export lists. */
count=0;
for (i=0; i<num_obj; i++){
if (newparts[i] != parts[i]){
/* export_global_ids[count] = global_ids[i]; */
ZOLTAN_SET_GID(zz, &((*export_global_ids)[count*zz->Num_GID]),
&global_ids[i*zz->Num_GID]);
if (local_ids)
/* export_local_ids[count] = local_ids[i]; */
ZOLTAN_SET_LID(zz, &((*export_local_ids)[count*zz->Num_LID]),
&local_ids[i*zz->Num_LID]);
/* Set new partition number. */
(*export_to_part)[count] = newparts[i];
/* Processor is derived from partition number. */
(*export_procs)[count] = Zoltan_LB_Part_To_Proc(zz,
(*export_to_part)[count], &global_ids[i*zz->Num_GID]);
++count;
}
}
End:
/* Free local memory, but not export lists. */
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_FREE(&parts);
ZOLTAN_FREE(&newparts);
if (wtflag) ZOLTAN_FREE(&wgts);
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions,
ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
int *npids, int nregions, int *c2)
{
char *yo = "Zoltan_Oct_migreg_migrate_regions";
int i; /* index counter */
int ierr = ZOLTAN_OK;
int n_import;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
Region *import_objs = NULL; /* Array of import objects used to request
the objs from other processors. */
ZOLTAN_ID_PTR import_gids = NULL; /* Array of global IDs of import_objs. */
ZOLTAN_ID_PTR import_lids = NULL; /* Array of local IDs of import_objs. */
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator,
MIGMIGREGCommCreate, &n_import);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
*c2 = n_import;
if (n_import > 0) {
import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region));
import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import);
import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import);
if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region),
(char *) import_objs);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids,
sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
if (num_lid_entries > 0) {
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids,
sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
}
for (i=0; i<n_import; i++) {
import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]);
import_objs[i].Local_ID = (num_lid_entries
? &(import_lids[i*num_lid_entries])
: NULL);
Zoltan_Oct_insert_orphan(zz, import_objs[i]);
}
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
return ierr;
}
static int local_HEs_from_import_lists(
ZZ *zz,
int remap_type, /* type of remapping to do: parts, procs, or none. */
int nobj, /* # objs the processor knows about (keep + imports) */
int *proc, /* On input, old processor assignment for each obj;
Upon return, remapped new proc assignment for
each obj. */
int *old_part, /* old partition assignments for each objs */
int *new_part, /* On input, new partition assignments for each objs.
Upon return, remapped new partition assignments */
int *HEcnt, /* # of HEs allocated. */
int **HEinfo /* Array of HE info; for each HE, two pins and
one edge weight. Stored as a single vector
to minimize communication calls. */
)
{
/* Routine to remap partitions (to new processors or new partition numbers)
* to reduce data movement.
* This routine assumes the load-balancing algorithm built import lists.
* Objects described are those that ENDED UP on my_proc due to load balancing.
* For all these objects, new_proc == my_proc.
*/
char *yo = "local_HEs_from_import_lists";
int ierr = ZOLTAN_OK;
int i, cnt, tmp;
int *tmp_HEinfo;
int old_size; /* # of old entries to remap to. If remapping
parts to processors, old_size = Num_Procs;
if renumbering partitions, old_size = old
num parts. */
int fp; /* First partition on this processor in new
decomposition. */
int np; /* # of partitions on this processor in new
decomposition. */
int my_proc = zz->Proc; /* This processor's rank. */
int minp, maxp; /* Lowest and highest partition numbers on this
processor in old decomposition;
partition numbers are assumed to be dense,
but no particular distribution is assumed. */
int HEwgt_size; /* # of HE weights allocated. */
int *HEwgt = NULL; /* Array of HE weights. Initially includes
zero weights; later zero-weights are removed.*/
if (remap_type == ZOLTAN_LB_REMAP_PROCESSORS) {
/* Renumber new processors to minimize changes in proc assignment. */
HEwgt_size = zz->Num_Proc;
HEwgt = (int *) ZOLTAN_CALLOC(HEwgt_size, sizeof(int));
if (!HEwgt) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < nobj; i++)
HEwgt[proc[i]]++; /* At this point, proc has old proc assignments */
*HEcnt = 0;
for (i = 0; i < HEwgt_size; i++)
if (HEwgt[i] != 0) (*HEcnt)++;
ierr = malloc_HEinfo(zz, *HEcnt, HEinfo);
if (ierr < 0)
goto End;
tmp_HEinfo = *HEinfo;
cnt = 0;
for (i = 0; i < HEwgt_size; i++) {
if (HEwgt[i] != 0) {
tmp = cnt * HEINFO_ENTRIES;
tmp_HEinfo[tmp] = i; /* Old processor number */
tmp_HEinfo[tmp+1] = my_proc; /* New processor number */
tmp_HEinfo[tmp+2] = HEwgt[i]; /* shift non-zero weights down. */
cnt++;
}
}
}
else { /* ZOLTAN_LB_REMAP_PARTS */
/* Renumber new partitions to minimize changes in partition assignment */
for (minp = INT_MAX, maxp = 0, i = 0; i < nobj; i++) {
if (old_part[i] < minp) minp = old_part[i];
if (old_part[i] > maxp) maxp = old_part[i];
}
/* Don't include old partition numbers that are greater than
* zz->LB.Num_Global_Parts - 1; they are not valid values for
* remapping of new partition numbers.
*/
if (minp >= zz->LB.Num_Global_Parts)
minp = zz->LB.Num_Global_Parts-1;
if (maxp >= zz->LB.Num_Global_Parts)
maxp = zz->LB.Num_Global_Parts-1;
old_size = maxp - minp + 1;
Zoltan_LB_Proc_To_Part(zz, my_proc, &np, &fp);
HEwgt_size = np * old_size;
if (HEwgt_size > 0) {
HEwgt = (int *) ZOLTAN_CALLOC(HEwgt_size, sizeof(int));
if (!HEwgt) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
for (i = 0; i < nobj; i++) {
if (old_part[i] < zz->LB.Num_Global_Parts) {
/* Include only HEs to old partitions numbered
* 0 to zz->LB.Num_Global_Parts-1; these are the only valid
* remapping values for the new partition numbers.
*/
tmp = (new_part[i]-fp) * old_size;
HEwgt[tmp + (old_part[i]-minp)]++;
}
}
*HEcnt = 0;
for (i = 0; i < HEwgt_size; i++)
if (HEwgt[i] != 0) (*HEcnt)++;
ierr = malloc_HEinfo(zz, *HEcnt, HEinfo);
if (ierr < 0)
goto End;
tmp_HEinfo = *HEinfo;
cnt = 0;
for (i = 0; i < HEwgt_size; i++) {
if (HEwgt[i] != 0) {
tmp = cnt * HEINFO_ENTRIES;
tmp_HEinfo[tmp] = i%old_size + minp; /* Old partition number */
tmp_HEinfo[tmp+1] = i/old_size + fp; /* New partition number */
tmp_HEinfo[tmp+2] = HEwgt[i]; /* shift non-zero weights down. */
cnt++;
}
}
}
End:
if (HEwgt) ZOLTAN_FREE(&HEwgt);
return ierr;
}
int
Zoltan_Matrix_Sym(ZZ* zz, Zoltan_matrix *matrix, int bipartite)
{
static char *yo = "Zoltan_Matrix_Sym";
int ierr = ZOLTAN_OK;
Zoltan_Arc *tr_tab = NULL;
int i, j, cnt;
ZOLTAN_ID_PTR yGID = NULL;
int *ypid=NULL;
float *pinwgt=NULL;
int * ybipart = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (bipartite || !matrix->opts.enforceSquare) {
matrix->bipartite = 1;
matrix->redist = 1;
}
if (matrix->ywgtdim != zz->Obj_Weight_Dim)
FATAL_ERROR("Cannot form bipartite graph: vertex and edge weights are not consistant");
matrix->opts.symmetrize = 1;
/* Update the data directories */
tr_tab = (Zoltan_Arc*) ZOLTAN_MALLOC(sizeof(Zoltan_Arc)*(matrix->nPins*2+matrix->nY));
if (matrix->nPins && tr_tab == NULL) MEMORY_ERROR;
pinwgt = (float*)ZOLTAN_MALLOC((matrix->nPins*2+matrix->nY)*matrix->pinwgtdim*sizeof(float));
for (i = 0 ; i < 2 ; ++i) /* Copy pin weights */
memcpy(pinwgt + i*matrix->nPins*matrix->pinwgtdim*sizeof(float),
matrix->pinwgt, matrix->nPins*matrix->pinwgtdim*sizeof(float));
for (i=0, cnt = 0 ; i < matrix->nY ; ++i) {
for (j = matrix->ystart[i] ; j < matrix->yend[i] ; ++j) {
tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */
tr_tab[cnt].GNO[1] = matrix->pinGNO[j];
memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim,
matrix->pinwgtdim*sizeof(float));
cnt ++;
tr_tab[cnt].GNO[0] = matrix->pinGNO[j]; /* Symmetric arc */
tr_tab[cnt].GNO[1] = matrix->yGNO[i] + bipartite*matrix->globalX; /* new ordering */
memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim,
matrix->pinwgtdim*sizeof(float));
cnt ++;
}
if (matrix->ystart[i] == matrix->yend[i]) { /* Singleton */
tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */
tr_tab[cnt].GNO[1] = -1;
cnt ++;
}
}
ZOLTAN_FREE(&matrix->pinwgt);
Zoltan_Matrix_Remove_DupArcs(zz, cnt, tr_tab, pinwgt, matrix);
ZOLTAN_FREE(&tr_tab);
ZOLTAN_FREE(&pinwgt);
if (bipartite) {
int endX;
int * yGNO = NULL;
/* Update data directories */
yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nY);
ypid = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int));
ybipart = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int));
for (endX = 0 ; endX < matrix->nY ; ++endX) {
if (matrix->yGNO[endX] >= matrix->globalX)
break;
ybipart[endX] = 0;
}
/* Get Informations about X */
Zoltan_DD_Find (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, NULL,
endX, NULL);
yGNO = (int*)ZOLTAN_MALLOC(endX*sizeof(int));
for (i = endX ; i < matrix->nY ; ++i) {
yGNO[i-endX] = matrix->yGNO[i] - matrix->globalX;
/* TODO: add a something to have the correct ypid */
ybipart[endX] = 1;
}
/* Get Informations about Y */
Zoltan_DD_Find (matrix->ddY, (ZOLTAN_ID_PTR)yGNO,
yGID + endX*zz->Num_GID,
NULL, NULL,
matrix->nY-endX, NULL);
if (matrix->ddY != matrix->ddX)
Zoltan_DD_Destroy (&matrix->ddY);
Zoltan_DD_Destroy (&matrix->ddX);
matrix->globalX += matrix->globalY;
matrix->globalY = matrix->globalX;
/* I store : xGNO, xGID, xpid, bipart */
ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID,
1, matrix->globalX/zz->Num_Proc, 0);
matrix->ddY = matrix->ddX;
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
/* Associate all the data with our xyGNO */
Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, ybipart,
matrix->nY);
}
End:
ZOLTAN_FREE(&ybipart);
ZOLTAN_FREE(&ypid);
ZOLTAN_FREE(&pinwgt);
ZOLTAN_FREE(&yGID);
ZOLTAN_FREE(&tr_tab);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
