int Zoltan_Oct_migrate_octants(ZZ *zz, int *newpids, pOctant *octs, int nocts, int *nrecocts) { int i,j = 0; int nsends = 0; int nreceives = 0; int *despid = NULL; OCTNEW_msg *snd_reply = NULL; OCTNEW_msg *rcv_reply = NULL; int ierr = ZOLTAN_OK; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ char *yo = "Zoltan_Oct_migrate_octants"; pOctant *newocts = NULL; /* New foreign octant pointers */ if((newocts = (pOctant *) ZOLTAN_MALLOC(sizeof(pOctant)*(nocts+10))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } for (i=0; i<nocts; i++) newocts[i]=NULL; /* count number of sends */ nsends=0; for (i=0; i<nocts; i++) if (newpids[i]!=zz->Proc) nsends++; /* build message array */ /* if(nsends > 0) { */ if((snd_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nsends + 1) * sizeof(OCTNEW_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((despid = (int *) ZOLTAN_MALLOC((nsends+10) * sizeof(int))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); return ZOLTAN_MEMERR; } /* } */ /* else { */ /* snd_reply = NULL; */ /* despid = NULL; */ /* } */ j = 0; for (i=0; i<nocts; i++) if (newpids[i]!=zz->Proc) { snd_reply[j].num = i; despid[j++] = newpids[i]; } /* send messages */ ierr = Zoltan_Comm_Create(&comm_plan, nsends, despid, zz->Communicator, MigOctCommCreate, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); return (ierr); } if((rcv_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nreceives + 1) * sizeof(OCTNEW_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); return ZOLTAN_MEMERR; } ierr = Zoltan_Comm_Do(comm_plan, MigOctCommDo, (char *) snd_reply, sizeof(OCTNEW_msg), (char *) rcv_reply); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); return (ierr); } /* Reply to malloc requests and Receive malloc replies */ for (i=0; i< nreceives; i++) { rcv_reply[i].ptr = Zoltan_Oct_POct_new((OCT_Global_Info *) (zz->LB.Data_Structure)); } ; ierr = Zoltan_Comm_Do_Reverse(comm_plan, MigOctCommReverse, (char *) rcv_reply, sizeof(OCTNEW_msg), NULL, (char *) snd_reply); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); return (ierr); } /* store remote pointers locally for future migration */ for (i=0; i<nsends; i++) { newocts[snd_reply[i].num] = snd_reply[i].ptr; } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); return (ierr); } ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); /* set return value */ *nrecocts = nreceives; ierr = Zoltan_Oct_Update_Connections(zz, octs, newpids, newocts, nocts); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); abort(); return (ierr); } ierr = Zoltan_Oct_Final_Migration(zz, octs,newpids,newocts,nocts, *nrecocts); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); abort(); return (ierr); } Zoltan_Oct_Update_Map(zz); ZOLTAN_FREE(&newocts); return ierr; }
int Zoltan_ParMetis_Order( ZZ *zz, /* Zoltan structure */ int num_obj, /* Number of (local) objects to order. */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */ int *iperm, ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */ ) { static char *yo = "Zoltan_ParMetis_Order"; int i, n, ierr; ZOLTAN_Output_Order ord; ZOLTAN_Third_Graph gr; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't want to risk letting external packages changing our communicator */ #endif indextype numflag = 0; int timer_p = 0; int get_times = 0; int use_timers = 0; double times[5]; ZOLTAN_ID_PTR l_gids = NULL; ZOLTAN_ID_PTR l_lids = NULL; indextype options[MAX_OPTIONS]; char alg[MAX_PARAM_STRING_LEN+1]; ZOLTAN_TRACE_ENTER(zz, yo); #ifdef ZOLTAN_PARMETIS #if TPL_USE_DATATYPE != TPL_METIS_DATATYPES #ifdef TPL_FLOAT_WEIGHT i = 1; #else i = 0; #endif if ((sizeof(indextype) != sizeof(idxtype)) || (sizeof(weighttype) != sizeof(idxtype)) || i){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Not supported: Multiple 3rd party libraries with incompatible " "data types."); return ZOLTAN_FATAL; } #endif #endif memset(&gr, 0, sizeof(ZOLTAN_Third_Graph)); memset(&ord, 0, sizeof(ZOLTAN_Output_Order)); memset(times, 0, sizeof(times)); ord.order_opt = order_opt; if (!order_opt){ /* If for some reason order_opt is NULL, allocate a new ZOOS here. */ /* This should really never happen. */ order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS)); strcpy(order_opt->method,"PARMETIS"); } ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord); /* ParMetis only computes the rank vector */ order_opt->return_args = RETURN_RANK; /* Check that num_obj equals the number of objects on this proc. */ /* This constraint may be removed in the future. */ n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){ /* Return error code */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error."); return(ZOLTAN_FATAL); } if (n != num_obj){ /* Currently this is a fatal error. */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the " "number of objects."); return(ZOLTAN_FATAL); } /* Do not use weights for ordering */ gr.obj_wgt_dim = -1; gr.edge_wgt_dim = -1; gr.num_obj = num_obj; /* Check what ordering type is requested */ if (order_opt){ SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */ #ifdef ZOLTAN_PARMETIS if ((strcmp(order_opt->method, "METIS") == 0)) #endif /* ZOLTAN_PARMETIS */ SET_LOCAL_GRAPH(&gr.graph_type); } gr.get_data = 1; if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: " "set ParMetis instead."); return(ZOLTAN_FATAL); } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL); return (ierr); } /* Allocate space for separator sizes */ if (IS_GLOBAL_GRAPH(gr.graph_type)) { if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) { /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype)); if (ord.sep_sizes == NULL) { Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */ } /* Allocate space for direct perm */ ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype)); if (!ord.rank){ /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } if (IS_LOCAL_GRAPH(gr.graph_type)){ /* Allocate space for inverse perm */ ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype)); if (!ord.iperm){ /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } } else ord.iperm = NULL; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); #ifdef ZOLTAN_PARMETIS if (IS_GLOBAL_GRAPH(gr.graph_type)){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library"); ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy, &numflag, options, ord.rank, ord.sep_sizes, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else #endif /* ZOLTAN_PARMETIS */ #if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS) if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */ indextype numobj = gr.num_obj; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library"); options[0] = 0; /* Use default options for METIS. */ order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */ METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options, ord.iperm, ord.rank); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */ int numbloc; int start; int leaf; int *converttab; int levelmax; levelmax = mylog2(zz->Num_Proc) + 1; converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int)); memset(converttab, 0, zz->Num_Proc*2*sizeof(int)); /* Determine the first node in each separator, store it in zz->TPL_Order.start */ for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) { int father; father = zz->Num_Proc + numbloc; converttab[start] = 2*numbloc; zz->TPL_Order.leaves[leaf++]=start; zz->TPL_Order.ancestor[start] = start + 2; converttab[start+1] = 2*numbloc+1; zz->TPL_Order.leaves[leaf++]=start+1; zz->TPL_Order.ancestor[start+1] = start + 2; start+=2; do { converttab[start] = father; if (father %2 == 0) { int nextoffset; int level; level = mylog2(2*zz->Num_Proc - 1 - father); nextoffset = (1<<(levelmax-level)); zz->TPL_Order.ancestor[start] = start+nextoffset; start++; break; } else { zz->TPL_Order.ancestor[start] = start+1; start++; father = zz->Num_Proc + father/2; } } while (father < 2*zz->Num_Proc - 1); } zz->TPL_Order.start[0] = 0; zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1; for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) { int oldblock=converttab[numbloc-1]; zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock]; } ZOLTAN_FREE(&converttab); ZOLTAN_FREE(&ord.sep_sizes); zz->TPL_Order.leaves[zz->Num_Proc] = -1; zz->TPL_Order.nbr_leaves = zz->Num_Proc; zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1; } else { /* No tree */ zz->TPL_Order.nbr_blocks = 0; zz->TPL_Order.start = NULL; zz->TPL_Order.ancestor = NULL; zz->TPL_Order.leaves = NULL; } /* Correct because no redistribution */ memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE)); memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE)); ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL); ZOLTAN_FREE(&l_gids); ZOLTAN_FREE(&l_lids); /* Get a time here */ if (get_times) times[3] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){ memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype)); } else{ for (i=0; i < gr.num_obj; i++){ rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i]; } } if ((ord.iperm != NULL) && (iperm != NULL)){ if (sizeof(indextype) == sizeof(int)){ memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype)); } else{ for (i=0; i < gr.num_obj; i++){ iperm[i] = (int)ord.iperm[i]; } } } if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm); ZOLTAN_FREE(&ord.rank); /* Free all other "graph" stuff */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); }
static int Zoltan_Oct_Final_Migration( ZZ *zz, pOctant *octs, /* octs[nocts] */ int *newpids, /* newpids[nocts] */ pOctant *newocts, /* newocts[nocts] */ int nocts, /* number of octants leaving this processor */ int nrecocts) /* number of octants received in this processor */ { int i; Migrate_msg *msnd = NULL, *mrcv = NULL; int remotecount; int nsends; int nreceives; int *despid = NULL; int ierr = ZOLTAN_OK; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ char *yo = "Zoltan_Oct_Final_Migration"; OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure; /* count number of sends */ nsends=0; for (i=0; i<nocts; i++) if (newpids[i]!=zz->Proc) nsends++; if((despid = (int *) ZOLTAN_MALLOC((nocts+10) * sizeof(int))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((msnd = (Migrate_msg *) ZOLTAN_MALLOC((nocts+10) * sizeof(Migrate_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); return ZOLTAN_MEMERR; } remotecount = 0; for (i=0; i<nocts; i++) /* Send and free */ if (newpids[i]!=zz->Proc) { FILLMIGRATEMSG(octs[i], newocts[i], msnd[remotecount], zz->Proc); /* bug */ despid[remotecount++] = newpids[i]; Zoltan_Oct_clearRegions(octs[i]); /* KDDKDDFREE Change oct to &oct to allow NULL from ZOLTAN_FREE * KDDKDDFREE to propagate back. */ Zoltan_Oct_POct_free(OCT_info, &(octs[i])); } ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator, MigFinCommCreate, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); return (ierr); } if((mrcv = (Migrate_msg *) ZOLTAN_MALLOC((nreceives+10) * sizeof(Migrate_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); return ZOLTAN_MEMERR; } ierr = Zoltan_Comm_Do(comm_plan, MigFinCommDo, (char *) msnd, sizeof(Migrate_msg), (char *) mrcv); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); ZOLTAN_FREE(&mrcv); return (ierr); } for (i=0; i<nreceives; i++) { /* Receive new parocts */ /* Zoltan_Oct_setID(mrcv[i].ptr,mrcv[i].id); */ /* Zoltan_Oct_POct_setparent(OCT_info, mrcv[i].ptr, mrcv[i].parent, mrcv[i].ppid); */ SETOCTFROMMIGRATEMSG(OCT_info, mrcv[i]); /* Zoltan_Oct_setchildnum(mrcv[i].ptr,mrcv[i].childnum); */ /* Zoltan_Oct_setchildren(mrcv[i].ptr, mrcv[i].children, mrcv[i].cpids); */ /* Zoltan_Oct_setbounds(mrcv[i].ptr, mrcv[i].min, mrcv[i].max); */ /* Zoltan_Oct_setDir(mrcv[i].ptr,mrcv[i].dir); */ /* Zoltan_Oct_setMapIdx(mrcv[i].ptr,mrcv[i].mapidx); */ } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); ZOLTAN_FREE(&mrcv); return (ierr); } ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); ZOLTAN_FREE(&mrcv); return ierr; }
static int Zoltan_Oct_build_global_rootlist(ZZ *zz,Migrate_msg **ret_rmsg, int *size) { int j, k = 0; int *despid = NULL; int nroots, nreceives; pRList RootList; /* list of the local roots */ pOctant RootOct; Migrate_msg *snd_rmsg = NULL; Migrate_msg *rcv_rmsg = NULL; OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); /*Map *array = OCT_info->map;*/ ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ int ierr = ZOLTAN_OK; char *yo = "Zoltan_Oct_build_global_rootlist"; nroots = RL_numRootOctants(Zoltan_Oct_POct_localroots(OCT_info)); if (nroots > 0) { /* KDDKDD -- Added test to prevent departure before comm */ if((despid = (int *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(int))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((snd_rmsg = (Migrate_msg *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(Migrate_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); return ZOLTAN_MEMERR; } k = 0; for (j=0; j<zz->Num_Proc; j++) { RootList = Zoltan_Oct_POct_localroots(OCT_info); while((RootOct = RL_nextRootOctant(&RootList))) { /* if(array[Zoltan_Oct_mapidx(RootOct)].npid > 0) { */ FILLMIGRATEMSG(RootOct, RootOct, snd_rmsg[k], zz->Proc); despid[k] = j; k++; /* } */ } } } /* KDDKDD */ ierr = Zoltan_Comm_Create(&comm_plan, k, despid, zz->Communicator, RootListCommCreate, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&snd_rmsg); return (ierr); } if (nreceives > 0) { if((rcv_rmsg = (Migrate_msg *) ZOLTAN_MALLOC(nreceives * sizeof(Migrate_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&snd_rmsg); return ZOLTAN_MEMERR; } } ierr = Zoltan_Comm_Do(comm_plan, RootListCommDo, (char *) snd_rmsg, sizeof(Migrate_msg), (char *) rcv_rmsg); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&snd_rmsg); ZOLTAN_FREE(&rcv_rmsg); return (ierr); } ZOLTAN_FREE(&despid); ZOLTAN_FREE(&snd_rmsg); ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } *ret_rmsg = rcv_rmsg; *size = nreceives; return ierr; }
int Zoltan_RCB_Build_Structure(ZZ *zz, int *num_obj, int *max_obj, int wgtflag, int use_ids) { /* * Function to build the geometry-based data structures for * Steve Plimpton's RCB implementation. */ char *yo = "Zoltan_RCB_Build_Structure"; RCB_STRUCT *rcb; /* Data structure for RCB. */ struct rcb_tree *treeptr; int i, ierr = 0; /* * Allocate an RCB data structure for this Zoltan structure. * If the previous data structure is still there, free the Dots and IDs first; * the other fields can be reused. */ if (zz->LB.Data_Structure == NULL) { rcb = (RCB_STRUCT *) ZOLTAN_MALLOC(sizeof(RCB_STRUCT)); if (rcb == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); return(ZOLTAN_MEMERR); } zz->LB.Data_Structure = (void *) rcb; rcb->Tree_Ptr = NULL; rcb->Box = NULL; rcb->Global_IDs = NULL; rcb->Local_IDs = NULL; rcb->Dots = NULL; Zoltan_Initialize_Transformation(&(rcb->Tran)); rcb->Tree_Ptr = (struct rcb_tree *) ZOLTAN_MALLOC(zz->LB.Num_Global_Parts * sizeof(struct rcb_tree)); rcb->Box = (struct rcb_box *) ZOLTAN_MALLOC(sizeof(struct rcb_box)); if (rcb->Tree_Ptr == NULL || rcb->Box == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_RCB_Free_Structure(zz); return(ZOLTAN_MEMERR); } /* initialize Tree_Ptr */ for (i = 0; i < zz->LB.Num_Global_Parts; i++) { treeptr = &(rcb->Tree_Ptr[i]); /* initialize dim to -1 to prevent use of cut */ treeptr->dim = -1; treeptr->cut = 0.0; treeptr->parent = treeptr->left_leaf = treeptr->right_leaf = 0; } } else { rcb = (RCB_STRUCT *) zz->LB.Data_Structure; ZOLTAN_FREE(&(rcb->Global_IDs)); ZOLTAN_FREE(&(rcb->Local_IDs)); ZOLTAN_FREE(&(rcb->Dots)); } ierr = Zoltan_RB_Build_Structure(zz, &(rcb->Global_IDs), &(rcb->Local_IDs), &(rcb->Dots), num_obj, max_obj, &(rcb->Num_Dim), wgtflag, use_ids); if (ierr) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_RB_Build_Structure."); Zoltan_RCB_Free_Structure(zz); return(ierr); } return(ZOLTAN_OK); }
static int Zoltan_Oct_Update_Connections( ZZ *zz, pOctant *octs, /* octs[nocts] */ int *newpids, /* newpids[nocts] */ pOctant *newocts, /* newocts[nocts] */ int nocts) /* number of octants leaving this processor */ { int i, j; int nsends; int nreceives; pOctant parent; pOctant child; int ppid; int cpid; int childnum; int *despid = NULL; Update_msg umsg; Update_msg *localumsg = NULL; Update_msg *remoteumsg = NULL; Update_msg *rcv_umsg = NULL; int localcount; int remotecount; int ierr = ZOLTAN_OK; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ char *yo = "Zoltan_Oct_Update_Connections"; OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure; localcount=0; remotecount=0; /* count number of sends */ nsends = 0; for (i=0; i<nocts; i++) if (newpids[i]!=zz->Proc) nsends++; if(nocts > 0) { if((remoteumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((localumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); return ZOLTAN_MEMERR; } if((despid = (int *) ZOLTAN_MALLOC((nocts+1) * sizeof(int)*9)) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); return ZOLTAN_MEMERR; } } else { remoteumsg = NULL; localumsg = NULL; despid = NULL; } localcount = 0; remotecount = 0; for (i=0; i<nocts; i++) /* Send connection updates */ if (newpids[i]!=zz->Proc) { parent = Zoltan_Oct_parent(octs[i]); ppid = Zoltan_Oct_Ppid(octs[i]); childnum = Zoltan_Oct_childnum(octs[i]); if (parent) { /* Let parent of oct[i] know that it's moving */ if (ppid==zz->Proc) { FILLUPDATEMSG(localumsg[localcount], parent, childnum, newocts[i], newpids[i]); localcount++; } else { FILLUPDATEMSG(remoteumsg[remotecount], parent, childnum, newocts[i], newpids[i]); despid[remotecount++] = ppid; } } for (j=0; j<8; j++) { child = Zoltan_Oct_child(octs[i],j); cpid = octs[i]->cpid[j]; /* Tell child of oct[i] that it is moving */ if (child) { if (cpid==zz->Proc) { /* NOTE: -1 signals PARENT */ FILLUPDATEMSG(localumsg[localcount], child, -1, newocts[i], newpids[i]); localcount++; } else { /* NOTE: -1 signals PARENT */ FILLUPDATEMSG(remoteumsg[remotecount], child, -1, newocts[i], newpids[i]); despid[remotecount++] = cpid; } } } } ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator, MigUpdCommCreate, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&despid); return (ierr); } /* if(nreceives > 0) { */ if((rcv_umsg = (Update_msg *) ZOLTAN_MALLOC((nreceives +1) * sizeof(Update_msg)*9)) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&despid); return ZOLTAN_MEMERR; } ierr = Zoltan_Comm_Do(comm_plan, MigUpdCommDo, (char *) remoteumsg, sizeof(Update_msg), (char *) rcv_umsg); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_umsg); return (ierr); } /* } */ /* else { */ /* rcv_umsg = NULL; */ /* } */ /* update new octants */ for (i=0; i< (localcount+nreceives); i++) { if (i<localcount) umsg=localumsg[i]; else umsg=rcv_umsg[i-localcount]; if (umsg.childnum>=0) { Zoltan_Oct_setchild(umsg.oct,umsg.childnum,umsg.newptr); Zoltan_Oct_setCpid(umsg.oct,umsg.childnum,umsg.newpid); } else { if((Zoltan_Oct_data_newpid(umsg.oct) == OCT_info->OCT_localpid) || ((Zoltan_Oct_data_newpid(umsg.oct) != OCT_info->OCT_localpid) && (umsg.newpid == OCT_info->OCT_localpid))) Zoltan_Oct_POct_setparent(OCT_info, umsg.oct,umsg.newptr,umsg.newpid); else { umsg.oct->ppid = umsg.newpid; umsg.oct->parent = umsg.newptr; } } } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_umsg); return (ierr); } ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&rcv_umsg); ZOLTAN_FREE(&despid); return ierr; }
/* This function needs a distribution : rows then cols to work properly */ int Zoltan_ZG_Build (ZZ* zz, ZG* graph, int local) { static char *yo = "Zoltan_ZG_Build"; int ierr = ZOLTAN_OK; int diag; int *diagarray=NULL; Zoltan_matrix_options opt; char symmetrization[MAX_PARAM_STRING_LEN+1]; char bipartite_type[MAX_PARAM_STRING_LEN+1]; char weigth_type[MAX_PARAM_STRING_LEN+1]; char matrix_build_type[MAX_PARAM_STRING_LEN+1]; int bipartite = 0; #ifdef CC_TIMERS double times[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */ double gtimes[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */ char *timenames[9]= {"", "setup", "matrix build", "diag", "symmetrize", "dist lin", "2D dist", "complete", "clean up"}; MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); #endif /* CC_TIMERS */ ZOLTAN_TRACE_ENTER(zz, yo); memset (graph, 0, sizeof(ZG)); /* Read graph build parameters */ Zoltan_Bind_Param(ZG_params, "GRAPH_SYMMETRIZE", (void *) &symmetrization); Zoltan_Bind_Param(ZG_params, "GRAPH_SYM_WEIGHT", (void *) &weigth_type); Zoltan_Bind_Param(ZG_params, "GRAPH_BIPARTITE_TYPE", (void *) &bipartite_type); Zoltan_Bind_Param(ZG_params, "GRAPH_BUILD_TYPE", (void*) &matrix_build_type); /* Set default values */ strncpy(symmetrization, "NONE", MAX_PARAM_STRING_LEN); strncpy(bipartite_type, "OBJ", MAX_PARAM_STRING_LEN); strncpy(weigth_type, "ADD", MAX_PARAM_STRING_LEN); strncpy(matrix_build_type, "NORMAL", MAX_PARAM_STRING_LEN); Zoltan_Assign_Param_Vals(zz->Params, ZG_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); Zoltan_Matrix2d_Init(&graph->mtx); graph->mtx.comm = (PHGComm*)ZOLTAN_MALLOC (sizeof(PHGComm)); if (graph->mtx.comm == NULL) MEMORY_ERROR; Zoltan_PHGComm_Init (graph->mtx.comm); memset(&opt, 0, sizeof(Zoltan_matrix_options)); opt.enforceSquare = 1; /* We want a graph: square matrix */ if (!strcasecmp(weigth_type, "ADD")) opt.pinwgtop = ADD_WEIGHT; else if (!strcasecmp(weigth_type, "MAX")) opt.pinwgtop = MAX_WEIGHT; else if (!strcasecmp(weigth_type, "CMP")) opt.pinwgtop = MAX_WEIGHT; opt.pinwgt = 1; opt.randomize = 0; opt.local = local; opt.keep_distribution = 1; if (strcasecmp(symmetrization, "NONE")) { opt.symmetrize = 1; } if (!strcasecmp(matrix_build_type, "FAST")) opt.speed = MATRIX_FAST; else if (!strcasecmp(matrix_build_type, "FAST_NO_DUP")) opt.speed = MATRIX_NO_REDIST; else opt.speed = MATRIX_FULL_DD; #ifdef CC_TIMERS times[1] = Zoltan_Time(zz->Timer); #endif ierr = Zoltan_Matrix_Build(zz, &opt, &graph->mtx.mtx); CHECK_IERR; #ifdef CC_TIMERS times[2] = Zoltan_Time(zz->Timer); #endif ierr = Zoltan_Matrix_Mark_Diag (zz, &graph->mtx.mtx, &diag, &diagarray); CHECK_IERR; if (diag) { /* Some Diagonal Terms have to be removed */ ierr = Zoltan_Matrix_Delete_nnz(zz, &graph->mtx.mtx, diag, diagarray); ZOLTAN_FREE(&diagarray); CHECK_IERR; } #ifdef CC_TIMERS times[3] = Zoltan_Time(zz->Timer); #endif if (opt.symmetrize) { if (!strcasecmp(symmetrization, "BIPARTITE")) bipartite = 1; ierr = Zoltan_Matrix_Sym(zz, &graph->mtx.mtx, bipartite); CHECK_IERR; } #ifdef CC_TIMERS times[4] = Zoltan_Time(zz->Timer); #endif ierr = Zoltan_Distribute_LinearY(zz, graph->mtx.comm); CHECK_IERR; #ifdef CC_TIMERS times[5] = Zoltan_Time(zz->Timer); MPI_Barrier(zz->Communicator); #endif ierr = Zoltan_Matrix2d_Distribute (zz, graph->mtx.mtx, &graph->mtx, 0); CHECK_IERR; #ifdef CC_TIMERS times[6] = Zoltan_Time(zz->Timer); #endif ierr = Zoltan_Matrix_Complete(zz, &graph->mtx.mtx); #ifdef CC_TIMERS times[7] = Zoltan_Time(zz->Timer); #endif if (bipartite) { int vertlno; int limit; int offset; graph->bipartite = 1; graph->fixed_vertices = graph->mtx.mtx.ybipart; /* graph->fixed_vertices = (int*) ZOLTAN_MALLOC(graph->mtx.mtx.nY*sizeof(int)); */ /* if (graph->mtx.mtx.nY && graph->fixed_vertices == NULL) MEMORY_ERROR; */ /* limit = graph->mtx.mtx.offsetY; */ /* /\* What kind of vertices do we want to keep ? *\/ */ /* graph->fixObj = !strcasecmp(bipartite_type, "OBJ"); /\* Non-zero value means "objects" *\/ */ /* offset = graph->mtx.mtx.offsetY - graph->mtx.dist_y[graph->mtx.comm->myProc_y]; */ /* if (graph->fixObj) /\* What kind of vertices do we want to keep ? *\/ */ /* for (vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++ vertlno) */ /* graph->fixed_vertices[vertlno] = (vertlno < offset); */ /* else */ /* for (vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++ vertlno) */ /* graph->fixed_vertices[vertlno] = (vertlno >= offset); */ } #ifdef CC_TIMERS MPI_Barrier(zz->Communicator); times[8] = Zoltan_Time(zz->Timer); MPI_Reduce(times, gtimes, 9, MPI_DOUBLE, MPI_MAX, 0, zz->Communicator); if (!zz->Proc) { int i; printf("Total Build Time in Proc-0: %.2lf Max: %.2lf\n", times[8]-times[0], gtimes[8]-times[0]); for (i=1; i<9; ++i) printf("%-13s in Proc-0: %8.2lf Max: %8.2lf\n", timenames[i], times[i]-times[i-1], gtimes[i]-gtimes[i-1]); } #endif End: ZOLTAN_FREE(&diagarray); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Order ( struct Zoltan_Struct *zz, int num_gid_entries, int num_obj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR permuted_global_ids ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids. * num_gid_entries * Output: * permuted_global_ids * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { char msg[253]; sprintf(msg, "num_gid_entries=%d is not equal to parameter setting " "NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ #ifdef HAVE_MPI strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "GLOBAL"); #else strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "LOCAL"); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* * Check that the user has allocated space for the return args. */ if (num_obj && !(gids && permuted_global_ids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "LOCAL_HSFC")) { Order_fn = Zoltan_LocalHSFC_Order; strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/ } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PTSCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_SCOTCH */ #ifdef ZOLTAN_HUND else if (!strcasecmp(opt.method, "HUND")) { ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL); goto End; } #endif /* ZOLTAN_HUND */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* TODO : Ask why useful ! */ /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /************************************ * Check for required query function ************************************/ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } /* TODO allocate all this stuff with the graph */ local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); /* * Call the actual ordering function. * Compute gid according to the local graph. */ ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 2, &local_gids, &local_rank); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* TODO: At this time, I consider rank == permuted_global_ids */ /* I store : GNO, rank, permuted GID */ /* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0); /* Hope a linear assignment will help a little */ if (local_num_obj) Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", permuted_global_ids[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } #ifdef ZOLTAN_HUND End: #endif /*ZOLTAN_HUND*/ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Comm_Do_Post( ZOLTAN_COMM_OBJ * plan, /* communication data structure */ int tag, /* message tag for communicating */ char *send_data, /* array of data I currently own */ int nbytes, /* multiplier for sizes */ char *recv_data) /* array of data I'll own after comm */ { char *send_buff; /* space to buffer outgoing data */ int my_proc; /* processor ID */ unsigned int self_recv_address = 0;/* where in recv_data self info starts */ int self_num=0; /* where in send list my_proc appears */ int offset; /* offset into array I'm copying into */ int self_index = 0; /* send offset for data I'm keeping */ int out_of_mem; /* am I out of memory? */ int nblocks; /* number of procs who need my data */ int proc_index; /* loop counter over procs to send to */ int i, j, k, jj; /* loop counters */ static char *yo = "Zoltan_Comm_Do_Post"; /* Check input parameters */ if (!plan) { MPI_Comm_rank(MPI_COMM_WORLD, &my_proc); ZOLTAN_COMM_ERROR("Communication plan = NULL", yo, my_proc); return ZOLTAN_FATAL; } /* If not point to point, currently we do synchroneous communications */ if (plan->maxed_recvs){ int status; status = Zoltan_Comm_Do_AlltoAll(plan, send_data, nbytes, recv_data); return (status); } MPI_Comm_rank(plan->comm, &my_proc); if ((plan->nsends + plan->self_msg) && !send_data) { int sum = 0; if (plan->sizes_to) /* Not an error if all sizes_to == 0 */ for (i = 0; i < (plan->nsends + plan->self_msg); i++) sum += plan->sizes_to[i]; if (!plan->sizes_to || (plan->sizes_to && sum)) { ZOLTAN_COMM_ERROR("nsends not zero, but send_data = NULL", yo, my_proc); return ZOLTAN_FATAL; } } if ((plan->nrecvs + plan->self_msg) && !recv_data) { int sum = 0; if (plan->sizes_from) /* Not an error if all sizes_from == 0 */ for (i = 0; i < (plan->nrecvs + plan->self_msg); i++) sum += plan->sizes_from[i]; if (!plan->sizes_from || (plan->sizes_from && sum)) { ZOLTAN_COMM_ERROR("nrecvs not zero, but recv_data = NULL", yo, my_proc); return ZOLTAN_FATAL; } } if (nbytes < 0) { ZOLTAN_COMM_ERROR("Scale factor nbytes is negative", yo, my_proc); return ZOLTAN_FATAL; } /* Post irecvs */ out_of_mem = 0; if (plan->indices_from == NULL) { /* Data can go directly into user space. */ plan->recv_buff = recv_data; } else { /* Need to buffer receive to reorder */ plan->recv_buff = (char *) ZOLTAN_MALLOC(plan->total_recv_size * nbytes); if (plan->recv_buff == NULL && plan->total_recv_size * nbytes != 0) out_of_mem = 1; } if (!out_of_mem) { if (plan->sizes == NULL) { /* All data the same size */ k = 0; for (i = 0; i < plan->nrecvs + plan->self_msg; i++) { if (plan->procs_from[i] != my_proc) { MPI_Irecv((void *) & plan->recv_buff[plan->starts_from[i] * nbytes], plan->lengths_from[i] * nbytes, (MPI_Datatype) MPI_BYTE, plan->procs_from[i], tag, plan->comm, &plan->request[k]); k++; } else { self_recv_address = plan->starts_from[i] * nbytes; } } } else { /* Data of varying sizes */ k = 0; for (i = 0; i < plan->nrecvs + plan->self_msg; i++) { if (plan->procs_from[i] != my_proc) { if (plan->sizes_from[i]) MPI_Irecv((void *) &plan->recv_buff[plan->starts_from_ptr[i] * nbytes], plan->sizes_from[i] * nbytes, (MPI_Datatype) MPI_BYTE, plan->procs_from[i], tag, plan->comm, &plan->request[k]); else plan->request[k] = MPI_REQUEST_NULL; k++; } else { self_recv_address = plan->starts_from_ptr[i] * nbytes; } } } } /* Do remaining allocation to check for any mem problems. */ if (plan->indices_to != NULL) { /* can't sent straight from input */ send_buff = (char *) ZOLTAN_MALLOC(plan->max_send_size * nbytes); if (send_buff == 0 && plan->max_send_size * nbytes != 0) out_of_mem = 1; } else send_buff = NULL; /* Barrier to ensure irecvs are posted before doing any sends. */ /* Simultaneously see if anyone out of memory */ MPI_Allreduce(&out_of_mem, &j, 1, MPI_INT, MPI_SUM, plan->comm); if (j > 0) { /* Some proc is out of memory -> Punt */ ZOLTAN_FREE(&send_buff); if (plan->indices_from != NULL) ZOLTAN_FREE(&plan->recv_buff); return (ZOLTAN_MEMERR); } /* Send out data */ /* Scan through procs_to list to start w/ higher numbered procs */ /* This should balance message traffic. */ nblocks = plan->nsends + plan->self_msg; proc_index = 0; while (proc_index < nblocks && plan->procs_to[proc_index] < my_proc) proc_index++; if (proc_index == nblocks) proc_index = 0; if (plan->sizes == NULL) { /* Data all of same size */ if (plan->indices_to == NULL) { /* data already blocked by processor. */ for (i = proc_index, j = 0; j < nblocks; j++) { if (plan->procs_to[i] != my_proc) { MPI_Rsend((void *) &send_data[plan->starts_to[i] * nbytes], plan->lengths_to[i] * nbytes, (MPI_Datatype) MPI_BYTE, plan->procs_to[i], tag, plan->comm); } else self_num = i; if (++i == nblocks) i = 0; } if (plan->self_msg) { /* Copy data to self. */ /* I use array+offset instead of &(array[offset]) because of a bug with PGI v9 */ /* I use memmove because I'm not sure that the pointer are not overlapped. */ memmove( plan->recv_buff+self_recv_address, send_data+(size_t)(plan->starts_to[self_num])*(size_t)nbytes, (size_t) (plan->lengths_to[self_num]) * (size_t) nbytes); } } else { /* Not blocked by processor. Need to buffer. */ for (i = proc_index, jj = 0; jj < nblocks; jj++) { if (plan->procs_to[i] != my_proc) { /* Need to pack message first. */ offset = 0; j = plan->starts_to[i]; for (k = 0; k < plan->lengths_to[i]; k++) { memcpy(&send_buff[offset], &send_data[plan->indices_to[j++] * nbytes], nbytes); offset += nbytes; } MPI_Rsend((void *) send_buff, plan->lengths_to[i] * nbytes, (MPI_Datatype) MPI_BYTE, plan->procs_to[i], tag, plan->comm); } else { self_num = i; self_index = plan->starts_to[i]; } if (++i == nblocks) i = 0; } if (plan->self_msg) { /* Copy data to self. */ for (k = 0; k < plan->lengths_to[self_num]; k++) { memcpy(&plan->recv_buff[self_recv_address], &send_data[plan->indices_to[self_index++] * nbytes], nbytes); self_recv_address += nbytes; } } ZOLTAN_FREE(&send_buff); } } else { /* Data of differing sizes */ if (plan->indices_to == NULL) { /* data already blocked by processor. */ for (i = proc_index, j = 0; j < nblocks; j++) { if (plan->procs_to[i] != my_proc) { if (plan->sizes_to[i]) { MPI_Rsend((void *) &send_data[plan->starts_to_ptr[i] * nbytes], plan->sizes_to[i] * nbytes, (MPI_Datatype) MPI_BYTE, plan->procs_to[i], tag, plan->comm); } } else self_num = i; if (++i == nblocks) i = 0; } if (plan->self_msg) { /* Copy data to self. */ if (plan->sizes_to[self_num]) { char* lrecv = &plan->recv_buff[self_recv_address]; char* lsend = &send_data[plan->starts_to_ptr[self_num] * nbytes]; int sindex = plan->sizes_to[self_num], idx; for (idx=0; idx<nbytes; idx++) { memcpy(lrecv, lsend, sindex); lrecv += sindex; lsend += sindex; } } } } else { /* Not blocked by processor. Need to buffer. */ for (i = proc_index, jj = 0; jj < nblocks; jj++) { if (plan->procs_to[i] != my_proc) { /* Need to pack message first. */ offset = 0; j = plan->starts_to[i]; for (k = 0; k < plan->lengths_to[i]; k++) { if (plan->sizes[plan->indices_to[j]]) { memcpy(&send_buff[offset], &send_data[plan->indices_to_ptr[j] * nbytes], plan->sizes[plan->indices_to[j]] * nbytes); offset += plan->sizes[plan->indices_to[j]] * nbytes; } j++; } if (plan->sizes_to[i]) { MPI_Rsend((void *) send_buff, plan->sizes_to[i] * nbytes, (MPI_Datatype) MPI_BYTE, plan->procs_to[i], tag, plan->comm); } } else self_num = i; if (++i == nblocks) i = 0; } if (plan->self_msg) { /* Copy data to self. */ if (plan->sizes_to[self_num]) { j = plan->starts_to[self_num]; for (k = 0; k < plan->lengths_to[self_num]; k++) { int kk = plan->indices_to_ptr[j]; char* lrecv = &plan->recv_buff[self_recv_address]; unsigned int send_idx = kk * nbytes; char* lsend = &send_data[send_idx]; int sindex = plan->sizes[plan->indices_to[j]], idx; for (idx=0; idx<nbytes; idx++) { memcpy(lrecv, lsend, sindex); lrecv += sindex; lsend += sindex; } self_recv_address += plan->sizes[plan->indices_to[j]] * nbytes; j++; } } } ZOLTAN_FREE(&send_buff); } } return (ZOLTAN_OK); }
int Zoltan_PHG_2ways_hyperedge_partition ( ZZ *zz, /* Input : Zoltan data structure */ HGraph *hg, Partition parts, Zoltan_PHG_Tree *tree, struct Zoltan_DD_Struct * gnoToGID, struct Zoltan_DD_Struct **dd, int *numParts, int **sizeParts ) { int ierr = ZOLTAN_OK; char *yo = "Zoltan_PHG_2ways_hyperedge_partition"; int nEdge, hEdge; int *interval; int *partnumber = NULL; int tree_size; ZOLTAN_ID_TYPE *rowpart =NULL; /* ZOLTAN_ID_TYPE because it's used in Zoltan_DD_* */ ZOLTAN_GNO_TYPE *rowGNO = NULL; ZOLTAN_ID_PTR rowGID=NULL; int index; int offset; ZOLTAN_TRACE_ENTER(zz, yo); nEdge = hg->nEdge; fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge); interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int)); if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR; tree_size = get_tree_size(tree) + 1; for (index = 0 ; index < nEdge ; ++index){ SET_MIN_NODE(interval, index, tree_size); SET_MAX_NODE(interval, index, -1); } /* Update interval with the local knowledge */ /* XXX: I loop on the hyperedges, as I think it's more cache friendly * and it allows me to discoupled the computation if a non blocking MPI_Reduce is * available */ for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){ int part; int max = -1; /* Trick : we use the initialized values */ int min = tree_size + 1; for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) { part = parts[hg->hvertex[index]]; max = MAX(max, part); min = MIN(min, part); } SET_MIN_NODE(interval, hEdge, min); SET_MAX_NODE(interval, hEdge, max); } /* Update results to view the complete hyperedges */ Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm); /* Now I have to compute the partition of hyperedges according to the "interval" * and the tree */ /* First, compute the partition number corresponding to the nodes in the tree */ partnumber = compute_part_number(tree); if (partnumber == NULL) { ierr = ZOLTAN_FATAL; goto End; } (*numParts) = get_tree_size(tree); rowpart = (ZOLTAN_ID_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_ID_TYPE)); if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR; rowGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_GNO_TYPE)); if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR; (*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int)); if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR; offset = hg->dist_y[hg->comm->myProc_y]; /* Then we search we is the hyperedge in the tree */ for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) { int node; node = find_interval_in_tree(tree, interval+2*hEdge); rowpart[hEdge] = partnumber[node]; (*sizeParts)[rowpart[hEdge]] ++; rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge); #if 0 fprintf (stderr, "%zd : " ZOLTAN_ID_SPEC " (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]); #endif } partnumber += 1; ZOLTAN_FREE(&partnumber); ZOLTAN_FREE(&interval); /* Compute number of elements per parts */ /* TODO: support processor which are not part of the distribution */ /* Update results to view the complete hyperedges */ Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm); /* Export results to data directory */ /* First, get the GIDs of our edges */ rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge); if (nEdge && rowGID == NULL) MEMORY_ERROR; ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL, nEdge, NULL); ZOLTAN_FREE(&rowGNO); ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, rowpart, NULL, NULL, nEdge); #ifdef CEDRIC_PRINT for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) { fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]); } #endif End: ZOLTAN_FREE(&rowGID); ZOLTAN_FREE(&rowGNO); ZOLTAN_FREE(&rowpart); if (partnumber != NULL) partnumber += 1; ZOLTAN_FREE(&partnumber); ZOLTAN_FREE(&interval); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Comm_Do_AlltoAll( ZOLTAN_COMM_OBJ * plan, /* communication data structure */ char *send_data, /* array of data I currently own */ int nbytes, /* multiplier for sizes */ char *recv_data) /* array of data I'll own after comm */ { static char *yo = "Zoltan_Comm_Do_AlltoAll"; char *outbuf=NULL, *inbuf=NULL, *buf=NULL; int *outbufCounts=NULL, *outbufOffsets=NULL; int *inbufCounts=NULL, *inbufOffsets=NULL; int nprocs, me, rc, i, j, k, p, sorted; int nSendMsgs, nSendItems, nRecvMsgs, nRecvItems; int length, offset, itemSize, outbufLen; int sm = (plan->self_msg > 0) ? 1 : 0; nSendMsgs = plan->nsends + sm; nRecvMsgs = plan->nrecvs + sm; for (i=0, nSendItems=0; i <nSendMsgs; i++){ nSendItems += plan->lengths_to[i]; } for (i=0, nRecvItems=0; i <nRecvMsgs; i++){ nRecvItems += plan->lengths_from[i]; } MPI_Comm_size(plan->comm, &nprocs); MPI_Comm_rank(plan->comm, &me); outbufCounts = (int *) ZOLTAN_CALLOC(nprocs , sizeof(int)); outbufOffsets = (int *) ZOLTAN_CALLOC(nprocs , sizeof(int)); inbufCounts = (int *) ZOLTAN_CALLOC(nprocs , sizeof(int)); inbufOffsets = (int *) ZOLTAN_CALLOC(nprocs , sizeof(int)); if (!outbufCounts || !outbufOffsets || !inbufCounts || !inbufOffsets){ ZOLTAN_COMM_ERROR("memory error", yo, me); } /* The *_to fields of the plan refer to the items in the send_data buffer, * and how to pull out the correct items for each receiver. The * *_from fields of the plan refer to the recv_data buffer. Items * arrive in process rank order, and these fields tell us where to * put them in the recv_data buffer. */ /* CREATE SEND BUFFER */ sorted = 0; if (plan->indices_to == NULL){ sorted = 1; for (i=1; i< nSendMsgs; i++){ if (plan->starts_to[i] < plan->starts_to[i-1]){ sorted = 0; break; } } } if (plan->sizes_to){ /* * Each message contains items for a process, and each item may be * a different size. */ for (i=0, outbufLen=0; i < nSendMsgs; i++){ outbufLen += plan->sizes_to[i]; } if (plan->indices_to){ /* * items are not grouped by message */ buf = outbuf = (char *)ZOLTAN_MALLOC(outbufLen * nbytes); if (outbufLen && nbytes && !outbuf){ ZOLTAN_COMM_ERROR("memory error", yo, me); } for (p=0, i=0, k=0; p < nprocs; p++){ length = 0; if (i < nSendMsgs){ if (plan->procs_to[i] == p){ /* procs_to is sorted */ for (j=0; j < plan->lengths_to[i]; j++,k++){ itemSize = plan->sizes[plan->indices_to[k]] * nbytes; offset = plan->indices_to_ptr[k] * nbytes; memcpy(buf, send_data + offset, itemSize); buf += itemSize; length += itemSize; } i++; } } outbufCounts[p] = length; if (p){ outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1]; } } } else{ /* * items are stored contiguously for each message */ if (!sorted || (plan->nvals > nSendItems) ){ buf = outbuf = (char *)ZOLTAN_MALLOC(outbufLen * nbytes); if (outbufLen && nbytes && !outbuf){ ZOLTAN_COMM_ERROR("memory error", yo, me); } } else{ /* All items in send_data are being sent, and they are sorted * in process rank order. */ outbuf = send_data; } for (p=0, i=0; p < nprocs; p++){ length = 0; if (i < nSendMsgs){ if (plan->procs_to[i] == p){ /* procs_to is sorted */ length = plan->sizes_to[i] * nbytes; offset = plan->starts_to_ptr[i] * nbytes; if ((!sorted || (plan->nvals > nSendItems)) && length){ memcpy(buf, send_data + offset, length); buf += length; } i++; } } outbufCounts[p] = length; if (p){ outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1]; } } } } else if (plan->indices_to){ /* * item sizes are constant, however the items belonging in a given * message may not be contiguous in send_data */ buf = outbuf = (char *)ZOLTAN_MALLOC(nSendItems * nbytes); if (nSendMsgs && nbytes && !outbuf){ ZOLTAN_COMM_ERROR("memory error", yo, me); } for (p=0, i=0, k=0; p < nprocs; p++){ length = 0; if (i < nSendMsgs){ if (plan->procs_to[i] == p){ /* procs_to is sorted */ for (j=0; j < plan->lengths_to[i]; j++,k++){ offset = plan->indices_to[k] * nbytes; memcpy(buf, send_data + offset, nbytes); buf += nbytes; } length = plan->lengths_to[i] * nbytes; i++; } } outbufCounts[p] = length; if (p){ outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1]; } } } else{ /* item sizes are constant, and items belonging to a * given message are always stored contiguously in send_data */ if (!sorted || (plan->nvals > nSendItems)){ buf = outbuf = (char *)ZOLTAN_MALLOC(nSendItems * nbytes); if (nSendItems && nbytes && !outbuf){ ZOLTAN_COMM_ERROR("memory error", yo, me); } } else{ /* send_data is sorted by process, and we don't skip * any of the data in the buffer, so we can use send_data * in the alltoall call */ outbuf = send_data; } for (p=0,i=0; p < nprocs; p++){ length = 0; if (i < nSendMsgs){ if (plan->procs_to[i] == p){ /* procs_to is sorted */ offset = plan->starts_to[i] * nbytes; length = plan->lengths_to[i] * nbytes; if ((!sorted || (plan->nvals > nSendItems)) && length){ memcpy(buf, send_data + offset, length); buf += length; } i++; } } outbufCounts[p] = length; if (p){ outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1]; } } } /* CREATE RECEIVE BUFFER */ sorted = 0; if (plan->indices_from == NULL){ sorted = 1; for (i=1; i< nRecvMsgs; i++){ if (plan->starts_from[i] < plan->starts_from[i-1]){ sorted = 0; break; } } } if (sorted){ /* Caller already expects received data to be ordered by * the sending process rank. */ inbuf = recv_data; } else{ inbuf = (char *)ZOLTAN_MALLOC(plan->total_recv_size * nbytes); if (plan->total_recv_size && nbytes && !inbuf){ ZOLTAN_COMM_ERROR("memory error", yo, me); } } for (p=0, i=0; p < nprocs; p++){ length = 0; if (i < nRecvMsgs){ if (plan->procs_from[i] == p){ if (plan->sizes == NULL){ length = plan->lengths_from[i] * nbytes; } else{ length = plan->sizes_from[i] * nbytes; } i++; } } inbufCounts[p] = length; if (p){ inbufOffsets[p] = inbufOffsets[p-1] + inbufCounts[p-1]; } } /* EXCHANGE DATA */ rc = MPI_Alltoallv(outbuf, outbufCounts, outbufOffsets, MPI_BYTE, inbuf, inbufCounts, inbufOffsets, MPI_BYTE, plan->comm); if (outbuf != send_data){ ZOLTAN_FREE(&outbuf); } ZOLTAN_FREE(&outbufCounts); ZOLTAN_FREE(&outbufOffsets); /* WRITE RECEIVED DATA INTO USER'S BUFFER WHERE IT'S EXPECTED */ if (!sorted){ buf = inbuf; if (plan->sizes == NULL){ /* each item in each message is nbytes long */ if (plan->indices_from == NULL){ for (i=0; i < nRecvMsgs; i++){ offset = plan->starts_from[i] * nbytes; length = plan->lengths_from[i] * nbytes; memcpy(recv_data + offset, buf, length); buf += length; } } else{ for (i=0,k=0; i < nRecvMsgs; i++){ for (j=0; j < plan->lengths_from[i]; j++,k++){ offset = plan->indices_from[k] * nbytes; memcpy(recv_data + offset, buf, nbytes); buf += nbytes; } } } } else{ /* (sizes!=NULL) && (indices_from!=NULL) not allowed by Zoltan_Comm_Resize */ /* items can be different sizes */ for (i=0; i < nRecvMsgs; i++){ offset = plan->starts_from_ptr[i] * nbytes; length = plan->sizes_from[i] * nbytes; memcpy(recv_data + offset, buf, length); buf += length; } } ZOLTAN_FREE(&inbuf); } ZOLTAN_FREE(&inbufCounts); ZOLTAN_FREE(&inbufOffsets); return ZOLTAN_OK; }
int Zoltan_Comm_Do_Wait( ZOLTAN_COMM_OBJ * plan, /* communication data structure */ int tag, /* message tag for communicating */ char *send_data, /* array of data I currently own */ int nbytes, /* multiplier for sizes */ char *recv_data) /* array of data I'll own after comm */ { MPI_Status status; /* return from Waitany */ int my_proc; /* processor ID */ int self_num; /* where in send list my_proc appears */ int i, j, k, jj; /* loop counters */ /* If not point to point, currently we do synchroneous communications */ if (plan->maxed_recvs){ /* Do nothing */ return (ZOLTAN_OK); } MPI_Comm_rank(plan->comm, &my_proc); /* Wait for messages to arrive & unpack them if necessary. */ /* Note: since request is in plan, could wait in later routine. */ if (plan->indices_from == NULL) { /* No copying required */ if (plan->nrecvs > 0) { MPI_Waitall(plan->nrecvs, plan->request, plan->status); } } else { /* Need to copy into recv_data. */ if (plan->self_msg) { /* Unpack own data before waiting */ for (self_num = 0; self_num < plan->nrecvs + plan->self_msg; self_num++) if (plan->procs_from[self_num] == my_proc) break; k = plan->starts_from[self_num]; if (!plan->sizes_from || plan->sizes_from[self_num]) { for (j = plan->lengths_from[self_num]; j; j--) { memcpy(&recv_data[plan->indices_from[k] * nbytes], &plan->recv_buff[k * nbytes], nbytes); k++; } } } else self_num = plan->nrecvs; for (jj = 0; jj < plan->nrecvs; jj++) { MPI_Waitany(plan->nrecvs, plan->request, &i, &status); if (i == MPI_UNDEFINED) break; /* No more receives */ if (i >= self_num) i++; k = plan->starts_from[i]; for (j = plan->lengths_from[i]; j; j--) { memcpy(&recv_data[plan->indices_from[k] * nbytes], &plan->recv_buff[k * nbytes], nbytes); k++; } } ZOLTAN_FREE(&plan->recv_buff); } return (ZOLTAN_OK); }
int Zoltan_LB_Get_Part_Sizes(ZZ *zz, int num_global_parts, int part_dim, float *part_sizes) { /* * Function to get the scaled partition sizes. * * Input: * zz -- The Zoltan structure to which this method * applies. * num_global_parts -- Number of global partitions. * (This usually equals lb->Num_Global_Parts) * part_dim -- The number of object weights per partition. * (This usually equals lb->Obj_Wgt_Dim.) * * Output: * part_sizes -- Array of floats that gives the set partition * sizes, scaled such that they sum to one. */ int i, j, nparts, fpart; float *temp_part_sizes=NULL, *sum=NULL; int error = ZOLTAN_OK; char msg[128]; static char *yo = "Zoltan_LB_Get_Part_Sizes"; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: num_global_parts = %d\n", zz->Proc, num_global_parts); /* Barrier to make sure all procs have finished Zoltan_LB_Set_Part_Sizes */ MPI_Barrier(zz->Communicator); /* For convenience, if no weights are used, set part_dim to 1 */ if (part_dim==0) part_dim = 1; if (part_sizes == NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument part_sizes is NULL."); error = ZOLTAN_FATAL; goto End; } /* Find max Part_Info_Len over all procs to see if they are all zero. */ MPI_Allreduce((void*) &(zz->LB.Part_Info_Len), (void*) &j, 1, MPI_INT, MPI_MAX, zz->Communicator); if (j == 0){ /* Uniform partition sizes. */ zz->LB.Uniform_Parts = 1; for (i = 0; i < num_global_parts*part_dim; i++) part_sizes[i] = 1.0 / (float)num_global_parts; } else { /* Get the partition sizes set by the user (application). * Each processor puts its data in a part_dim * num_global_parts * array. Then we gather all the data across processors. * Out-of-range partition size data is ignored. */ zz->LB.Uniform_Parts = 0; /* Pack LB.Part_Info into temp array */ temp_part_sizes = (float *)ZOLTAN_MALLOC(num_global_parts*part_dim *sizeof(float)); sum = (float *)ZOLTAN_MALLOC(part_dim*sizeof(float)); if ((!temp_part_sizes) || (!sum)){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); error = ZOLTAN_MEMERR; goto End; } for (i = 0; i < num_global_parts*part_dim; i++){ temp_part_sizes[i] = -1.0; } for (i = 0; i < zz->LB.Part_Info_Len; i++){ /* Only assemble partition sizes for partitions and weights in the requested range. */ if (zz->LB.Part_Info[i].Idx < part_dim){ j = zz->LB.Part_Info[i].Part_id; if (zz->LB.Part_Info[i].Global_num == 0) { Zoltan_LB_Proc_To_Part(zz, zz->Proc, &nparts, &fpart); j += fpart; } if (j >= num_global_parts){ sprintf(msg, "Partition number %d is >= num_global_parts %d.", j, num_global_parts); ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); error = ZOLTAN_WARN; } else temp_part_sizes[j*part_dim + zz->LB.Part_Info[i].Idx] = zz->LB.Part_Info[i].Size; } } /* Reduce over all procs */ MPI_Allreduce((void*) temp_part_sizes, (void*) part_sizes, num_global_parts*part_dim, MPI_FLOAT, MPI_MAX, zz->Communicator); /* Check for errors. Scale the sizes so they sum to one for each weight. */ for (j = 0; j < part_dim; j++) sum[j] = 0.0; for (i = 0; i < num_global_parts; i++){ for (j = 0; j < part_dim; j++){ if (part_sizes[i*part_dim+j]<0) part_sizes[i*part_dim+j] = 1.0; /* default value if not set */ sum[j] += part_sizes[i*part_dim+j]; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){ printf("[%1d] In %s: Partition size %1d (before scaling) = ", zz->Proc, yo, i); for (j = 0; j < part_dim; j++) printf("%f, ", part_sizes[i*part_dim+j]); printf("\n"); } } /* Check for sum[j] == 0 (error). */ for (j = 0; j < part_dim; j++) { if (sum[j] == 0.0) { sprintf(msg, "Sum of weights (component %1d) is zero.", j); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); error = ZOLTAN_FATAL; goto End; } } /* Normalize partition sizes */ for (i = 0; i < num_global_parts; i++) for (j = 0; j < part_dim; j++) part_sizes[i*part_dim+j] /= sum[j]; } End: if (temp_part_sizes) ZOLTAN_FREE(&temp_part_sizes); if (sum) ZOLTAN_FREE(&sum); ZOLTAN_TRACE_EXIT(zz, yo); return error; }
int Zoltan_ParMetis( ZZ *zz, /* Zoltan structure */ float *part_sizes, /* Input: Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ int *num_imp, /* number of objects to be imported */ ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */ ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */ int **imp_procs, /* list of processors to import from */ int **imp_to_part, /* list of partitions to which imported objects are assigned. */ int *num_exp, /* number of objects to be exported */ ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */ ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */ int **exp_procs, /* list of processors to export to */ int **exp_to_part /* list of partitions to which exported objects are assigned. */ ) { char *yo = "Zoltan_ParMetis"; int ierr; ZOLTAN_Third_Graph gr; ZOLTAN_Third_Geom *geo = NULL; ZOLTAN_Third_Vsize vsp; ZOLTAN_Third_Part prt; ZOLTAN_Output_Part part; ZOLTAN_ID_PTR global_ids = NULL; ZOLTAN_ID_PTR local_ids = NULL; int use_timers = 0; int timer_p = -1; int get_times = 0; double times[5]; double pmv3_itr = 0.0; realtype itr = 0.0; indextype options[MAX_OPTIONS]; char alg[MAX_PARAM_STRING_LEN+1]; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */ /* change our zz struct. */ #endif indextype i; realtype *imb_tols; indextype ncon; indextype edgecut; indextype wgtflag; indextype numflag = 0; indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */ ZOLTAN_TRACE_ENTER(zz, yo); Zoltan_Third_Init(&gr, &prt, &vsp, &part, imp_gids, imp_lids, imp_procs, imp_to_part, exp_gids, exp_lids, exp_procs, exp_to_part); if (sizeof(realtype) != sizeof(float)) { int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1); realtype sum = 0.; prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype)); for (i = 0; i < tmp; i++) { prt.input_part_sizes[i] = (realtype) part_sizes[i]; sum += prt.input_part_sizes[i]; } if (sum != (realtype) 1.0) { /* rescale part sizes in case of roundoff in conversion; * ParMETIS requires sum of part sizes to be 1.0 */ for (i = 0; i < tmp; i++) { prt.input_part_sizes[i] /= sum; } } prt.part_sizes = prt.input_part_sizes; } else prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes; ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } gr.graph_type = 0; #ifdef ZOLTAN_PARMETIS SET_GLOBAL_GRAPH(&gr.graph_type); /* Select type of graph, negative because we impose them */ /* TODO: add a parameter to select the type, shared with Scotch */ /* if (strcmp (graph_type, "GLOBAL") != 0) { */ /* gr.graph_type = - LOCAL_GRAPH; */ /* if (zz->Num_Proc > 1) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */ /* gr.graph_type = - GLOBAL_GRAPH; */ /* retval = ZOLTAN_WARN; */ /* } */ /* } */ #else /* graph is local */ SET_LOCAL_GRAPH(&gr.graph_type); #endif /* ZOLTAN_PARMETIS */ /* Some algorithms use geometry data */ if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */ geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom)); memset (geo, 0, sizeof(ZOLTAN_Third_Geom)); /* ParMETIS will crash if geometric method and some procs have no nodes. */ /* Avoid fatal crash by setting scatter to level 2 or higher. */ gr.scatter_min = 2; if (geo == NULL) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory."); return (ZOLTAN_MEMERR); } if (strcmp(alg, "PARTGEOM") == 0) { gr.get_data = 0; } } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } vsp.vsize_malloc = 0; #ifdef PARMETIS31_ALWAYS_FREES_VSIZE if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) { /* ParMETIS will free this memory; use malloc to allocate so ZOLTAN_MALLOC counters don't show an error. */ vsp.vsize_malloc = 1 ; } #endif /* PARMETIS31_ALWAYS_FREES_VSIZE */ ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr, geo, &prt, &vsp); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } /* Get object sizes if requested */ if (options[PMV3_OPT_USE_OBJ_SIZE] && (zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) && (!strcmp(alg, "ADAPTIVEREPART") || gr.final_output)) gr.showMoveVol = 1; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); /* Get ready to call ParMETIS */ edgecut = -1; wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0); numflag = 0; ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1); if (!prt.part_sizes){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL."); } if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) { for (i=0; i<num_part; i++){ indextype j; printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i); for (j=0; j<ncon; j++) printf("%f ", prt.part_sizes[i*ncon+j]); printf("\n"); } } /* if (strcmp(alg, "ADAPTIVEREPART") == 0) */ for (i = 0; i < num_part*ncon; i++) if (prt.part_sizes[i] == 0) ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! " "ParMETIS 3.x will likely fail. Please use a " "different method, or remove the zero-sized " "parts from the problem."); /* Set Imbalance Tolerance for each weight component. */ imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype)); if (!imb_tols){ /* Not enough memory */ ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } for (i=0; i<ncon; i++) imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]); /* Now we can call ParMetis */ /* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */ #ifdef ZOLTAN_PARMETIS if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */ /* First check for ParMetis 3 routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartKway"); ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOMKWAY") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeomKway"); ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts, &wgtflag, &numflag, &ndims, geo->xyz, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOM") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeom"); ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "ADAPTIVEREPART") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_AdaptiveRepart"); ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, &itr, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "REFINEKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_RefineKway"); ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown ParMetis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_METIS /* TODO: I don't know how to set balance ! */ if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Check for Metis routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library "); /* Use default options for METIS */ options[0] = 0; #if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5 METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &num_part, prt.part_sizes, options, &edgecut, prt.part); #else METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown Metis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (gr.final_output) { /* Do final output now because after the data will not be coherent: unscatter only unscatter part data, not graph */ ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr); } /* Ignore the timings of Final Ouput */ if (get_times) times[3] = Zoltan_Time(zz->Timer); ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr, geo, &prt, &vsp, NULL, &part); Zoltan_Third_Export_User(&part, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part); /* Get a time here */ if (get_times) times[4] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers && timer_p >= 0) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL); if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols); if (geo != NULL) ZOLTAN_FREE(&geo); ZOLTAN_FREE(&global_ids); ZOLTAN_FREE(&local_ids); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Oct_migreg_migrate_orphans(ZZ *zz, pRegion RegionList, int nregions, int level, OMap *array, int *c1, int *c2) { int i, j, k; /* index counters */ pRegion ptr; /* region in the mesh */ COORD origin; /* centroid coordinate information */ pRegion *regions = NULL; /* an array of regions */ int *npids = NULL; Region *regions2 = NULL; /* an array of regions */ int *npids2 = NULL; int nreg; /* number of regions */ COORD min, /* minimum bounds of an octant */ max; /* maximum bounds of an octant */ COORD cmin, /* minimum bounds of a child octant */ cmax; /* maximum bounds of a child octant */ COORD rmin, /* minimum bounds of a remote octant */ rmax; /* maximum bounds of a remote octant */ int new_num; int n; int dir = 0; pRList RootList; pOctant RootOct; OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); char *yo = "Zoltan_Oct_migreg_migrate_orphans_static"; int ierr = ZOLTAN_OK; ZOLTAN_ID_PTR gids2, lids2; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; if(nregions > 0) { /* create the array of messages to be sent to other processors */ /* Array = (Message *) ZOLTAN_MALLOC(nregions * sizeof(Message)); */ if((regions = (pRegion *) ZOLTAN_MALLOC(nregions * sizeof(pRegion))) == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((npids = (int *) ZOLTAN_MALLOC(nregions * sizeof(int))) == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(®ions); return ZOLTAN_MEMERR; } } ptr = RegionList; n = nreg = 0; while((ptr != NULL) && (nregions > 0)) { if(ptr->attached == 1) { /* if region already attached to an octant, then skip to next region */ ptr = ptr->next; continue; } /* region not attached, have to find which processor to send to */ j=0; dir = 0; vector_set(min, OCT_info->OCT_gmin); vector_set(max, OCT_info->OCT_gmax); /* * for each level of refinement, find which child region belongs to. * translate which child to which entry in map array. */ for(i=0; i<level; i++) { Zoltan_Oct_bounds_to_origin(min, max, origin); if(OCT_info->OCT_dimension == 2) j = j * 4; else j = j * 8; k = Zoltan_Oct_child_which(OCT_info,origin, ptr->Coord); new_num = Zoltan_Oct_convert_idx_from_map(OCT_info, dir, k); dir = Zoltan_Oct_get_child_dir(OCT_info, dir, new_num); j += new_num; Zoltan_Oct_child_bounds(min, max, origin, k, cmin, cmax); vector_set(min, cmin); vector_set(max, cmax); } /* inform message which processor to send to */ npids[n] = array[j].npid; RootList = array[j].list; while((RootOct = RL_nextRootOctant(&RootList))) { Zoltan_Oct_bounds(RootOct,rmin,rmax); if (Zoltan_Oct_in_box_closure(OCT_info, ptr->Coord ,rmin, rmax)) { npids[n] = RootOct->npid; break; } } if((npids[n] != -1) && (npids[n] != zz->Proc)) { Zoltan_Oct_copy_info(zz, ptr, &(regions[n++])); } else { Zoltan_Oct_insert_orphan(zz, *ptr); } nreg++; /* increment region counter */ ptr = ptr->next; /* look at next region */ } /* * if regions looked at != number of regions in region list, * then there is an error */ if (nreg!=nregions) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "regions found != to expected number of regions"); return ZOLTAN_FATAL; } regions2 = (Region *) ZOLTAN_MALLOC(n * sizeof(Region)); gids2 = ZOLTAN_MALLOC_GID_ARRAY(zz, n); lids2 = ZOLTAN_MALLOC_LID_ARRAY(zz, n); npids2 = (int *) ZOLTAN_MALLOC(n * sizeof(int)); for(i=0; i<n; i++) { npids2[i] = npids[i]; vector_set(regions2[i].Coord, regions[i]->Coord); regions2[i].Weight = regions[i]->Weight; regions2[i].Global_ID = &(gids2[i*num_gid_entries]); regions2[i].Local_ID = (num_lid_entries ? &(lids2[i*num_lid_entries]) : NULL); ZOLTAN_SET_GID(zz, &(gids2[i*num_gid_entries]), regions[i]->Global_ID); ZOLTAN_SET_LID(zz, &(lids2[i*num_lid_entries]), regions[i]->Local_ID); regions2[i].Proc = regions[i]->Proc; regions2[i].attached = 0; } *c1 = n; /* migrate the orphan regions according to the message array */ Zoltan_Oct_migreg_migrate_regions(zz, regions2, gids2, lids2, npids2, n, c2); for (i=0; i < n; i++) { ZOLTAN_FREE(&(regions[i]->Global_ID)); ZOLTAN_FREE(&(regions[i]->Local_ID)); ZOLTAN_FREE(&(regions[i])); } ZOLTAN_FREE(®ions); ZOLTAN_FREE(&npids); ZOLTAN_FREE(®ions2); ZOLTAN_FREE(&gids2); ZOLTAN_FREE(&lids2); ZOLTAN_FREE(&npids2); return ierr; }
int Zoltan_Block( ZZ *zz, /* The Zoltan structure. */ float *part_sizes, /* Input: Array of size zz->LB.Num_Global_Parts containing the percentage of work to be assigned to each partition. */ int *num_import, /* Return -1. We use only export lists. */ ZOLTAN_ID_PTR *import_global_ids, /* Not used. */ ZOLTAN_ID_PTR *import_local_ids, /* Not used. */ int **import_procs, /* Not used. */ int **import_to_part, /* Not used. */ int *num_export, /* Output: Number of objects to export. */ ZOLTAN_ID_PTR *export_global_ids, /* Output: GIDs to export. */ ZOLTAN_ID_PTR *export_local_ids, /* Output: LIDs to export. */ int **export_procs, /* Output: Processsors to export to. */ int **export_to_part /* Output: Partitions to export to. */ ) { int ierr = ZOLTAN_OK; int i, count, num_obj; int wtflag; ZOLTAN_ID_PTR global_ids = NULL; ZOLTAN_ID_PTR local_ids = NULL; int *parts = NULL; int *newparts = NULL; float *wgts = NULL; static char *yo = "Zoltan_Block"; ZOLTAN_TRACE_ENTER(zz, yo); /* No import lists computed. */ *num_import = -1; *export_global_ids = *export_local_ids = NULL; *export_procs = *export_to_part = NULL; /* Get list of local objects. */ if (zz->Obj_Weight_Dim > 1) { ierr = ZOLTAN_FATAL; ZOLTAN_PRINT_ERROR(zz->Proc, yo, "OBJ_WEIGHT_DIM > 1 not supported by LB_METHOD BLOCK."); goto End; } wtflag = (zz->Obj_Weight_Dim>0 ? 1 : 0); ierr = Zoltan_Get_Obj_List(zz, &num_obj, &global_ids, &local_ids, wtflag, &wgts, &parts); /* Compute the new partition numbers. */ newparts = (int *) ZOLTAN_MALLOC(num_obj * sizeof(int)); if (num_obj && (!newparts)){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } block_part(zz, num_obj, wtflag, wgts, part_sizes, newparts); /* Check how many partition numbers changed. */ count=0; for (i=0; i<num_obj; i++){ if (newparts[i] != parts[i]) ++count; } (*num_export) = count; /* Allocate export lists. */ if ((*num_export) > 0) { if (!Zoltan_Special_Malloc(zz, (void **)export_global_ids, (*num_export), ZOLTAN_SPECIAL_MALLOC_GID) || !Zoltan_Special_Malloc(zz, (void **)export_local_ids, (*num_export), ZOLTAN_SPECIAL_MALLOC_LID) || !Zoltan_Special_Malloc(zz, (void **)export_procs, (*num_export), ZOLTAN_SPECIAL_MALLOC_INT) || !Zoltan_Special_Malloc(zz, (void **)export_to_part, (*num_export), ZOLTAN_SPECIAL_MALLOC_INT)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* Loop over objects and fill export lists. */ count=0; for (i=0; i<num_obj; i++){ if (newparts[i] != parts[i]){ /* export_global_ids[count] = global_ids[i]; */ ZOLTAN_SET_GID(zz, &((*export_global_ids)[count*zz->Num_GID]), &global_ids[i*zz->Num_GID]); if (local_ids) /* export_local_ids[count] = local_ids[i]; */ ZOLTAN_SET_LID(zz, &((*export_local_ids)[count*zz->Num_LID]), &local_ids[i*zz->Num_LID]); /* Set new partition number. */ (*export_to_part)[count] = newparts[i]; /* Processor is derived from partition number. */ (*export_procs)[count] = Zoltan_LB_Part_To_Proc(zz, (*export_to_part)[count], &global_ids[i*zz->Num_GID]); ++count; } } End: /* Free local memory, but not export lists. */ ZOLTAN_FREE(&global_ids); ZOLTAN_FREE(&local_ids); ZOLTAN_FREE(&parts); ZOLTAN_FREE(&newparts); if (wtflag) ZOLTAN_FREE(&wgts); ZOLTAN_TRACE_EXIT(zz, yo); return ierr; }
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int *npids, int nregions, int *c2) { char *yo = "Zoltan_Oct_migreg_migrate_regions"; int i; /* index counter */ int ierr = ZOLTAN_OK; int n_import; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ Region *import_objs = NULL; /* Array of import objects used to request the objs from other processors. */ ZOLTAN_ID_PTR import_gids = NULL; /* Array of global IDs of import_objs. */ ZOLTAN_ID_PTR import_lids = NULL; /* Array of local IDs of import_objs. */ int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator, MIGMIGREGCommCreate, &n_import); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create"); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } *c2 = n_import; if (n_import > 0) { import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region)); import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import); import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import); if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region), (char *) import_objs); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids, sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } if (num_lid_entries > 0) { ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids, sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } } for (i=0; i<n_import; i++) { import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]); import_objs[i].Local_ID = (num_lid_entries ? &(import_lids[i*num_lid_entries]) : NULL); Zoltan_Oct_insert_orphan(zz, import_objs[i]); } ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } return ierr; }
static int local_HEs_from_import_lists( ZZ *zz, int remap_type, /* type of remapping to do: parts, procs, or none. */ int nobj, /* # objs the processor knows about (keep + imports) */ int *proc, /* On input, old processor assignment for each obj; Upon return, remapped new proc assignment for each obj. */ int *old_part, /* old partition assignments for each objs */ int *new_part, /* On input, new partition assignments for each objs. Upon return, remapped new partition assignments */ int *HEcnt, /* # of HEs allocated. */ int **HEinfo /* Array of HE info; for each HE, two pins and one edge weight. Stored as a single vector to minimize communication calls. */ ) { /* Routine to remap partitions (to new processors or new partition numbers) * to reduce data movement. * This routine assumes the load-balancing algorithm built import lists. * Objects described are those that ENDED UP on my_proc due to load balancing. * For all these objects, new_proc == my_proc. */ char *yo = "local_HEs_from_import_lists"; int ierr = ZOLTAN_OK; int i, cnt, tmp; int *tmp_HEinfo; int old_size; /* # of old entries to remap to. If remapping parts to processors, old_size = Num_Procs; if renumbering partitions, old_size = old num parts. */ int fp; /* First partition on this processor in new decomposition. */ int np; /* # of partitions on this processor in new decomposition. */ int my_proc = zz->Proc; /* This processor's rank. */ int minp, maxp; /* Lowest and highest partition numbers on this processor in old decomposition; partition numbers are assumed to be dense, but no particular distribution is assumed. */ int HEwgt_size; /* # of HE weights allocated. */ int *HEwgt = NULL; /* Array of HE weights. Initially includes zero weights; later zero-weights are removed.*/ if (remap_type == ZOLTAN_LB_REMAP_PROCESSORS) { /* Renumber new processors to minimize changes in proc assignment. */ HEwgt_size = zz->Num_Proc; HEwgt = (int *) ZOLTAN_CALLOC(HEwgt_size, sizeof(int)); if (!HEwgt) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } for (i = 0; i < nobj; i++) HEwgt[proc[i]]++; /* At this point, proc has old proc assignments */ *HEcnt = 0; for (i = 0; i < HEwgt_size; i++) if (HEwgt[i] != 0) (*HEcnt)++; ierr = malloc_HEinfo(zz, *HEcnt, HEinfo); if (ierr < 0) goto End; tmp_HEinfo = *HEinfo; cnt = 0; for (i = 0; i < HEwgt_size; i++) { if (HEwgt[i] != 0) { tmp = cnt * HEINFO_ENTRIES; tmp_HEinfo[tmp] = i; /* Old processor number */ tmp_HEinfo[tmp+1] = my_proc; /* New processor number */ tmp_HEinfo[tmp+2] = HEwgt[i]; /* shift non-zero weights down. */ cnt++; } } } else { /* ZOLTAN_LB_REMAP_PARTS */ /* Renumber new partitions to minimize changes in partition assignment */ for (minp = INT_MAX, maxp = 0, i = 0; i < nobj; i++) { if (old_part[i] < minp) minp = old_part[i]; if (old_part[i] > maxp) maxp = old_part[i]; } /* Don't include old partition numbers that are greater than * zz->LB.Num_Global_Parts - 1; they are not valid values for * remapping of new partition numbers. */ if (minp >= zz->LB.Num_Global_Parts) minp = zz->LB.Num_Global_Parts-1; if (maxp >= zz->LB.Num_Global_Parts) maxp = zz->LB.Num_Global_Parts-1; old_size = maxp - minp + 1; Zoltan_LB_Proc_To_Part(zz, my_proc, &np, &fp); HEwgt_size = np * old_size; if (HEwgt_size > 0) { HEwgt = (int *) ZOLTAN_CALLOC(HEwgt_size, sizeof(int)); if (!HEwgt) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } for (i = 0; i < nobj; i++) { if (old_part[i] < zz->LB.Num_Global_Parts) { /* Include only HEs to old partitions numbered * 0 to zz->LB.Num_Global_Parts-1; these are the only valid * remapping values for the new partition numbers. */ tmp = (new_part[i]-fp) * old_size; HEwgt[tmp + (old_part[i]-minp)]++; } } *HEcnt = 0; for (i = 0; i < HEwgt_size; i++) if (HEwgt[i] != 0) (*HEcnt)++; ierr = malloc_HEinfo(zz, *HEcnt, HEinfo); if (ierr < 0) goto End; tmp_HEinfo = *HEinfo; cnt = 0; for (i = 0; i < HEwgt_size; i++) { if (HEwgt[i] != 0) { tmp = cnt * HEINFO_ENTRIES; tmp_HEinfo[tmp] = i%old_size + minp; /* Old partition number */ tmp_HEinfo[tmp+1] = i/old_size + fp; /* New partition number */ tmp_HEinfo[tmp+2] = HEwgt[i]; /* shift non-zero weights down. */ cnt++; } } } End: if (HEwgt) ZOLTAN_FREE(&HEwgt); return ierr; }
int Zoltan_Matrix_Sym(ZZ* zz, Zoltan_matrix *matrix, int bipartite) { static char *yo = "Zoltan_Matrix_Sym"; int ierr = ZOLTAN_OK; Zoltan_Arc *tr_tab = NULL; int i, j, cnt; ZOLTAN_ID_PTR yGID = NULL; int *ypid=NULL; float *pinwgt=NULL; int * ybipart = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (bipartite || !matrix->opts.enforceSquare) { matrix->bipartite = 1; matrix->redist = 1; } if (matrix->ywgtdim != zz->Obj_Weight_Dim) FATAL_ERROR("Cannot form bipartite graph: vertex and edge weights are not consistant"); matrix->opts.symmetrize = 1; /* Update the data directories */ tr_tab = (Zoltan_Arc*) ZOLTAN_MALLOC(sizeof(Zoltan_Arc)*(matrix->nPins*2+matrix->nY)); if (matrix->nPins && tr_tab == NULL) MEMORY_ERROR; pinwgt = (float*)ZOLTAN_MALLOC((matrix->nPins*2+matrix->nY)*matrix->pinwgtdim*sizeof(float)); for (i = 0 ; i < 2 ; ++i) /* Copy pin weights */ memcpy(pinwgt + i*matrix->nPins*matrix->pinwgtdim*sizeof(float), matrix->pinwgt, matrix->nPins*matrix->pinwgtdim*sizeof(float)); for (i=0, cnt = 0 ; i < matrix->nY ; ++i) { for (j = matrix->ystart[i] ; j < matrix->yend[i] ; ++j) { tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */ tr_tab[cnt].GNO[1] = matrix->pinGNO[j]; memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim, matrix->pinwgtdim*sizeof(float)); cnt ++; tr_tab[cnt].GNO[0] = matrix->pinGNO[j]; /* Symmetric arc */ tr_tab[cnt].GNO[1] = matrix->yGNO[i] + bipartite*matrix->globalX; /* new ordering */ memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim, matrix->pinwgtdim*sizeof(float)); cnt ++; } if (matrix->ystart[i] == matrix->yend[i]) { /* Singleton */ tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */ tr_tab[cnt].GNO[1] = -1; cnt ++; } } ZOLTAN_FREE(&matrix->pinwgt); Zoltan_Matrix_Remove_DupArcs(zz, cnt, tr_tab, pinwgt, matrix); ZOLTAN_FREE(&tr_tab); ZOLTAN_FREE(&pinwgt); if (bipartite) { int endX; int * yGNO = NULL; /* Update data directories */ yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nY); ypid = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int)); ybipart = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int)); for (endX = 0 ; endX < matrix->nY ; ++endX) { if (matrix->yGNO[endX] >= matrix->globalX) break; ybipart[endX] = 0; } /* Get Informations about X */ Zoltan_DD_Find (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, NULL, endX, NULL); yGNO = (int*)ZOLTAN_MALLOC(endX*sizeof(int)); for (i = endX ; i < matrix->nY ; ++i) { yGNO[i-endX] = matrix->yGNO[i] - matrix->globalX; /* TODO: add a something to have the correct ypid */ ybipart[endX] = 1; } /* Get Informations about Y */ Zoltan_DD_Find (matrix->ddY, (ZOLTAN_ID_PTR)yGNO, yGID + endX*zz->Num_GID, NULL, NULL, matrix->nY-endX, NULL); if (matrix->ddY != matrix->ddX) Zoltan_DD_Destroy (&matrix->ddY); Zoltan_DD_Destroy (&matrix->ddX); matrix->globalX += matrix->globalY; matrix->globalY = matrix->globalX; /* I store : xGNO, xGID, xpid, bipart */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID, 1, matrix->globalX/zz->Num_Proc, 0); matrix->ddY = matrix->ddX; /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xyGNO */ Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, ybipart, matrix->nY); } End: ZOLTAN_FREE(&ybipart); ZOLTAN_FREE(&ypid); ZOLTAN_FREE(&pinwgt); ZOLTAN_FREE(&yGID); ZOLTAN_FREE(&tr_tab); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }