static int Compute_Adjpart(
ZZ *zz,
int nvtx, /* Input: # vtxs in this processor */
indextype *vtxdist, /* Input: Distribution of vertices across processors */
indextype *xadj, /* Input: Index of adjncy: adjncy[xadj[i]] to
adjncy[xadj[i]+1] are all edge nbors of vtx i. */
indextype *adjncy, /* Input: Array of nbor vertices. */
int *adjproc, /* Input: adjproc[j] == processor owning adjncy[j]. */
indextype *part, /* Input: Partition assignments of vtxs. */
int *adjpart /* Output: adjpart[j] == partition owning adjncy[j] */
)
{
/* Given an adjacency list adjncy, find the partition number of each
* vertex in adjncy. Return it in adjpart.
*/
ZOLTAN_COMM_OBJ *plan;
int i;
indextype start = vtxdist[zz->Proc]; /* First vertex on this processor */
indextype *recv_gno= NULL;
int *send_int=NULL;
int nrecv;
int tag = 24542;
Zoltan_Comm_Create(&plan, (int)xadj[nvtx], adjproc, zz->Communicator, tag++, &nrecv);
if (nrecv){
recv_gno = (indextype *) ZOLTAN_MALLOC(nrecv * sizeof(indextype));
send_int = (int *) ZOLTAN_MALLOC(nrecv * sizeof(int));
if (!recv_gno || !send_int) {
Zoltan_Comm_Destroy(&plan);
ZOLTAN_FREE(&recv_gno);
ZOLTAN_FREE(&send_int);
return ZOLTAN_MEMERR;
}
}
Zoltan_Comm_Do(plan, tag++, (char *) adjncy, sizeof(indextype), (char *) recv_gno);
for (i = 0; i < nrecv; i++){
send_int[i] = part[recv_gno[i] - start];
}
ZOLTAN_FREE(&recv_gno);
Zoltan_Comm_Do_Reverse(plan, tag, (char *)send_int, sizeof(int), NULL, (char *) adjpart);
ZOLTAN_FREE(&send_int);
Zoltan_Comm_Destroy(&plan);
return ZOLTAN_OK;
}
static int
Zoltan_Postprocess_UnScatter_Graph (ZZ *zz,
ZOLTAN_Third_Graph *gr,
ZOLTAN_Third_Part *prt,
indextype **rank)
{
static char * yo = "Zoltan_Postprocess_UnScatter_Graph";
int ierr = ZOLTAN_FATAL;
indextype *src;
indextype *dst;
if (gr->scatter >0){
gr->num_obj = gr->num_obj_orig;
if (*rank) { /* We have to project back rank */
dst = (indextype*) ZOLTAN_MALLOC(gr->num_obj*sizeof(indextype));
src = *rank;
}
else {
dst = prt->part_orig;
src = prt->part;
}
ierr = Zoltan_Comm_Do_Reverse(gr->comm_plan, TAG2, (char *) src,
sizeof(indextype), NULL, (char *) dst);
if ((ierr == ZOLTAN_FATAL) || (ierr == ZOLTAN_MEMERR)){
ZOLTAN_THIRD_ERROR(ierr, "Zoltan_Comm_Do_Reverse returned error.");
}
Zoltan_Comm_Destroy(&gr->comm_plan); /* Destroy the comm. plan */
/* We don't need the partition array with the scattered distribution
* any more */
ZOLTAN_FREE(&src);
if (prt) {
/* part is now the new partition array under the original distribution */
prt->part = prt->part_orig;
prt->part_orig = NULL;
}
else {
*rank = dst;
}
}
return (ierr);
}
/* Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
ZZ *zz, /* Zoltan data structure */
HGraph *hg, /* Input hypergraph to be partitioned */
int p, /* Input: number partitions to be generated */
float *part_sizes, /* Input: array of length p containing percentages
of work to be assigned to each partition */
Partition parts, /* Input: initial partition #s; aligned with vtx
arrays.
Output: computed partition #s */
PHGPartParams *hgp, /* Input: parameters for hgraph partitioning. */
int level)
{
PHGComm *hgc = hg->comm;
VCycle *vcycle=NULL, *del=NULL;
int i, err = ZOLTAN_OK;
int prevVcnt = 2*hg->dist_x[hgc->nProc_x];
int prevVedgecnt = 2*hg->dist_y[hgc->nProc_y];
char *yo = "Zoltan_PHG_Partition";
static int timer_match = -1, /* Timers for various stages */
timer_coarse = -1, /* Declared static so we can accumulate */
timer_refine = -1, /* times over calls to Zoltan_PHG_Partition */
timer_coarsepart = -1,
timer_project = -1,
timer_vcycle = -1; /* times everything in Vcycle not included
in above timers */
int do_timing = (hgp->use_timers > 1);
int vcycle_timing = (hgp->use_timers > 4);
ZOLTAN_TRACE_ENTER(zz, yo);
if (do_timing) {
if (timer_vcycle < 0)
timer_vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
if (timer_match < 0)
timer_match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
if (timer_coarse < 0)
timer_coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
if (timer_coarsepart < 0)
timer_coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
"Coarse_Partition");
if (timer_refine < 0)
timer_refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
if (timer_project < 0)
timer_project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
return ZOLTAN_MEMERR;
}
/****** Coarsening ******/
#define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */
while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > hg->redl)
&& ((hg->dist_x[hgc->nProc_x] < (int) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5))
|| (hg->dist_y[hgc->nProc_y] < (int) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5)))
&& hg->dist_y[hgc->nProc_y] && hgp->matching) {
int *match = NULL;
VCycle *coarser=NULL;
prevVcnt = hg->dist_x[hgc->nProc_x];
prevVedgecnt = hg->dist_y[hgc->nProc_y];
#ifdef _DEBUG
/* UVC: load balance stats */
Zoltan_PHG_LoadBalStat(zz, hg);
#endif
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc,
"START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_match, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_match < 0) {
char str[80];
sprintf(str, "VC Matching %d", hg->info);
vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
}
/* Allocate and initialize Matching Array */
if (hg->nVtx && !(match = (int*) ZOLTAN_MALLOC (hg->nVtx*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
match[i] = i;
/* Calculate matching (packing or grouping) */
err = Zoltan_PHG_Matching (zz, hg, match, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_FREE ((void**) &match);
goto End;
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_match, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_coarse, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_coarse < 0) {
char str[80];
sprintf(str, "VC Coarsening %d", hg->info);
vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
}
if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
ZOLTAN_FREE ((void**) &match);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
goto End;
}
/* Construct coarse hypergraph and LevelMap */
err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
&vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData,
&vcycle->comm_plan, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_coarse, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
ZOLTAN_FREE ((void**) &match);
if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
goto End;
vcycle = coarser;
hg = vcycle->hg;
}
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
/* free array that may have been allocated in matching */
if (hgp->vtx_scal) ZOLTAN_FREE(&(hgp->vtx_scal));
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_coarsepart, hgc->Communicator);
}
/****** Coarse Partitioning ******/
err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_coarsepart, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
del = vcycle;
/****** Uncoarsening/Refinement ******/
while (vcycle) {
VCycle *finer = vcycle->finer;
hg = vcycle->hg;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_refine, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_refine < 0) {
char str[80];
sprintf(str, "VC Refinement %d", hg->info);
vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
}
err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_refine, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
if (hgp->output_level >= PHG_DEBUG_LIST)
uprintf(hgc,
"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, p, vcycle->Part),
Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
"partitioned plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_project, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_project < 0) {
char str[80];
sprintf(str, "VC Project Up %d", hg->info);
vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
}
/* Project coarse partition to fine partition */
if (finer) {
int *rbuffer;
/* easy to undo internal matches */
for (i = 0; i < finer->hg->nVtx; i++)
if (finer->LevelMap[i] >= 0)
finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
/* fill sendbuffer with part data for external matches I owned */
for (i = 0; i < finer->LevelCnt; i++) {
++i; /* skip return lno */
finer->LevelData[i] = finer->Part[finer->LevelData[i]];
}
/* allocate rec buffer */
rbuffer = NULL;
if (finer->LevelSndCnt > 0) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
return ZOLTAN_MEMERR;
}
}
/* get partition assignments from owners of externally matchted vtxs */
Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1,
(char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);
/* process data to undo external matches */
for (i = 0; i < 2 * finer->LevelSndCnt;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_project, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
vcycle = finer;
} /* while (vcycle) */
End:
vcycle = del;
while (vcycle) {
if (vcycle_timing) {
Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
Zoltan_Timer_Destroy(&vcycle->timer);
}
if (vcycle->finer) { /* cleanup by level */
Zoltan_HG_HGraph_Free (vcycle->hg);
Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part, &vcycle->LevelMap,
&vcycle->LevelData, &vcycle->hg);
}
else /* cleanup top level */
Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
&vcycle->LevelData);
del = vcycle;
vcycle = vcycle->finer;
ZOLTAN_FREE(&del);
}
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TRACE_EXIT(zz, yo) ;
return err;
}
int Zoltan_DD_Find (
Zoltan_DD_Directory *dd, /* contains directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */
ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */
char *data, /* Outgoing optional corresponding user data */
int *partition, /* Outgoing optional partition information */
int count, /* Count of GIDs in above list (in) */
int *owner) /* Outgoing optional list of data owners */
{
ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */
char *rbuff = NULL; /* receive buffer */
char *rbufftmp = NULL; /* pointer into receive buffer */
char *sbuff = NULL; /* send buffer */
char *sbufftmp = NULL; /* pointer into send buffer */
int *procs = NULL; /* list of processors to contact */
DD_Find_Msg *ptr = NULL;
int i;
int nrec; /* number of messages to receive */
int err = ZOLTAN_OK; /* return error condition */
int errcount; /* count of GIDs not found */
char *yo = "Zoltan_DD_Find";
/* input sanity check */
if (dd == NULL || count < 0 || (gid == NULL && count > 0)) {
ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
"Invalid input argument");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* allocate memory for processors to contact for directory info */
if (count) {
procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
if (procs == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
}
/* allocate memory for DD_Find_Msg send buffer */
if (count) {
sbuff = (char*) ZOLTAN_CALLOC (count, dd->find_msg_size);
if (sbuff == NULL) {
ZOLTAN_FREE (&procs);
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs");
/* for each GID, fill DD_Find_Msg buffer and contact list */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc,
dd->hashdata, dd->hashfn);
ptr = (DD_Find_Msg*) sbufftmp;
sbufftmp += dd->find_msg_size;
ptr->index = i;
ptr->proc = procs[i];
ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length);
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill");
/* create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_FIND_MSG_TAG, &nrec);
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create");
if (err != ZOLTAN_OK)
goto fini;
/* allocate receive buffer */
if (nrec) {
rbuff = (char*) ZOLTAN_MALLOC ((size_t)nrec*(size_t)(dd->find_msg_size));
if (rbuff == NULL) {
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* send out find messages across entire system */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
dd->find_msg_size, rbuff);
if (err != ZOLTAN_OK)
goto fini;
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do");
/* get find messages directed to me, fill in return information */
errcount = 0;
rbufftmp = rbuff;
for (i = 0; i < nrec; i++) {
ptr = (DD_Find_Msg*) rbufftmp;
rbufftmp += dd->find_msg_size;
err = DD_Find_Local (dd, ptr->id, ptr->id,
(char *)(ptr->id + dd->max_id_length),
&ptr->partition, &ptr->proc);
if (err == ZOLTAN_WARN)
++errcount;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info");
/* send return information back to requester */
err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
dd->find_msg_size, NULL, sbuff);
if (err != ZOLTAN_OK)
goto fini;
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse");
/* fill in user supplied lists with returned information */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
ptr = (DD_Find_Msg*) sbufftmp;
sbufftmp += dd->find_msg_size;
if (owner)
owner[ptr->index] = ptr->proc;
if (partition)
partition[ptr->index] = ptr->partition ;
if (lid)
ZOLTAN_SET_ID(dd->lid_length,lid+ptr->index*dd->lid_length,ptr->id);
if (data)
memcpy(data + (size_t)(ptr->index) * (size_t)(dd->user_data_length),
ptr->id + dd->max_id_length, dd->user_data_length);
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists");
/* err = ZOLTAN_OK; */
MPI_Allreduce(&errcount, &err, 1, MPI_INT, MPI_SUM, dd->comm);
err = (err) ? ZOLTAN_WARN : ZOLTAN_OK;
/* if at least one GID was not found, potentially notify caller of error */
if (dd->debug_level > 0) {
char str[100]; /* diagnostic message string */
sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount);
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
}
fini:
ZOLTAN_FREE (&sbuff);
ZOLTAN_FREE (&rbuff);
ZOLTAN_FREE (&procs) ;
Zoltan_Comm_Destroy (&plan);
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err;
}
static int Zoltan_PHG_Output_Parts (
ZZ *zz,
ZHG *zhg,
Partition hg_parts /* Output partitions relative to the 2D distribution
of zhg->HG */
)
{
/* Function to map the computed partition from the distribution in HGraph
* to the input distribution
*/
static char *yo = "Zoltan_PHG_Output_Parts";
int i;
int msg_tag = 31000;
int ierr = ZOLTAN_OK;
int nObj = zhg->nObj;
int *outparts = NULL;
int *sendbuf = NULL;
HGraph *phg = &(zhg->HG);
zhg->Output_Parts = outparts
= (int*) ZOLTAN_MALLOC (nObj * sizeof(int));
if (zhg->VtxPlan != NULL) {
/* Get the partition information from the 2D decomposition back to the
* original owning processor for each GID. */
sendbuf = (int*) ZOLTAN_MALLOC(zhg->nRecv_GNOs * sizeof(int));
for (i = 0; i < zhg->nRecv_GNOs; i++)
sendbuf[i] = hg_parts[VTX_GNO_TO_LNO(phg, zhg->Recv_GNOs[i])];
ierr = Zoltan_Comm_Do_Reverse(zhg->VtxPlan, msg_tag, (char*) sendbuf,
sizeof(int), NULL, (char *) outparts);
if (ierr) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error from Zoltan_Comm_Do_Reverse");
goto End;
}
ZOLTAN_FREE(&sendbuf);
Zoltan_Comm_Destroy(&(zhg->VtxPlan));
}
else {
for (i = 0; i < zhg->nRecv_GNOs; i++)
outparts[i] = hg_parts[zhg->Recv_GNOs[i]];
}
if (zz->LB.Remap_Flag) {
int new_map;
int *newproc = (int *) ZOLTAN_MALLOC(nObj * sizeof(int));
int num_gid_entries = zz->Num_GID;
if (nObj && !newproc) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < nObj; i++){
newproc[i] = Zoltan_LB_Part_To_Proc(zz, outparts[i],
&(zhg->GIDs[i*num_gid_entries]));
if (newproc[i]<0){
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Zoltan_LB_Part_To_Proc returned invalid processor number.");
ierr = ZOLTAN_FATAL;
ZOLTAN_FREE(&newproc);
goto End;
}
}
ierr = Zoltan_LB_Remap(zz, &new_map, nObj, newproc, zhg->Input_Parts,
outparts, 1);
if (ierr < 0)
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_LB_Remap");
ZOLTAN_FREE(&newproc);
}
End:
if (zhg->Recv_GNOs) ZOLTAN_FREE(&(zhg->Recv_GNOs));
zhg->nRecv_GNOs = 0;
return ierr;
}
int main(int argc, char *argv[])
{
FILE *in_file = NULL; /* file with data for problems */
ZOLTAN_COMM_OBJ *plan; /* communication data structure pointer */
struct Params params; /* parameters describing a problem */
struct Data data; /* data describing a problem instance */
struct Data my_send_data; /* data I initially own */
struct Answer true_answer; /* expected outcome of exchange */
int nvals_recv; /* number of vals I own after exchange */
float *recv_data; /* values I own after exchange */
float *reverse_data; /* values I own after reversing communication */
char file_name[100]; /* name of input file */
int nbytes; /* size of objects */
int my_proc; /* my processor ID */
int nprocs; /* total number of processors */
int flag; /* return code from comm operations */
int more_problems; /* are there more problems to do? */
int i, j;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &my_proc);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
Zoltan_Memory_Debug(2);
if (argc > 1) strcpy(file_name, argv[1]);
else strcpy(file_name,"comm_input.dat");
if (my_proc == 0) {
in_file = fopen(file_name, "r");
if (in_file == NULL) {
printf("No input file `%s' found.\n", file_name);
}
}
/* Read some problem descriptors from a file */
more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc);
while (more_problems) {
/* Generate full data at random on each proc */
gen_comm_data(¶ms, &data, nprocs);
if (out_level > 2) if (my_proc == 0) print_data("DATA", &data);
/* Figure out from the data what what to expect to receive */
extract_comm_answer(&data, &true_answer, my_proc, nprocs);
/* Extract what to send */
set_up_comm_from_send(&data, &my_send_data, params.blocked, my_proc, nprocs);
if (out_level > 2) print_data("MY_DATA", &my_send_data);
if (out_level > 1) printf("%d: About to call comm_create\n", my_proc);
/* Call comm routines */
Zoltan_Comm_Create(&plan, my_send_data.nvals, my_send_data.proc_dest,
MPI_COMM_WORLD, 1, &nvals_recv);
if (out_level > 1) printf("%d: About to call comm_info\n", my_proc);
check_comm_info(plan, &my_send_data, nvals_recv, my_proc);
if (out_level > 2) print_plan("BEFORE RESIZE", plan, my_proc);
/* "4" reflects the max_sizes value in gen_comm_data */
recv_data = (float *) ZOLTAN_MALLOC(nvals_recv * 4 * sizeof(float));
reverse_data = (float *) ZOLTAN_MALLOC(my_send_data.nvals * 4 * sizeof(float));
if (my_send_data.sizes != NULL) {
if (out_level > 1) printf("%d: About to call comm_resize\n", my_proc);
Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
Zoltan_Comm_Resize(plan, NULL, 43, NULL);
Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
if (out_level > 2) print_plan("AFTER RESIZE", plan, my_proc);
}
if (my_send_data.sizes == NULL) nbytes = my_send_data.same_size * sizeof(float);
else nbytes = sizeof(float);
if (out_level > 1) printf("%d: About to call comm_do\n", my_proc);
flag = Zoltan_Comm_Do(plan, 2, (char *) my_send_data.vals, nbytes,
(char *) recv_data);
if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer\n", my_proc);
/* Check answers */
check_comm_answer(&true_answer, recv_data, my_proc);
}
else {
printf("%d: Comm_Do returned error code %d\n", my_proc, flag);
}
if (out_level > 1) printf("%d: About to call comm_do_reverse\n", my_proc);
i = (true_answer.sizes != NULL && plan->indices_to != NULL);
MPI_Allreduce(&i, &j, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (j == 0) flag = Zoltan_Comm_Do_Reverse(plan, 2, (char *) recv_data,
nbytes, true_answer.sizes, (char *) reverse_data);
else {
if (my_proc == 0)
printf(">> Non-blocked, variable-sized recvs not supported\n");
flag = ZOLTAN_FATAL;
}
if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer_reverse\n", my_proc);
/* Check answers */
check_comm_answer_reverse(&my_send_data, reverse_data, my_proc);
}
else {
if (out_level > 1) printf("%d: Comm_Do_Reverse returned error code %d\n", my_proc, flag);
}
/* Free up data structures */
ZOLTAN_FREE(&reverse_data);
ZOLTAN_FREE(&recv_data);
free_comm_data(&data, &my_send_data, &true_answer);
Zoltan_Comm_Destroy(&plan);
/* Read some problem descriptors from a file */
more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc);
}
Zoltan_Memory_Stats();
MPI_Finalize();
return(0);
}
/* Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
ZZ *zz, /* Zoltan data structure */
HGraph *hg, /* Input hypergraph to be partitioned */
int p, /* Input: number partitions to be generated */
float *part_sizes, /* Input: array of length p containing percentages
of work to be assigned to each partition */
Partition parts, /* Input: initial partition #s; aligned with vtx
arrays.
Output: computed partition #s */
PHGPartParams *hgp) /* Input: parameters for hgraph partitioning. */
{
PHGComm *hgc = hg->comm;
VCycle *vcycle=NULL, *del=NULL;
int i, err = ZOLTAN_OK, middle;
ZOLTAN_GNO_TYPE origVpincnt; /* for processor reduction test */
ZOLTAN_GNO_TYPE prevVcnt = 2*hg->dist_x[hgc->nProc_x]; /* initialized so that the */
ZOLTAN_GNO_TYPE prevVedgecnt = 2*hg->dist_y[hgc->nProc_y]; /* while loop will be entered
before any coarsening */
ZOLTAN_GNO_TYPE tot_nPins, local_nPins;
MPI_Datatype zoltan_gno_mpi_type;
char *yo = "Zoltan_PHG_Partition";
int do_timing = (hgp->use_timers > 1);
int fine_timing = (hgp->use_timers > 2);
int vcycle_timing = (hgp->use_timers > 4 && hgp->ProRedL == 0);
short refine = 0;
struct phg_timer_indices *timer = Zoltan_PHG_LB_Data_timers(zz);
int reset_geometric_matching = 0;
char reset_geometric_string[4];
ZOLTAN_TRACE_ENTER(zz, yo);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
if (do_timing) {
if (timer->vcycle < 0)
timer->vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
if (timer->procred < 0)
timer->procred = Zoltan_Timer_Init(zz->ZTime, 0, "Processor Reduction");
if (timer->match < 0)
timer->match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
if (timer->coarse < 0)
timer->coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
if (timer->coarsepart < 0)
timer->coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
"Coarse_Partition");
if (timer->refine < 0)
timer->refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
if (timer->project < 0)
timer->project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
MPI_Allreduce(&local_nPins,&tot_nPins,1,zoltan_gno_mpi_type,MPI_SUM,hgc->Communicator);
origVpincnt = tot_nPins;
if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
/* For geometric coarsening, hgp->matching pointer and string are reset
* after geometric_levels of coarsening. Will need to reset them after
* this vcycle is completed. Capture that fact now! */
if (!strcasecmp(hgp->redm_str, "rcb") || !strcasecmp(hgp->redm_str, "rib")) {
reset_geometric_matching = 1;
strcpy(reset_geometric_string, hgp->redm_str);
}
/****** Coarsening ******/
#define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */
while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > (ZOLTAN_GNO_TYPE)hg->redl)
&& ((hg->dist_x[hgc->nProc_x] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5)) /* prevVcnt initialized to 2*hg->dist_x[hgc->nProc_x] */
|| (hg->dist_y[hgc->nProc_y] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5))) /* prevVedgecnt initialized to 2*hg->dist_y[hgc->nProc_y] */
&& hg->dist_y[hgc->nProc_y] && hgp->matching) {
ZOLTAN_GNO_TYPE *match = NULL;
VCycle *coarser=NULL, *redistributed=NULL;
prevVcnt = hg->dist_x[hgc->nProc_x];
prevVedgecnt = hg->dist_y[hgc->nProc_y];
#ifdef _DEBUG
/* UVC: load balance stats */
Zoltan_PHG_LoadBalStat(zz, hg);
#endif
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc,
"START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->match, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_match < 0) {
char str[80];
sprintf(str, "VC Matching %d", hg->info);
vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
}
/* Allocate and initialize Matching Array */
if (hg->nVtx && !(match = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC (hg->nVtx*sizeof(ZOLTAN_GNO_TYPE)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
match[i] = i;
/* Calculate matching (packing or grouping) */
err = Zoltan_PHG_Matching (zz, hg, match, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_FREE (&match);
goto End;
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->match, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->coarse, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_coarse < 0) {
char str[80];
sprintf(str, "VC Coarsening %d", hg->info);
vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
}
if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
ZOLTAN_FREE (&match);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
goto End;
}
/* Construct coarse hypergraph and LevelMap */
err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
&vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData,
&vcycle->comm_plan, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarse, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
ZOLTAN_FREE (&match);
if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
goto End;
vcycle = coarser;
hg = vcycle->hg;
if (hgc->nProc > 1 && hgp->ProRedL > 0) {
local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
MPI_Allreduce(&local_nPins, &tot_nPins, 1, zoltan_gno_mpi_type, MPI_SUM,
hgc->Communicator);
if (tot_nPins < (ZOLTAN_GNO_TYPE)(hgp->ProRedL * origVpincnt + 0.5)) {
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->procred, hgc->Communicator);
}
/* redistribute to half the processors */
origVpincnt = tot_nPins; /* update for processor reduction test */
if(hg->nVtx&&!(hg->vmap=(int*)ZOLTAN_MALLOC(hg->nVtx*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: hg->vmap");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
hg->vmap[i] = i;
middle = (int)((float) (hgc->nProc-1) * hgp->ProRedL);
if (hgp->nProc_x_req!=1&&hgp->nProc_y_req!=1) { /* Want 2D decomp */
if ((middle+1) > SMALL_PRIME && Zoltan_PHG_isPrime(middle+1))
--middle; /* if it was prime just use one less #procs (since
it should be bigger than SMALL_PRIME it is safe to
decrement) */
}
if (!(hgc = (PHGComm*) ZOLTAN_MALLOC (sizeof(PHGComm)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: PHGComm");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!(redistributed=newVCycle(zz,NULL,NULL,vcycle,vcycle_timing))) {
ZOLTAN_FREE (&hgc);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "redistributed is NULL.");
goto End;
}
Zoltan_PHG_Redistribute(zz,hgp,hg,0,middle,hgc, redistributed->hg,
&vcycle->vlno,&vcycle->vdest);
if (hgp->UseFixedVtx || hgp->UsePrefPart)
redistributed->hg->bisec_split = hg->bisec_split;
if ((err=allocVCycle(redistributed))!= ZOLTAN_OK)
goto End;
vcycle = redistributed;
if (hgc->myProc < 0)
/* I'm not in the redistributed part so I should go to uncoarsening
refinement and wait */ {
if (fine_timing) {
if (timer->cpgather < 0)
timer->cpgather = Zoltan_Timer_Init(zz->ZTime, 1, "CP Gather");
if (timer->cprefine < 0)
timer->cprefine =Zoltan_Timer_Init(zz->ZTime, 0, "CP Refine");
if (timer->cpart < 0)
timer->cpart = Zoltan_Timer_Init(zz->ZTime, 0, "CP Part");
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
goto Refine;
}
hg = vcycle->hg;
hg->redl = hgp->redl; /* not set with hg creation */
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
}
}
}
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
/* free array that may have been allocated in matching */
if (hgp->vtx_scal) {
hgp->vtx_scal_size = 0;
ZOLTAN_FREE(&(hgp->vtx_scal));
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->coarsepart, hgc->Communicator);
}
/****** Coarse Partitioning ******/
err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarsepart, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
Refine:
del = vcycle;
refine = 1;
/****** Uncoarsening/Refinement ******/
while (vcycle) {
VCycle *finer = vcycle->finer;
hg = vcycle->hg;
if (refine && hgc->myProc >= 0) {
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->refine, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_refine < 0) {
char str[80];
sprintf(str, "VC Refinement %d", hg->info);
vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
}
err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->refine, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
if (hgp->output_level >= PHG_DEBUG_LIST)
uprintf(hgc,
"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, 0, p,
vcycle->Part),
Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
"partitioned plot");
}
if (finer) {
int *rbuffer;
/* Project coarse partition to fine partition */
if (finer->comm_plan) {
refine = 1;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->project, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_project < 0) {
char str[80];
sprintf(str, "VC Project Up %d", hg->info);
vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
}
/* easy to assign partitions to internal matches */
for (i = 0; i < finer->hg->nVtx; i++)
if (finer->LevelMap[i] >= 0) /* if considers only the local vertices */
finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
/* now that the course partition assignments have been propagated */
/* upward to the finer level for the local vertices, we need to */
/* fill the LevelData (matched pairs of a local vertex with a */
/* off processor vertex) with the partition assignment of the */
/* local vertex - can be done totally in the finer level! */
for (i = 0; i < finer->LevelCnt; i++) {
++i; /* skip over off processor lno */
finer->LevelData[i] = finer->Part[finer->LevelData[i]];
}
/* allocate rec buffer to exchange LevelData information */
rbuffer = NULL;
if (finer->LevelSndCnt > 0) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
/* get partition assignments from owners of externally matched vtxs */
Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1,
(char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);
/* process data to assign partitions to expernal matches */
for (i = 0; i < 2 * finer->LevelSndCnt;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->project, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
} else {
int *sendbuf = NULL, size;
refine = 0;
/* ints local and partition numbers */
if (finer->vlno) {
sendbuf = (int*) ZOLTAN_MALLOC (2 * hg->nVtx * sizeof(int));
if (!sendbuf) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; ++i) {
sendbuf[2 * i] = finer->vlno[i]; /* assign local numbers */
sendbuf[2 * i + 1] = vcycle->Part[i];/* assign partition numbers */
}
}
ZOLTAN_FREE (&hgc);
hgc = finer->hg->comm; /* updating hgc is required when the processors
change */
/* Create comm plan to unredistributed processors */
err = Zoltan_Comm_Create(&finer->comm_plan, finer->vlno ? hg->nVtx : 0,
finer->vdest, hgc->Communicator, COMM_TAG+2,
&size);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(hgc->myProc, yo, "Zoltan_Comm_Create failed.");
goto End;
}
/* allocate rec buffer to exchange sendbuf information */
rbuffer = NULL;
if (finer->hg->nVtx) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->hg->nVtx * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
/* Use plan to send partitions to the unredistributed processors */
Zoltan_Comm_Do(finer->comm_plan, COMM_TAG+3, (char *) sendbuf,
2*sizeof(int), (char *) rbuffer);
MPI_Bcast(rbuffer, 2*finer->hg->nVtx, MPI_INT, 0, hgc->col_comm);
/* process data to assign partitions to unredistributed processors */
for (i = 0; i < 2 * finer->hg->nVtx;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
if (finer->vlno)
ZOLTAN_FREE (&sendbuf);
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
}
}
vcycle = finer;
} /* while (vcycle) */
End:
vcycle = del;
while (vcycle) {
if (vcycle_timing) {
Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
Zoltan_Timer_Destroy(&vcycle->timer);
}
if (vcycle->finer) { /* cleanup by level */
Zoltan_HG_HGraph_Free (vcycle->hg);
if (vcycle->LevelData)
Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part,
&vcycle->LevelMap, &vcycle->LevelData, &vcycle->hg);
else if (vcycle->vlno)
Zoltan_Multifree (__FILE__, __LINE__, 5, &vcycle->Part, &vcycle->vdest,
&vcycle->vlno, &vcycle->LevelMap, &vcycle->hg);
else
Zoltan_Multifree (__FILE__, __LINE__, 3, &vcycle->Part,
&vcycle->LevelMap, &vcycle->hg);
}
else /* cleanup top level */
Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
&vcycle->LevelData);
del = vcycle;
vcycle = vcycle->finer;
ZOLTAN_FREE(&del);
}
if (reset_geometric_matching) {
strcpy(hgp->redm_str, reset_geometric_string);
Zoltan_PHG_Set_Matching_Fn(hgp);
}
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TRACE_EXIT(zz, yo) ;
return err;
}
int Zoltan_Order_Get_GID_Order(
struct Zoltan_Struct *zz,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR order_ids
)
{
int * proctab;
ZOLTAN_ID_PTR sendgidtab;
int * sendtab, *recvtab;
int i;
int nrecv;
struct Zoltan_Comm_Obj *plan;
int ierr = ZOLTAN_OK;
int *vtxdist;
int tag = 24061986;
int offset;
zz->Order.gidrank = order_ids;
if (!strcmp(zz->Order.order_type, "LOCAL")) {
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
order_ids[i] = global_ids[zz->Order.rank[i] - zz->Order.start_index];
}
return (ZOLTAN_OK);
}
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr != ZOLTAN_OK) return (ierr);
proctab = (int*) ZOLTAN_MALLOC(3*zz->Order.nbr_objects*sizeof(int));
sendgidtab = ZOLTAN_MALLOC_GID_ARRAY(zz, 2*zz->Order.nbr_objects);
if (proctab == NULL) {
ZOLTAN_FREE(&vtxdist);
return (ZOLTAN_MEMERR);
}
if (sendgidtab == NULL) {
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ZOLTAN_MEMERR);
}
sendtab = proctab + zz->Order.nbr_objects;
recvtab = sendtab + zz->Order.nbr_objects;
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
proctab[i] = Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, zz->Order.rank[i] - zz->Order.start_index);
sendtab[i] = zz->Order.rank[i] - zz->Order.start_index;
}
ierr = Zoltan_Comm_Create(&plan, zz->Order.nbr_objects, proctab, zz->Communicator, tag++,
&nrecv);
if (ierr != ZOLTAN_OK) {
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
ierr = Zoltan_Comm_Do(plan, tag++, (char *) sendtab, sizeof(int), (char *) recvtab);
if (ierr != ZOLTAN_OK) {
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
offset = vtxdist[zz->Proc];
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
int position;
position = recvtab[i]-offset;
ZOLTAN_SET_GID(zz, &sendgidtab[i], &global_ids[position]);
}
ierr = Zoltan_Comm_Do_Reverse(plan, tag++, (char *) sendgidtab, zz->Num_GID*sizeof(int), NULL,
(char *) order_ids);
Zoltan_Comm_Destroy(&plan);
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
int Zoltan_DD_Find (
Zoltan_DD_Directory *dd, /* contains directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */
ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */
ZOLTAN_ID_PTR data, /* Outgoing optional corresponding user data */
int *partition, /* Outgoing optional partition information */
int count, /* Count of GIDs in above list (in) */
int *owner) /* Outgoing corresponding list of data locations */
{
ZOLTAN_COMM_OBJ *plan = NULL ; /* efficient MPI communication */
char *rbuff = NULL ; /* receive buffer */
char *sbuff = NULL ; /* send buffer */
int *procs = NULL ; /* list of processors to contact */
DD_Find_Msg *ptr = NULL ;
int i ;
int nrec ; /* number of messages to receive */
int err ; /* return error condition */
int errcount ; /* count of GIDs not found */
char *yo = "Zoltan_DD_Find" ;
if (dd != NULL && dd->debug_level > 1)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* input sanity check */
if (dd == NULL || count < 0 || ((owner == NULL || gid == NULL) && count > 0))
{
ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
yo, "Invalid input argument.") ;
if (dd != NULL && dd->debug_level > 1)
ZOLTAN_TRACE_OUT((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
yo, NULL);
return ZOLTAN_DD_INPUT_ERROR ;
}
/* allocate memory for processors to contact for directory info */
if (count > 0)
{
procs = (int *) ZOLTAN_MALLOC (sizeof (int) * count) ;
if (procs == NULL)
{
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list.") ;
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_DD_MEMORY_ERROR ;
}
}
/* allocate memory for DD_Find_Msg send buffer */
if (count > 0)
{
sbuff = (char *) ZOLTAN_MALLOC (dd->find_msg_size * count) ;
if (sbuff == NULL)
{
ZOLTAN_FREE (&procs) ;
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer.") ;
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_DD_MEMORY_ERROR ;
}
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs.");
/* for each GID, fill DD_Find_Msg buffer and contact list */
for (i = 0 ; i < count ; i++)
{
procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc) ;
ptr = (DD_Find_Msg *) (sbuff + i * dd->find_msg_size) ;
ptr->index = i ;
ptr->proc = procs[i] ;
ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length) ;
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill.");
/* create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_FIND_MSG_TAG, &nrec) ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create.");
if (err != ZOLTAN_OK)
goto fini ;
/* allocate receive buffer */
if (nrec > 0)
{
rbuff = (char *) ZOLTAN_MALLOC (nrec * dd->find_msg_size) ;
if (rbuff == NULL)
{
err = ZOLTAN_DD_MEMORY_ERROR ;
goto fini ;
}
}
/* send out find messages across entire system */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
dd->find_msg_size, rbuff) ;
if (err != ZOLTAN_OK)
goto fini ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do.");
/* get find messages directed to me, fill in return information */
errcount = 0 ;
for (i = 0 ; i < nrec ; i++)
{
ptr = (DD_Find_Msg *) (rbuff + i*dd->find_msg_size) ;
err = DD_Find_Local (dd, ptr->id, ptr->id, ptr->id + dd->max_id_length,
(partition) ? (&ptr->partition) : NULL, &ptr->proc) ;
if (err == ZOLTAN_DD_GID_NOT_FOUND_ERROR)
errcount++ ;
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info.");
/* send return information back to requester */
err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
dd->find_msg_size, NULL, sbuff) ;
if (err != ZOLTAN_OK)
goto fini ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse.");
/* fill in user supplied lists with returned information */
for (i = 0 ; i < count ; i++)
{
ptr = (DD_Find_Msg *) (sbuff + i*dd->find_msg_size) ;
owner[ptr->index] = ptr->proc ;
if (partition != NULL)
partition[ptr->index] = ptr->partition ;
if (lid != NULL)
ZOLTAN_SET_ID (dd->lid_length, lid + ptr->index * dd->lid_length,
ptr->id) ;
if (data != NULL)
ZOLTAN_SET_ID (dd->user_data_length, data + ptr->index
* dd->user_data_length, ptr->id + dd->max_id_length) ;
}
/* if at least one GID was not found, notify caller of error */
err = (errcount == 0) ? ZOLTAN_DD_NORMAL_RETURN
: ZOLTAN_DD_GID_NOT_FOUND_ERROR ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists.");
fini:
ZOLTAN_FREE (&sbuff) ;
ZOLTAN_FREE (&rbuff) ;
ZOLTAN_FREE (&procs) ;
Zoltan_Comm_Destroy (&plan) ;
if (err != ZOLTAN_DD_NORMAL_RETURN)
ZOLTAN_PRINT_WARN (dd->my_proc, yo, "Return is not normal.") ;
if (dd->debug_level > 0)
{
char str[100] ; /* diagnostic message string */
sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount) ;
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ;
}
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err ;
}
int Zoltan_Oct_migrate_octants(ZZ *zz, int *newpids, pOctant *octs, int nocts, int *nrecocts) {
int i,j = 0;
int nsends = 0;
int nreceives = 0;
int *despid = NULL;
OCTNEW_msg *snd_reply = NULL;
OCTNEW_msg *rcv_reply = NULL;
int ierr = ZOLTAN_OK;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
char *yo = "Zoltan_Oct_migrate_octants";
pOctant *newocts = NULL; /* New foreign octant pointers */
if((newocts = (pOctant *) ZOLTAN_MALLOC(sizeof(pOctant)*(nocts+10))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i=0; i<nocts; i++)
newocts[i]=NULL;
/* count number of sends */
nsends=0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc)
nsends++;
/* build message array */
/* if(nsends > 0) { */
if((snd_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nsends + 1) * sizeof(OCTNEW_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((despid = (int *) ZOLTAN_MALLOC((nsends+10) * sizeof(int))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
return ZOLTAN_MEMERR;
}
/* } */
/* else { */
/* snd_reply = NULL; */
/* despid = NULL; */
/* } */
j = 0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc) {
snd_reply[j].num = i;
despid[j++] = newpids[i];
}
/* send messages */
ierr = Zoltan_Comm_Create(&comm_plan, nsends, despid, zz->Communicator, MigOctCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
return (ierr);
}
if((rcv_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nreceives + 1) * sizeof(OCTNEW_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return ZOLTAN_MEMERR;
}
ierr = Zoltan_Comm_Do(comm_plan, MigOctCommDo, (char *) snd_reply,
sizeof(OCTNEW_msg), (char *) rcv_reply);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
/* Reply to malloc requests and Receive malloc replies */
for (i=0; i< nreceives; i++) {
rcv_reply[i].ptr = Zoltan_Oct_POct_new((OCT_Global_Info *) (zz->LB.Data_Structure));
}
;
ierr = Zoltan_Comm_Do_Reverse(comm_plan, MigOctCommReverse, (char *) rcv_reply,
sizeof(OCTNEW_msg), NULL, (char *) snd_reply);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
/* store remote pointers locally for future migration */
for (i=0; i<nsends; i++) {
newocts[snd_reply[i].num] = snd_reply[i].ptr;
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
/* set return value */
*nrecocts = nreceives;
ierr = Zoltan_Oct_Update_Connections(zz, octs, newpids, newocts, nocts);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
abort();
return (ierr);
}
ierr = Zoltan_Oct_Final_Migration(zz, octs,newpids,newocts,nocts, *nrecocts);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
abort();
return (ierr);
}
Zoltan_Oct_Update_Map(zz);
ZOLTAN_FREE(&newocts);
return ierr;
}
