int main(int argc, char *argv[])
{
struct Zoltan_Timer *zt1, *zt2, *zt3, *zt4;
int i, me;
const int MAINLOOP=20;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &me);
zt1 = Zoltan_Timer_Create(ZOLTAN_TIME_WALL);
zt2 = Zoltan_Timer_Create(ZOLTAN_TIME_USER);
zt3 = Zoltan_Timer_Create(ZOLTAN_TIME_WALL);
for (i = 0; i < MAINLOOP; i++) {
if (me == 0) printf("\n\n\t****Beginning first test****\n");
first_test(zt1);
if (me == 0) printf("\n\n\t****Beginning second test****\n");
second_test(zt2);
}
if (me == 0) printf("\n\nFINAL RESULTS -- FIRST TEST:\n");
Zoltan_Timer_PrintAll(zt1, 0, MPI_COMM_WORLD, stdout);
if (me == 0) printf("\n\nFINAL RESULTS -- SECOND TEST:\n");
Zoltan_Timer_PrintAll(zt2, 0, MPI_COMM_WORLD, stdout);
/* Copy tests */
Zoltan_Timer_Copy_To(&zt3, zt1);
zt4 = Zoltan_Timer_Copy(zt2);
for (i = 0; i < MAINLOOP; i++) {
if (me == 0) printf("\n\n\t****Beginning first copy test****\n");
first_test(zt3);
if (me == 0) printf("\n\n\t****Beginning second copy test****\n");
second_test(zt4);
}
if (me == 0) printf("\n\nFINAL RESULTS -- FIRST COPY TEST:\n");
Zoltan_Timer_PrintAll(zt3, 0, MPI_COMM_WORLD, stdout);
if (me == 0) printf("\n\nFINAL RESULTS -- SECOND COPY TEST:\n");
Zoltan_Timer_PrintAll(zt4, 0, MPI_COMM_WORLD, stdout);
/* Test printing while timer is still running. */
if (me == 0) printf("\n\n\t****Intermediate print test****\n");
third_test(zt1);
if (me == 0) printf("\n\nFINAL RESULTS -- INTERMEDIATE PRINT TEST:\n");
Zoltan_Timer_PrintAll(zt1, 0, MPI_COMM_WORLD, stdout);
if (me == 0) printf("\n\nTHE END\n");
Zoltan_Timer_Destroy(&zt1);
Zoltan_Timer_Destroy(&zt2);
MPI_Finalize();
return 0;
}
void first_test(struct Zoltan_Timer *zt)
{
/* First test of Timer: This test accrues times through
* separate calls to first_test.
* The time for timer two should be roughly twice that of timer one.
* The time for timer three should be roughly four times that of timer one.
*/
int i, j, me;
static int firsttime=1;
const int LOOP1=1000,
LOOP2=2000,
LOOP3=4000;
const int MAINLOOP=100;
const int USE_BARRIER=1;
static int t1=-1, t2=-1, t3=-1;
MPI_Comm_rank(MPI_COMM_WORLD, &me);
for (i = 0; i < MAINLOOP; i++) {
if (firsttime)
t1 = Zoltan_Timer_Init(zt, USE_BARRIER, "Loop 1");
ZOLTAN_TIMER_START(zt, t1, MPI_COMM_WORLD);
for (j = 0; j < LOOP1; j++) {
double a;
a = sqrt((double) (j * LOOP1));
}
ZOLTAN_TIMER_STOP(zt, t1, MPI_COMM_WORLD);
if (firsttime)
t2 = Zoltan_Timer_Init(zt, USE_BARRIER, "Loop 2");
ZOLTAN_TIMER_START(zt, t2, MPI_COMM_WORLD);
for (j = 0; j < LOOP2; j++) {
double a;
a = sqrt((double) (j * LOOP2));
}
ZOLTAN_TIMER_STOP(zt, t2, MPI_COMM_WORLD);
if (firsttime)
t3 = Zoltan_Timer_Init(zt, USE_BARRIER, "Loop 3");
ZOLTAN_TIMER_START(zt, t3, MPI_COMM_WORLD);
for (j = 0; j < LOOP3; j++) {
double a;
a = sqrt((double) (j * LOOP3));
}
ZOLTAN_TIMER_STOP(zt, t3, MPI_COMM_WORLD);
firsttime=0;
}
Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stdout);
}
void third_test(struct Zoltan_Timer *zt)
{
/* Third test of Timer: This test accrues times through
* separate calls to third_test.
* The time for timer two should be roughly twice that of timer one.
* The time for timer three should be roughly four times that of timer one.
* Intermediate print statements are included (i.e., prints while the timer
* is still running).
*/
int i, j, me;
const int LOOP1=1000,
LOOP2=2000,
LOOP3=4000;
const int MAINLOOP=100;
const int t1=0, t2=1, t3=2;
MPI_Comm_rank(MPI_COMM_WORLD, &me);
for (i = 0; i < MAINLOOP; i++) {
ZOLTAN_TIMER_START(zt, t1, MPI_COMM_WORLD);
for (j = 0; j < LOOP1; j++) {
double a;
a = sqrt((double) (j * LOOP1));
if (!(j%1000)) {
if (me == 0) printf("LOOP1 %d: \n", j);
Zoltan_Timer_Print(zt, t1, 0, MPI_COMM_WORLD, stdout);
}
}
ZOLTAN_TIMER_STOP(zt, t1, MPI_COMM_WORLD);
ZOLTAN_TIMER_START(zt, t2, MPI_COMM_WORLD);
for (j = 0; j < LOOP2; j++) {
double a;
a = sqrt((double) (j * LOOP2));
if (!(j%1000)) {
if (me == 0) printf("LOOP2 %d: \n", j);
Zoltan_Timer_Print(zt, t2, 0, MPI_COMM_WORLD, stdout);
}
}
ZOLTAN_TIMER_STOP(zt, t2, MPI_COMM_WORLD);
ZOLTAN_TIMER_START(zt, t3, MPI_COMM_WORLD);
for (j = 0; j < LOOP3; j++) {
double a;
a = sqrt((double) (j * LOOP3));
if (!(j%1000)) {
if (me == 0) printf("LOOP3 %d: \n", j);
Zoltan_Timer_Print(zt, t3, 0, MPI_COMM_WORLD, stdout);
}
}
ZOLTAN_TIMER_STOP(zt, t3, MPI_COMM_WORLD);
}
Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stdout);
}
void second_test(struct Zoltan_Timer *zt)
{
/* Second test of Timer: This test does not accrue times through
* separate function calls. It exercises the REALLOC when more timers
* than INITLENGTH are requested.
* Computation is similar to first test. Timer_flag used is different.
*/
int i, j, me;
const int LOOP1=1000,
LOOP2=2000,
LOOP3=4000;
const int MAINLOOP=100;
const int USE_BARRIER=1;
int t1=-1, t2=-1, t3=-1;
char str[200];
static int cnt = 0;
MPI_Comm_rank(MPI_COMM_WORLD, &me);
sprintf(str, "STLoop 1 %d", cnt);
t1 = Zoltan_Timer_Init(zt, USE_BARRIER, str);
sprintf(str, "STLoop 2 %d", cnt);
t2 = Zoltan_Timer_Init(zt, USE_BARRIER, str);
sprintf(str, "STLoop 3 %d", cnt);
t3 = Zoltan_Timer_Init(zt, USE_BARRIER, str);
cnt++;
for (i = 0; i < MAINLOOP; i++) {
ZOLTAN_TIMER_START(zt, t1, MPI_COMM_WORLD);
for (j = 0; j < LOOP1; j++) {
double a;
a = sqrt((double) (j * LOOP1));
}
ZOLTAN_TIMER_STOP(zt, t1, MPI_COMM_WORLD);
ZOLTAN_TIMER_START(zt, t2, MPI_COMM_WORLD);
for (j = 0; j < LOOP2; j++) {
double a;
a = sqrt((double) (j * LOOP2));
}
ZOLTAN_TIMER_STOP(zt, t2, MPI_COMM_WORLD);
ZOLTAN_TIMER_START(zt, t3, MPI_COMM_WORLD);
for (j = 0; j < LOOP3; j++) {
double a;
a = sqrt((double) (j * LOOP3));
}
ZOLTAN_TIMER_STOP(zt, t3, MPI_COMM_WORLD);
}
Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stdout);
}
/* Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
ZZ *zz, /* Zoltan data structure */
HGraph *hg, /* Input hypergraph to be partitioned */
int p, /* Input: number partitions to be generated */
float *part_sizes, /* Input: array of length p containing percentages
of work to be assigned to each partition */
Partition parts, /* Input: initial partition #s; aligned with vtx
arrays.
Output: computed partition #s */
PHGPartParams *hgp, /* Input: parameters for hgraph partitioning. */
int level)
{
PHGComm *hgc = hg->comm;
VCycle *vcycle=NULL, *del=NULL;
int i, err = ZOLTAN_OK;
int prevVcnt = 2*hg->dist_x[hgc->nProc_x];
int prevVedgecnt = 2*hg->dist_y[hgc->nProc_y];
char *yo = "Zoltan_PHG_Partition";
static int timer_match = -1, /* Timers for various stages */
timer_coarse = -1, /* Declared static so we can accumulate */
timer_refine = -1, /* times over calls to Zoltan_PHG_Partition */
timer_coarsepart = -1,
timer_project = -1,
timer_vcycle = -1; /* times everything in Vcycle not included
in above timers */
int do_timing = (hgp->use_timers > 1);
int vcycle_timing = (hgp->use_timers > 4);
ZOLTAN_TRACE_ENTER(zz, yo);
if (do_timing) {
if (timer_vcycle < 0)
timer_vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
if (timer_match < 0)
timer_match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
if (timer_coarse < 0)
timer_coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
if (timer_coarsepart < 0)
timer_coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
"Coarse_Partition");
if (timer_refine < 0)
timer_refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
if (timer_project < 0)
timer_project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
return ZOLTAN_MEMERR;
}
/****** Coarsening ******/
#define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */
while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > hg->redl)
&& ((hg->dist_x[hgc->nProc_x] < (int) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5))
|| (hg->dist_y[hgc->nProc_y] < (int) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5)))
&& hg->dist_y[hgc->nProc_y] && hgp->matching) {
int *match = NULL;
VCycle *coarser=NULL;
prevVcnt = hg->dist_x[hgc->nProc_x];
prevVedgecnt = hg->dist_y[hgc->nProc_y];
#ifdef _DEBUG
/* UVC: load balance stats */
Zoltan_PHG_LoadBalStat(zz, hg);
#endif
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc,
"START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_match, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_match < 0) {
char str[80];
sprintf(str, "VC Matching %d", hg->info);
vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
}
/* Allocate and initialize Matching Array */
if (hg->nVtx && !(match = (int*) ZOLTAN_MALLOC (hg->nVtx*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
match[i] = i;
/* Calculate matching (packing or grouping) */
err = Zoltan_PHG_Matching (zz, hg, match, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_FREE ((void**) &match);
goto End;
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_match, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_coarse, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_coarse < 0) {
char str[80];
sprintf(str, "VC Coarsening %d", hg->info);
vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
}
if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
ZOLTAN_FREE ((void**) &match);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
goto End;
}
/* Construct coarse hypergraph and LevelMap */
err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
&vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData,
&vcycle->comm_plan, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_coarse, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
ZOLTAN_FREE ((void**) &match);
if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
goto End;
vcycle = coarser;
hg = vcycle->hg;
}
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
/* free array that may have been allocated in matching */
if (hgp->vtx_scal) ZOLTAN_FREE(&(hgp->vtx_scal));
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_coarsepart, hgc->Communicator);
}
/****** Coarse Partitioning ******/
err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_coarsepart, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
del = vcycle;
/****** Uncoarsening/Refinement ******/
while (vcycle) {
VCycle *finer = vcycle->finer;
hg = vcycle->hg;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_refine, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_refine < 0) {
char str[80];
sprintf(str, "VC Refinement %d", hg->info);
vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
}
err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_refine, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
if (hgp->output_level >= PHG_DEBUG_LIST)
uprintf(hgc,
"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, p, vcycle->Part),
Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
"partitioned plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_project, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_project < 0) {
char str[80];
sprintf(str, "VC Project Up %d", hg->info);
vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
}
/* Project coarse partition to fine partition */
if (finer) {
int *rbuffer;
/* easy to undo internal matches */
for (i = 0; i < finer->hg->nVtx; i++)
if (finer->LevelMap[i] >= 0)
finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
/* fill sendbuffer with part data for external matches I owned */
for (i = 0; i < finer->LevelCnt; i++) {
++i; /* skip return lno */
finer->LevelData[i] = finer->Part[finer->LevelData[i]];
}
/* allocate rec buffer */
rbuffer = NULL;
if (finer->LevelSndCnt > 0) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
return ZOLTAN_MEMERR;
}
}
/* get partition assignments from owners of externally matchted vtxs */
Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1,
(char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);
/* process data to undo external matches */
for (i = 0; i < 2 * finer->LevelSndCnt;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_project, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
vcycle = finer;
} /* while (vcycle) */
End:
vcycle = del;
while (vcycle) {
if (vcycle_timing) {
Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
Zoltan_Timer_Destroy(&vcycle->timer);
}
if (vcycle->finer) { /* cleanup by level */
Zoltan_HG_HGraph_Free (vcycle->hg);
Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part, &vcycle->LevelMap,
&vcycle->LevelData, &vcycle->hg);
}
else /* cleanup top level */
Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
&vcycle->LevelData);
del = vcycle;
vcycle = vcycle->finer;
ZOLTAN_FREE(&del);
}
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TRACE_EXIT(zz, yo) ;
return err;
}
int main (int argc, char *argv[])
{
Zoltan_DD_Directory *dd;
ZTIMER *zt;
int myproc; /* MPI rank, my processor's MPI ID */
int nproc; /* MPI size, number of processors */
ZOLTAN_ID_PTR glist = NULL; /* list of GIDs */
ZOLTAN_ID_PTR llist = NULL; /* list of LIDs */
ZOLTAN_ID_PTR ulist = NULL; /* list of user data of type
ZOLTAN_ID_TYPE */
int *plist = NULL; /* list of partitions */
int *olist = NULL; /* list of owners */
Param param ; /* program's adjustable parameters */
char *store = NULL; /* non directory storage of test data */
Data *data = NULL; /* pointer into store */
/* these are all temporary variables */
int new_owner;
int count;
int i;
char *p;
int err;
int errcount;
int loop;
char str[100]; /* for building output messages */
static int timer[7] = {-1,-1,-1,-1,-1,-1,-1};
char *yo = "DD_Main";
/* initialize MPI communications, ignore errors */
MPI_Init (&argc, &argv);
MPI_Comm_rank (MPI_COMM_WORLD, &myproc);
MPI_Comm_size (MPI_COMM_WORLD, &nproc);
get_params (¶m); /* read input parameters */
zt = Zoltan_Timer_Create(1);
MACRO_TIMER_START(5, "Total time", 0);
MACRO_TIMER_START(0, "DD_Create time", 0);
err = Zoltan_DD_Create (&dd, MPI_COMM_WORLD, param.glen, param.llen,
param.ulen, param.tlen, param.debug_level);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Create");
MACRO_TIMER_STOP(0);
param.slen = sizeof (Data) + sizeof(ZOLTAN_ID_TYPE)
* (param.glen + param.llen + param.ulen);
param.slen = Zoltan_Align(param.slen);
store = (char*) ZOLTAN_MALLOC (param.count * param.slen);
/* allocate storage for various lists */
glist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC(sizeof(ZOLTAN_ID_TYPE) * param.count
* param.glen);
plist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count);
olist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count);
if (param.llen)
llist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE)
* param.count * param.llen);
if (param.ulen)
ulist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE)
* param.count * param.ulen) ;
if (store == NULL || glist == NULL || (param.llen != 0 && llist == NULL)
|| (param.ulen != 0 && ulist == NULL) || plist == NULL || olist == NULL) {
ZOLTAN_PRINT_ERROR (myproc, yo, "Unable to malloc storage lists");
return ZOLTAN_MEMERR;
}
initialize_data (¶m, store, nproc);
/* create & update directory with myproc's initial simulated GIDs */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data *)p)->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data*)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data *)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen,
((Data *)p)->id + (param.glen + param.llen));
plist [count] = ((Data *)p)->partition;
++count;
}
MACRO_TIMER_START (1, "DD_Update timer", 0);
err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
i = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen) {
ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id);
++i;
}
MACRO_TIMER_START(2, "DD_Find timer", 0);
err = Zoltan_DD_Find (dd, glist, llist, ulist, plist, param.count, olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
errcount = 0;
for (i = 0; i < param.count; i++)
if (olist[i] != ((Data *) (store + i * param.slen))->new_owner)
++errcount;
if (errcount)
sprintf (str, "FIRST TEST FAILED, errcount is %d", errcount);
else
sprintf (str, "FIRST TEST SUCCESSFUL");
ZOLTAN_PRINT_INFO (myproc, yo, str);
/* repeatedly simulate moving "dots" around the system */
for (loop = 0; loop < param.nloops; loop++)
{
for (p = store; p < store + param.count * param.slen; p += param.slen)
((Data*) p)->old_owner = ((Data*) p)->new_owner;
/* randomly exchange some dots and randomly reassign others */
for (i = 0; i < (param.pmove * param.count)/100; i++) {
Data *d1, *d2;
d1 = (Data*) (store + param.slen * (rand() % param.count));
d2 = (Data*) (store + param.slen * (rand() % param.count));
new_owner = d1->new_owner;
d1->new_owner = d2->new_owner;
d2->new_owner = new_owner;
}
for (i = 0; i < (param.count * param.pscatter)/100; i++)
((Data*) (store + param.slen *(rand() % param.count)))->new_owner
= rand() % nproc;
/* determine which new GIDs myproc gained, and update directory */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == myproc) {
((Data*)p)->id[param.glen] = count; /* set LID */
ZOLTAN_SET_ID (param.glen, glist + count * param.glen,
((Data *)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data*)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen,
((Data*)p)->id + (param.glen + param.llen));
plist [count] = ((Data *)p)->partition;
++count;
}
MACRO_TIMER_START(1, "DD_Update", 0);
err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
/* use directory to "find" GIDs */
i = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen) {
ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id);
++i;
}
MACRO_TIMER_START(2, "DD_Find timer", 0);
if (loop % 5 == 0)
err = Zoltan_DD_Find(dd,glist,NULL,ulist,plist,param.count,olist);
else if (loop % 7 == 0)
err = Zoltan_DD_Find(dd,glist,llist,NULL,plist,param.count,olist);
else if (loop % 9 == 0)
err = Zoltan_DD_Find(dd,glist,llist,ulist,NULL,param.count,olist);
else if (loop % 2 == 0)
err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,NULL);
else
err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
if (loop % 2) {
errcount = 0 ;
for (i = 0; i < param.count; i++)
if (olist[i] != ((Data *)(store + i * param.slen))->new_owner)
++errcount;
if (errcount)
sprintf (str,"LOOP %d TEST FAILED, errcount is %d",loop,errcount);
else
sprintf (str, "LOOP %d TEST SUCCESSFUL", loop);
ZOLTAN_PRINT_INFO (myproc, yo, str) ;
}
else {
sprintf (str, "LOOP %d Completed", loop);
ZOLTAN_PRINT_INFO (myproc, yo, str);
}
/* randomly remove a percentage of GIDs from the directory */
count = 0;
for (i = 0; i < (param.count * param.pdelete)/100; i++) {
data = (Data*) (store + param.slen * (rand() % param.count));
if (data->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id);
++count;
}
data->new_owner = NO_PROC;
}
MACRO_TIMER_START(3, "DD_Remove timer", 0);
err = Zoltan_DD_Remove (dd, glist, count);
MACRO_TIMER_STOP(3);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Remove");
/* update directory (put directory entries back in) */
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == NO_PROC)
((Data*)p)->new_owner = loop % nproc; /* place in new location */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == myproc) {
ZOLTAN_SET_ID(param.glen,glist+count*param.glen,((Data*)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data*)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID(param.ulen, ulist + count * param.ulen,
((Data *)p)->id + (param.glen + param.llen));
plist [count] = ((Data*)p)->partition;
++count;
}
MACRO_TIMER_START(1, "DD_Update timer", 0);
err = Zoltan_DD_Update (dd, glist, NULL, NULL, NULL, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
}
/* now Find directory info for GIDs myproc now owns and validate */
count = 0;
for (i = 0; i < param.count; i++) {
data = (Data *) (store + i * param.slen);
if (data->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id);
++count;
}
}
MACRO_TIMER_START(2, "DD_Find", 0);
err = Zoltan_DD_Find (dd, glist, NULL, NULL, NULL, count, olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
errcount = 0;
for (i = 0; i < count; i++)
if (olist[i] != myproc)
++errcount;
if (errcount) {
sprintf (str, "errcount is %d", errcount);
ZOLTAN_PRINT_ERROR (myproc, yo, str);
}
else
ZOLTAN_PRINT_INFO (myproc, yo, "TEST SUCCESSFUL");
Zoltan_DD_Stats (dd);
/* done, now free memory, stop MPI & directory, return */
ZOLTAN_FREE (&store);
ZOLTAN_FREE (&glist);
ZOLTAN_FREE (&plist);
ZOLTAN_FREE (&olist);
if (param.llen) ZOLTAN_FREE (&llist);
if (param.ulen) ZOLTAN_FREE (&ulist);
ZOLTAN_PRINT_INFO (myproc, yo, "Completing program");
MACRO_TIMER_START(4, "DD_Destroy", 0);
Zoltan_DD_Destroy (&dd);
MACRO_TIMER_STOP(4);
MACRO_TIMER_STOP(5);
Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stderr);
MPI_Finalize ();
return ZOLTAN_OK;
}
int Zoltan_PHG(
ZZ *zz, /* The Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work assigned
to each partition. */
int *num_imp, /* not computed */
ZOLTAN_ID_PTR *imp_gids, /* not computed */
ZOLTAN_ID_PTR *imp_lids, /* not computed */
int **imp_procs, /* not computed */
int **imp_to_part, /* not computed */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part ) /* list of partitions to which exported objs
are assigned. */
{
char *yo = "Zoltan_PHG";
ZHG *zoltan_hg = NULL;
PHGPartParams hgp; /* Hypergraph parameters. */
HGraph *hg = NULL; /* Hypergraph itself */
Partition parts = NULL; /* Partition assignments in
2D distribution. */
int err = ZOLTAN_OK, p=0;
struct phg_timer_indices *timer = NULL;
int do_timing = 0;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Initialization of return arguments. */
*num_imp = *num_exp = -1;
*imp_gids = *exp_gids = NULL;
*imp_lids = *exp_lids = NULL;
*imp_procs = *exp_procs = NULL;
/* Initialize HG parameters. */
err = Zoltan_PHG_Initialize_Params(zz, part_sizes, &hgp);
if (err != ZOLTAN_OK)
goto End;
if (hgp.use_timers) {
if (!zz->LB.Data_Structure) {
zz->LB.Data_Structure = (struct phg_timer_indices *)
ZOLTAN_MALLOC(sizeof(struct phg_timer_indices));
initialize_timer_indices((struct phg_timer_indices *)zz->LB.Data_Structure);
}
timer = zz->LB.Data_Structure;
if (timer->all < 0)
timer->all = Zoltan_Timer_Init(zz->ZTime, 1, "Zoltan_PHG");
}
if (hgp.use_timers > 1) {
do_timing = 1;
if (timer->build < 0)
timer->build = Zoltan_Timer_Init(zz->ZTime, 1, "Build");
if (timer->setupvmap < 0)
timer->setupvmap = Zoltan_Timer_Init(zz->ZTime, 0, "Vmaps");
}
if (hgp.use_timers)
ZOLTAN_TIMER_START(zz->ZTime, timer->all, zz->Communicator);
if (do_timing)
ZOLTAN_TIMER_START(zz->ZTime, timer->build, zz->Communicator);
/* build initial Zoltan hypergraph from callback functions. */
err = Zoltan_PHG_Build_Hypergraph (zz, &zoltan_hg, &parts, &hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error building hypergraph.");
goto End;
}
if (zoltan_hg->GnObj == 0){
/* degenerate case - no objects to partition */
hgp.final_output = 0;
goto End;
}
hg = &zoltan_hg->HG;
p = zz->LB.Num_Global_Parts;
zoltan_hg->HG.redl = MAX(hgp.redl, p); /* redl needs to be dynamic */
/* RTHRTH -- redl may need to be scaled by number of procs */
/* EBEB -- at least make sure redl > #procs */
if (hgp.UseFixedVtx)
hg->bisec_split = 1; /* this will be used only #parts=2
otherwise rdivide will set to appropriate
value */
if (hgp.UsePrefPart || hgp.UseFixedVtx) { /* allocate memory for pref_part */
if (hg->nVtx &&
!(hg->pref_part = (int*) ZOLTAN_MALLOC (sizeof(int) * hg->nVtx))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error building hypergraph.");
goto End;
}
}
if (hgp.UsePrefPart) /* copy input parts as pref_part;
UVCUVC: TODO: this code (and alloc of pref_part)
should go to Build_Hypergraph later */
memcpy(hg->pref_part, parts, sizeof(int) * hg->nVtx);
if (hgp.UseFixedVtx) {
int i;
for (i=0; i<hg->nVtx; ++i)
if (hg->fixed_part[i]>=0)
hg->pref_part[i] = hg->fixed_part[i];
else if (!hgp.UsePrefPart)
hg->pref_part[i] = -1;
}
hgp.UsePrefPart |= hgp.UseFixedVtx;
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->build, zz->Communicator);
/*
UVCUVC DEBUG PRINT
uprintf(hg->comm, "Zoltan_PHG kway=%d #parts=%d\n", hgp.kway, zz->LB.Num_Global_Parts);
*/
if (!strcasecmp(hgp.hgraph_pkg, "PARKWAY")){
if (do_timing) {
if (timer->parkway < 0)
timer->parkway = Zoltan_Timer_Init(zz->ZTime, 0, "PHG_ParKway");
ZOLTAN_TIMER_START(zz->ZTime, timer->parkway, zz->Communicator);
}
err = Zoltan_PHG_ParKway(zz, hg, p,
parts, &hgp);
if (err != ZOLTAN_OK)
goto End;
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->parkway, zz->Communicator);
} else if (!strcasecmp(hgp.hgraph_pkg, "PATOH")){
if (hgp.use_timers > 1) {
if (timer->patoh < 0)
timer->patoh = Zoltan_Timer_Init(zz->ZTime, 0, "HG_PaToH");
ZOLTAN_TIMER_START(zz->ZTime, timer->patoh, zz->Communicator);
}
err = Zoltan_PHG_PaToH(zz, hg, p,
parts, &hgp);
if (err != ZOLTAN_OK)
goto End;
if (hgp.use_timers > 1)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->patoh, zz->Communicator);
}
else { /* it must be PHG */
/* UVC: if it is bisection anyways; no need to create vmap etc;
rdivide is going to call Zoltan_PHG_Partition anyways... */
if (hgp.globalcomm.Communicator != MPI_COMM_NULL) {
/* This processor is part of the 2D data distribution; it should
participate in partitioning. */
if (hgp.kway || zz->LB.Num_Global_Parts == 2) {
/* call main V cycle routine */
err = Zoltan_PHG_Partition(zz, hg, p,
hgp.part_sizes, parts, &hgp);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error partitioning hypergraph.");
goto End;
}
}
else {
int i;
if (do_timing)
ZOLTAN_TIMER_START(zz->ZTime, timer->setupvmap, zz->Communicator);
/* vmap associates original vertices to sub hypergraphs */
if (hg->nVtx &&
!(hg->vmap = (int*) ZOLTAN_MALLOC(hg->nVtx*sizeof (int)))) {
err = ZOLTAN_MEMERR;
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
goto End;
}
for (i = 0; i < hg->nVtx; ++i)
hg->vmap[i] = i;
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->setupvmap, zz->Communicator);
/* partition hypergraph */
err = Zoltan_PHG_rdivide (0, p-1, parts, zz, hg, &hgp, 0);
if (hgp.output_level >= PHG_DEBUG_LIST)
uprintf(hg->comm, "FINAL %3d |V|=%6d |E|=%6d #pins=%6d %s/%s/%s/%s p=%d "
"bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins,
hgp.convert_str, hgp.redm_str,
hgp.coarsepartition_str, hgp.refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, hgp.part_sizes, 0,
p, parts),
Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, p, &err));
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error partitioning hypergraph.");
goto End;
}
ZOLTAN_FREE (&hg->vmap);
}
#ifdef CHECK_LEFTALONE_VERTICES
findAndSaveLeftAloneVertices(zz, hg, p, parts, &hgp);
#endif
}
}
if (!strcasecmp(hgp.hgraph_method, "REPARTITION")) {
Zoltan_PHG_Remove_Repart_Data(zz, zoltan_hg, hg, &hgp);
}
/* UVC DEBUG PRINT
if (!strcasecmp(hgp->hgraph_method, "REFINE")){
uprintf(hg->comm,
"UVC ATTHEEND |V|=%6d |E|=%6d #pins=%6d p=%d bal=%.2f cutl=%.2f\n",
hg->nVtx, hg->nEdge, hg->nPins, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, p, parts),
Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, p, &err));
detailed_balance_info(zz, hg, part_sizes, p, parts);
}
*/
/* Initialize these timers here so their output is near end of printout */
if (do_timing)
if (timer->retlist < 0)
timer->retlist = Zoltan_Timer_Init(zz->ZTime, 1, "Return_Lists");
if (hgp.use_timers)
if (timer->finaloutput < 0)
timer->finaloutput = Zoltan_Timer_Init(zz->ZTime, 1, "Final_Output");
if (do_timing)
ZOLTAN_TIMER_START(zz->ZTime, timer->retlist, zz->Communicator);
/* Build Zoltan's Output_Parts, mapped from 2D distribution
to input distribution. */
Zoltan_PHG_Output_Parts(zz, zoltan_hg, parts);
/* Build Zoltan's return arguments. */
Zoltan_PHG_Return_Lists(zz, zoltan_hg, num_exp, exp_gids,
exp_lids, exp_procs, exp_to_part);
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->retlist, zz->Communicator);
End:
if (err == ZOLTAN_MEMERR)
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Memory error.")
else if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Error partitioning hypergraph.")
/* KDDKDD The following code prints a final quality result even when
* KDDKDD phg_output_level is zero. It is useful for our tests and
* KDDKDD data collection. */
if ((err == ZOLTAN_OK) && hgp.final_output) {
static int nRuns=0;
static double balsum = 0.0, cutlsum = 0.0, cutnsum = 0.0, movesum = 0.0, repartsum = 0.0;
static double balmax = 0.0, cutlmax = 0.0, cutnmax = 0.0, movemax = 0.0, repartmax = 0.0;
static double balmin = 1e100, cutlmin = 1e100, cutnmin = 1e100, movemin = 1e100, repartmin = 1e100;
double bal = 0.;
double cutl = 0.; /* Connnectivity cuts: sum_over_edges((npart-1)*ewgt) */
double cutn = 0.; /* Net cuts: sum_over_edges((nparts>1)*ewgt) */
double rlocal[2]; /* local cut stats for removed edges */
double rglobal[2]; /* global cut stats for removed edges */
int gnremove, i;
double move=0.0, gmove; /* local and global migration costs */
double repart=0.0; /* total repartitioning cost: comcost x multiplier + migration_cost */
if (hgp.use_timers) {
/* Do not include final output time in partitioning time */
ZOLTAN_TIMER_STOP(zz->ZTime, timer->all, zz->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->finaloutput, zz->Communicator);
}
if (hgp.globalcomm.Communicator != MPI_COMM_NULL) {
/* Processor participated in partitioning */
bal = Zoltan_PHG_Compute_Balance(zz, hg, hgp.part_sizes, 0,
zz->LB.Num_Global_Parts, parts);
cutl= Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts,
zz->LB.Num_Global_Parts, &err);
cutn = Zoltan_PHG_Compute_NetCut(hg->comm, hg, parts,
zz->LB.Num_Global_Parts);
for (i = 0; i < zoltan_hg->nObj; ++i) {
/* uprintf(hg->comm, " obj[%d] = %d in=%d out=%d\n", i, zoltan_hg->AppObjSizes[i], zoltan_hg->Input_Parts[i], zoltan_hg->Output_Parts[i]); */
if (zoltan_hg->Input_Parts[i] != zoltan_hg->Output_Parts[i])
move += (double) ((zoltan_hg->AppObjSizes) ? zoltan_hg->AppObjSizes[i] : 1.0);
}
}
if (!err) {
/* Add in cut contributions from removed edges */
MPI_Allreduce(&(zoltan_hg->nRemove), &gnremove, 1, MPI_INT, MPI_SUM,
zz->Communicator);
if (gnremove) {
err = Zoltan_PHG_Removed_Cuts(zz, zoltan_hg, rlocal);
MPI_Allreduce(rlocal, rglobal, 2, MPI_DOUBLE,MPI_SUM,zz->Communicator);
cutl += rglobal[0];
cutn += rglobal[1];
}
MPI_Allreduce(&move, &gmove, 1, MPI_DOUBLE, MPI_SUM, zz->Communicator);
repart = cutl*hgp.RepartMultiplier + gmove;
repartsum += repart;
if (repart > repartmax) repartmax = repart;
if (repart < repartmin) repartmin = repart;
movesum += gmove;
if (gmove > movemax) movemax = gmove;
if (gmove < movemin) movemin = gmove;
cutlsum += cutl;
if (cutl > cutlmax) cutlmax = cutl;
if (cutl < cutlmin) cutlmin = cutl;
cutnsum += cutn;
if (cutn > cutnmax) cutnmax = cutn;
if (cutn < cutnmin) cutnmin = cutn;
balsum += bal;
if (bal > balmax) balmax = bal;
if (bal < balmin) balmin = bal;
nRuns++;
if (zz->Proc == 0) {
uprintf(hg->comm,
"STATS Runs %d bal CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, bal, balmax, balmin, balsum/nRuns);
uprintf(hg->comm,
"STATS Runs %d cutl CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, cutl, cutlmax, cutlmin, cutlsum/nRuns);
uprintf(hg->comm,
"STATS Runs %d cutn CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, cutn, cutnmax, cutnmin, cutnsum/nRuns);
uprintf(hg->comm,
"STATS Runs %d %s CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, (zoltan_hg->showMoveVol) ? "moveVol" : "moveCnt", gmove, movemax, movemin, movesum/nRuns);
if (zoltan_hg->showMoveVol)
uprintf(hg->comm,
"STATS Runs %d repart CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, repart, repartmax, repartmin, repartsum/nRuns);
}
}
if (hgp.use_timers) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->finaloutput, zz->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->all, zz->Communicator);
}
}
/* KDDKDD End of printing section. */
ZOLTAN_FREE(&parts);
if (zoltan_hg != NULL) {
Zoltan_PHG_Free_Hypergraph_Data(zoltan_hg);
ZOLTAN_FREE (&zoltan_hg);
}
if (hgp.use_timers) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->all, zz->Communicator);
if (hgp.globalcomm.Communicator != MPI_COMM_NULL)
Zoltan_Timer_PrintAll(zz->ZTime, 0, hgp.globalcomm.Communicator, stdout);
}
if (hgp.globalcomm.row_comm != MPI_COMM_NULL)
MPI_Comm_free(&(hgp.globalcomm.row_comm));
if (hgp.globalcomm.col_comm != MPI_COMM_NULL)
MPI_Comm_free(&(hgp.globalcomm.col_comm));
if (hgp.globalcomm.Communicator != MPI_COMM_NULL)
MPI_Comm_free(&(hgp.globalcomm.Communicator));
/* Free part_sizes if created new due to ADD_OBJ_WEIGHT */
if (hgp.part_sizes != part_sizes)
ZOLTAN_FREE(&hgp.part_sizes);
ZOLTAN_TRACE_EXIT(zz, yo);
return err;
}
/* Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
ZZ *zz, /* Zoltan data structure */
HGraph *hg, /* Input hypergraph to be partitioned */
int p, /* Input: number partitions to be generated */
float *part_sizes, /* Input: array of length p containing percentages
of work to be assigned to each partition */
Partition parts, /* Input: initial partition #s; aligned with vtx
arrays.
Output: computed partition #s */
PHGPartParams *hgp) /* Input: parameters for hgraph partitioning. */
{
PHGComm *hgc = hg->comm;
VCycle *vcycle=NULL, *del=NULL;
int i, err = ZOLTAN_OK, middle;
ZOLTAN_GNO_TYPE origVpincnt; /* for processor reduction test */
ZOLTAN_GNO_TYPE prevVcnt = 2*hg->dist_x[hgc->nProc_x]; /* initialized so that the */
ZOLTAN_GNO_TYPE prevVedgecnt = 2*hg->dist_y[hgc->nProc_y]; /* while loop will be entered
before any coarsening */
ZOLTAN_GNO_TYPE tot_nPins, local_nPins;
MPI_Datatype zoltan_gno_mpi_type;
char *yo = "Zoltan_PHG_Partition";
int do_timing = (hgp->use_timers > 1);
int fine_timing = (hgp->use_timers > 2);
int vcycle_timing = (hgp->use_timers > 4 && hgp->ProRedL == 0);
short refine = 0;
struct phg_timer_indices *timer = Zoltan_PHG_LB_Data_timers(zz);
int reset_geometric_matching = 0;
char reset_geometric_string[4];
ZOLTAN_TRACE_ENTER(zz, yo);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
if (do_timing) {
if (timer->vcycle < 0)
timer->vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
if (timer->procred < 0)
timer->procred = Zoltan_Timer_Init(zz->ZTime, 0, "Processor Reduction");
if (timer->match < 0)
timer->match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
if (timer->coarse < 0)
timer->coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
if (timer->coarsepart < 0)
timer->coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
"Coarse_Partition");
if (timer->refine < 0)
timer->refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
if (timer->project < 0)
timer->project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
MPI_Allreduce(&local_nPins,&tot_nPins,1,zoltan_gno_mpi_type,MPI_SUM,hgc->Communicator);
origVpincnt = tot_nPins;
if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
/* For geometric coarsening, hgp->matching pointer and string are reset
* after geometric_levels of coarsening. Will need to reset them after
* this vcycle is completed. Capture that fact now! */
if (!strcasecmp(hgp->redm_str, "rcb") || !strcasecmp(hgp->redm_str, "rib")) {
reset_geometric_matching = 1;
strcpy(reset_geometric_string, hgp->redm_str);
}
/****** Coarsening ******/
#define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */
while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > (ZOLTAN_GNO_TYPE)hg->redl)
&& ((hg->dist_x[hgc->nProc_x] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5)) /* prevVcnt initialized to 2*hg->dist_x[hgc->nProc_x] */
|| (hg->dist_y[hgc->nProc_y] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5))) /* prevVedgecnt initialized to 2*hg->dist_y[hgc->nProc_y] */
&& hg->dist_y[hgc->nProc_y] && hgp->matching) {
ZOLTAN_GNO_TYPE *match = NULL;
VCycle *coarser=NULL, *redistributed=NULL;
prevVcnt = hg->dist_x[hgc->nProc_x];
prevVedgecnt = hg->dist_y[hgc->nProc_y];
#ifdef _DEBUG
/* UVC: load balance stats */
Zoltan_PHG_LoadBalStat(zz, hg);
#endif
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc,
"START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->match, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_match < 0) {
char str[80];
sprintf(str, "VC Matching %d", hg->info);
vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
}
/* Allocate and initialize Matching Array */
if (hg->nVtx && !(match = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC (hg->nVtx*sizeof(ZOLTAN_GNO_TYPE)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
match[i] = i;
/* Calculate matching (packing or grouping) */
err = Zoltan_PHG_Matching (zz, hg, match, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_FREE (&match);
goto End;
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->match, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->coarse, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_coarse < 0) {
char str[80];
sprintf(str, "VC Coarsening %d", hg->info);
vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
}
if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
ZOLTAN_FREE (&match);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
goto End;
}
/* Construct coarse hypergraph and LevelMap */
err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
&vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData,
&vcycle->comm_plan, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarse, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
ZOLTAN_FREE (&match);
if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
goto End;
vcycle = coarser;
hg = vcycle->hg;
if (hgc->nProc > 1 && hgp->ProRedL > 0) {
local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
MPI_Allreduce(&local_nPins, &tot_nPins, 1, zoltan_gno_mpi_type, MPI_SUM,
hgc->Communicator);
if (tot_nPins < (ZOLTAN_GNO_TYPE)(hgp->ProRedL * origVpincnt + 0.5)) {
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->procred, hgc->Communicator);
}
/* redistribute to half the processors */
origVpincnt = tot_nPins; /* update for processor reduction test */
if(hg->nVtx&&!(hg->vmap=(int*)ZOLTAN_MALLOC(hg->nVtx*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: hg->vmap");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
hg->vmap[i] = i;
middle = (int)((float) (hgc->nProc-1) * hgp->ProRedL);
if (hgp->nProc_x_req!=1&&hgp->nProc_y_req!=1) { /* Want 2D decomp */
if ((middle+1) > SMALL_PRIME && Zoltan_PHG_isPrime(middle+1))
--middle; /* if it was prime just use one less #procs (since
it should be bigger than SMALL_PRIME it is safe to
decrement) */
}
if (!(hgc = (PHGComm*) ZOLTAN_MALLOC (sizeof(PHGComm)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: PHGComm");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!(redistributed=newVCycle(zz,NULL,NULL,vcycle,vcycle_timing))) {
ZOLTAN_FREE (&hgc);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "redistributed is NULL.");
goto End;
}
Zoltan_PHG_Redistribute(zz,hgp,hg,0,middle,hgc, redistributed->hg,
&vcycle->vlno,&vcycle->vdest);
if (hgp->UseFixedVtx || hgp->UsePrefPart)
redistributed->hg->bisec_split = hg->bisec_split;
if ((err=allocVCycle(redistributed))!= ZOLTAN_OK)
goto End;
vcycle = redistributed;
if (hgc->myProc < 0)
/* I'm not in the redistributed part so I should go to uncoarsening
refinement and wait */ {
if (fine_timing) {
if (timer->cpgather < 0)
timer->cpgather = Zoltan_Timer_Init(zz->ZTime, 1, "CP Gather");
if (timer->cprefine < 0)
timer->cprefine =Zoltan_Timer_Init(zz->ZTime, 0, "CP Refine");
if (timer->cpart < 0)
timer->cpart = Zoltan_Timer_Init(zz->ZTime, 0, "CP Part");
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
goto Refine;
}
hg = vcycle->hg;
hg->redl = hgp->redl; /* not set with hg creation */
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
}
}
}
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
/* free array that may have been allocated in matching */
if (hgp->vtx_scal) {
hgp->vtx_scal_size = 0;
ZOLTAN_FREE(&(hgp->vtx_scal));
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->coarsepart, hgc->Communicator);
}
/****** Coarse Partitioning ******/
err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarsepart, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
Refine:
del = vcycle;
refine = 1;
/****** Uncoarsening/Refinement ******/
while (vcycle) {
VCycle *finer = vcycle->finer;
hg = vcycle->hg;
if (refine && hgc->myProc >= 0) {
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->refine, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_refine < 0) {
char str[80];
sprintf(str, "VC Refinement %d", hg->info);
vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
}
err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->refine, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
if (hgp->output_level >= PHG_DEBUG_LIST)
uprintf(hgc,
"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, 0, p,
vcycle->Part),
Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
"partitioned plot");
}
if (finer) {
int *rbuffer;
/* Project coarse partition to fine partition */
if (finer->comm_plan) {
refine = 1;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->project, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_project < 0) {
char str[80];
sprintf(str, "VC Project Up %d", hg->info);
vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
}
/* easy to assign partitions to internal matches */
for (i = 0; i < finer->hg->nVtx; i++)
if (finer->LevelMap[i] >= 0) /* if considers only the local vertices */
finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
/* now that the course partition assignments have been propagated */
/* upward to the finer level for the local vertices, we need to */
/* fill the LevelData (matched pairs of a local vertex with a */
/* off processor vertex) with the partition assignment of the */
/* local vertex - can be done totally in the finer level! */
for (i = 0; i < finer->LevelCnt; i++) {
++i; /* skip over off processor lno */
finer->LevelData[i] = finer->Part[finer->LevelData[i]];
}
/* allocate rec buffer to exchange LevelData information */
rbuffer = NULL;
if (finer->LevelSndCnt > 0) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
/* get partition assignments from owners of externally matched vtxs */
Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1,
(char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);
/* process data to assign partitions to expernal matches */
for (i = 0; i < 2 * finer->LevelSndCnt;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->project, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
} else {
int *sendbuf = NULL, size;
refine = 0;
/* ints local and partition numbers */
if (finer->vlno) {
sendbuf = (int*) ZOLTAN_MALLOC (2 * hg->nVtx * sizeof(int));
if (!sendbuf) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; ++i) {
sendbuf[2 * i] = finer->vlno[i]; /* assign local numbers */
sendbuf[2 * i + 1] = vcycle->Part[i];/* assign partition numbers */
}
}
ZOLTAN_FREE (&hgc);
hgc = finer->hg->comm; /* updating hgc is required when the processors
change */
/* Create comm plan to unredistributed processors */
err = Zoltan_Comm_Create(&finer->comm_plan, finer->vlno ? hg->nVtx : 0,
finer->vdest, hgc->Communicator, COMM_TAG+2,
&size);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(hgc->myProc, yo, "Zoltan_Comm_Create failed.");
goto End;
}
/* allocate rec buffer to exchange sendbuf information */
rbuffer = NULL;
if (finer->hg->nVtx) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->hg->nVtx * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
/* Use plan to send partitions to the unredistributed processors */
Zoltan_Comm_Do(finer->comm_plan, COMM_TAG+3, (char *) sendbuf,
2*sizeof(int), (char *) rbuffer);
MPI_Bcast(rbuffer, 2*finer->hg->nVtx, MPI_INT, 0, hgc->col_comm);
/* process data to assign partitions to unredistributed processors */
for (i = 0; i < 2 * finer->hg->nVtx;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
if (finer->vlno)
ZOLTAN_FREE (&sendbuf);
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
}
}
vcycle = finer;
} /* while (vcycle) */
End:
vcycle = del;
while (vcycle) {
if (vcycle_timing) {
Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
Zoltan_Timer_Destroy(&vcycle->timer);
}
if (vcycle->finer) { /* cleanup by level */
Zoltan_HG_HGraph_Free (vcycle->hg);
if (vcycle->LevelData)
Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part,
&vcycle->LevelMap, &vcycle->LevelData, &vcycle->hg);
else if (vcycle->vlno)
Zoltan_Multifree (__FILE__, __LINE__, 5, &vcycle->Part, &vcycle->vdest,
&vcycle->vlno, &vcycle->LevelMap, &vcycle->hg);
else
Zoltan_Multifree (__FILE__, __LINE__, 3, &vcycle->Part,
&vcycle->LevelMap, &vcycle->hg);
}
else /* cleanup top level */
Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
&vcycle->LevelData);
del = vcycle;
vcycle = vcycle->finer;
ZOLTAN_FREE(&del);
}
if (reset_geometric_matching) {
strcpy(hgp->redm_str, reset_geometric_string);
Zoltan_PHG_Set_Matching_Fn(hgp);
}
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TRACE_EXIT(zz, yo) ;
return err;
}
