/*
* gather_by_list
* input: a list of processors, a message to be sent to these processors,
* and a receive buffer to get data back.
*
* output: concatenated messages from other processors in rbuff, with
* appropriate size info in rbuff_size.
*/
int gather_by_list(int procs_length, int *procs,
int sbuff_size, char *sbuff,
int *rbuff_size, char **rbuff, MPI_Comm *comm)
{
int i, receive_size, mtag, err, myProc;
char *send;
ZOLTAN_COMM_OBJ *plan;
static char *yo = "gather_by_list";
MPI_Comm_rank(*comm, &myProc);
if (!(send = (char*) ZOLTAN_MALLOC(procs_length * sizeof(int)))) {
ZOLTAN_PRINT_ERROR(myProc, yo, "Insufficient memory");
return ZOLTAN_MEMERR;
}
mtag = 0;
for (i = 0; i < procs_length; ++i)
((int*)send)[i] = sbuff_size;
/* create and resize communication plan */
err = Zoltan_Comm_Create(&plan, procs_length, procs, *comm,
mtag++, &receive_size);
if (err < 0)
ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Create failed.");
err = Zoltan_Comm_Resize(plan, (int*)send, mtag++, rbuff_size);
if (err < 0)
ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Resize failed.");
ZOLTAN_FREE(&send);
/* allocate send and receive buffer */
if ((!(*rbuff = (char*) ZOLTAN_MALLOC(*rbuff_size)) && (*rbuff_size != 0))
||!( send = (char*) ZOLTAN_MALLOC(sbuff_size * procs_length))) {
printf("failed allocating %d + %d * %d bytes. . .", *rbuff_size, sbuff_size, procs_length);
fflush(NULL);
ZOLTAN_PRINT_ERROR(myProc, yo, "Insufficient Memory");
return ZOLTAN_MEMERR;
}
/* copy message for each processor */
for (i = 0; i < procs_length; ++i)
memcpy(send + i * sbuff_size, sbuff, sbuff_size);
/* finally, we can do the communication */
err = Zoltan_Comm_Do(plan, mtag++, send, 1, *rbuff);
if (err < 0)
ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Do failed.");
/* clean up */
Zoltan_Comm_Destroy(&plan);
ZOLTAN_FREE(&send);
return err;
}
static int Compute_Adjpart(
ZZ *zz,
int nvtx, /* Input: # vtxs in this processor */
indextype *vtxdist, /* Input: Distribution of vertices across processors */
indextype *xadj, /* Input: Index of adjncy: adjncy[xadj[i]] to
adjncy[xadj[i]+1] are all edge nbors of vtx i. */
indextype *adjncy, /* Input: Array of nbor vertices. */
int *adjproc, /* Input: adjproc[j] == processor owning adjncy[j]. */
indextype *part, /* Input: Partition assignments of vtxs. */
int *adjpart /* Output: adjpart[j] == partition owning adjncy[j] */
)
{
/* Given an adjacency list adjncy, find the partition number of each
* vertex in adjncy. Return it in adjpart.
*/
ZOLTAN_COMM_OBJ *plan;
int i;
indextype start = vtxdist[zz->Proc]; /* First vertex on this processor */
indextype *recv_gno= NULL;
int *send_int=NULL;
int nrecv;
int tag = 24542;
Zoltan_Comm_Create(&plan, (int)xadj[nvtx], adjproc, zz->Communicator, tag++, &nrecv);
if (nrecv){
recv_gno = (indextype *) ZOLTAN_MALLOC(nrecv * sizeof(indextype));
send_int = (int *) ZOLTAN_MALLOC(nrecv * sizeof(int));
if (!recv_gno || !send_int) {
Zoltan_Comm_Destroy(&plan);
ZOLTAN_FREE(&recv_gno);
ZOLTAN_FREE(&send_int);
return ZOLTAN_MEMERR;
}
}
Zoltan_Comm_Do(plan, tag++, (char *) adjncy, sizeof(indextype), (char *) recv_gno);
for (i = 0; i < nrecv; i++){
send_int[i] = part[recv_gno[i] - start];
}
ZOLTAN_FREE(&recv_gno);
Zoltan_Comm_Do_Reverse(plan, tag, (char *)send_int, sizeof(int), NULL, (char *) adjpart);
ZOLTAN_FREE(&send_int);
Zoltan_Comm_Destroy(&plan);
return ZOLTAN_OK;
}
static int Zoltan_Oct_Final_Migration(
ZZ *zz,
pOctant *octs, /* octs[nocts] */
int *newpids, /* newpids[nocts] */
pOctant *newocts, /* newocts[nocts] */
int nocts, /* number of octants leaving this processor */
int nrecocts) /* number of octants received in this processor */
{
int i;
Migrate_msg *msnd = NULL, *mrcv = NULL;
int remotecount;
int nsends;
int nreceives;
int *despid = NULL;
int ierr = ZOLTAN_OK;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
char *yo = "Zoltan_Oct_Final_Migration";
OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure;
/* count number of sends */
nsends=0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc)
nsends++;
if((despid = (int *) ZOLTAN_MALLOC((nocts+10) * sizeof(int))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((msnd = (Migrate_msg *) ZOLTAN_MALLOC((nocts+10) * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
return ZOLTAN_MEMERR;
}
remotecount = 0;
for (i=0; i<nocts; i++) /* Send and free */
if (newpids[i]!=zz->Proc)
{
FILLMIGRATEMSG(octs[i], newocts[i], msnd[remotecount], zz->Proc); /* bug */
despid[remotecount++] = newpids[i];
Zoltan_Oct_clearRegions(octs[i]);
/* KDDKDDFREE Change oct to &oct to allow NULL from ZOLTAN_FREE
* KDDKDDFREE to propagate back. */
Zoltan_Oct_POct_free(OCT_info, &(octs[i]));
}
ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator,
MigFinCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
return (ierr);
}
if((mrcv = (Migrate_msg *) ZOLTAN_MALLOC((nreceives+10) * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
return ZOLTAN_MEMERR;
}
ierr = Zoltan_Comm_Do(comm_plan, MigFinCommDo, (char *) msnd,
sizeof(Migrate_msg), (char *) mrcv);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
ZOLTAN_FREE(&mrcv);
return (ierr);
}
for (i=0; i<nreceives; i++) { /* Receive new parocts */
/* Zoltan_Oct_setID(mrcv[i].ptr,mrcv[i].id); */
/* Zoltan_Oct_POct_setparent(OCT_info, mrcv[i].ptr, mrcv[i].parent, mrcv[i].ppid); */
SETOCTFROMMIGRATEMSG(OCT_info, mrcv[i]);
/* Zoltan_Oct_setchildnum(mrcv[i].ptr,mrcv[i].childnum); */
/* Zoltan_Oct_setchildren(mrcv[i].ptr, mrcv[i].children, mrcv[i].cpids); */
/* Zoltan_Oct_setbounds(mrcv[i].ptr, mrcv[i].min, mrcv[i].max); */
/* Zoltan_Oct_setDir(mrcv[i].ptr,mrcv[i].dir); */
/* Zoltan_Oct_setMapIdx(mrcv[i].ptr,mrcv[i].mapidx); */
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
ZOLTAN_FREE(&mrcv);
return (ierr);
}
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&msnd);
ZOLTAN_FREE(&mrcv);
return ierr;
}
int Zoltan_DD_Remove (
Zoltan_DD_Directory *dd, /* directory state infomation */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to remove */
int count) /* Number of GIDs in removal list */
{
int *procs = NULL; /* list of processors to contact */
DD_Remove_Msg *ptr = NULL;
ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */
char *sbuff = NULL; /* send buffer */
char *sbufftmp = NULL;/* pointer into send buffer */
char *rbuff = NULL; /* receive buffer */
char *rbufftmp = NULL;/* pointer into receive buffer */
int nrec; /* number of receives to expect */
int i;
int err; /* error condition to return */
int errcount; /* count of GIDs not found */
char str[100]; /* string to build error messages */
char *yo = "Zoltan_DD_Remove";
/* input sanity checks */
if (dd == NULL || count < 0 || (gid == NULL && count > 0)) {
ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
"Invalid input argument");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL);
/* allocate memory for processor contact list */
if (count) {
procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
if (procs == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
}
/* allocate memory for DD_Remove_Msg send buffer */
if (count) {
sbuff = (char*)ZOLTAN_MALLOC((size_t)(dd->remove_msg_size)*(size_t)count);
if (sbuff == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
err = ZOLTAN_MEMERR;
goto fini;
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After proc & sbuff mallocs");
/* for each GID, fill in contact list and then message structure */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc,
dd->hashdata, dd->hashfn);
ptr = (DD_Remove_Msg*) sbufftmp;
sbufftmp += dd->remove_msg_size;
ptr->owner = dd->my_proc;
ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length);
}
/* now create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_REMOVE_MSG_TAG, &nrec);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error");
goto fini;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Create");
/* allocate receive buffer for nrec DD_Remove_Msg structures */
if (nrec) {
rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->remove_msg_size));
if (rbuff == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed");
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* send my remove messages & receive removes directed to me */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff,
dd->remove_msg_size, rbuff);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error");
goto fini;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Do");
/* for each message rec'd, remove local directory info */
errcount = 0;
rbufftmp = rbuff;
for (i = 0; i < nrec; i++) {
ptr = (DD_Remove_Msg*) rbufftmp;
rbufftmp += dd->remove_msg_size;
err = DD_Remove_Local (dd, ptr->gid);
if (err == ZOLTAN_WARN)
++errcount;
}
err = ZOLTAN_OK;
if (dd->debug_level) {
sprintf (str, "Processed %d GIDs (%d local), %d GIDs not found",
count, nrec, errcount);
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK;
}
/* done, now free up things and return */
fini:
ZOLTAN_FREE (&procs);
ZOLTAN_FREE (&sbuff);
ZOLTAN_FREE (&rbuff);
Zoltan_Comm_Destroy (&plan);
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL);
return err;
}
/*
* void malloc_new_objects();
*
* gets the tags being imported into this processor, and sets up the
* import_tags array, and the nrectags array.
*/
static int malloc_new_objects(ZZ *zz, int nsentags, pRegion exported_tags,
ZOLTAN_ID_PTR exported_gids,
ZOLTAN_ID_PTR exported_lids, int *tag_pids,
int *nstags, pRegion *ex_tags,
pRegion prev_tags, ZOLTAN_ID_PTR prev_gids,
ZOLTAN_ID_PTR prev_lids, int npimtags,
float *c3)
{
char *yo = "malloc_new_objects";
int i; /* index counter */
int nreceives; /* number of messages received */
pRegion t_b_exp; /* array of tags to be exported */
pRegion tmp = NULL;
ZOLTAN_ID_PTR tmp_gids = NULL;
ZOLTAN_ID_PTR tmp_lids = NULL;
int msgtag, msgtag2;
int j;
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
float im_load;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
im_load = 0;
msgtag = 32767;
ierr = Zoltan_Comm_Create(&comm_plan, nsentags, tag_pids, zz->Communicator,
msgtag, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_Comm_Create.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (nreceives > 0) {
tmp = (pRegion) ZOLTAN_MALLOC(nreceives * sizeof(Region));
tmp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, nreceives);
tmp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, nreceives);
if(tmp == NULL || !tmp_gids || (num_lid_entries && !tmp_lids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_tags,
sizeof(Region), (char *) tmp);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_gids,
sizeof(ZOLTAN_ID_TYPE)*num_gid_entries,
(char *) tmp_gids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
fprintf(stderr, "OCT %s Error %s returned from Zoltan_Comm_Do\n", yo,
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
if (num_lid_entries > 0) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_lids,
sizeof(ZOLTAN_ID_TYPE)*num_lid_entries,
(char *) tmp_lids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_Comm_Destroy.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
return(ierr);
}
/* get each message sent, and store region in import array */
j=0;
for (i=0; i<nreceives; i++) {
im_load += tmp[i].Weight;
if(tmp[i].newProc != zz->Proc) {
j++;
}
}
if((j + npimtags) != 0) { /* malloc import array */
if((t_b_exp = (pRegion)ZOLTAN_MALLOC((j+npimtags)*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
}
else
t_b_exp = NULL;
/* setup return pointer */
(*ex_tags) = t_b_exp;
j=0;
for (i=0; i<nreceives; i++) {
if(tmp[i].newProc != zz->Proc) {
t_b_exp[j] = tmp[i];
t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz);
if (!(t_b_exp[j].Global_ID) ||
(num_lid_entries && !(t_b_exp[j].Local_ID))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
&(tmp_gids[i*num_gid_entries]));
ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(tmp_lids[i*num_lid_entries]));
j++;
}
}
if(npimtags > 0) {
for(i=0; i<npimtags; i++) {
t_b_exp[j] = prev_tags[i];
t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz);
if (!(t_b_exp[j].Global_ID) ||
(num_lid_entries && !(t_b_exp[j].Local_ID))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
&(prev_gids[i*num_gid_entries]));
ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID,
&(prev_lids[i*num_lid_entries]));
j++;
}
}
*nstags = j;
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
if((*nstags == 0) && (*ex_tags != NULL)) {
ZOLTAN_TRACE_DETAIL(zz, yo,
"Fatal error, import tags not empty but no tags received\n");
return ZOLTAN_FATAL;
}
*c3 = im_load;
return ierr;
}
int main(int argc, char *argv[])
{
int rc, i, ngids, maxcol, ncolors;
float ver;
struct Zoltan_Struct *zz=NULL;
#ifdef ZOLTANV31
int numGidEntries, numLidEntries;
#else
ZOLTAN_GRAPH_EVAL graph;
#endif
int *color;
ZOLTAN_ID_PTR gid_list;
UZData guz, *uz=&guz;
int msg_tag = 9999;
int nlvtx, next, maxdeg=0;
double times[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
double gtimes[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &uz->myRank);
MPI_Comm_size(MPI_COMM_WORLD, &uz->numProcs);
MPI_Barrier(MPI_COMM_WORLD);
times[0] = u_wseconds();
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
fprintf(stderr, "Sorry Zoltan initialize failed...\n");
goto End;
}
zz = Zoltan_Create(MPI_COMM_WORLD);
if (argc<3 && !uz->myRank) {
fprintf(stderr, "usage: %s [meshR] [meshC] [X-point stencil] [procR] [procC] [ws-beta] [<ZoltanParam>=<Val>] ...\n\n", argv[0]);
fprintf(stderr, "ws-beta: is the probablity of adding an edge to a vertex to generate Watts-Strogatz graphs\n");
fprintf(stderr, "Valid values for Stencil are 5, 7 and 9\n");
fprintf(stderr, "Zoltan Coloring Parameters and values are\n");
fprintf(stderr, "\tDISTANCE : 1 or 2\n");
fprintf(stderr, "\tSUPERSTEP_SIZE : suggested >= 100\n");
fprintf(stderr, "\tCOMM_PATTERN : S or A\n");
fprintf(stderr, "\tCOLOR_ORDER : I, B, U\n");
fprintf(stderr, "\tCOLORING_METHOD : F (for now)\n");
fprintf(stderr, "\n");
}
uz->procR = uz->procC = 0;
uz->meshR = uz->meshC = 1024;
uz->stencil = 9;
if (argc>1)
uz->meshR = atoi(argv[1]);
if (argc>2)
uz->meshC = atoi(argv[2]);
if (argc>3)
uz->stencil = atoi(argv[3]);
if (uz->stencil!=5 && uz->stencil!=7 && uz->stencil!=9) {
fprintf(stderr, "\t invalid stencil value. Valid values are 5, 7 and 9. Assumed 9.\n");
uz->stencil = 9;
}
--uz->stencil;
if (argc>4)
uz->procR = atoi(argv[4]);
if (argc>5)
uz->procC = atoi(argv[5]);
if (uz->procR <= 0 || uz->procC <= 0)
computeProcMesh(uz);
if (uz->procR*uz->procC!=uz->numProcs) {
fprintf(stderr, "#Procs=%d but requested %dx%d Proc Mesh Partitioning...\n", uz->numProcs, uz->procR, uz->procC);
goto End;
}
if (argc>6)
uz->beta = atof(argv[6]);
else
uz->beta = 0.0;
/* compute which part of mesh I will compute */
uz->myR = uz->myRank / uz->procC;
uz->myC = uz->myRank % uz->procC;
uz->_sr = uz->myR * (uz->meshR / uz->procR);
uz->_er = (uz->myR+1) * (uz->meshR / uz->procR);
if (uz->_er>uz->meshR)
uz->_er = uz->meshR;
uz->_sc = uz->myC * (uz->meshC / uz->procC);
uz->_ec = (uz->myC+1) * (uz->meshC / uz->procC);
if (uz->_ec>uz->meshC)
uz->_ec = uz->meshC;
if ( (uz->meshR % uz->procR) !=0 || (uz->meshC % uz->procC)!=0) {
printf("Mesh dimensions are not divisible with proc mesh.\nRequested mesh is %dx%d and proc mesh is %d x %d\n", uz->meshR, uz->meshC, uz->procR, uz->procC);
exit(1);
}
nlvtx= (uz->_er-uz->_sr) * (uz->_ec-uz->_sc);
if (uz->myRank==0)
printf("Running %s on %d x %d processor mesh, generating %d-point %d x %d mesh with beta=%.3lf\n", argv[0], uz->procR, uz->procC, uz->stencil+1, uz->meshR, uz->meshC, uz->beta);
times[1] = u_wseconds();
uz->numredge = 0;
uz->redgeto = NULL;
if (uz->beta>0) { /* create random edges for WS graph */
int ngvtx=uz->meshC*uz->meshR, trsh=(int) (uz->beta*100.0);
int ierr=0;
int *edges=NULL, *redges=NULL, *proclist=NULL, nedge;
ZOLTAN_COMM_OBJ *plan;
uz->redgeto = (int *) malloc(nlvtx*sizeof(int));
for (i=0; i<nlvtx; ++i) {
int rv = Zoltan_Rand_InRange(NULL, 100);
if ( rv < trsh) {
if ((uz->redgeto[i] = Zoltan_Rand_InRange(NULL, ngvtx))==gIDfLID(i)) /* is it a self edge */
uz->redgeto[i] = -1;
else
++uz->numredge;
} else
uz->redgeto[i] = -1;
}
edges = (int *) malloc(sizeof(int)*2*uz->numredge);
proclist = (int *) malloc(sizeof(int)*uz->numredge);
next = 0;
for (i=0; i<nlvtx; ++i)
if (uz->redgeto[i]>0) {
edges[2*next] = uz->redgeto[i];
edges[2*next+1] = gIDfLID(i);
proclist[next] = pIDfGID(uz->redgeto[i]);
++next;
}
ierr = Zoltan_Comm_Create(&plan, uz->numredge, proclist, MPI_COMM_WORLD,
msg_tag, &nedge);
redges = (int *) malloc(sizeof(int)*2*nedge);
--msg_tag;
ierr |= Zoltan_Comm_Do(plan, msg_tag, (char *) edges, 2*sizeof(int),
(char *) redges);
ierr |= Zoltan_Comm_Destroy(&plan);
free(proclist);
free(edges);
if (ierr) {
printf("error while communicating edges!\n");
exit(1);
}
xadj = (int *) calloc(1+nlvtx, sizeof(int));
adj = (int *) malloc(sizeof(int)*nedge);
for (i=0; i<nedge; ++i) {
if (redges[2*i] < gID(uz->_sr, uz->_sc) || redges[2*i] >= gID(uz->_er, uz->_ec)) {
printf("[%d/%d] received gid=%d doesn't blong to processor range [%d, %d) should go to proc %d\n", uz->myRank, uz->numProcs, redges[2*i], gID(uz->_sr, uz->_sc), gID(uz->_er, uz->_ec), pIDfGID(redges[2*i]));
}
++xadj[lIDfGID(redges[2*i])];
}
xadj[nlvtx] = nedge;
maxdeg = xadj[0];
for (i=1; i<nlvtx; ++i) {
maxdeg = xadj[i]>maxdeg ? xadj[i] : maxdeg;
xadj[i] += xadj[i-1];
}
for (i=0; i<nedge; ++i) {
int u = lIDfGID(redges[2*i]);
int v = redges[2*i+1];
adj[--xadj[u]] = v;
}
free(redges);
}
maxdeg += uz->stencil+1;
adjTemp = (int *) malloc(sizeof(int)*2*maxdeg);
times[2] = u_wseconds();
/*
printf("My rank %d/%d at proc-mesh loc (%d, %d) generating [%d, %d) x [%d, %d) + %d random edges TotEdge=%d\n", uz->myRank, uz->numProcs, uz->myR, uz->myC, uz->_sr, uz->_er, uz->_sc, uz->_ec, uz->numredge, xadj[nlvtx]); */
printStats("Number of Vertices ", nlvtx, uz->myRank, uz->numProcs);
if (xadj)
printStats("Number of Rand Edges", xadj[nlvtx], uz->myRank, uz->numProcs);
/* General parameters */
#ifndef ZOLTANV31
Zoltan_Set_Param(zz, "GRAPH_BUILD_TYPE", "FAST_NO_DUP");
#endif
/* General parameters */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "3");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");
/* coloring parameters */
Zoltan_Set_Param(zz, "SUPERSTEP_SIZE", "500"); /* let's make S=500 default */
for (i=7; i<argc; ++i) {
char param[256], *eq;
if (!uz->myRank)
printf("processing argv[%d]='%s'\n", i, argv[i]);
strncpy(param, argv[i], sizeof(param));
eq = strchr(param, '=');
if (!eq) {
fprintf(stderr, "invalid argument '%s', Zoltan Paramters should be in the format <ZoltanParam>=<Val>\n", param);
goto End;
}
*eq = 0;
Zoltan_Set_Param(zz, param, eq+1);
}
#if 0
/* Graph parameters */
Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
#endif
/* set call backs */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, uz);
Zoltan_Set_Obj_List_Fn(zz, get_object_list, uz);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, uz);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, uz);
#if 0
#ifndef ZOLTANV31
Zoltan_LB_Eval_Graph(zz, 0, &graph);
if (!uz->myRank) {
printf("EdgeCut Min=%8.0f Max=%8.0f Sum=%8.0f\n", graph.cuts[EVAL_GLOBAL_MIN], graph.cuts[EVAL_GLOBAL_MAX], graph.cuts[EVAL_GLOBAL_SUM]);
printf("#Vertices Min=%8.0f Max=%8.0f Sum=%8.0f imbal=%.2f\n", graph.nobj[EVAL_GLOBAL_MIN], graph.nobj[EVAL_GLOBAL_MAX], graph.nobj[EVAL_GLOBAL_SUM], graph.obj_imbalance);
}
#endif
#endif
/* now color */
ngids = get_number_of_objects(uz, &rc);
gid_list = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * ngids);
#ifndef ZOLTANV31
next = 0;
for (i=uz->_sr; i<uz->_er; ++i) {
int j;
for (j=uz->_sc; j<uz->_ec; ++j) {
gid_list[next++] = i*uz->meshC + j;
}
}
#endif
color = (int *) malloc(sizeof(int) * ngids);
MPI_Barrier(MPI_COMM_WORLD);
times[3] = u_wseconds();
#ifdef ZOLTANV31
rc = Zoltan_Color(zz, /* input (all remaining fields are output) */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
ngids, /* #objects to color in this proc */
gid_list, /* global ids of colored vertices */
NULL, /* we ignore local ids */
color); /* result color */
#else
rc = Zoltan_Color(zz, /* input (all remaining fields are output) */
1, /* Number of integers used for a global ID */
ngids, /* #objects to color in this proc */
gid_list, /* global ids of colored vertices */
color); /* result color */
#endif
MPI_Barrier(MPI_COMM_WORLD);
times[4] = u_wseconds();
MPI_Reduce(times, gtimes, 5, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (rc != ZOLTAN_OK)
fprintf(stderr, "Zoltan_Color failed with return code %d...\n", rc);
for (maxcol=i=0; i<ngids; ++i)
if (color[i] > maxcol)
maxcol = color[i];
MPI_Reduce(&maxcol, &ncolors, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
if (uz->myRank==0) {
struct rusage usage;
printf("%s setup Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[1]-times[0], gtimes[1]-gtimes[0]);
printf("%s gen rand edges Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[2]-times[1], gtimes[2]-gtimes[1]);
printf("%s set gids Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[3]-times[2], gtimes[3]-gtimes[2]);
printf("%s Zoltan_Color call Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[4]-times[3], gtimes[4]-gtimes[3]);
printf("%s Coloring Time : %.2lf # Colors used : %d\n", argv[0], gtimes[4]-gtimes[0], ncolors);
getrusage(RUSAGE_SELF, &usage);
printf("%s maxrss=%ld minflt=%ld majflt=%ld nswap=%ld\n", argv[0], usage.ru_maxrss, usage.ru_minflt, usage.ru_majflt, usage.ru_nswap);
}
#ifdef _DEBUG
saveColor(argv[0], uz, (int *) gid_list, color, ngids);
#endif
/******************************************************************
** Clean up
******************************************************************/
if (gid_list)
free(gid_list);
if (color)
free(color);
if (xadj)
free(xadj);
if (adj)
free(adj);
if (adjTemp)
free(adjTemp);
if (uz->redgeto)
free(uz->redgeto);
End:
Zoltan_Destroy(&zz);
MPI_Finalize();
return 0;
}
int Zoltan_Invert_Lists(
ZZ *zz, /* Zoltan structure. */
int num_in, /* Number of objects in the input lists. */
ZOLTAN_ID_PTR in_global_ids, /* Array of input global IDs. */
ZOLTAN_ID_PTR in_local_ids, /* Array of input local IDs. */
int *in_procs, /* Array of processor IDs of processors owning
the input objects. */
int *in_to_part, /* Optional: Array of partition numbers to
which input objects should be assigned. */
int *num_out, /* Returned value: Number of output objs. */
ZOLTAN_ID_PTR *out_global_ids,/* Returned value: Array of global IDs of
output objects. */
ZOLTAN_ID_PTR *out_local_ids, /* Returned value: Array of local IDs of
output objects. */
int **out_procs, /* Returned value: Array of processor IDs
to which output objects are assigned. */
int **out_to_part /* Optional: Returned value: Array of
partition numbers to which output
objects should be assigned. */
)
{
/*
* Routine to compute the inverse map. Can be used in two ways:
* 1. Given, for each processor, a list of objects to be received by the
* processor, compute the list of objects that the processor needs to send
* to other processors to satisfy their needs.
* 2. Given, for each processor, a list of objects to be sent to other
* processors, compute the list of objects that the processor needs to receive
* to satisfy its needs.
*/
char *yo = "Zoltan_Invert_Lists";
char msg[256];
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned communication routines */
int msgtag, msgtag2; /* Message tags for communication routines */
int num_gid_entries, num_lid_entries; /* Length of global and local ids */
int include_parts; /* Flag indicating whether to compute
inverse list for partitions. */
int ierr, ret_ierr = ZOLTAN_OK;
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Check that all procs use the same id types.
*/
ierr = check_invert_input(zz, num_in, in_procs, in_to_part,
&num_gid_entries, &num_lid_entries, &include_parts);
if (ierr != ZOLTAN_OK) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
/* Initialize returned arrays. */
*out_global_ids = NULL;
*out_local_ids = NULL;
*out_procs = NULL;
if (include_parts) *out_to_part = NULL;
/*
* Compute communication map and num_out, the number of objs this
* processor has to out to establish the new decomposition.
*/
msgtag = 32767;
ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator,
msgtag, num_out);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Create.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
goto End;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create");
/*
* Allocate space for the object tags that need to be outed. Communicate
* to get the list of objects to be outed.
*/
if (*num_out > 0) {
if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out,
ZOLTAN_SPECIAL_MALLOC_GID)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out,
ZOLTAN_SPECIAL_MALLOC_LID)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (include_parts) {
if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
}
}
/*
* Use the communication plan to send global IDs, local IDs, and processor
* numbers. Do in separate communications to avoid a memory copy and to
* simplify implementation when a data type is added to the comm. package
* (to support heterogeneous computing).
*/
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) *out_global_ids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) *out_local_ids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
}
Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, *out_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part,
(int) sizeof(int), (char *) *out_to_part);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do");
End:
Zoltan_Comm_Destroy(&comm_plan);
if (ret_ierr == ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT);
if (include_parts)
Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ret_ierr);
}
static int read_comm_map_info(int pexoid, int Proc, PROB_INFO_PTR prob,
MESH_INFO_PTR mesh)
{
/* Local declarations. */
char *yo = "read_comm_map_info";
int ielem, imap, loc_elem, iblk, max_len, offset, index;
int nnodei, nnodeb, nnodee, nelemi, nelemb, nncmap;
int *int_elem, *bor_elem;
int *proc_ids;
int *gids, *my_procs, *recv_procs;
int ierr, nrecv;
int msg = 200;
int sid;
ELEM_INFO_PTR elements = mesh->elements;
ZOLTAN_COMM_OBJ *comm_obj;
E_Type etype;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
if (ne_get_loadbal_param(pexoid, &nnodei, &nnodeb, &nnodee,
&nelemi, &nelemb, &nncmap, &(mesh->necmap), Proc) < 0) {
Gen_Error(0, "fatal: Error returned from ne_get_loadbal_param");
return 0;
}
/*
* get the list of the border elements in order to set
* the border flag in the element structures
*/
int_elem = (int *) malloc ((nelemi + nelemb) * sizeof(int));
if (!int_elem) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
bor_elem = int_elem + nelemi;
if (ne_get_elem_map(pexoid, int_elem, bor_elem, Proc) < 0) {
Gen_Error(0, "fatal: Error returned from ne_get_elem_map");
return 0;
}
for (ielem = 0; ielem < nelemb; ielem++) {
elements[bor_elem[ielem]-1].border = 1;
}
free(int_elem);
/*
* For now, only get the elemental communication maps,
* since, in the driver, elements are only considered
* adjacent if they share a face (same definition used
* in element communication maps). Eventually, the ability
* to consider elements that are connected by any nodes
* adjacent will have to be added. When that happens,
* the nodal communication maps will be needed.
*/
mesh->ecmap_cnt = (int *) malloc (mesh->necmap * sizeof(int));
mesh->ecmap_id = (int *) malloc(mesh->necmap * sizeof(int));
if (!mesh->ecmap_cnt || !mesh->ecmap_id) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (ne_get_cmap_params(pexoid, NULL, NULL, mesh->ecmap_id,
mesh->ecmap_cnt, Proc) < 0) {
Gen_Error(0, "fatal: Error returned from ne_get_cmap_params");
return 0;
}
max_len = 0;
for (imap = 0; imap < mesh->necmap; imap++)
max_len += mesh->ecmap_cnt[imap];
proc_ids = (int *) malloc(4 * max_len * sizeof(int));
gids = proc_ids + max_len;
my_procs = gids + max_len;
recv_procs = my_procs + max_len;
mesh->ecmap_elemids = (int *) malloc(max_len * sizeof(int));
mesh->ecmap_sideids = (int *) malloc(max_len * sizeof(int));
mesh->ecmap_neighids = (int *) malloc(max_len * sizeof(int));
if (!mesh->ecmap_elemids || !mesh->ecmap_sideids || !mesh->ecmap_neighids) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
offset = 0;
for (imap = 0; imap < mesh->necmap; imap++) {
if(ne_get_elem_cmap(pexoid, mesh->ecmap_id[imap],
&(mesh->ecmap_elemids[offset]),
&(mesh->ecmap_sideids[offset]),
&(proc_ids[offset]), Proc) < 0) {
Gen_Error(0, "fatal: Error returned from ne_get_elem_cmap");
return 0;
}
offset += mesh->ecmap_cnt[imap];
} /* End: "for (imap = 0; imap < mesh->necmap; imap++)" */
/*
* Decrement the ecmap_elemids by one for zero-based local numbering.
* Convert the element ids to global ids to send to
* the neighboring processor.
*/
for (ielem = 0; ielem < max_len; ielem++) {
mesh->ecmap_elemids[ielem]--;
gids[ielem] = elements[mesh->ecmap_elemids[ielem]].globalID;
my_procs[ielem] = Proc;
}
/*
* Now communicate with other processor to get global IDs
* for the adjacent elements in this communication map.
*/
ierr = Zoltan_Comm_Create(&comm_obj, max_len, proc_ids, MPI_COMM_WORLD,
msg, &nrecv);
if (ierr != ZOLTAN_OK) {
Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Create");
return 0;
}
if (nrecv != max_len) {
/* Sanity check; this should never happen. */
Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Create");
return 0;
}
/* Exchange ids to neighbors.
* Assuming messages will be stored in order of processor number in
* ecmap_neighids.
*/
ierr = Zoltan_Comm_Do(comm_obj, msg+1, (char *) gids, sizeof(int),
(char *) (mesh->ecmap_neighids));
/* Exchange sanity check information.
* Allows to check assumption that messages are stored in order of
* processor number.
*/
if (ierr == ZOLTAN_OK)
ierr = Zoltan_Comm_Do(comm_obj, msg+2, (char *) my_procs, sizeof(int),
(char *) recv_procs);
if (ierr != ZOLTAN_OK) {
Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Do");
return 0;
}
ierr = Zoltan_Comm_Destroy(&comm_obj);
/* Sanity check: messages stored in order of processor number. */
for (ielem = 0; ielem < max_len; ielem++) {
if (proc_ids[ielem] != recv_procs[ielem]) {
Gen_Error(0, "fatal: Sanity check failed; assumption wrong");
return 0;
}
}
/* now process all of the element ids that have been received */
offset = 0;
for (imap = 0; imap < mesh->necmap; imap++) {
for (ielem = 0; ielem < mesh->ecmap_cnt[imap]; ielem++) {
index = ielem + offset;
/* translate from element id in the communication map to local elem id */
loc_elem = mesh->ecmap_elemids[index];
iblk = elements[loc_elem].elem_blk;
etype = (E_Type) (mesh->eb_etypes[iblk]);
(elements[loc_elem].nadj)++;
if(elements[loc_elem].nadj > elements[loc_elem].adj_len) {
/* Shouldn't happen as long as only side adjacencies are used. */
/* adj_len == number of sides. */
/* Space should already be allocated for the adjacencies read */
/* from the communication maps. */
Gen_Error(0, "fatal: Number of adj greater than adj_len");
return 0;
}
/* Store adjacency info in the adj entry corresponding to this side. */
sid = mesh->ecmap_sideids[index] - 1;
elements[loc_elem].adj[sid] = mesh->ecmap_neighids[index];
elements[loc_elem].adj_proc[sid] = mesh->ecmap_id[imap];
elements[loc_elem].edge_wgt[sid] =
(float) get_elem_info(NSNODES, etype, mesh->ecmap_sideids[index]);
} /* End: "for (ielem = 0; ielem < mesh->ecmap_cnt[imap]; ielem++)" */
offset += mesh->ecmap_cnt[imap];
} /* End: "for for (imap = 0; imap < mesh->necmap; imap++)" */
free (proc_ids);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
static int
Zoltan_Preprocess_Scatter_Graph (ZZ *zz,
ZOLTAN_Third_Graph *gr,
ZOLTAN_Third_Part *prt,
ZOLTAN_Third_Geom *geo,
ZOLTAN_Third_Vsize *vsp)
{
int ierr = ZOLTAN_OK;
int tmp;
if ((gr->scatter>0) && (gr->scatter<3)){
int j;
/* Decide if the data imbalance is so bad that we should scatter the graph. */
/* scatter==1: Scatter if all the objects are on a single processor. */
/* scatter==2: Scatter if any processor has no objects. */
gr->num_obj_orig = gr->num_obj; /* Save value for later. */
if (gr->num_obj==0)
j = 1;
else
j = 0;
MPI_Allreduce(&j, &tmp, 1, MPI_INT, MPI_SUM, zz->Communicator);
if (gr->scatter == 1){
if (tmp < zz->Num_Proc-1)
gr->scatter = 0;
}
else if (gr->scatter==2){
if (tmp == 0)
gr->scatter = 0;
}
}
/* We need to make sure we don't scatter the graph
* if graph_type = LOCAL_GRAPH, i.e. METIS is used.
*/
if (gr->scatter && (gr->graph_type == LOCAL_GRAPH)){
gr->scatter = 0;
ZOLTAN_PRINT_WARN(zz->Proc, __func__, "Setting scatter_graph=0 since the graph"
" is local on each proc");
ierr = ZOLTAN_WARN;
}
if (gr->scatter){
if (geo)
ierr = Zoltan_Scatter_Graph(&gr->vtxdist, &gr->xadj, &gr->adjncy, &gr->vwgt, &vsp->vsize,
&gr->ewgts, &geo->xyz, geo->ndims, gr->obj_wgt_dim, zz, &gr->comm_plan);
else {
float* xyz = NULL;
ierr = Zoltan_Scatter_Graph(&gr->vtxdist, &gr->xadj, &gr->adjncy, &gr->vwgt, &vsp->vsize,
&gr->ewgts, &xyz, 0, gr->obj_wgt_dim, zz, &gr->comm_plan);
}
if ((ierr == ZOLTAN_FATAL) || (ierr == ZOLTAN_MEMERR)){
ZOLTAN_THIRD_ERROR(ierr, "Zoltan_Scatter_Graph returned error.");
}
gr->num_obj = gr->vtxdist[zz->Proc+1]-gr->vtxdist[zz->Proc];
if (prt) {
prt->part_orig = prt->part;
prt->part = (indextype *) ZOLTAN_MALLOC((gr->num_obj+1) * sizeof(indextype));
Zoltan_Comm_Do(gr->comm_plan, TAG1, (char *) prt->part_orig, sizeof(indextype),
(char *) prt->part);
}
}
return ierr;
}
int Zoltan_Order_Get_GID_Order(
struct Zoltan_Struct *zz,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR order_ids
)
{
int * proctab;
ZOLTAN_ID_PTR sendgidtab;
int * sendtab, *recvtab;
int i;
int nrecv;
struct Zoltan_Comm_Obj *plan;
int ierr = ZOLTAN_OK;
int *vtxdist;
int tag = 24061986;
int offset;
zz->Order.gidrank = order_ids;
if (!strcmp(zz->Order.order_type, "LOCAL")) {
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
order_ids[i] = global_ids[zz->Order.rank[i] - zz->Order.start_index];
}
return (ZOLTAN_OK);
}
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr != ZOLTAN_OK) return (ierr);
proctab = (int*) ZOLTAN_MALLOC(3*zz->Order.nbr_objects*sizeof(int));
sendgidtab = ZOLTAN_MALLOC_GID_ARRAY(zz, 2*zz->Order.nbr_objects);
if (proctab == NULL) {
ZOLTAN_FREE(&vtxdist);
return (ZOLTAN_MEMERR);
}
if (sendgidtab == NULL) {
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ZOLTAN_MEMERR);
}
sendtab = proctab + zz->Order.nbr_objects;
recvtab = sendtab + zz->Order.nbr_objects;
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
proctab[i] = Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, zz->Order.rank[i] - zz->Order.start_index);
sendtab[i] = zz->Order.rank[i] - zz->Order.start_index;
}
ierr = Zoltan_Comm_Create(&plan, zz->Order.nbr_objects, proctab, zz->Communicator, tag++,
&nrecv);
if (ierr != ZOLTAN_OK) {
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
ierr = Zoltan_Comm_Do(plan, tag++, (char *) sendtab, sizeof(int), (char *) recvtab);
if (ierr != ZOLTAN_OK) {
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
offset = vtxdist[zz->Proc];
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
int position;
position = recvtab[i]-offset;
ZOLTAN_SET_GID(zz, &sendgidtab[i], &global_ids[position]);
}
ierr = Zoltan_Comm_Do_Reverse(plan, tag++, (char *) sendgidtab, zz->Num_GID*sizeof(int), NULL,
(char *) order_ids);
Zoltan_Comm_Destroy(&plan);
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
/* if !copy, inmat is not usable after this call */
int
Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */
Zoltan_matrix_2d *outmat, int copy)
{
static char *yo = "Zoltan_Matrix2d_Distribute";
int ierr = ZOLTAN_OK;
int nProc_x, nProc_y;
int myProc_x, myProc_y;
int i, j, cnt;
int *proclist = NULL;
Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL;
ZOLTAN_GNO_TYPE *perm_y = NULL;
float *wgtarray = NULL;
float *tmpwgtarray = NULL;
int msg_tag = 1021982;
ZOLTAN_COMM_OBJ *plan;
MPI_Comm communicator = MPI_COMM_NULL;
int nProc;
ZOLTAN_GNO_TYPE *yGNO = NULL;
ZOLTAN_GNO_TYPE *pinGNO = NULL;
ZOLTAN_GNO_TYPE tmp_gno;
void *partdata = NULL;
MPI_Datatype zoltan_gno_mpi_type;
ZOLTAN_TRACE_ENTER(zz, yo);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix));
if(copy) {
/* TODO: We need to copy the arrays also */
Zoltan_Matrix_Reset (&outmat->mtx);
/* Copy also directories */
outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX);
if (inmat.ddY == inmat.ddX)
outmat->mtx.ddY = outmat->mtx.ddX;
else
outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY);
}
communicator = outmat->comm->Communicator;
nProc = outmat->comm->nProc;
nProc_x = outmat->comm->nProc_x;
nProc_y = outmat->comm->nProc_y;
myProc_x = outmat->comm->myProc_x;
myProc_y = outmat->comm->myProc_y;
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_Duplicates");
ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx);
/* KDDKDDKDD FIX INDENTATION OF THIS BLOCK */
if (inmat.opts.speed != MATRIX_NO_REDIST) {
if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) {
/* I need to know the original distribution */
if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */
outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear;
}
else {
int *cmember = NULL;
cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int));
if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR;
Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL,
outmat->mtx.nY, NULL);
KDDKDDKDD(zz->Proc, " Zoltan_Distribute_Partition_Register");
partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO,
cmember, zz->Num_Proc, zz->Num_Proc);
ZOLTAN_FREE(&cmember);
Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata);
}
}
/*
* Build comm plan for sending non-zeros to their target processors in
* 2D data distribution.
*/
/* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */
proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int));
sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc));
if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR;
wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float));
if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR;
yGNO = outmat->mtx.yGNO;
pinGNO = outmat->mtx.pinGNO;
KDDKDDKDD(zz->Proc, " CommPlan Hash");
cnt = 0;
for (i = 0; i < outmat->mtx.nY; i++) {
ZOLTAN_GNO_TYPE edge_gno=-1;
/* processor row for the edge */
edge_gno = yGNO[i];
for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) {
ZOLTAN_GNO_TYPE vtx_gno=-1;
/* processor column for the vertex */
vtx_gno = pinGNO[j];
proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData,
&sendbuf[cnt].part_y);
if (proclist[cnt] < 0) /* Discard this nnz */
continue;
sendbuf[cnt].GNO[0] = edge_gno;
sendbuf[cnt].GNO[1] = vtx_gno;
memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim,
outmat->mtx.pinwgtdim*sizeof(float));
cnt++;
}
if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) {
proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData,
&sendbuf[cnt].part_y);
if (proclist[cnt] < 0) /* Discard this nnz */
continue;
sendbuf[cnt].GNO[0] = edge_gno;
sendbuf[cnt].GNO[1] = -1;
memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float));
cnt++;
}
}
if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin)
Zoltan_Distribute_Partition_Free(&outmat->hashDistData);
if (outmat->mtx.yend != outmat->mtx.ystart + 1)
ZOLTAN_FREE(&outmat->mtx.yend);
outmat->mtx.yend = NULL;
ZOLTAN_FREE(&outmat->mtx.ystart);
ZOLTAN_FREE(&outmat->mtx.yGNO);
ZOLTAN_FREE(&outmat->mtx.pinGNO);
ZOLTAN_FREE(&outmat->mtx.pinwgt);
ZOLTAN_FREE(&outmat->mtx.yGID);
/*
* Send pins to their target processors.
* They become non-zeros in the 2D data distribution.
*/
KDDKDDKDD(zz->Proc, " CommPlan Create");
msg_tag--;
ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins);
ZOLTAN_FREE(&proclist);
nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc));
if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR;
msg_tag--;
Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros);
ZOLTAN_FREE(&sendbuf);
if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */
tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float));
if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR;
msg_tag--;
Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float),
(char *) tmpwgtarray);
ZOLTAN_FREE(&wgtarray);
}
Zoltan_Comm_Destroy(&plan);
/* Unpack the non-zeros received. */
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_DupArcs");
/* TODO: do take care about singletons */
Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray,
&outmat->mtx);
}
/* Now we just have to change numbering */
outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE));
outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE));
if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR;
/* FIXME: Work only in 1D */
tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY;
MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator);
for (i = 1 ; i <= nProc_y ; i ++) {
outmat->dist_y[i] += outmat->dist_y[i-1];
}
outmat->dist_x[1] = outmat->mtx.globalX;
perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE));
if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR;
for (i = 0 ; i < outmat->mtx.nY ; ++i){
perm_y[i] = i + outmat->dist_y[myProc_y];
}
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute");
Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y);
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute done");
End:
ZOLTAN_FREE(&perm_y);
ZOLTAN_FREE(&proclist);
ZOLTAN_FREE(&sendbuf);
ZOLTAN_FREE(&nonzeros);
ZOLTAN_FREE(&tmpwgtarray);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_DD_Find (
Zoltan_DD_Directory *dd, /* contains directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */
ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */
ZOLTAN_ID_PTR data, /* Outgoing optional corresponding user data */
int *partition, /* Outgoing optional partition information */
int count, /* Count of GIDs in above list (in) */
int *owner) /* Outgoing corresponding list of data locations */
{
ZOLTAN_COMM_OBJ *plan = NULL ; /* efficient MPI communication */
char *rbuff = NULL ; /* receive buffer */
char *sbuff = NULL ; /* send buffer */
int *procs = NULL ; /* list of processors to contact */
DD_Find_Msg *ptr = NULL ;
int i ;
int nrec ; /* number of messages to receive */
int err ; /* return error condition */
int errcount ; /* count of GIDs not found */
char *yo = "Zoltan_DD_Find" ;
if (dd != NULL && dd->debug_level > 1)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* input sanity check */
if (dd == NULL || count < 0 || ((owner == NULL || gid == NULL) && count > 0))
{
ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
yo, "Invalid input argument.") ;
if (dd != NULL && dd->debug_level > 1)
ZOLTAN_TRACE_OUT((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
yo, NULL);
return ZOLTAN_DD_INPUT_ERROR ;
}
/* allocate memory for processors to contact for directory info */
if (count > 0)
{
procs = (int *) ZOLTAN_MALLOC (sizeof (int) * count) ;
if (procs == NULL)
{
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list.") ;
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_DD_MEMORY_ERROR ;
}
}
/* allocate memory for DD_Find_Msg send buffer */
if (count > 0)
{
sbuff = (char *) ZOLTAN_MALLOC (dd->find_msg_size * count) ;
if (sbuff == NULL)
{
ZOLTAN_FREE (&procs) ;
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer.") ;
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_DD_MEMORY_ERROR ;
}
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs.");
/* for each GID, fill DD_Find_Msg buffer and contact list */
for (i = 0 ; i < count ; i++)
{
procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc) ;
ptr = (DD_Find_Msg *) (sbuff + i * dd->find_msg_size) ;
ptr->index = i ;
ptr->proc = procs[i] ;
ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length) ;
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill.");
/* create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_FIND_MSG_TAG, &nrec) ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create.");
if (err != ZOLTAN_OK)
goto fini ;
/* allocate receive buffer */
if (nrec > 0)
{
rbuff = (char *) ZOLTAN_MALLOC (nrec * dd->find_msg_size) ;
if (rbuff == NULL)
{
err = ZOLTAN_DD_MEMORY_ERROR ;
goto fini ;
}
}
/* send out find messages across entire system */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
dd->find_msg_size, rbuff) ;
if (err != ZOLTAN_OK)
goto fini ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do.");
/* get find messages directed to me, fill in return information */
errcount = 0 ;
for (i = 0 ; i < nrec ; i++)
{
ptr = (DD_Find_Msg *) (rbuff + i*dd->find_msg_size) ;
err = DD_Find_Local (dd, ptr->id, ptr->id, ptr->id + dd->max_id_length,
(partition) ? (&ptr->partition) : NULL, &ptr->proc) ;
if (err == ZOLTAN_DD_GID_NOT_FOUND_ERROR)
errcount++ ;
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info.");
/* send return information back to requester */
err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
dd->find_msg_size, NULL, sbuff) ;
if (err != ZOLTAN_OK)
goto fini ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse.");
/* fill in user supplied lists with returned information */
for (i = 0 ; i < count ; i++)
{
ptr = (DD_Find_Msg *) (sbuff + i*dd->find_msg_size) ;
owner[ptr->index] = ptr->proc ;
if (partition != NULL)
partition[ptr->index] = ptr->partition ;
if (lid != NULL)
ZOLTAN_SET_ID (dd->lid_length, lid + ptr->index * dd->lid_length,
ptr->id) ;
if (data != NULL)
ZOLTAN_SET_ID (dd->user_data_length, data + ptr->index
* dd->user_data_length, ptr->id + dd->max_id_length) ;
}
/* if at least one GID was not found, notify caller of error */
err = (errcount == 0) ? ZOLTAN_DD_NORMAL_RETURN
: ZOLTAN_DD_GID_NOT_FOUND_ERROR ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists.");
fini:
ZOLTAN_FREE (&sbuff) ;
ZOLTAN_FREE (&rbuff) ;
ZOLTAN_FREE (&procs) ;
Zoltan_Comm_Destroy (&plan) ;
if (err != ZOLTAN_DD_NORMAL_RETURN)
ZOLTAN_PRINT_WARN (dd->my_proc, yo, "Return is not normal.") ;
if (dd->debug_level > 0)
{
char str[100] ; /* diagnostic message string */
sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount) ;
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ;
}
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err ;
}
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions,
ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
int *npids, int nregions, int *c2)
{
char *yo = "Zoltan_Oct_migreg_migrate_regions";
int i; /* index counter */
int ierr = ZOLTAN_OK;
int n_import;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
Region *import_objs = NULL; /* Array of import objects used to request
the objs from other processors. */
ZOLTAN_ID_PTR import_gids = NULL; /* Array of global IDs of import_objs. */
ZOLTAN_ID_PTR import_lids = NULL; /* Array of local IDs of import_objs. */
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator,
MIGMIGREGCommCreate, &n_import);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
*c2 = n_import;
if (n_import > 0) {
import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region));
import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import);
import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import);
if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region),
(char *) import_objs);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids,
sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
if (num_lid_entries > 0) {
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids,
sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
}
for (i=0; i<n_import; i++) {
import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]);
import_objs[i].Local_ID = (num_lid_entries
? &(import_lids[i*num_lid_entries])
: NULL);
Zoltan_Oct_insert_orphan(zz, import_objs[i]);
}
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
return ierr;
}
int Zoltan_Oct_migrate_octants(ZZ *zz, int *newpids, pOctant *octs, int nocts, int *nrecocts) {
int i,j = 0;
int nsends = 0;
int nreceives = 0;
int *despid = NULL;
OCTNEW_msg *snd_reply = NULL;
OCTNEW_msg *rcv_reply = NULL;
int ierr = ZOLTAN_OK;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
char *yo = "Zoltan_Oct_migrate_octants";
pOctant *newocts = NULL; /* New foreign octant pointers */
if((newocts = (pOctant *) ZOLTAN_MALLOC(sizeof(pOctant)*(nocts+10))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i=0; i<nocts; i++)
newocts[i]=NULL;
/* count number of sends */
nsends=0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc)
nsends++;
/* build message array */
/* if(nsends > 0) { */
if((snd_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nsends + 1) * sizeof(OCTNEW_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((despid = (int *) ZOLTAN_MALLOC((nsends+10) * sizeof(int))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
return ZOLTAN_MEMERR;
}
/* } */
/* else { */
/* snd_reply = NULL; */
/* despid = NULL; */
/* } */
j = 0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc) {
snd_reply[j].num = i;
despid[j++] = newpids[i];
}
/* send messages */
ierr = Zoltan_Comm_Create(&comm_plan, nsends, despid, zz->Communicator, MigOctCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
return (ierr);
}
if((rcv_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nreceives + 1) * sizeof(OCTNEW_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return ZOLTAN_MEMERR;
}
ierr = Zoltan_Comm_Do(comm_plan, MigOctCommDo, (char *) snd_reply,
sizeof(OCTNEW_msg), (char *) rcv_reply);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
/* Reply to malloc requests and Receive malloc replies */
for (i=0; i< nreceives; i++) {
rcv_reply[i].ptr = Zoltan_Oct_POct_new((OCT_Global_Info *) (zz->LB.Data_Structure));
}
;
ierr = Zoltan_Comm_Do_Reverse(comm_plan, MigOctCommReverse, (char *) rcv_reply,
sizeof(OCTNEW_msg), NULL, (char *) snd_reply);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
/* store remote pointers locally for future migration */
for (i=0; i<nsends; i++) {
newocts[snd_reply[i].num] = snd_reply[i].ptr;
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
return (ierr);
}
ZOLTAN_FREE(&snd_reply);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_reply);
/* set return value */
*nrecocts = nreceives;
ierr = Zoltan_Oct_Update_Connections(zz, octs, newpids, newocts, nocts);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
abort();
return (ierr);
}
ierr = Zoltan_Oct_Final_Migration(zz, octs,newpids,newocts,nocts, *nrecocts);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
abort();
return (ierr);
}
Zoltan_Oct_Update_Map(zz);
ZOLTAN_FREE(&newocts);
return ierr;
}
static int Zoltan_Oct_build_global_rootlist(ZZ *zz,Migrate_msg **ret_rmsg, int *size) {
int j, k = 0;
int *despid = NULL;
int nroots, nreceives;
pRList RootList; /* list of the local roots */
pOctant RootOct;
Migrate_msg *snd_rmsg = NULL;
Migrate_msg *rcv_rmsg = NULL;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
/*Map *array = OCT_info->map;*/
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_Oct_build_global_rootlist";
nroots = RL_numRootOctants(Zoltan_Oct_POct_localroots(OCT_info));
if (nroots > 0) { /* KDDKDD -- Added test to prevent departure before comm */
if((despid = (int *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(int))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((snd_rmsg = (Migrate_msg *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
return ZOLTAN_MEMERR;
}
k = 0;
for (j=0; j<zz->Num_Proc; j++) {
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
/* if(array[Zoltan_Oct_mapidx(RootOct)].npid > 0) { */
FILLMIGRATEMSG(RootOct, RootOct, snd_rmsg[k], zz->Proc);
despid[k] = j;
k++;
/* } */
}
}
} /* KDDKDD */
ierr = Zoltan_Comm_Create(&comm_plan, k, despid, zz->Communicator,
RootListCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
return (ierr);
}
if (nreceives > 0) {
if((rcv_rmsg = (Migrate_msg *) ZOLTAN_MALLOC(nreceives * sizeof(Migrate_msg))) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
return ZOLTAN_MEMERR;
}
}
ierr = Zoltan_Comm_Do(comm_plan, RootListCommDo, (char *) snd_rmsg, sizeof(Migrate_msg), (char *) rcv_rmsg);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
ZOLTAN_FREE(&rcv_rmsg);
return (ierr);
}
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&snd_rmsg);
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
*ret_rmsg = rcv_rmsg;
*size = nreceives;
return ierr;
}
int Zoltan_Scatter_Graph(
indextype **vtxdist,
indextype **xadj,
indextype **adjncy,
weighttype **vwgt,
indextype **vsize,
weighttype **adjwgt,
realtype **xyz,
int ndims, /* # dimensions of xyz geometry data */
int vwgt_dim,
ZZ *zz,
ZOLTAN_COMM_OBJ **plan
)
{
static char *yo = "Zoltan_Scatter_Graph";
char msg[256];
indextype *old_vtxdist, *old_adjncy;
indextype *old_xadj;
indextype *old_vsize;
weighttype *old_vwgt, *old_adjwgt;
realtype *old_xyz;
int *ptr, *proclist = NULL, *proclist2 = NULL;
int i, j, num_obj, old_num_obj, num_edges, nrecv;
int use_graph; /* do we use graph data, or only the geometry? */
int use_vsize; /* do we use the vsize array? */
int ewgt_dim= zz->Edge_Weight_Dim;
ZOLTAN_COMM_OBJ *plan2;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Save pointers to "old" data distribution */
old_adjncy = NULL;
old_xadj = NULL;
old_vwgt = old_adjwgt = NULL;
old_vsize = NULL;
old_xyz = NULL;
old_vtxdist = *vtxdist;
if (xadj)
old_xadj = *xadj;
if (adjncy)
old_adjncy = *adjncy;
if (vwgt)
old_vwgt = *vwgt;
if (vsize)
old_vsize = *vsize;
if (adjwgt)
old_adjwgt = *adjwgt;
if (xyz)
old_xyz = *xyz;
old_num_obj = (int)(old_vtxdist[zz->Proc+1] - old_vtxdist[zz->Proc]);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Old number of objects = %d\n", zz->Proc, old_num_obj);
/* Compute new distribution, *vtxdist */
(*vtxdist) = (indextype *)ZOLTAN_MALLOC((zz->Num_Proc+1)* sizeof(indextype));
for (i=0; i<=zz->Num_Proc; i++){
(*vtxdist)[i] = (i*old_vtxdist[zz->Num_Proc])/zz->Num_Proc;
}
/* Check if any proc has graph data */
i = (old_xadj != NULL);
MPI_Allreduce(&i, &use_graph, 1, MPI_INT, MPI_LOR, zz->Communicator);
j = (old_vsize != NULL);
MPI_Allreduce(&j, &use_vsize, 1, MPI_INT, MPI_LOR, zz->Communicator);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: use_graph = %1d, use_vsize = %1d\n", zz->Proc,
use_graph, use_vsize);
/* Reset all data pointers to NULL for now */
*xadj = NULL;
*adjncy = NULL;
*vwgt = *adjwgt = NULL;
*xyz = NULL;
if (use_vsize) *vsize = NULL;
/* Convert the xdj array so that it contains the degree of each vertex */
if (use_graph){
for (i=0; i<old_num_obj; i++){
old_xadj[i] = old_xadj[i+1] - old_xadj[i];
}
}
/* Allocate new space for vertex data */
num_obj = (int)((*vtxdist)[zz->Proc+1] - (*vtxdist)[zz->Proc]);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: New number of objects = %d\n", zz->Proc, num_obj);
if (use_graph)
*xadj = (indextype *) ZOLTAN_MALLOC((num_obj+1)*sizeof(indextype));
if (vwgt_dim)
*vwgt = (weighttype *) ZOLTAN_MALLOC(vwgt_dim*num_obj*sizeof(weighttype));
if (use_vsize)
*vsize = (indextype *) ZOLTAN_MALLOC(num_obj*sizeof(indextype));
if (ndims)
*xyz = (realtype *) ZOLTAN_MALLOC(ndims*num_obj*sizeof(realtype));
if (old_num_obj > 0) {
/* Set up the communication plan for the vertex data */
proclist = (int *) ZOLTAN_MALLOC(old_num_obj * sizeof(int));
/* Let j be the new owner of vertex old_vtxdist[zz->Proc]+i */
j = 0;
while (old_vtxdist[zz->Proc] >= (*vtxdist)[j+1]) j++;
for (i=0; i<old_num_obj; i++){
if (old_vtxdist[zz->Proc]+i >= (*vtxdist)[j+1]) j++;
proclist[i] = j;
}
}
Zoltan_Comm_Create(plan, old_num_obj, proclist, zz->Communicator, TAG1, &nrecv);
if (nrecv != num_obj){
sprintf(msg,"Proc %d received %d object but expected %d.",
zz->Proc, nrecv, num_obj);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
/* Free data */
ZOLTAN_FREE(&proclist);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_FATAL;
}
/* Do the communication. To save memory, we do not pack all the data into
* a buffer, but send directly from the old arrays to the new arrays.
* We use the vertex communication plan for all the vertex-based arrays
* and the edge communication plan for all the edge-based arrays.
*/
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Starting vertex-based communication.\n", zz->Proc);
if (use_graph){
Zoltan_Comm_Do( *plan, TAG2, (char *) old_xadj, sizeof(indextype), (char *) *xadj);
}
if (vwgt_dim){
Zoltan_Comm_Do( *plan, TAG3, (char *) old_vwgt, vwgt_dim*sizeof(weighttype), (char *) *vwgt);
}
if (use_vsize){
Zoltan_Comm_Do( *plan, TAG4, (char *) old_vsize, sizeof(indextype), (char *) *vsize);
}
if (ndims){
Zoltan_Comm_Do( *plan, TAG5, (char *) old_xyz, ndims*sizeof(realtype), (char *) *xyz);
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Finished vertex-based communication.\n", zz->Proc);
if (use_graph){
/* Rebuild xadj from degrees */
for (i=1; i<num_obj; i++)
(*xadj)[i] += (*xadj)[i-1];
for (i=num_obj; i>0; i--)
(*xadj)[i] = (*xadj)[i-1];
(*xadj)[0] = 0;
/* Allocate space for new edge data structures */
num_edges = (*xadj)[num_obj];
*adjncy = (indextype *) ZOLTAN_MALLOC(num_edges*sizeof(indextype));
if (ewgt_dim)
*adjwgt = (weighttype *) ZOLTAN_MALLOC(ewgt_dim*num_edges*sizeof(weighttype));
/* Set up the communication plan for the edge data. */
ptr = proclist2 = (int *) ZOLTAN_MALLOC(old_xadj[old_num_obj] * sizeof(int));
for (i=0; i<old_num_obj; i++)
for (j=0; j<old_xadj[i]; j++)
*ptr++ = proclist[i];
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] Debug: Allocated proclist of length " TPL_IDX_SPEC " for edges.\n",
zz->Proc, old_xadj[old_num_obj]);
}
Zoltan_Comm_Create(&plan2, (int)old_xadj[old_num_obj], proclist2, zz->Communicator,
TAG1, &nrecv);
if (nrecv != num_edges){
sprintf(msg,"Proc %d received %d edges but expected %d.",
zz->Proc, nrecv, num_edges);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
/* Free data */
ZOLTAN_FREE(&proclist);
ZOLTAN_FREE(&proclist2);
ZOLTAN_FREE(&old_vtxdist);
ZOLTAN_FREE(&old_xadj);
ZOLTAN_FREE(&old_adjncy);
ZOLTAN_FREE(&old_vwgt);
ZOLTAN_FREE(&old_vsize);
ZOLTAN_FREE(&old_adjwgt);
ZOLTAN_FREE(&old_xyz);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_FATAL;
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Starting edge-based communication.\n", zz->Proc);
/* Do the communication. */
Zoltan_Comm_Do( plan2, TAG2, (char *) old_adjncy, sizeof(indextype), (char *) *adjncy);
if (ewgt_dim){
Zoltan_Comm_Do( plan2, TAG3, (char *) old_adjwgt, ewgt_dim*sizeof(weighttype), (char *) *adjwgt);
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: Finished edge-based communication.\n", zz->Proc);
/* Free the comm. plan for edge data */
Zoltan_Comm_Destroy(&plan2);
} /* end of use_graph */
/* Free data structures */
ZOLTAN_FREE(&proclist);
ZOLTAN_FREE(&proclist2);
ZOLTAN_FREE(&old_vtxdist);
ZOLTAN_FREE(&old_xadj);
ZOLTAN_FREE(&old_adjncy);
ZOLTAN_FREE(&old_vwgt);
ZOLTAN_FREE(&old_vsize);
ZOLTAN_FREE(&old_adjwgt);
ZOLTAN_FREE(&old_xyz);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_OK;
}
int Zoltan_DD_Find (
Zoltan_DD_Directory *dd, /* contains directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */
ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */
char *data, /* Outgoing optional corresponding user data */
int *partition, /* Outgoing optional partition information */
int count, /* Count of GIDs in above list (in) */
int *owner) /* Outgoing optional list of data owners */
{
ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */
char *rbuff = NULL; /* receive buffer */
char *rbufftmp = NULL; /* pointer into receive buffer */
char *sbuff = NULL; /* send buffer */
char *sbufftmp = NULL; /* pointer into send buffer */
int *procs = NULL; /* list of processors to contact */
DD_Find_Msg *ptr = NULL;
int i;
int nrec; /* number of messages to receive */
int err = ZOLTAN_OK; /* return error condition */
int errcount; /* count of GIDs not found */
char *yo = "Zoltan_DD_Find";
/* input sanity check */
if (dd == NULL || count < 0 || (gid == NULL && count > 0)) {
ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo,
"Invalid input argument");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* allocate memory for processors to contact for directory info */
if (count) {
procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
if (procs == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
}
/* allocate memory for DD_Find_Msg send buffer */
if (count) {
sbuff = (char*) ZOLTAN_CALLOC (count, dd->find_msg_size);
if (sbuff == NULL) {
ZOLTAN_FREE (&procs);
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_MEMERR;
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs");
/* for each GID, fill DD_Find_Msg buffer and contact list */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc,
dd->hashdata, dd->hashfn);
ptr = (DD_Find_Msg*) sbufftmp;
sbufftmp += dd->find_msg_size;
ptr->index = i;
ptr->proc = procs[i];
ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length);
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill");
/* create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_FIND_MSG_TAG, &nrec);
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create");
if (err != ZOLTAN_OK)
goto fini;
/* allocate receive buffer */
if (nrec) {
rbuff = (char*) ZOLTAN_MALLOC ((size_t)nrec*(size_t)(dd->find_msg_size));
if (rbuff == NULL) {
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* send out find messages across entire system */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
dd->find_msg_size, rbuff);
if (err != ZOLTAN_OK)
goto fini;
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do");
/* get find messages directed to me, fill in return information */
errcount = 0;
rbufftmp = rbuff;
for (i = 0; i < nrec; i++) {
ptr = (DD_Find_Msg*) rbufftmp;
rbufftmp += dd->find_msg_size;
err = DD_Find_Local (dd, ptr->id, ptr->id,
(char *)(ptr->id + dd->max_id_length),
&ptr->partition, &ptr->proc);
if (err == ZOLTAN_WARN)
++errcount;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info");
/* send return information back to requester */
err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
dd->find_msg_size, NULL, sbuff);
if (err != ZOLTAN_OK)
goto fini;
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse");
/* fill in user supplied lists with returned information */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
ptr = (DD_Find_Msg*) sbufftmp;
sbufftmp += dd->find_msg_size;
if (owner)
owner[ptr->index] = ptr->proc;
if (partition)
partition[ptr->index] = ptr->partition ;
if (lid)
ZOLTAN_SET_ID(dd->lid_length,lid+ptr->index*dd->lid_length,ptr->id);
if (data)
memcpy(data + (size_t)(ptr->index) * (size_t)(dd->user_data_length),
ptr->id + dd->max_id_length, dd->user_data_length);
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists");
/* err = ZOLTAN_OK; */
MPI_Allreduce(&errcount, &err, 1, MPI_INT, MPI_SUM, dd->comm);
err = (err) ? ZOLTAN_WARN : ZOLTAN_OK;
/* if at least one GID was not found, potentially notify caller of error */
if (dd->debug_level > 0) {
char str[100]; /* diagnostic message string */
sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount);
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
}
fini:
ZOLTAN_FREE (&sbuff);
ZOLTAN_FREE (&rbuff);
ZOLTAN_FREE (&procs) ;
Zoltan_Comm_Destroy (&plan);
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err;
}
/* Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
ZZ *zz, /* Zoltan data structure */
HGraph *hg, /* Input hypergraph to be partitioned */
int p, /* Input: number partitions to be generated */
float *part_sizes, /* Input: array of length p containing percentages
of work to be assigned to each partition */
Partition parts, /* Input: initial partition #s; aligned with vtx
arrays.
Output: computed partition #s */
PHGPartParams *hgp) /* Input: parameters for hgraph partitioning. */
{
PHGComm *hgc = hg->comm;
VCycle *vcycle=NULL, *del=NULL;
int i, err = ZOLTAN_OK, middle;
ZOLTAN_GNO_TYPE origVpincnt; /* for processor reduction test */
ZOLTAN_GNO_TYPE prevVcnt = 2*hg->dist_x[hgc->nProc_x]; /* initialized so that the */
ZOLTAN_GNO_TYPE prevVedgecnt = 2*hg->dist_y[hgc->nProc_y]; /* while loop will be entered
before any coarsening */
ZOLTAN_GNO_TYPE tot_nPins, local_nPins;
MPI_Datatype zoltan_gno_mpi_type;
char *yo = "Zoltan_PHG_Partition";
int do_timing = (hgp->use_timers > 1);
int fine_timing = (hgp->use_timers > 2);
int vcycle_timing = (hgp->use_timers > 4 && hgp->ProRedL == 0);
short refine = 0;
struct phg_timer_indices *timer = Zoltan_PHG_LB_Data_timers(zz);
int reset_geometric_matching = 0;
char reset_geometric_string[4];
ZOLTAN_TRACE_ENTER(zz, yo);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
if (do_timing) {
if (timer->vcycle < 0)
timer->vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
if (timer->procred < 0)
timer->procred = Zoltan_Timer_Init(zz->ZTime, 0, "Processor Reduction");
if (timer->match < 0)
timer->match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
if (timer->coarse < 0)
timer->coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
if (timer->coarsepart < 0)
timer->coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
"Coarse_Partition");
if (timer->refine < 0)
timer->refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
if (timer->project < 0)
timer->project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
MPI_Allreduce(&local_nPins,&tot_nPins,1,zoltan_gno_mpi_type,MPI_SUM,hgc->Communicator);
origVpincnt = tot_nPins;
if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
/* For geometric coarsening, hgp->matching pointer and string are reset
* after geometric_levels of coarsening. Will need to reset them after
* this vcycle is completed. Capture that fact now! */
if (!strcasecmp(hgp->redm_str, "rcb") || !strcasecmp(hgp->redm_str, "rib")) {
reset_geometric_matching = 1;
strcpy(reset_geometric_string, hgp->redm_str);
}
/****** Coarsening ******/
#define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */
while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > (ZOLTAN_GNO_TYPE)hg->redl)
&& ((hg->dist_x[hgc->nProc_x] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5)) /* prevVcnt initialized to 2*hg->dist_x[hgc->nProc_x] */
|| (hg->dist_y[hgc->nProc_y] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5))) /* prevVedgecnt initialized to 2*hg->dist_y[hgc->nProc_y] */
&& hg->dist_y[hgc->nProc_y] && hgp->matching) {
ZOLTAN_GNO_TYPE *match = NULL;
VCycle *coarser=NULL, *redistributed=NULL;
prevVcnt = hg->dist_x[hgc->nProc_x];
prevVedgecnt = hg->dist_y[hgc->nProc_y];
#ifdef _DEBUG
/* UVC: load balance stats */
Zoltan_PHG_LoadBalStat(zz, hg);
#endif
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc,
"START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->match, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_match < 0) {
char str[80];
sprintf(str, "VC Matching %d", hg->info);
vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
}
/* Allocate and initialize Matching Array */
if (hg->nVtx && !(match = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC (hg->nVtx*sizeof(ZOLTAN_GNO_TYPE)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
match[i] = i;
/* Calculate matching (packing or grouping) */
err = Zoltan_PHG_Matching (zz, hg, match, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_FREE (&match);
goto End;
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->match, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->coarse, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_coarse < 0) {
char str[80];
sprintf(str, "VC Coarsening %d", hg->info);
vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
}
if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
ZOLTAN_FREE (&match);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
goto End;
}
/* Construct coarse hypergraph and LevelMap */
err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
&vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData,
&vcycle->comm_plan, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarse, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
ZOLTAN_FREE (&match);
if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
goto End;
vcycle = coarser;
hg = vcycle->hg;
if (hgc->nProc > 1 && hgp->ProRedL > 0) {
local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
MPI_Allreduce(&local_nPins, &tot_nPins, 1, zoltan_gno_mpi_type, MPI_SUM,
hgc->Communicator);
if (tot_nPins < (ZOLTAN_GNO_TYPE)(hgp->ProRedL * origVpincnt + 0.5)) {
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->procred, hgc->Communicator);
}
/* redistribute to half the processors */
origVpincnt = tot_nPins; /* update for processor reduction test */
if(hg->nVtx&&!(hg->vmap=(int*)ZOLTAN_MALLOC(hg->nVtx*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: hg->vmap");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
hg->vmap[i] = i;
middle = (int)((float) (hgc->nProc-1) * hgp->ProRedL);
if (hgp->nProc_x_req!=1&&hgp->nProc_y_req!=1) { /* Want 2D decomp */
if ((middle+1) > SMALL_PRIME && Zoltan_PHG_isPrime(middle+1))
--middle; /* if it was prime just use one less #procs (since
it should be bigger than SMALL_PRIME it is safe to
decrement) */
}
if (!(hgc = (PHGComm*) ZOLTAN_MALLOC (sizeof(PHGComm)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: PHGComm");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!(redistributed=newVCycle(zz,NULL,NULL,vcycle,vcycle_timing))) {
ZOLTAN_FREE (&hgc);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "redistributed is NULL.");
goto End;
}
Zoltan_PHG_Redistribute(zz,hgp,hg,0,middle,hgc, redistributed->hg,
&vcycle->vlno,&vcycle->vdest);
if (hgp->UseFixedVtx || hgp->UsePrefPart)
redistributed->hg->bisec_split = hg->bisec_split;
if ((err=allocVCycle(redistributed))!= ZOLTAN_OK)
goto End;
vcycle = redistributed;
if (hgc->myProc < 0)
/* I'm not in the redistributed part so I should go to uncoarsening
refinement and wait */ {
if (fine_timing) {
if (timer->cpgather < 0)
timer->cpgather = Zoltan_Timer_Init(zz->ZTime, 1, "CP Gather");
if (timer->cprefine < 0)
timer->cprefine =Zoltan_Timer_Init(zz->ZTime, 0, "CP Refine");
if (timer->cpart < 0)
timer->cpart = Zoltan_Timer_Init(zz->ZTime, 0, "CP Part");
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
goto Refine;
}
hg = vcycle->hg;
hg->redl = hgp->redl; /* not set with hg creation */
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
}
}
}
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
/* free array that may have been allocated in matching */
if (hgp->vtx_scal) {
hgp->vtx_scal_size = 0;
ZOLTAN_FREE(&(hgp->vtx_scal));
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->coarsepart, hgc->Communicator);
}
/****** Coarse Partitioning ******/
err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarsepart, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
Refine:
del = vcycle;
refine = 1;
/****** Uncoarsening/Refinement ******/
while (vcycle) {
VCycle *finer = vcycle->finer;
hg = vcycle->hg;
if (refine && hgc->myProc >= 0) {
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->refine, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_refine < 0) {
char str[80];
sprintf(str, "VC Refinement %d", hg->info);
vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
}
err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->refine, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
if (hgp->output_level >= PHG_DEBUG_LIST)
uprintf(hgc,
"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, 0, p,
vcycle->Part),
Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
"partitioned plot");
}
if (finer) {
int *rbuffer;
/* Project coarse partition to fine partition */
if (finer->comm_plan) {
refine = 1;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->project, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_project < 0) {
char str[80];
sprintf(str, "VC Project Up %d", hg->info);
vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
}
/* easy to assign partitions to internal matches */
for (i = 0; i < finer->hg->nVtx; i++)
if (finer->LevelMap[i] >= 0) /* if considers only the local vertices */
finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
/* now that the course partition assignments have been propagated */
/* upward to the finer level for the local vertices, we need to */
/* fill the LevelData (matched pairs of a local vertex with a */
/* off processor vertex) with the partition assignment of the */
/* local vertex - can be done totally in the finer level! */
for (i = 0; i < finer->LevelCnt; i++) {
++i; /* skip over off processor lno */
finer->LevelData[i] = finer->Part[finer->LevelData[i]];
}
/* allocate rec buffer to exchange LevelData information */
rbuffer = NULL;
if (finer->LevelSndCnt > 0) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
/* get partition assignments from owners of externally matched vtxs */
Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1,
(char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);
/* process data to assign partitions to expernal matches */
for (i = 0; i < 2 * finer->LevelSndCnt;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer->project, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
} else {
int *sendbuf = NULL, size;
refine = 0;
/* ints local and partition numbers */
if (finer->vlno) {
sendbuf = (int*) ZOLTAN_MALLOC (2 * hg->nVtx * sizeof(int));
if (!sendbuf) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; ++i) {
sendbuf[2 * i] = finer->vlno[i]; /* assign local numbers */
sendbuf[2 * i + 1] = vcycle->Part[i];/* assign partition numbers */
}
}
ZOLTAN_FREE (&hgc);
hgc = finer->hg->comm; /* updating hgc is required when the processors
change */
/* Create comm plan to unredistributed processors */
err = Zoltan_Comm_Create(&finer->comm_plan, finer->vlno ? hg->nVtx : 0,
finer->vdest, hgc->Communicator, COMM_TAG+2,
&size);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(hgc->myProc, yo, "Zoltan_Comm_Create failed.");
goto End;
}
/* allocate rec buffer to exchange sendbuf information */
rbuffer = NULL;
if (finer->hg->nVtx) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->hg->nVtx * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
/* Use plan to send partitions to the unredistributed processors */
Zoltan_Comm_Do(finer->comm_plan, COMM_TAG+3, (char *) sendbuf,
2*sizeof(int), (char *) rbuffer);
MPI_Bcast(rbuffer, 2*finer->hg->nVtx, MPI_INT, 0, hgc->col_comm);
/* process data to assign partitions to unredistributed processors */
for (i = 0; i < 2 * finer->hg->nVtx;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
if (finer->vlno)
ZOLTAN_FREE (&sendbuf);
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
}
}
vcycle = finer;
} /* while (vcycle) */
End:
vcycle = del;
while (vcycle) {
if (vcycle_timing) {
Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
Zoltan_Timer_Destroy(&vcycle->timer);
}
if (vcycle->finer) { /* cleanup by level */
Zoltan_HG_HGraph_Free (vcycle->hg);
if (vcycle->LevelData)
Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part,
&vcycle->LevelMap, &vcycle->LevelData, &vcycle->hg);
else if (vcycle->vlno)
Zoltan_Multifree (__FILE__, __LINE__, 5, &vcycle->Part, &vcycle->vdest,
&vcycle->vlno, &vcycle->LevelMap, &vcycle->hg);
else
Zoltan_Multifree (__FILE__, __LINE__, 3, &vcycle->Part,
&vcycle->LevelMap, &vcycle->hg);
}
else /* cleanup top level */
Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
&vcycle->LevelData);
del = vcycle;
vcycle = vcycle->finer;
ZOLTAN_FREE(&del);
}
if (reset_geometric_matching) {
strcpy(hgp->redm_str, reset_geometric_string);
Zoltan_PHG_Set_Matching_Fn(hgp);
}
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
ZOLTAN_TRACE_EXIT(zz, yo) ;
return err;
}
int Zoltan_Verify_Graph(MPI_Comm comm, indextype *vtxdist, indextype *xadj,
indextype *adjncy, weighttype *vwgt, weighttype *adjwgt,
int vwgt_dim, int ewgt_dim,
int graph_type, int check_graph, int output_level)
{
int ierr, flag, cross_edges = 0, mesg_size, sum;
int nprocs, proc, *proclist, errors, global_errors;
int *perm=NULL;
int free_adjncy_sort=0;
ZOLTAN_COMM_OBJ *comm_plan;
static char *yo = "Zoltan_Verify_Graph";
char msg[256];
ZOLTAN_GNO_TYPE num_obj;
int nrecv;
indextype *ptr, *ptr1, *ptr2;
indextype global_i, global_j;
indextype *sendgno=NULL, *recvgno=NULL, *adjncy_sort=NULL;
weighttype *sendwgt, *recvwgt;
ZOLTAN_GNO_TYPE num_duplicates, num_singletons;
ZOLTAN_GNO_TYPE num_selfs, nedges, global_sum, num_zeros;
ZOLTAN_GNO_TYPE i, j, ii, k;
MPI_Datatype zoltan_gno_mpi_type;
ierr = ZOLTAN_OK;
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
/* Make sure all procs have same value of check_graph. */
MPI_Allreduce(&check_graph, &i, 1, MPI_INT, MPI_MAX, comm);
check_graph = i;
if (check_graph == 0) /* perform no error checking at all */
return ierr;
/* Get number of procs and my rank */
MPI_Comm_size(comm, &nprocs);
MPI_Comm_rank(comm, &proc);
/* Check number of vertices (objects) */
num_obj = (ZOLTAN_GNO_TYPE)(vtxdist[proc+1] - vtxdist[proc]);
MPI_Reduce(&num_obj, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum==0)){
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
if (output_level>0)
ZOLTAN_PRINT_WARN(proc, yo, "No vertices in graph.");
}
/* Verify that vertex weights are non-negative */
num_zeros = 0;
if (vwgt_dim){
for (i=0; i<num_obj; i++){
sum = 0;
for (k=0; k<vwgt_dim; k++){
if (vwgt[i*vwgt_dim+k] < 0) {
sprintf(msg, "Negative object weight of " TPL_WGT_SPEC " for object " ZOLTAN_GNO_SPEC ".",
vwgt[i*vwgt_dim+k], i);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
sum += vwgt[i*vwgt_dim+k];
}
if (sum == 0){
num_zeros++;
if (output_level>1) {
sprintf(msg, "Zero vertex (object) weights for object " ZOLTAN_GNO_SPEC ".", i);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
}
}
MPI_Reduce(&num_zeros, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " objects have zero weights.", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
}
/* Check number of edges */
nedges = (ZOLTAN_GNO_TYPE)xadj[num_obj];
MPI_Reduce(&nedges, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum==0)){
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
if (output_level>0)
ZOLTAN_PRINT_WARN(proc, yo, "No edges in graph.");
}
/* Verify that edge weights are non-negative */
num_zeros = 0;
if (ewgt_dim){
for (j=0; j<nedges; j++){
sum = 0;
for (k=0; k<ewgt_dim; k++){
if (adjwgt[j*ewgt_dim+k] < 0) {
sprintf(msg, "Negative edge weight of " TPL_WGT_SPEC " in edge " ZOLTAN_GNO_SPEC ".",
adjwgt[j*ewgt_dim+k], j);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
sum += adjwgt[j*ewgt_dim+k];
}
if (sum == 0){
num_zeros++;
if (output_level>1) {
sprintf(msg, "Zero edge (communication) weights for edge " ZOLTAN_GNO_SPEC ".", j);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
}
}
MPI_Reduce(&num_zeros, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " edges have zero weights.", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
}
/* Verify that the graph is symmetric (edge weights, too) */
/* Also check for self-edges and multi-edges */
/* Pre-processing: Check if edge lists are sorted. If not, */
/* make a copy and sort so we can save time in the lookups. */
flag = 0; /* Assume sorted. */
for (i=0; (i<num_obj) && (flag==0); i++){
for (ii=xadj[i]; ii<xadj[i+1]-1; ii++){
if (adjncy[ii] > adjncy[ii+1]){
flag = 1; /* Not sorted. */
break;
}
}
}
if (flag){ /* Need to sort. */
adjncy_sort = (indextype *) ZOLTAN_MALLOC(nedges*sizeof(indextype));
perm = (int *) ZOLTAN_MALLOC(nedges*sizeof(int));
free_adjncy_sort = 1;
if (nedges && (!adjncy_sort || !perm)){
/* Out of memory. */
ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory.");
ierr = ZOLTAN_MEMERR;
}
for (k=0; k<nedges; k++){
adjncy_sort[k] = adjncy[k];
perm[k] = k;
}
if (sizeof(indextype) == sizeof(short)){
for (i=0; i<num_obj; i++)
Zoltan_quicksort_list_inc_short((short *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
}
else if (sizeof(indextype) == sizeof(int)){
for (i=0; i<num_obj; i++)
Zoltan_quicksort_list_inc_int((int *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
}
else if (sizeof(indextype) == sizeof(long)){
for (i=0; i<num_obj; i++)
Zoltan_quicksort_list_inc_long((long *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
}
else if (sizeof(indextype) == sizeof(int64_t)){
for (i=0; i<num_obj; i++)
Zoltan_quicksort_list_inc_long_long((int64_t*)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
}
else{
ZOLTAN_PRINT_ERROR(proc, yo, "Error in third party library data type support.");
ierr = ZOLTAN_MEMERR;
}
}
else { /* Already sorted. */
adjncy_sort = adjncy;
}
/* First pass: Check on-processor edges and count # off-proc edges */
cross_edges = 0;
num_selfs = 0;
num_duplicates = 0;
num_singletons = 0;
for (i=0; i<num_obj; i++){
if (IS_GLOBAL_GRAPH(graph_type)){
global_i = vtxdist[proc]+i;
}
else{ /* graph_type == LOCAL_GRAPH */
global_i = i; /* A bit confusingly, global_i = i for local graphs */
}
/* Singleton? */
if (xadj[i] == xadj[i+1]){
num_singletons++;
if (output_level>1){
sprintf(msg, "Vertex " TPL_IDX_SPEC " has no edges.", global_i);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
for (ii=xadj[i]; ii<xadj[i+1]; ii++){
global_j = adjncy_sort[ii];
/* Valid vertex number? */
if ((IS_GLOBAL_GRAPH(graph_type) &&
((global_j < vtxdist[0]) || (global_j >= vtxdist[nprocs])))
|| (IS_LOCAL_GRAPH(graph_type) &&
((global_j < 0) || (global_j >= num_obj)))){
sprintf(msg, "Edge to invalid vertex " TPL_IDX_SPEC " detected.", global_j);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
/* Self edge? */
if (global_j == global_i){
num_selfs++;
if (output_level>1){
sprintf(msg, "Self edge for vertex " TPL_IDX_SPEC " detected.", global_i);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
/* Duplicate edge? */
if ((ii+1<xadj[i+1]) && (adjncy_sort[ii]==adjncy_sort[ii+1])){
num_duplicates++;
if (output_level>1){
sprintf(msg, "Duplicate edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") detected.", global_i, global_j);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
/* Is global_j a local vertex? */
if (IS_LOCAL_GRAPH(graph_type) || (IS_GLOBAL_GRAPH(graph_type) &&
(global_j >= vtxdist[proc]) && (global_j < vtxdist[proc+1]))){
/* Check if (global_j, global_i) is an edge */
if (IS_GLOBAL_GRAPH(graph_type))
j = global_j - vtxdist[proc];
else /* graph_type == LOCAL_GRAPH */
j = global_j;
/* Binary search for edge (global_j, global_i) */
ptr = (indextype *)bsearch(&global_i, &adjncy_sort[xadj[j]], (int)(xadj[j+1]-xadj[j]),
sizeof(indextype), Zoltan_Compare_Indextypes);
if (ptr){
/* OK, found edge (global_j, global_i) */
if ((adjncy_sort==adjncy) && ewgt_dim){
/* Compare weights */
/* EBEB For now, don't compare weights if we sorted edge lists. */
flag = 0;
for (k=0; k<ewgt_dim; k++){
if (adjwgt[(ptr-adjncy_sort)*ewgt_dim+k] !=
adjwgt[ii*ewgt_dim+k]){
/* Numerically nonsymmetric */
flag = -1;
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
}
}
if (flag<0 && output_level>0){
sprintf(msg, "Graph is numerically nonsymmetric "
"in edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ")", global_i, global_j);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
}
else { /* bsearch failed */
sprintf(msg, "Graph is not symmetric. "
"Edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") exists, but no edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ").",
global_i, global_j, global_j, global_i);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
}
else {
cross_edges++;
}
}
}
/* Sum up warnings so far. */
MPI_Reduce(&num_selfs, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " self-edges in graph.", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
MPI_Reduce(&num_duplicates, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " duplicate edges in graph.", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
MPI_Reduce(&num_singletons, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " vertices in the graph are singletons (have no edges).", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
/* Check if any processor has encountered an error so far */
errors = 0;
if (ierr == ZOLTAN_WARN)
errors |= 1;
if (ierr == ZOLTAN_MEMERR)
errors |= 2;
if (ierr == ZOLTAN_FATAL)
errors |= 4;
MPI_Allreduce(&errors, &global_errors, 1, MPI_INT, MPI_BOR, comm);
if (global_errors & 4){
/* Fatal error: return now */
if (free_adjncy_sort) ZOLTAN_FREE(&adjncy_sort);
if (free_adjncy_sort) ZOLTAN_FREE(&perm);
return ZOLTAN_FATAL;
}
if ((IS_GLOBAL_GRAPH(graph_type)) && (check_graph >= 2)) {
/* Test for consistency across processors. */
/* Allocate space for off-proc data */
mesg_size = (2*sizeof(indextype)) + (ewgt_dim * sizeof(weighttype));
sendgno = (indextype *) ZOLTAN_MALLOC(cross_edges*mesg_size);
recvgno = (indextype *) ZOLTAN_MALLOC(cross_edges*mesg_size);
proclist = (int *) ZOLTAN_MALLOC(cross_edges*sizeof(int));
if (cross_edges && !(sendgno && recvgno && proclist)){
ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory.");
ierr = ZOLTAN_MEMERR;
}
/* Second pass: Copy data to send buffer */
nedges = 0;
ptr1 = (indextype *) sendgno;
for (i=0; i<num_obj; i++){
global_i = vtxdist[proc]+i;
for (ii=xadj[i]; ii<xadj[i+1]; ii++){
global_j = adjncy[ii];
/* Is global_j off-proc? */
if ((global_j < vtxdist[proc]) || (global_j >= vtxdist[proc+1])){
/* Add to list */
k=0;
while (global_j >= vtxdist[k+1]) k++;
proclist[nedges++] = k;
/* Copy (global_i, global_j) and corresponding weights to sendgno */
*ptr1++ = global_i;
*ptr1++ = global_j;
for (k=0; k<ewgt_dim; k++){
*ptr1++ = adjwgt[ii*ewgt_dim+k];
}
}
}
}
/* Do the irregular communication */
ierr = Zoltan_Comm_Create(&comm_plan, cross_edges, proclist, comm, TAG1, &nrecv);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Create.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(proc, yo, msg);
}
else {
if (nrecv != cross_edges){
sprintf(msg, "Incorrect number of edges to/from proc %d.", proc);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
ierr = Zoltan_Comm_Do(comm_plan, TAG2, (char *)sendgno, mesg_size, (char *)recvgno);
Zoltan_Comm_Destroy(&comm_plan);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(proc, yo, msg);
}
else {
/* Third pass: Compare on-proc data to the off-proc data we received */
/* sendgno and recvgno should contain the same data except (i,j) is */
/* (j,i) */
for (i=0, ptr1=sendgno; i<cross_edges; i++){
flag = 0;
sendwgt = (weighttype *)(ptr1 + 2);
for (j=0, ptr2=recvgno; j<cross_edges; j++){
recvwgt = (weighttype *)(ptr2 + 2);
if ((ptr2[0] == ptr1[1]) && (ptr2[1] == ptr1[0])){
/* Found matching edge */
flag = 1;
/* Check weights */
for (k=0; k<ewgt_dim; k++){
if (sendwgt[k] != recvwgt[k]){
flag = -1;
ierr = ZOLTAN_WARN;
}
}
if (flag<0 && output_level>0){
sprintf(msg, "Edge weight (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") is not symmetric",
ptr1[0], ptr1[1]);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
ptr2 = (indextype *)(recvwgt + ewgt_dim);
}
if (!flag){
sprintf(msg, "Graph is not symmetric. "
"Edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") exists, but not (" TPL_IDX_SPEC "," TPL_IDX_SPEC ").",
ptr1[0], ptr1[1], ptr1[1], ptr1[0]);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
ptr1 = (indextype *)(sendwgt + ewgt_dim);
}
}
}
/* Free memory */
ZOLTAN_FREE(&sendgno);
ZOLTAN_FREE(&recvgno);
ZOLTAN_FREE(&proclist);
}
if (free_adjncy_sort) ZOLTAN_FREE(&adjncy_sort);
if (free_adjncy_sort) ZOLTAN_FREE(&perm);
/* Compute global error code */
errors = 0;
if (ierr == ZOLTAN_WARN)
errors |= 1;
if (ierr == ZOLTAN_MEMERR)
errors |= 2;
if (ierr == ZOLTAN_FATAL)
errors |= 4;
MPI_Allreduce(&errors, &global_errors, 1, MPI_INT, MPI_BOR, comm);
if (global_errors & 4){
return ZOLTAN_FATAL;
}
else if (global_errors & 2){
return ZOLTAN_MEMERR;
}
else if (global_errors & 1){
return ZOLTAN_WARN;
}
else {
if (proc==0 && output_level>0){
printf("ZOLTAN %s: The graph is valid with check_graph = %1d\n",
yo, check_graph);
}
return ZOLTAN_OK;
}
}
int main(int argc, char *argv[])
{
FILE *in_file = NULL; /* file with data for problems */
ZOLTAN_COMM_OBJ *plan; /* communication data structure pointer */
struct Params params; /* parameters describing a problem */
struct Data data; /* data describing a problem instance */
struct Data my_send_data; /* data I initially own */
struct Answer true_answer; /* expected outcome of exchange */
int nvals_recv; /* number of vals I own after exchange */
float *recv_data; /* values I own after exchange */
float *reverse_data; /* values I own after reversing communication */
char file_name[100]; /* name of input file */
int nbytes; /* size of objects */
int my_proc; /* my processor ID */
int nprocs; /* total number of processors */
int flag; /* return code from comm operations */
int more_problems; /* are there more problems to do? */
int i, j;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &my_proc);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
Zoltan_Memory_Debug(2);
if (argc > 1) strcpy(file_name, argv[1]);
else strcpy(file_name,"comm_input.dat");
if (my_proc == 0) {
in_file = fopen(file_name, "r");
if (in_file == NULL) {
printf("No input file `%s' found.\n", file_name);
}
}
/* Read some problem descriptors from a file */
more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc);
while (more_problems) {
/* Generate full data at random on each proc */
gen_comm_data(¶ms, &data, nprocs);
if (out_level > 2) if (my_proc == 0) print_data("DATA", &data);
/* Figure out from the data what what to expect to receive */
extract_comm_answer(&data, &true_answer, my_proc, nprocs);
/* Extract what to send */
set_up_comm_from_send(&data, &my_send_data, params.blocked, my_proc, nprocs);
if (out_level > 2) print_data("MY_DATA", &my_send_data);
if (out_level > 1) printf("%d: About to call comm_create\n", my_proc);
/* Call comm routines */
Zoltan_Comm_Create(&plan, my_send_data.nvals, my_send_data.proc_dest,
MPI_COMM_WORLD, 1, &nvals_recv);
if (out_level > 1) printf("%d: About to call comm_info\n", my_proc);
check_comm_info(plan, &my_send_data, nvals_recv, my_proc);
if (out_level > 2) print_plan("BEFORE RESIZE", plan, my_proc);
/* "4" reflects the max_sizes value in gen_comm_data */
recv_data = (float *) ZOLTAN_MALLOC(nvals_recv * 4 * sizeof(float));
reverse_data = (float *) ZOLTAN_MALLOC(my_send_data.nvals * 4 * sizeof(float));
if (my_send_data.sizes != NULL) {
if (out_level > 1) printf("%d: About to call comm_resize\n", my_proc);
Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
Zoltan_Comm_Resize(plan, NULL, 43, NULL);
Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
if (out_level > 2) print_plan("AFTER RESIZE", plan, my_proc);
}
if (my_send_data.sizes == NULL) nbytes = my_send_data.same_size * sizeof(float);
else nbytes = sizeof(float);
if (out_level > 1) printf("%d: About to call comm_do\n", my_proc);
flag = Zoltan_Comm_Do(plan, 2, (char *) my_send_data.vals, nbytes,
(char *) recv_data);
if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer\n", my_proc);
/* Check answers */
check_comm_answer(&true_answer, recv_data, my_proc);
}
else {
printf("%d: Comm_Do returned error code %d\n", my_proc, flag);
}
if (out_level > 1) printf("%d: About to call comm_do_reverse\n", my_proc);
i = (true_answer.sizes != NULL && plan->indices_to != NULL);
MPI_Allreduce(&i, &j, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (j == 0) flag = Zoltan_Comm_Do_Reverse(plan, 2, (char *) recv_data,
nbytes, true_answer.sizes, (char *) reverse_data);
else {
if (my_proc == 0)
printf(">> Non-blocked, variable-sized recvs not supported\n");
flag = ZOLTAN_FATAL;
}
if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer_reverse\n", my_proc);
/* Check answers */
check_comm_answer_reverse(&my_send_data, reverse_data, my_proc);
}
else {
if (out_level > 1) printf("%d: Comm_Do_Reverse returned error code %d\n", my_proc, flag);
}
/* Free up data structures */
ZOLTAN_FREE(&reverse_data);
ZOLTAN_FREE(&recv_data);
free_comm_data(&data, &my_send_data, &true_answer);
Zoltan_Comm_Destroy(&plan);
/* Read some problem descriptors from a file */
more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc);
}
Zoltan_Memory_Stats();
MPI_Finalize();
return(0);
}
static int local_HEs_from_export_lists(
ZZ *zz,
int remap_type, /* type of remapping to do: parts, procs, or none. */
int nobj, /* # objs the processor knows about (keep + exports) */
int *new_proc, /* On input, new processor assignment for each obj;
Upon return, remapped new proc assignment for
each obj. */
int *old_part, /* old partition assignments for each objs */
int *new_part, /* On input, new partition assignments for each objs.
Upon return, remapped new partition assignments */
int *HEcnt, /* # of HEs allocated. */
int **HEinfo /* Array of HE info; for each HE, two pins and
one edge weight. Stored as a single vector
to minimize communication calls. */
)
{
/* Routine to remap partitions (to new processors or new partition numbers)
* to reduce data movement.
* This routine assumes the load-balancing algorithm built export lists.
* Objects described are those that STARTED on zz->Proc due to load balancing.
* For all these objects, old_proc == zz->Proc.
*/
char *yo = "local_HEs_from_export_lists";
int ierr = ZOLTAN_OK;
int i, cnt, tmp;
int *tmp_HEinfo;
int my_proc = zz->Proc; /* This processor's rank. */
int nimp = 0;
int *imp_proc = NULL, /* Temporary arrays if inversion of export to */
*imp_old_part = NULL, /* import lists is needed. */
*imp_new_part = NULL;
int HEwgt_size; /* # of HE weights allocated. */
int *HEwgt = NULL; /* Array of HE weights. Initially includes
zero weights; later zero-weights are removed.*/
if (remap_type == ZOLTAN_LB_REMAP_PROCESSORS) {
/* Build HEs based on processor assignment.
* We know the old processor for all objects we are keeping and all
* export objects -- it is my_proc!
* We also know the new processor number for all objects initially on
* my_proc (since we built export lists.)
* This case is a special case of partition remapping; it is easy to
* build the hyperedges in this special case.
*/
HEwgt_size = zz->Num_Proc;
HEwgt = (int *) ZOLTAN_CALLOC(HEwgt_size, sizeof(int));
if (!HEwgt) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < nobj; i++)
HEwgt[new_proc[i]]++;
*HEcnt = 0;
for (i = 0; i < HEwgt_size; i++)
if (HEwgt[i] != 0) (*HEcnt)++;
ierr = malloc_HEinfo(zz, *HEcnt, HEinfo);
if (ierr < 0)
goto End;
tmp_HEinfo = *HEinfo;
cnt = 0;
for (i = 0; i < HEwgt_size; i++) {
if (HEwgt[i] != 0) {
tmp = cnt * HEINFO_ENTRIES;
tmp_HEinfo[tmp] = my_proc; /* Old processor number */
tmp_HEinfo[tmp+1] = i; /* New processor number */
tmp_HEinfo[tmp+2] = HEwgt[i]; /* shift non-zero weights down. */
cnt++;
}
}
}
else { /* ZOLTAN_LB_REMAP_PARTS */
/* Cannot renumber partitions given export lists without summing HE weights
* across processors. This summation is not straightforward. Also, a
* potentially large number of HEs may exist
* (max_old_partition_number * zz->Num_Global_Parts). Rather than build
* this large matrix, just compute import lists from the export lists
* and run the import-list algorithm.
*/
ZOLTAN_COMM_OBJ *plan;
int msg_tag = 22345;
ierr = Zoltan_Comm_Create(&plan, nobj, new_proc, zz->Communicator,
msg_tag, &nimp);
if (nimp > 0) {
imp_proc = (int *) ZOLTAN_MALLOC(3 * nimp * sizeof(int));
imp_old_part = imp_proc + nimp;
imp_new_part = imp_old_part + nimp;
if (!imp_proc) {
ierr = ZOLTAN_MEMERR;
ZOLTAN_PRINT_ERROR(my_proc, yo, "Memory error.");
goto End;
}
}
ierr = Zoltan_Comm_Info(plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, imp_proc, NULL);
msg_tag++;
ierr = Zoltan_Comm_Do(plan, msg_tag, (char *) old_part, sizeof(int),
(char *) imp_old_part);
msg_tag++;
ierr = Zoltan_Comm_Do(plan, msg_tag, (char *) new_part, sizeof(int),
(char *) imp_new_part);
Zoltan_Comm_Destroy(&plan);
ierr = local_HEs_from_import_lists(zz, remap_type, nimp, imp_proc,
imp_old_part, imp_new_part,
HEcnt, HEinfo);
}
End:
if (HEwgt) ZOLTAN_FREE(&HEwgt);
if (imp_proc) ZOLTAN_FREE(&imp_proc);
return ierr;
}
void check_comm_info(
ZOLTAN_COMM_OBJ *plan,
struct Data *my_send_data,
int nvals_recv,
int my_proc
)
{
int *info_tmp = NULL, /* Temp bufs for verifying Zoltan_Comm_Info */
*info_tmp_send = NULL,
*info_tmp_recv = NULL;
int *info_send_list = NULL,
*info_send_procs = NULL,
*info_send_lengths = NULL;
int info_nsend = 0,
info_send_nvals = 0,
info_send_size = 0,
info_max_send_size = 0;
int *info_recv_list = NULL,
*info_recv_procs = NULL,
*info_recv_lengths = NULL;
int info_nrecv = 0,
info_recv_nvals = 0,
info_recv_size = 0,
info_total_recv_size = 0;
int info_self_msg = 0;
int i;
Zoltan_Comm_Info(plan, &info_nsend, NULL, NULL,
&info_send_nvals, NULL, NULL, &info_nrecv, NULL,
NULL, &info_recv_nvals, NULL, NULL, &info_self_msg);
if (info_send_nvals != my_send_data->nvals)
printf("%d Error in Zoltan_Comm_Info info_send_nvals %d != %d\n",
my_proc, info_send_nvals, my_send_data->nvals);
if (info_recv_nvals != nvals_recv)
printf("%d Error in Zoltan_Comm_Info info_recv_nvals %d != %d\n",
my_proc, info_recv_nvals, nvals_recv);
info_send_size = 2 * (info_nsend + info_self_msg) + info_send_nvals;
info_send_procs = (int *) ZOLTAN_MALLOC(info_send_size * sizeof(int));
info_send_lengths = info_send_procs + (info_nsend + info_self_msg);
info_send_list = info_send_lengths + (info_nsend + info_self_msg);
info_recv_size = 2 * (info_nrecv + info_self_msg) + info_recv_nvals;
info_recv_procs = (int *) ZOLTAN_MALLOC(info_recv_size * sizeof(int));
info_recv_lengths = info_recv_procs + (info_nrecv + info_self_msg);
info_recv_list = info_recv_lengths + (info_nrecv + info_self_msg);
Zoltan_Comm_Info(plan, &info_nsend, info_send_procs, info_send_lengths,
&info_send_nvals, &info_max_send_size, info_send_list,
&info_nrecv, info_recv_procs, info_recv_lengths,
&info_recv_nvals, &info_total_recv_size,
info_recv_list, &info_self_msg);
for (i = 0; i < info_send_nvals; i++)
if (info_send_list[i] != my_send_data->proc_dest[i])
printf("%d Error in Zoltan_Comm_Info send_list[%d]: %d != %d\n",
my_proc, i, info_send_list[i], my_send_data->proc_dest[i]);
info_tmp = (int *) ZOLTAN_MALLOC((info_send_nvals + info_recv_nvals)
* sizeof(int));
info_tmp_send = info_tmp;
info_tmp_recv = info_tmp_send + info_send_nvals;
for (i = 0; i < info_send_nvals; i++)
info_tmp_send[i] = my_proc;
Zoltan_Comm_Do(plan, 12, (char *) info_tmp_send, sizeof(int),
(char *) info_tmp_recv);
for (i = 0; i < info_recv_nvals; i++)
if (info_recv_list[i] != info_tmp_recv[i])
printf("%d Error in Zoltan_Comm_Info recv_list[%d]: %d != %d\n",
my_proc, i, info_recv_list[i], info_tmp_recv[i]);
ZOLTAN_FREE(&info_tmp);
ZOLTAN_FREE(&info_recv_procs);
ZOLTAN_FREE(&info_send_procs);
}
int Zoltan_DD_Update (
Zoltan_DD_Directory *dd, /* directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to update */
ZOLTAN_ID_PTR lid, /* Incoming corresponding LIDs (optional) */
char *user, /* Incoming list of user data (optional) */
int *partition, /* Optional, grouping of GIDs to partitions */
int count) /* Number of GIDs in update list */
{
int *procs = NULL; /* list of processors to contact */
DD_Update_Msg *ptr = NULL;
ZOLTAN_COMM_OBJ *plan = NULL; /* for efficient MPI communication */
char *sbuff = NULL; /* send buffer */
char *sbufftmp = NULL;/* pointer into send buffer */
char *rbuff = NULL; /* receive buffer */
char *rbufftmp = NULL;/* pointer into receive buffer */
int nrec = 0; /* number of receives to expect */
int i;
int err;
int errcount = 0; /* count of GIDs not found, added */
char str[100]; /* build error message string */
char *yo = "Zoltan_DD_Update";
/* input sanity checking */
if (dd == NULL || count < 0 || (gid == NULL && count > 0)) {
ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo,
"Invalid input argument");
return ZOLTAN_FATAL;
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* part of initializing the error checking process */
/* for each linked list head, walk its list resetting errcheck */
if (dd->debug_level)
for (i = 0; i < dd->table_length; i++) {
DD_NodeIdx nodeidx;
for (nodeidx = dd->table[i]; nodeidx != -1;
nodeidx = dd->nodelist[nodeidx].next)
dd->nodelist[nodeidx].errcheck = ZOLTAN_DD_NO_PROC;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After reset errcheck");
/* allocate memory for list of processors to contact */
if (count) {
procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count);
if (procs == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list");
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* allocate memory for DD_Update_Msg send buffer */
if (count) {
sbuff = (char*) ZOLTAN_CALLOC (count, dd->update_msg_size);
if (sbuff == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer");
err = ZOLTAN_MEMERR;
goto fini;
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs");
/* for each GID given, fill in contact list and then message structure */
sbufftmp = sbuff;
for (i = 0; i < count; i++) {
procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc,
dd->hashdata, dd->hashfn);
ptr = (DD_Update_Msg*) sbufftmp;
sbufftmp += dd->update_msg_size;
ptr->lid_flag = (lid) ? 1 : 0;
ptr->user_flag = (user) ? 1 : 0;
ptr->partition_flag = (partition) ? 1 : 0;
ptr->partition = (partition) ? *(partition + i) : -1;
ptr->owner = dd->my_proc;
ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length);
if (lid) {
ZOLTAN_SET_ID(dd->lid_length, ptr->gid + dd->gid_length,
lid + i * dd->lid_length);
}
else {
memset(ptr->gid + dd->gid_length, 0, dd->lid_length);
}
if (user) {
memcpy(ptr->gid + (dd->gid_length + dd->lid_length),
user + (size_t)i * (size_t)(dd->user_data_length),
dd->user_data_length);
}
else {
memset(ptr->gid + (dd->gid_length + dd->lid_length), 0,
dd->user_data_length);
}
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill contact list");
/* now create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_UPDATE_MSG_TAG, &nrec);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error");
goto fini;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create");
/* If dd has no nodes allocated (e.g., first call to DD_Update;
* create the nodelist and freelist
*/
if (nrec && dd->nodelistlen == 0) {
DD_Memory_Alloc_Nodelist(dd, (DD_NodeIdx) nrec, 0.);
/* TODO Add overalloc parameter */
}
/* allocate receive buffer for nrec DD_Update_Msg structures */
if (nrec) {
rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->update_msg_size));
if (rbuff == NULL) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed");
err = ZOLTAN_MEMERR;
goto fini;
}
}
/* send my update messages & receive updates directed to me */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff,
dd->update_msg_size, rbuff);
if (err != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error");
goto fini;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do");
/* for each message rec'd, update local directory information */
errcount = 0;
rbufftmp = rbuff;
for (i = 0; i < nrec; i++) {
ptr = (DD_Update_Msg *) rbufftmp;
rbufftmp += dd->update_msg_size;
err = DD_Update_Local (dd, ptr->gid,
(ptr->lid_flag) ? (ptr->gid + dd->gid_length) : NULL,
(char*)((ptr->user_flag)?(ptr->gid+(dd->gid_length+dd->lid_length)):NULL),
(ptr->partition_flag) ? (ptr->partition) : -1, /* illegal partition */
ptr->owner);
if (err != ZOLTAN_OK)
++errcount;
}
if (dd->debug_level > 6)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Local update");
err = ZOLTAN_OK;
if (dd->debug_level) /* overwrite error return if extra checking is on */
err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK;
fini:
ZOLTAN_FREE (&procs);
ZOLTAN_FREE (&sbuff);
ZOLTAN_FREE (&rbuff);
Zoltan_Comm_Destroy (&plan);
if (dd->debug_level) {
sprintf (str, "Processed %d GIDs (%d local), %d GID errors", count,
nrec, errcount);
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str);
}
if (dd->debug_level > 4)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err;
}
int
Zoltan_Postprocess_FinalOutput (ZZ* zz, ZOLTAN_Third_Graph *gr,
ZOLTAN_Third_Part *prt, ZOLTAN_Third_Vsize *vsp,
int use_timers, realtype itr)
{
#define FOMAXDIM 10
static char * yo = "Zoltan_Postprocess_FinalOutput";
static int nRuns=0;
static double balsum[FOMAXDIM], cutesum[FOMAXDIM];
static double balmax[FOMAXDIM], cutemax[FOMAXDIM];
static double balmin[FOMAXDIM], cutemin[FOMAXDIM];
/* following variables are defined double to avoid overflow */
static double cutlsum = 0.0, cutnsum = 0.0, movesum = 0.0, repartsum = 0.0;
static float cutlmax = 0, cutnmax = 0;
static double movemax = 0, repartmax = 0;
static float cutlmin = INT_MAX, cutnmin = INT_MAX;
static double movemin = 1e100, repartmin = 1e100;
static int timer_final_output = -1;
double bal[FOMAXDIM]; /* Balance: max / avg */
double cute[FOMAXDIM]; /* Traditional weighted graph edge cuts */
float cutl; /* Connnectivity cuts: sum_over_edges((npart-1)) */
float cutn; /* Net cuts: sum_over_edges((nparts>1)) */
double move = 0.0, gmove =0.0; /* migration cost */
double repart; /* total repartitioning cost; cutl x multiplier + move */
int *adjpart = NULL;
int vdim;
int edim;
indextype *vsizeBACKUP = NULL;
indextype *input_part;
int i,rc;
/* #define UVC_DORUK_COMP_OBJSIZE */
#ifdef UVC_DORUK_COMP_OBJSIZE
double minD, maxD, gminD, gmaxD;
#endif
if (use_timers) {
if (timer_final_output < 0)
timer_final_output = Zoltan_Timer_Init(zz->ZTime, 1, "Final_Output");
ZOLTAN_TIMER_START(zz->ZTime, timer_final_output, zz->Communicator);
}
if (nRuns == 0) {
for (i = 0; i < FOMAXDIM; i++) {
/* Initialize first time */
balsum[i] = cutesum[i] = 0.0;
balmax[i] = cutemax[i] = 0.0;
balmin[i] = cutemin[i] = 1e100;
}
}
if (vsp && vsp->vsizeBACKUP)
vsizeBACKUP = vsp->vsizeBACKUP;
vdim = MAX(gr->obj_wgt_dim,1);
edim = MAX(zz->Edge_Weight_Dim,1);
if (gr->obj_wgt_dim < FOMAXDIM && zz->Edge_Weight_Dim < FOMAXDIM) {
if (gr->xadj[gr->num_obj]){
adjpart = (int *) ZOLTAN_MALLOC(gr->xadj[gr->num_obj] * sizeof(int));
if (!adjpart){
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR,
"Error 1 returned from Zoltan_Postprocess_FinalOutput");
}
}
Compute_Bal(zz, gr->num_obj, gr->vwgt, gr->obj_wgt_dim, prt->part, bal);
rc = Compute_Adjpart(zz, gr->num_obj, gr->vtxdist, gr->xadj, gr->adjncy,
gr->adjproc, prt->part, adjpart);
if (rc != ZOLTAN_OK){
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR,
"Error 2 returned from Zoltan_Postprocess_FinalOutput");
}
Compute_EdgeCut(zz, gr->num_obj, gr->xadj, gr->float_ewgts,
prt->part, adjpart, cute);
cutl = Compute_ConCut(zz, gr->num_obj, gr->xadj, gr->float_ewgts,
prt->part, adjpart);
cutn = Compute_NetCut(zz, gr->num_obj, gr->xadj, gr->float_ewgts,
prt->part, adjpart);
#ifdef UVC_DORUK_COMP_OBJSIZE
if (vsizeBACKUP) {
minD = vsizeBACKUP[0];
maxD = vsizeBACKUP[0];
}
#endif
if (gr->scatter != 0) { /* Project input part in the scatter graph */
input_part = (indextype*) ZOLTAN_MALLOC(gr->num_obj * sizeof(int));
Zoltan_Comm_Do(gr->comm_plan, TAG1, (char *) prt->input_part, sizeof(indextype),
(char *) input_part);
}
else
input_part = prt->input_part;
for (i=0; i<gr->num_obj; i++) {
/*printf("obj[%d] = %d\n", i, vsize[i]);*/
if (prt->part[i] != input_part[i]) {
move += (double) ((vsizeBACKUP) ? vsizeBACKUP[i] : 1.0);
}
#ifdef UVC_DORUK_COMP_OBJSIZE
if (vsizeBACKUP) {
minD = minD < vsizeBACKUP[i] ? minD : vsizeBACKUP[i];
maxD = maxD > vsizeBACKUP[i] ? maxD : vsizeBACKUP[i];
}
#endif
}
if (gr->scatter != 0)
ZOLTAN_FREE(&input_part);
#ifdef UVC_DORUK_COMP_OBJSIZE
if (gr->showMoveVol) {
MPI_Allreduce(&minD, &gminD, 1, MPI_DOUBLE, MPI_MIN, zz->Communicator);
MPI_Allreduce(&maxD, &gmaxD, 1, MPI_DOUBLE, MPI_MAX, zz->Communicator);
if (zz->Proc == 0)
printf("minD: %f, maxD: %f, gminD: %f, gmaxD: %f\n", minD, maxD, gminD, gmaxD);
}
#endif
MPI_Allreduce(&move, &gmove, 1, MPI_DOUBLE, MPI_SUM, zz->Communicator);
repart = (itr) * (double) cutl + gmove;
repartsum += repart;
if (repart > repartmax) repartmax = repart;
if (repart < repartmin) repartmin = repart;
movesum += gmove;
if (gmove > movemax) movemax = gmove;
if (gmove < movemin) movemin = gmove;
cutlsum += cutl;
if (cutl > cutlmax) cutlmax = cutl;
if (cutl < cutlmin) cutlmin = cutl;
cutnsum += cutn;
if (cutn > cutnmax) cutnmax = cutn;
if (cutn < cutnmin) cutnmin = cutn;
for (i = 0; i < vdim; i++) {
balsum[i] += bal[i];
if (bal[i] > balmax[i]) balmax[i] = bal[i];
if (bal[i] < balmin[i]) balmin[i] = bal[i];
}
for (i = 0; i < edim; i++) {
cutesum[i] += cute[i];
if (cute[i] > cutemax[i]) cutemax[i] = cute[i];
if (cute[i] < cutemin[i]) cutemin[i] = cute[i];
}
nRuns++;
if (zz->Proc == 0) {
for (i = 0; i < vdim; i++)
printf("STATS Runs %d bal[%d] CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, i, bal[i], balmax[i], balmin[i], balsum[i]/nRuns);
printf("STATS Runs %d cutl CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, cutl, cutlmax, cutlmin, cutlsum/nRuns);
printf("STATS Runs %d cutn CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, cutn, cutnmax, cutnmin, cutnsum/nRuns);
for (i = 0; i < edim; i++)
printf("STATS Runs %d cute[%d] CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, i, cute[i], cutemax[i], cutemin[i], cutesum[i]/nRuns);
printf("STATS Runs %d %s CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, gr->showMoveVol ? "moveVol" : "moveCnt", gmove,
movemax, movemin, movesum/nRuns);
if (gr->showMoveVol)
printf("STATS Runs %d repart CURRENT %f MAX %f MIN %f AVG %f\n",
nRuns, repart, repartmax, repartmin, repartsum/nRuns);
}
ZOLTAN_FREE(&adjpart);
}
#undef FOMAXDIM
if (use_timers)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_final_output, zz->Communicator);
return (ZOLTAN_OK);
}
int Zoltan_Migrate(
ZZ *zz, /* Zoltan structure. */
int num_import, /* Number of non-local objects assigned to the
processor in the new decomposition. */
ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects
assigned to this processor in the new
decomposition; this field can be NULL if
the application doesn't provide import IDs.*/
ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects
assigned to the processor in the new
decomposition; this field can be NULL if the
application does not provide import IDs. */
int *import_procs, /* Array of processor IDs of processors owning
the non-local objects that are assigned to
this processor in the new decomposition; this
field can be NULL if the application does
not provide import IDs. */
int *import_to_part, /* Array of partition numbers to which imported
objects should be assigned. */
int num_export, /* Number of objs to be exported
to other processors to establish the new
decomposition. */
ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of
objects to be exported to other processors
to establish the new decomposition. */
ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of
objects to be exported to other processors
to establish the new decomposition. */
int *export_procs, /* Array of processor IDs
to which objects will be exported
to establish the new decomposition. */
int *export_to_part /* Array of partition numbers to which exported
objects should be assigned. */
)
{
/*
* Routine to help perform migration. If migration pre-processing routine
* (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn.
* It then calls a function to obtain the size of the migrating objects
* (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified
* object packing routine (ZOLTAN_PACK_OBJ_FN) for each object
* to be exported. It develops the needed communication map to move the
* objects to other processors. It performs the communication according
* to the map, and then calls an application-specified object unpacking
* routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported.
*/
char *yo = "Zoltan_Migrate";
int num_gid_entries, num_lid_entries; /* lengths of global & local ids */
int *sizes = NULL; /* sizes (in bytes) of the object data for export. */
int id_size; /* size (in bytes) of ZOLTAN_GID + padding for
alignment */
int tag_size; /* size (in bytes) of ZOLTAN_GID + one int
(for message size) */
char *export_buf = NULL; /* buffer for packing export data. */
char *import_buf = NULL; /* buffer for receiving imported data. */
char *tmp; /* temporary pointer into buffers. */
int i; /* loop counter. */
int tmp_size; /* size of a single object's data. */
int *idx = NULL; /* index used for multi-fn packs and unpacks. */
int idx_cnt = 0; /* index counter for idx array. */
ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm
buf */
ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass
NULL to query functions when NUM_LID_ENTRIES=0. */
ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */
ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */
int msgtag, msgtag2; /* Tags for communication routines */
int total_send_size; /* Total size of outcoming message (in #items) */
int total_recv_size; /* Total size of incoming message (in #items) */
int aligned_int; /* size of an int padded for alignment */
int dest; /* temporary destination partition. */
int include_parts = 0; /* flag indicating whether partition info is
provided */
int ierr = ZOLTAN_OK;
int actual_num_exp = 0;
int actual_exp_allocated = 0;
ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */
int *actual_exp_procs = NULL; /* are changing partition but not */
int *actual_exp_to_part = NULL; /* processor may not be included. */
int actual_num_imp = 0;
int actual_imp_allocated = 0;
ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */
int *actual_imp_procs = NULL; /* are changing partition but not */
int *actual_imp_to_part = NULL; /* processor may not be included. */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
goto End;
}
/*
* Check that all procs use the same id types.
*/
ierr = check_input(zz,
((num_export >= 0 && export_to_part) ||
(num_import >= 0 && import_to_part)),
&include_parts);
if (ierr != ZOLTAN_OK)
goto End;
num_gid_entries = zz->Num_GID;
num_lid_entries = zz->Num_LID;
/*
* Check that all necessary query functions are available.
*/
if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (num_export >= 0) {
/* Build the actual export arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_export, export_global_ids, export_local_ids,
export_procs, export_to_part,
&actual_num_exp, &actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part,
&actual_exp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on actual exports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs,
zz->Communicator, msgtag, &actual_num_imp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
}
else if (num_import >= 0) {
/* Build the actual import arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_import, import_global_ids, import_local_ids,
import_procs, import_to_part,
&actual_num_imp, &actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part,
&actual_imp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on imports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs,
zz->Communicator, msgtag, &actual_num_exp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
/* Compute actual export lists for packing objects */
if (actual_num_exp > 0) {
actual_exp_allocated = 1;
actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp);
actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp);
actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp);
if (include_parts)
actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp);
if (actual_exp_gids == NULL ||
(num_lid_entries && actual_exp_lids == NULL) ||
actual_exp_procs == NULL ||
(import_to_part != NULL && actual_exp_to_part == NULL)) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
ierr = ZOLTAN_MEMERR;
goto End;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) actual_exp_gids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) actual_exp_lids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part,
(int) sizeof(int), (char *) actual_exp_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
/* Create inverse plan (i.e., plan based on exports) so can set
* variable sizes.
* (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable
* but does not tell how large to allocate receive buffer.
*/
ierr = Zoltan_Comm_Invert_Plan(&imp_plan);
exp_plan = imp_plan;
imp_plan = NULL;
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Migrate.Pre_Migrate_PP != NULL) {
zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Pre_Migrate != NULL) {
zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_FN function.");
goto End;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing");
id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE));
/* Note that alignment is not strictly necessary
when ZOLTAN_ID_TYPE is int or unsigned int. */
aligned_int = Zoltan_Align(sizeof(int));
tag_size = id_size + aligned_int;
/*
* For each object, allow space for its global ID and its data plus
* one int (for the object data size).
* Zoltan will pack the global IDs; the application must pack the data
* through the pack routine. Zoltan needs the global IDs for unpacking,
* as the order of the data received during communication is not
* necessarily the same order as import_global_ids[].
* Zoltan also needs to communicate the sizes of the objects because
* only the sender knows the size of each object.
*/
if (actual_num_exp > 0) {
sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!sizes) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_Size_Multi != NULL) {
zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids, sizes, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE_MULTI function.");
goto End;
}
}
else {
for (i = 0; i < actual_num_exp; i++) {
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]),
lid, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE function.");
goto End;
}
}
}
total_send_size = 0;
for (i = 0; i < actual_num_exp; i++) {
sizes[i] = Zoltan_Align(sizes[i]);
total_send_size += sizes[i] + tag_size;
}
export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char));
if (!export_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Pack_Obj_Multi != NULL) {
/* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */
idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Pack the objects for export.
*/
idx_cnt = 0;
tmp = export_buf;
for (i = 0; i < actual_num_exp; i++) {
/* Pack the object's global ID */
tmp_id = (ZOLTAN_ID_PTR) tmp;
ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries]));
tmp += id_size;
/* Pack the object's size */
*((int *)tmp) = sizes[i];
tmp += aligned_int;
/* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Pack_Obj_Multi != NULL) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n",
zz->Proc, yo);
}
zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids,
(actual_exp_to_part!=NULL ? actual_exp_to_part
: actual_exp_procs),
sizes, idx, export_buf, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ_MULTI function.");
goto End;
}
}
else {
tmp = export_buf + tag_size;
for (i = 0; i < actual_num_exp; i++) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries]));
printf("size = %d bytes\n", sizes[i]);
}
/* Pack the object's data */
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i]
: actual_exp_procs[i]);
zz->Pack_Obj(zz->Pack_Obj_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]), lid, dest,
sizes[i], tmp, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ function.");
goto End;
}
tmp += sizes[i] + tag_size;
}
}
ZOLTAN_FREE(&idx);
tmp_id = NULL;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects");
/* Modify sizes[] to contain message sizes, not object sizes */
for (i=0; i<actual_num_exp; i++) {
sizes[i] += tag_size;
}
msgtag--;
ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize.");
goto End;
}
if (actual_num_imp > 0) {
import_buf = (char *) ZOLTAN_MALLOC(total_recv_size);
if (!import_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Send the export data using the communication plan.
*/
msgtag2 = 32765;
ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
/*
* Free whatever memory we can.
*/
Zoltan_Comm_Destroy(&exp_plan);
ZOLTAN_FREE(&export_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication");
/*
* Perform application-specified processing before unpacking the data.
*/
if (zz->Migrate.Mid_Migrate_PP != NULL) {
zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Mid_Migrate != NULL) {
zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_FN function.");
goto End;
}
}
/*
* Unpack the object data.
*/
if (actual_num_imp > 0) {
if (zz->Unpack_Obj_Multi != NULL) {
/* Allocate and fill input arrays for Unpack_Obj_Multi. */
sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp);
idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
if (!sizes || !tmp_id || !idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
tmp = import_buf;
idx_cnt = 0;
for (i = 0; i < actual_num_imp; i++) {
/* Unpack the object's global ID */
ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp);
tmp += id_size;
/* Unpack the object's size */
sizes[i] = *((int *)tmp);
tmp += aligned_int;
/* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n",
zz->Proc,yo);
}
zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries,
actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr);
ZOLTAN_FREE(&import_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_FREE(&tmp_id);
ZOLTAN_FREE(&idx);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_MULTI_FN.");
goto End;
}
}
else {
tmp = import_buf;
for (i = 0; i < actual_num_imp; i++) {
tmp_size = *((int *)(tmp + id_size));
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp);
printf("size = %d bytes\n", tmp_size);
}
/* Unpack the object's data */
zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries,
(ZOLTAN_ID_PTR) tmp, tmp_size,
tmp + tag_size, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_FN.");
goto End;
}
tmp += (tmp_size + tag_size);
}
ZOLTAN_FREE(&import_buf);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects");
if (zz->Migrate.Post_Migrate_PP != NULL) {
zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Post_Migrate != NULL) {
zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_FN function.");
goto End;
}
}
End:
if (actual_exp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
}
if (actual_imp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part);
}
if (ierr < 0) {
if (exp_plan) Zoltan_Comm_Destroy(&exp_plan);
Zoltan_Multifree(__FILE__, __LINE__, 5,
&import_buf, &tmp_id, &sizes, &idx, &export_buf);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Inverse_Perm(
ZZ *zz, /* Input: Zoltan struct */
int *perm, /* Input: Permutation to invert. */
int *inv_perm, /* Output: Inverse permutation of perm. */
int *vtxdist, /* Input: Distribution of the vectors. */
char *order_type, /* Input: Local or global ordering? */
int start_index /* Input: Do permutations start with 0 or 1? */
)
{
int i, ierr, num_obj, nrecv, offset;
int *proclist, *sendlist, *recvlist;
ZOLTAN_COMM_OBJ *comm_plan;
char msg[256];
char *yo = "Zoltan_Inverse_Perm";
ierr = ZOLTAN_OK;
proclist = sendlist = recvlist = NULL;
comm_plan = NULL;
/* num_obj = local number of objects (elements in the perm vectors) */
num_obj = vtxdist[(zz->Proc)+1] - vtxdist[zz->Proc];
/* Verify that input permutation is really a permutation */
/* Also check that start_index is correct. */
/* Convert permutation vector to 0-base if necessary */
if (start_index>0){
for (i=0; i<num_obj; i++)
perm[i] -= start_index;
}
if (strcmp(order_type, "LOCAL")==0){
/* Local inverse */
for (i=0; i<num_obj; i++)
inv_perm[perm[i]] = i;
}
else if (strcmp(order_type, "GLOBAL")==0){
/* Global inverse; use Zoltan Comm package */
proclist = (int *) ZOLTAN_MALLOC (5*num_obj*sizeof(int));
sendlist = &proclist[num_obj];
recvlist = &proclist[3*num_obj];
/* Set up comm plan. We know where to send. */
/* Send pairs of (i, perm[i]) to other procs */
offset = vtxdist[zz->Proc];
for (i=0; i<num_obj; i++){
sendlist[2*i] = offset+i;
sendlist[2*i+1] = perm[i];
proclist[i] = Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, perm[i]);
}
ierr = Zoltan_Comm_Create(&comm_plan, num_obj, proclist,
zz->Communicator, TAG1, &nrecv);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in Zoltan_Comm_Create");
goto error;
}
if (nrecv != num_obj){
/* This should never happen. */
sprintf(msg, "Internal error: nrecv (%3d) != num_obj (%3d). Invalid permutation.\n", nrecv, num_obj);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ierr = ZOLTAN_FATAL;
goto error;
}
/* Do the communication. */
ierr = Zoltan_Comm_Do(comm_plan, TAG2, (char *)sendlist, 2*sizeof(int), (char *) recvlist);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in Zoltan_Comm_Do");
goto error;
}
/* Permute data locally. */
for (i=0; i<num_obj; i++){
/* inv_perm[perm[i]] = i; */
inv_perm[recvlist[2*i+1]-offset] = recvlist[2*i];
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown order_type.");
ierr = ZOLTAN_FATAL;
goto error;
}
/* Convert permutation vectors back to their right start_index */
if (start_index>0){
for (i=0; i<num_obj; i++){
perm[i] += start_index;
inv_perm[i] += start_index;
}
}
error:
/* Free the comm_plan, proclist, sendlist, recvlist. */
if (comm_plan) Zoltan_Comm_Destroy( &comm_plan);
if (proclist ) ZOLTAN_FREE(&proclist);
return (ierr);
}
static int Zoltan_PHG_Redistribute_Hypergraph(
ZZ *zz,
PHGPartParams *hgp, /* Input: parameters; used only for UseFixedVtx */
HGraph *ohg, /* Input: Local part of distributed hypergraph */
int firstproc, /* Input: rank (in ocomm) of the first proc of
the ncomm*/
int *v2Col, /* Input: Vertex to processor Column Mapping */
int *n2Row, /* Input: Net to processor Row Mapping */
PHGComm *ncomm, /* Input: communicators of new distribution */
HGraph *nhg, /* Output: Newly redistributed hypergraph */
int **vmap, /* Output: allocated with the size nhg->nVtx and
vertex map from nhg to ohg's local vertex number*/
int **vdest /* Output: allocated with the size nhg->nVtx and
stores dest proc in ocomm */
)
{
char * yo = "Zoltan_PHG_Redistribute_Hypergraph";
PHGComm *ocomm = ohg->comm;
int ierr=ZOLTAN_OK;
int i, v, n, nPins, nsend, elemsz, nVtx, nEdge;
int msg_tag = 9999;
int *proclist=NULL, *sendbuf=NULL;
int *vno=NULL, *nno=NULL, *dist_x=NULL, *dist_y=NULL,
*vsn=NULL, *nsn=NULL, *pins=NULL, *cnt=NULL;
ZOLTAN_COMM_OBJ *plan;
Zoltan_HG_HGraph_Init (nhg);
nhg->comm = ncomm;
nhg->dist_x = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int));
nhg->dist_y = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int));
dist_x = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int));
dist_y = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int));
vsn = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int));
nsn = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int));
vno = (int *) ZOLTAN_MALLOC(ohg->nVtx * sizeof(int));
nno = (int *) ZOLTAN_MALLOC(ohg->nEdge * sizeof(int));
if (!nhg->dist_x || !nhg->dist_y || !dist_x || !dist_y ||
!vsn || !nsn || (ohg->nVtx && !vno) || (ohg->nEdge && !nno) ) {
uprintf(ocomm, " new comm nProcx=%d nProcy=%d nvtx=%d nedge=%d", ncomm->nProc_x, ncomm->nProc_y, ohg->nVtx, ohg->nEdge);
MEMORY_ERROR;
}
for (v = 0; v < ohg->nVtx; ++v)
++dist_x[v2Col[v]];
for (n = 0; n < ohg->nEdge; ++n)
++dist_y[n2Row[n]];
/* UVCUVC: CHECK ASSUMPTION
This code assumes that the objects in the receive buffer of
Zoltan_Comm_Do function are
1- in the increasing processor order,
2- order of the items send by a processor is preserved.
*/
/* compute prefix sum to find new vertex start numbers; for each processor */
MPI_Scan(dist_x, vsn, ncomm->nProc_x, MPI_INT, MPI_SUM, ocomm->row_comm);
/* All reduce to compute how many each processor will have */
MPI_Allreduce(dist_x, &(nhg->dist_x[1]), ncomm->nProc_x, MPI_INT, MPI_SUM,
ocomm->row_comm);
nhg->dist_x[0] = 0;
for (i=1; i<=ncomm->nProc_x; ++i)
nhg->dist_x[i] += nhg->dist_x[i-1];
MPI_Scan(dist_y, nsn, ncomm->nProc_y, MPI_INT, MPI_SUM, ocomm->col_comm);
MPI_Allreduce(dist_y, &(nhg->dist_y[1]), ncomm->nProc_y, MPI_INT, MPI_SUM, ocomm->col_comm);
nhg->dist_y[0] = 0;
for (i=1; i<=ncomm->nProc_y; ++i)
nhg->dist_y[i] += nhg->dist_y[i-1];
#ifdef _DEBUG1
PrintArr(ocomm, "vsn", vsn, ncomm->nProc_x);
PrintArr(ocomm, "nsn", nsn, ncomm->nProc_y);
#endif
/* find mapping of current LOCAL vertex no (in my node)
to "new" vertex no LOCAL to dest node*/
for (v = ohg->nVtx-1; v>=0; --v)
vno[v] = --vsn[v2Col[v]];
for (n = ohg->nEdge-1; n>=0; --n)
nno[n] = --nsn[n2Row[n]];
nsend = MAX(MAX(ohg->nPins, ohg->nVtx), ohg->nEdge);
elemsz = MAX(MAX(2, ohg->VtxWeightDim), ohg->EdgeWeightDim);
elemsz = (sizeof(float)>sizeof(int)) ? sizeof(float)*elemsz : sizeof(int)*elemsz;
proclist = (int *) ZOLTAN_MALLOC(nsend * sizeof(int));
sendbuf = (int *) ZOLTAN_MALLOC(nsend * elemsz);
/* first communicate pins */
nPins = 0;
for (v = 0; v < ohg->nVtx; ++v) {
for (i = ohg->vindex[v]; i < ohg->vindex[v+1]; ++i) {
#ifdef _DEBUG1
if ((n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v])<0 ||
(n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v])>=ocomm->nProc)
errexit("vertex %d vedge[%d]=%d n2Row=%d #Proc_x=%d v2Col=%d", i, ohg->vedge[i], n2Row[ohg->vedge[i]], ncomm->nProc_x , v2Col[v]);
#endif
proclist[nPins] = firstproc + n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v];
sendbuf[2*nPins] = vno[v];
sendbuf[2*nPins+1]= nno[ohg->vedge[i]];
++nPins;
}
}
#ifdef _DEBUG1
if (nPins!=ohg->nPins) {
uprintf(ocomm, "sanity check failed nPins(%d)!=hg->nPins(%d)\n", nPins, ohg->nPins);
errexit("terminating");
}
#endif
--msg_tag;
ierr |= Zoltan_Comm_Create(&plan, ohg->nPins, proclist, ocomm->Communicator,
msg_tag, &nPins);
#ifdef _DEBUG1
if (ncomm->myProc==-1 && nPins>1) { /* this processor is not in new comm but receiving data?*/
uprintf(ocomm, "Something wrong; why I'm receiving data nPins=%d\n", nPins);
errexit("terminating");
}
#endif
if (nPins && (pins = (int *) ZOLTAN_MALLOC(nPins * 2 * sizeof(int)))==NULL)
MEMORY_ERROR;
--msg_tag;
Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, 2*sizeof(int),
(char *) pins);
Zoltan_Comm_Destroy(&plan);
/* now communicate vertex map */
nsend = 0;
if (!ocomm->myProc_y) { /* only first row sends to the first row of ncomm */
for (v = 0; v < ohg->nVtx; ++v) {
proclist[nsend] = firstproc+v2Col[v];
sendbuf[nsend++] = ohg->vmap[v];
}
}
--msg_tag;
ierr |= Zoltan_Comm_Create(&plan, nsend, proclist, ocomm->Communicator,
msg_tag, &nVtx);
#ifdef _DEBUG1
if (ncomm->myProc==-1 && nVtx>1) { /* this processor is not in new comm but receiving data?*/
uprintf(ocomm, "Something wrong; why I'm receiving data nVtx=%d\n", nVtx);
errexit("terminating");
}
#endif
/* those are only needed in the first row of ncomm */
*vmap = *vdest = NULL;
if (!ncomm->myProc_y && nVtx &&
(!(*vmap = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int))) ||
!(*vdest = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int)))))
MEMORY_ERROR;
--msg_tag;
Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(int),
(char *) *vmap);
if (!ocomm->myProc_y) { /* only first row sends to the first row of ncomm */
for (v = 0; v < ohg->nVtx; ++v)
sendbuf[v] = ocomm->myProc;
}
--msg_tag;
Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(int),
(char *) *vdest);
if (ncomm->myProc!=-1) { /* I'm in the new comm */
/* ncomm's first row now bcast to other rows */
MPI_Bcast(&nVtx, 1, MPI_INT, 0, ncomm->col_comm);
#ifdef _DEBUG1
if (nVtx!=(nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x]))
errexit("nVtx(%d)!= nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x](%d)", nVtx, nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x]);
#endif
if (nVtx && (nhg->vmap = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int)))==NULL)
MEMORY_ERROR;
for (i=0; i<nVtx; ++i)
nhg->vmap[i] = i;
}
/* now communicate vertex weights */
if (ohg->VtxWeightDim) {
if (nVtx)
nhg->vwgt = (float*) ZOLTAN_MALLOC(nVtx*ohg->VtxWeightDim*sizeof(float));
--msg_tag;
Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->vwgt,
ohg->VtxWeightDim*sizeof(float), (char *) nhg->vwgt);
if (ncomm->myProc!=-1) /* ncomm's first row now bcast to other rows */
MPI_Bcast(nhg->vwgt, nVtx*ohg->VtxWeightDim, MPI_FLOAT, 0, ncomm->col_comm);
}
/* communicate fixed vertices, if any */
if (hgp->UseFixedVtx) {
if (nVtx)
nhg->fixed_part = (int *) ZOLTAN_MALLOC(nVtx*sizeof(int));
--msg_tag;
Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->fixed_part,
sizeof(int), (char *) nhg->fixed_part);
if (ncomm->myProc!=-1) /* ncomm's first row now bcast to other rows */
MPI_Bcast(nhg->fixed_part, nVtx, MPI_INT, 0, ncomm->col_comm);
}
/* communicate pref parts, if any */
if (hgp->UsePrefPart) {
if (nVtx)
nhg->pref_part = (int *) ZOLTAN_MALLOC(nVtx*sizeof(int));
--msg_tag;
Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->pref_part,
sizeof(int), (char *) nhg->pref_part);
if (ncomm->myProc!=-1) /* ncomm's first row now bcast to other rows */
MPI_Bcast(nhg->pref_part, nVtx, MPI_INT, 0, ncomm->col_comm);
}
/* this comm plan is no longer needed. */
Zoltan_Comm_Destroy(&plan);
if (ohg->EdgeWeightDim) { /* now communicate edge weights */
nsend = 0;
if (!ocomm->myProc_x) /* only first column sends to first column of ncomm */
for (n = 0; n < ohg->nEdge; ++n)
proclist[nsend++] = firstproc + n2Row[n]*ncomm->nProc_x;
--msg_tag;
ierr |= Zoltan_Comm_Create(&plan, nsend, proclist, ocomm->Communicator,
msg_tag, &nEdge);
#ifdef _DEBUG1
if (ncomm->myProc==-1 && nEdge>1) { /* this processor is not in new comm but receiving data?*/
uprintf(ocomm, "Something wrong; why I'm receiving data nEdge=%d\n", nEdge);
errexit("terminating");
}
#endif
if (ncomm->myProc!=-1) { /* if we're in the new comm */
/* ncomm's first column now bcast to other columns */
MPI_Bcast(&nEdge, 1, MPI_INT, 0, ncomm->row_comm);
#ifdef _DEBUG1
if (nEdge != (nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y]))
errexit("nEdge(%d)!=nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y](%d)", nEdge, nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y]);
#endif
}
if (nEdge)
nhg->ewgt = (float*) ZOLTAN_MALLOC(nEdge*ohg->EdgeWeightDim*sizeof(float));
--msg_tag;
Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->ewgt,
ohg->EdgeWeightDim*sizeof(float), (char *) nhg->ewgt);
if (ncomm->myProc!=-1) { /* if we're in the new comm */
/* ncomm's first column now bcast to other columns */
if (nEdge)
MPI_Bcast(nhg->ewgt, nEdge*ohg->EdgeWeightDim, MPI_FLOAT, 0,
ncomm->row_comm);
}
Zoltan_Comm_Destroy(&plan);
} else
nEdge = (ncomm->myProc==-1)
? 0
: nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y];
if (ncomm->myProc==-1) {
#ifdef _DEBUG1
if (nPins || nVtx || nEdge)
errexit("I should not have any data: hey nPins=%d nVtx=%d nEdge=%d\n", nPins, nVtx, nEdge);
#endif
nhg->nEdge = nhg->nVtx = nhg->nPins = 0;
} else {
nhg->nEdge = nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y];
nhg->nVtx = nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x];
nhg->nPins = nPins;
/* Unpack the pins received. */
cnt = (int *) ZOLTAN_CALLOC(nhg->nVtx + 1, sizeof(int));
nhg->vindex = (int *) ZOLTAN_CALLOC(nhg->nVtx + 1, sizeof(int));
nhg->vedge = (int *) ZOLTAN_MALLOC(nhg->nPins * sizeof(int));
if (!cnt || !nhg->vindex || (nPins && !nhg->vedge))
MEMORY_ERROR;
/* Count the number of pins per vertex */
for (i = 0; i < nPins; ++i)
++cnt[pins[2*i]];
/* Compute prefix sum to represent hindex correctly. */
for (i = 0; i < nhg->nVtx; ++i) {
nhg->vindex[i+1] = nhg->vindex[i] + cnt[i];
cnt[i] = nhg->vindex[i];
}
for (i = 0; i < nPins; ++i)
nhg->vedge[cnt[pins[2*i]]++] = pins[2*i+1];
nhg->info = ohg->info;
nhg->VtxWeightDim = ohg->VtxWeightDim;
nhg->EdgeWeightDim = ohg->EdgeWeightDim;
ierr = Zoltan_HG_Create_Mirror(zz, nhg);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN)
MEMORY_ERROR;
}
End:
Zoltan_Multifree(__FILE__, __LINE__, 10,
&proclist, &sendbuf, &pins, &cnt,
&vno, &nno, &dist_x, &dist_y, &vsn, &nsn
);
return ierr;
}
static int Zoltan_Oct_Update_Connections(
ZZ *zz,
pOctant *octs, /* octs[nocts] */
int *newpids, /* newpids[nocts] */
pOctant *newocts, /* newocts[nocts] */
int nocts) /* number of octants leaving this processor */
{
int i, j;
int nsends;
int nreceives;
pOctant parent;
pOctant child;
int ppid;
int cpid;
int childnum;
int *despid = NULL;
Update_msg umsg;
Update_msg *localumsg = NULL;
Update_msg *remoteumsg = NULL;
Update_msg *rcv_umsg = NULL;
int localcount;
int remotecount;
int ierr = ZOLTAN_OK;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
char *yo = "Zoltan_Oct_Update_Connections";
OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure;
localcount=0;
remotecount=0;
/* count number of sends */
nsends = 0;
for (i=0; i<nocts; i++)
if (newpids[i]!=zz->Proc)
nsends++;
if(nocts > 0) {
if((remoteumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((localumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
return ZOLTAN_MEMERR;
}
if((despid = (int *) ZOLTAN_MALLOC((nocts+1) * sizeof(int)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
return ZOLTAN_MEMERR;
}
}
else {
remoteumsg = NULL;
localumsg = NULL;
despid = NULL;
}
localcount = 0;
remotecount = 0;
for (i=0; i<nocts; i++) /* Send connection updates */
if (newpids[i]!=zz->Proc) {
parent = Zoltan_Oct_parent(octs[i]);
ppid = Zoltan_Oct_Ppid(octs[i]);
childnum = Zoltan_Oct_childnum(octs[i]);
if (parent) { /* Let parent of oct[i] know that it's moving */
if (ppid==zz->Proc) {
FILLUPDATEMSG(localumsg[localcount], parent, childnum, newocts[i], newpids[i]);
localcount++;
}
else {
FILLUPDATEMSG(remoteumsg[remotecount], parent, childnum, newocts[i], newpids[i]);
despid[remotecount++] = ppid;
}
}
for (j=0; j<8; j++) {
child = Zoltan_Oct_child(octs[i],j);
cpid = octs[i]->cpid[j];
/* Tell child of oct[i] that it is moving */
if (child) {
if (cpid==zz->Proc) {
/* NOTE: -1 signals PARENT */
FILLUPDATEMSG(localumsg[localcount], child, -1, newocts[i], newpids[i]);
localcount++;
}
else {
/* NOTE: -1 signals PARENT */
FILLUPDATEMSG(remoteumsg[remotecount], child, -1, newocts[i], newpids[i]);
despid[remotecount++] = cpid;
}
}
}
}
ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator,
MigUpdCommCreate, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
return (ierr);
}
/* if(nreceives > 0) { */
if((rcv_umsg = (Update_msg *) ZOLTAN_MALLOC((nreceives +1) * sizeof(Update_msg)*9)) == NULL) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
return ZOLTAN_MEMERR;
}
ierr = Zoltan_Comm_Do(comm_plan, MigUpdCommDo, (char *) remoteumsg,
sizeof(Update_msg), (char *) rcv_umsg);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_umsg);
return (ierr);
}
/* } */
/* else { */
/* rcv_umsg = NULL; */
/* } */
/* update new octants */
for (i=0; i< (localcount+nreceives); i++) {
if (i<localcount)
umsg=localumsg[i];
else
umsg=rcv_umsg[i-localcount];
if (umsg.childnum>=0) {
Zoltan_Oct_setchild(umsg.oct,umsg.childnum,umsg.newptr);
Zoltan_Oct_setCpid(umsg.oct,umsg.childnum,umsg.newpid);
}
else {
if((Zoltan_Oct_data_newpid(umsg.oct) == OCT_info->OCT_localpid) ||
((Zoltan_Oct_data_newpid(umsg.oct) != OCT_info->OCT_localpid) && (umsg.newpid == OCT_info->OCT_localpid)))
Zoltan_Oct_POct_setparent(OCT_info, umsg.oct,umsg.newptr,umsg.newpid);
else {
umsg.oct->ppid = umsg.newpid;
umsg.oct->parent = umsg.newptr;
}
}
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&despid);
ZOLTAN_FREE(&rcv_umsg);
return (ierr);
}
ZOLTAN_FREE(&remoteumsg);
ZOLTAN_FREE(&localumsg);
ZOLTAN_FREE(&rcv_umsg);
ZOLTAN_FREE(&despid);
return ierr;
}