/* * gather_by_list * input: a list of processors, a message to be sent to these processors, * and a receive buffer to get data back. * * output: concatenated messages from other processors in rbuff, with * appropriate size info in rbuff_size. */ int gather_by_list(int procs_length, int *procs, int sbuff_size, char *sbuff, int *rbuff_size, char **rbuff, MPI_Comm *comm) { int i, receive_size, mtag, err, myProc; char *send; ZOLTAN_COMM_OBJ *plan; static char *yo = "gather_by_list"; MPI_Comm_rank(*comm, &myProc); if (!(send = (char*) ZOLTAN_MALLOC(procs_length * sizeof(int)))) { ZOLTAN_PRINT_ERROR(myProc, yo, "Insufficient memory"); return ZOLTAN_MEMERR; } mtag = 0; for (i = 0; i < procs_length; ++i) ((int*)send)[i] = sbuff_size; /* create and resize communication plan */ err = Zoltan_Comm_Create(&plan, procs_length, procs, *comm, mtag++, &receive_size); if (err < 0) ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Create failed."); err = Zoltan_Comm_Resize(plan, (int*)send, mtag++, rbuff_size); if (err < 0) ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Resize failed."); ZOLTAN_FREE(&send); /* allocate send and receive buffer */ if ((!(*rbuff = (char*) ZOLTAN_MALLOC(*rbuff_size)) && (*rbuff_size != 0)) ||!( send = (char*) ZOLTAN_MALLOC(sbuff_size * procs_length))) { printf("failed allocating %d + %d * %d bytes. . .", *rbuff_size, sbuff_size, procs_length); fflush(NULL); ZOLTAN_PRINT_ERROR(myProc, yo, "Insufficient Memory"); return ZOLTAN_MEMERR; } /* copy message for each processor */ for (i = 0; i < procs_length; ++i) memcpy(send + i * sbuff_size, sbuff, sbuff_size); /* finally, we can do the communication */ err = Zoltan_Comm_Do(plan, mtag++, send, 1, *rbuff); if (err < 0) ZOLTAN_PRINT_ERROR(myProc, yo, "Zoltan_Comm_Do failed."); /* clean up */ Zoltan_Comm_Destroy(&plan); ZOLTAN_FREE(&send); return err; }
static int Compute_Adjpart( ZZ *zz, int nvtx, /* Input: # vtxs in this processor */ indextype *vtxdist, /* Input: Distribution of vertices across processors */ indextype *xadj, /* Input: Index of adjncy: adjncy[xadj[i]] to adjncy[xadj[i]+1] are all edge nbors of vtx i. */ indextype *adjncy, /* Input: Array of nbor vertices. */ int *adjproc, /* Input: adjproc[j] == processor owning adjncy[j]. */ indextype *part, /* Input: Partition assignments of vtxs. */ int *adjpart /* Output: adjpart[j] == partition owning adjncy[j] */ ) { /* Given an adjacency list adjncy, find the partition number of each * vertex in adjncy. Return it in adjpart. */ ZOLTAN_COMM_OBJ *plan; int i; indextype start = vtxdist[zz->Proc]; /* First vertex on this processor */ indextype *recv_gno= NULL; int *send_int=NULL; int nrecv; int tag = 24542; Zoltan_Comm_Create(&plan, (int)xadj[nvtx], adjproc, zz->Communicator, tag++, &nrecv); if (nrecv){ recv_gno = (indextype *) ZOLTAN_MALLOC(nrecv * sizeof(indextype)); send_int = (int *) ZOLTAN_MALLOC(nrecv * sizeof(int)); if (!recv_gno || !send_int) { Zoltan_Comm_Destroy(&plan); ZOLTAN_FREE(&recv_gno); ZOLTAN_FREE(&send_int); return ZOLTAN_MEMERR; } } Zoltan_Comm_Do(plan, tag++, (char *) adjncy, sizeof(indextype), (char *) recv_gno); for (i = 0; i < nrecv; i++){ send_int[i] = part[recv_gno[i] - start]; } ZOLTAN_FREE(&recv_gno); Zoltan_Comm_Do_Reverse(plan, tag, (char *)send_int, sizeof(int), NULL, (char *) adjpart); ZOLTAN_FREE(&send_int); Zoltan_Comm_Destroy(&plan); return ZOLTAN_OK; }
static int Zoltan_Oct_Final_Migration( ZZ *zz, pOctant *octs, /* octs[nocts] */ int *newpids, /* newpids[nocts] */ pOctant *newocts, /* newocts[nocts] */ int nocts, /* number of octants leaving this processor */ int nrecocts) /* number of octants received in this processor */ { int i; Migrate_msg *msnd = NULL, *mrcv = NULL; int remotecount; int nsends; int nreceives; int *despid = NULL; int ierr = ZOLTAN_OK; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ char *yo = "Zoltan_Oct_Final_Migration"; OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure; /* count number of sends */ nsends=0; for (i=0; i<nocts; i++) if (newpids[i]!=zz->Proc) nsends++; if((despid = (int *) ZOLTAN_MALLOC((nocts+10) * sizeof(int))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((msnd = (Migrate_msg *) ZOLTAN_MALLOC((nocts+10) * sizeof(Migrate_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); return ZOLTAN_MEMERR; } remotecount = 0; for (i=0; i<nocts; i++) /* Send and free */ if (newpids[i]!=zz->Proc) { FILLMIGRATEMSG(octs[i], newocts[i], msnd[remotecount], zz->Proc); /* bug */ despid[remotecount++] = newpids[i]; Zoltan_Oct_clearRegions(octs[i]); /* KDDKDDFREE Change oct to &oct to allow NULL from ZOLTAN_FREE * KDDKDDFREE to propagate back. */ Zoltan_Oct_POct_free(OCT_info, &(octs[i])); } ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator, MigFinCommCreate, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); return (ierr); } if((mrcv = (Migrate_msg *) ZOLTAN_MALLOC((nreceives+10) * sizeof(Migrate_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); return ZOLTAN_MEMERR; } ierr = Zoltan_Comm_Do(comm_plan, MigFinCommDo, (char *) msnd, sizeof(Migrate_msg), (char *) mrcv); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); ZOLTAN_FREE(&mrcv); return (ierr); } for (i=0; i<nreceives; i++) { /* Receive new parocts */ /* Zoltan_Oct_setID(mrcv[i].ptr,mrcv[i].id); */ /* Zoltan_Oct_POct_setparent(OCT_info, mrcv[i].ptr, mrcv[i].parent, mrcv[i].ppid); */ SETOCTFROMMIGRATEMSG(OCT_info, mrcv[i]); /* Zoltan_Oct_setchildnum(mrcv[i].ptr,mrcv[i].childnum); */ /* Zoltan_Oct_setchildren(mrcv[i].ptr, mrcv[i].children, mrcv[i].cpids); */ /* Zoltan_Oct_setbounds(mrcv[i].ptr, mrcv[i].min, mrcv[i].max); */ /* Zoltan_Oct_setDir(mrcv[i].ptr,mrcv[i].dir); */ /* Zoltan_Oct_setMapIdx(mrcv[i].ptr,mrcv[i].mapidx); */ } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); ZOLTAN_FREE(&mrcv); return (ierr); } ZOLTAN_FREE(&despid); ZOLTAN_FREE(&msnd); ZOLTAN_FREE(&mrcv); return ierr; }
int Zoltan_DD_Remove ( Zoltan_DD_Directory *dd, /* directory state infomation */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to remove */ int count) /* Number of GIDs in removal list */ { int *procs = NULL; /* list of processors to contact */ DD_Remove_Msg *ptr = NULL; ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL;/* pointer into send buffer */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL;/* pointer into receive buffer */ int nrec; /* number of receives to expect */ int i; int err; /* error condition to return */ int errcount; /* count of GIDs not found */ char str[100]; /* string to build error messages */ char *yo = "Zoltan_DD_Remove"; /* input sanity checks */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN (dd->my_proc, yo, NULL); /* allocate memory for processor contact list */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } /* allocate memory for DD_Remove_Msg send buffer */ if (count) { sbuff = (char*)ZOLTAN_MALLOC((size_t)(dd->remove_msg_size)*(size_t)count); if (sbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); err = ZOLTAN_MEMERR; goto fini; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After proc & sbuff mallocs"); /* for each GID, fill in contact list and then message structure */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Remove_Msg*) sbufftmp; sbufftmp += dd->remove_msg_size; ptr->owner = dd->my_proc; ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length); } /* now create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_REMOVE_MSG_TAG, &nrec); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Create"); /* allocate receive buffer for nrec DD_Remove_Msg structures */ if (nrec) { rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->remove_msg_size)); if (rbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed"); err = ZOLTAN_MEMERR; goto fini; } } /* send my remove messages & receive removes directed to me */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff, dd->remove_msg_size, rbuff); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO (dd->my_proc, yo, "After Zoltan_Comm_Do"); /* for each message rec'd, remove local directory info */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Remove_Msg*) rbufftmp; rbufftmp += dd->remove_msg_size; err = DD_Remove_Local (dd, ptr->gid); if (err == ZOLTAN_WARN) ++errcount; } err = ZOLTAN_OK; if (dd->debug_level) { sprintf (str, "Processed %d GIDs (%d local), %d GIDs not found", count, nrec, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK; } /* done, now free up things and return */ fini: ZOLTAN_FREE (&procs); ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); Zoltan_Comm_Destroy (&plan); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT (dd->my_proc, yo, NULL); return err; }
/* * void malloc_new_objects(); * * gets the tags being imported into this processor, and sets up the * import_tags array, and the nrectags array. */ static int malloc_new_objects(ZZ *zz, int nsentags, pRegion exported_tags, ZOLTAN_ID_PTR exported_gids, ZOLTAN_ID_PTR exported_lids, int *tag_pids, int *nstags, pRegion *ex_tags, pRegion prev_tags, ZOLTAN_ID_PTR prev_gids, ZOLTAN_ID_PTR prev_lids, int npimtags, float *c3) { char *yo = "malloc_new_objects"; int i; /* index counter */ int nreceives; /* number of messages received */ pRegion t_b_exp; /* array of tags to be exported */ pRegion tmp = NULL; ZOLTAN_ID_PTR tmp_gids = NULL; ZOLTAN_ID_PTR tmp_lids = NULL; int msgtag, msgtag2; int j; int ierr = ZOLTAN_OK; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; float im_load; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ im_load = 0; msgtag = 32767; ierr = Zoltan_Comm_Create(&comm_plan, nsentags, tag_pids, zz->Communicator, msgtag, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create."); ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); } if (nreceives > 0) { tmp = (pRegion) ZOLTAN_MALLOC(nreceives * sizeof(Region)); tmp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, nreceives); tmp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, nreceives); if(tmp == NULL || !tmp_gids || (num_lid_entries && !tmp_lids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } msgtag2 = 32766; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_tags, sizeof(Region), (char *) tmp); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_gids, sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) tmp_gids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); fprintf(stderr, "OCT %s Error %s returned from Zoltan_Comm_Do\n", yo, (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } if (num_lid_entries > 0) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_lids, sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) tmp_lids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Destroy."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); return(ierr); } /* get each message sent, and store region in import array */ j=0; for (i=0; i<nreceives; i++) { im_load += tmp[i].Weight; if(tmp[i].newProc != zz->Proc) { j++; } } if((j + npimtags) != 0) { /* malloc import array */ if((t_b_exp = (pRegion)ZOLTAN_MALLOC((j+npimtags)*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } } else t_b_exp = NULL; /* setup return pointer */ (*ex_tags) = t_b_exp; j=0; for (i=0; i<nreceives; i++) { if(tmp[i].newProc != zz->Proc) { t_b_exp[j] = tmp[i]; t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz); t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz); if (!(t_b_exp[j].Global_ID) || (num_lid_entries && !(t_b_exp[j].Local_ID))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID, &(tmp_gids[i*num_gid_entries])); ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(tmp_lids[i*num_lid_entries])); j++; } } if(npimtags > 0) { for(i=0; i<npimtags; i++) { t_b_exp[j] = prev_tags[i]; t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz); t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz); if (!(t_b_exp[j].Global_ID) || (num_lid_entries && !(t_b_exp[j].Local_ID))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID, &(prev_gids[i*num_gid_entries])); ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(prev_lids[i*num_lid_entries])); j++; } } *nstags = j; ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); if((*nstags == 0) && (*ex_tags != NULL)) { ZOLTAN_TRACE_DETAIL(zz, yo, "Fatal error, import tags not empty but no tags received\n"); return ZOLTAN_FATAL; } *c3 = im_load; return ierr; }
int main(int argc, char *argv[]) { int rc, i, ngids, maxcol, ncolors; float ver; struct Zoltan_Struct *zz=NULL; #ifdef ZOLTANV31 int numGidEntries, numLidEntries; #else ZOLTAN_GRAPH_EVAL graph; #endif int *color; ZOLTAN_ID_PTR gid_list; UZData guz, *uz=&guz; int msg_tag = 9999; int nlvtx, next, maxdeg=0; double times[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */ double gtimes[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */ /****************************************************************** ** Initialize MPI and Zoltan ******************************************************************/ MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &uz->myRank); MPI_Comm_size(MPI_COMM_WORLD, &uz->numProcs); MPI_Barrier(MPI_COMM_WORLD); times[0] = u_wseconds(); rc = Zoltan_Initialize(argc, argv, &ver); if (rc != ZOLTAN_OK){ fprintf(stderr, "Sorry Zoltan initialize failed...\n"); goto End; } zz = Zoltan_Create(MPI_COMM_WORLD); if (argc<3 && !uz->myRank) { fprintf(stderr, "usage: %s [meshR] [meshC] [X-point stencil] [procR] [procC] [ws-beta] [<ZoltanParam>=<Val>] ...\n\n", argv[0]); fprintf(stderr, "ws-beta: is the probablity of adding an edge to a vertex to generate Watts-Strogatz graphs\n"); fprintf(stderr, "Valid values for Stencil are 5, 7 and 9\n"); fprintf(stderr, "Zoltan Coloring Parameters and values are\n"); fprintf(stderr, "\tDISTANCE : 1 or 2\n"); fprintf(stderr, "\tSUPERSTEP_SIZE : suggested >= 100\n"); fprintf(stderr, "\tCOMM_PATTERN : S or A\n"); fprintf(stderr, "\tCOLOR_ORDER : I, B, U\n"); fprintf(stderr, "\tCOLORING_METHOD : F (for now)\n"); fprintf(stderr, "\n"); } uz->procR = uz->procC = 0; uz->meshR = uz->meshC = 1024; uz->stencil = 9; if (argc>1) uz->meshR = atoi(argv[1]); if (argc>2) uz->meshC = atoi(argv[2]); if (argc>3) uz->stencil = atoi(argv[3]); if (uz->stencil!=5 && uz->stencil!=7 && uz->stencil!=9) { fprintf(stderr, "\t invalid stencil value. Valid values are 5, 7 and 9. Assumed 9.\n"); uz->stencil = 9; } --uz->stencil; if (argc>4) uz->procR = atoi(argv[4]); if (argc>5) uz->procC = atoi(argv[5]); if (uz->procR <= 0 || uz->procC <= 0) computeProcMesh(uz); if (uz->procR*uz->procC!=uz->numProcs) { fprintf(stderr, "#Procs=%d but requested %dx%d Proc Mesh Partitioning...\n", uz->numProcs, uz->procR, uz->procC); goto End; } if (argc>6) uz->beta = atof(argv[6]); else uz->beta = 0.0; /* compute which part of mesh I will compute */ uz->myR = uz->myRank / uz->procC; uz->myC = uz->myRank % uz->procC; uz->_sr = uz->myR * (uz->meshR / uz->procR); uz->_er = (uz->myR+1) * (uz->meshR / uz->procR); if (uz->_er>uz->meshR) uz->_er = uz->meshR; uz->_sc = uz->myC * (uz->meshC / uz->procC); uz->_ec = (uz->myC+1) * (uz->meshC / uz->procC); if (uz->_ec>uz->meshC) uz->_ec = uz->meshC; if ( (uz->meshR % uz->procR) !=0 || (uz->meshC % uz->procC)!=0) { printf("Mesh dimensions are not divisible with proc mesh.\nRequested mesh is %dx%d and proc mesh is %d x %d\n", uz->meshR, uz->meshC, uz->procR, uz->procC); exit(1); } nlvtx= (uz->_er-uz->_sr) * (uz->_ec-uz->_sc); if (uz->myRank==0) printf("Running %s on %d x %d processor mesh, generating %d-point %d x %d mesh with beta=%.3lf\n", argv[0], uz->procR, uz->procC, uz->stencil+1, uz->meshR, uz->meshC, uz->beta); times[1] = u_wseconds(); uz->numredge = 0; uz->redgeto = NULL; if (uz->beta>0) { /* create random edges for WS graph */ int ngvtx=uz->meshC*uz->meshR, trsh=(int) (uz->beta*100.0); int ierr=0; int *edges=NULL, *redges=NULL, *proclist=NULL, nedge; ZOLTAN_COMM_OBJ *plan; uz->redgeto = (int *) malloc(nlvtx*sizeof(int)); for (i=0; i<nlvtx; ++i) { int rv = Zoltan_Rand_InRange(NULL, 100); if ( rv < trsh) { if ((uz->redgeto[i] = Zoltan_Rand_InRange(NULL, ngvtx))==gIDfLID(i)) /* is it a self edge */ uz->redgeto[i] = -1; else ++uz->numredge; } else uz->redgeto[i] = -1; } edges = (int *) malloc(sizeof(int)*2*uz->numredge); proclist = (int *) malloc(sizeof(int)*uz->numredge); next = 0; for (i=0; i<nlvtx; ++i) if (uz->redgeto[i]>0) { edges[2*next] = uz->redgeto[i]; edges[2*next+1] = gIDfLID(i); proclist[next] = pIDfGID(uz->redgeto[i]); ++next; } ierr = Zoltan_Comm_Create(&plan, uz->numredge, proclist, MPI_COMM_WORLD, msg_tag, &nedge); redges = (int *) malloc(sizeof(int)*2*nedge); --msg_tag; ierr |= Zoltan_Comm_Do(plan, msg_tag, (char *) edges, 2*sizeof(int), (char *) redges); ierr |= Zoltan_Comm_Destroy(&plan); free(proclist); free(edges); if (ierr) { printf("error while communicating edges!\n"); exit(1); } xadj = (int *) calloc(1+nlvtx, sizeof(int)); adj = (int *) malloc(sizeof(int)*nedge); for (i=0; i<nedge; ++i) { if (redges[2*i] < gID(uz->_sr, uz->_sc) || redges[2*i] >= gID(uz->_er, uz->_ec)) { printf("[%d/%d] received gid=%d doesn't blong to processor range [%d, %d) should go to proc %d\n", uz->myRank, uz->numProcs, redges[2*i], gID(uz->_sr, uz->_sc), gID(uz->_er, uz->_ec), pIDfGID(redges[2*i])); } ++xadj[lIDfGID(redges[2*i])]; } xadj[nlvtx] = nedge; maxdeg = xadj[0]; for (i=1; i<nlvtx; ++i) { maxdeg = xadj[i]>maxdeg ? xadj[i] : maxdeg; xadj[i] += xadj[i-1]; } for (i=0; i<nedge; ++i) { int u = lIDfGID(redges[2*i]); int v = redges[2*i+1]; adj[--xadj[u]] = v; } free(redges); } maxdeg += uz->stencil+1; adjTemp = (int *) malloc(sizeof(int)*2*maxdeg); times[2] = u_wseconds(); /* printf("My rank %d/%d at proc-mesh loc (%d, %d) generating [%d, %d) x [%d, %d) + %d random edges TotEdge=%d\n", uz->myRank, uz->numProcs, uz->myR, uz->myC, uz->_sr, uz->_er, uz->_sc, uz->_ec, uz->numredge, xadj[nlvtx]); */ printStats("Number of Vertices ", nlvtx, uz->myRank, uz->numProcs); if (xadj) printStats("Number of Rand Edges", xadj[nlvtx], uz->myRank, uz->numProcs); /* General parameters */ #ifndef ZOLTANV31 Zoltan_Set_Param(zz, "GRAPH_BUILD_TYPE", "FAST_NO_DUP"); #endif /* General parameters */ Zoltan_Set_Param(zz, "DEBUG_LEVEL", "3"); Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0"); /* coloring parameters */ Zoltan_Set_Param(zz, "SUPERSTEP_SIZE", "500"); /* let's make S=500 default */ for (i=7; i<argc; ++i) { char param[256], *eq; if (!uz->myRank) printf("processing argv[%d]='%s'\n", i, argv[i]); strncpy(param, argv[i], sizeof(param)); eq = strchr(param, '='); if (!eq) { fprintf(stderr, "invalid argument '%s', Zoltan Paramters should be in the format <ZoltanParam>=<Val>\n", param); goto End; } *eq = 0; Zoltan_Set_Param(zz, param, eq+1); } #if 0 /* Graph parameters */ Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); #endif /* set call backs */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, uz); Zoltan_Set_Obj_List_Fn(zz, get_object_list, uz); Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, uz); Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, uz); #if 0 #ifndef ZOLTANV31 Zoltan_LB_Eval_Graph(zz, 0, &graph); if (!uz->myRank) { printf("EdgeCut Min=%8.0f Max=%8.0f Sum=%8.0f\n", graph.cuts[EVAL_GLOBAL_MIN], graph.cuts[EVAL_GLOBAL_MAX], graph.cuts[EVAL_GLOBAL_SUM]); printf("#Vertices Min=%8.0f Max=%8.0f Sum=%8.0f imbal=%.2f\n", graph.nobj[EVAL_GLOBAL_MIN], graph.nobj[EVAL_GLOBAL_MAX], graph.nobj[EVAL_GLOBAL_SUM], graph.obj_imbalance); } #endif #endif /* now color */ ngids = get_number_of_objects(uz, &rc); gid_list = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * ngids); #ifndef ZOLTANV31 next = 0; for (i=uz->_sr; i<uz->_er; ++i) { int j; for (j=uz->_sc; j<uz->_ec; ++j) { gid_list[next++] = i*uz->meshC + j; } } #endif color = (int *) malloc(sizeof(int) * ngids); MPI_Barrier(MPI_COMM_WORLD); times[3] = u_wseconds(); #ifdef ZOLTANV31 rc = Zoltan_Color(zz, /* input (all remaining fields are output) */ &numGidEntries, /* Number of integers used for a global ID */ &numLidEntries, /* Number of integers used for a local ID */ ngids, /* #objects to color in this proc */ gid_list, /* global ids of colored vertices */ NULL, /* we ignore local ids */ color); /* result color */ #else rc = Zoltan_Color(zz, /* input (all remaining fields are output) */ 1, /* Number of integers used for a global ID */ ngids, /* #objects to color in this proc */ gid_list, /* global ids of colored vertices */ color); /* result color */ #endif MPI_Barrier(MPI_COMM_WORLD); times[4] = u_wseconds(); MPI_Reduce(times, gtimes, 5, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); if (rc != ZOLTAN_OK) fprintf(stderr, "Zoltan_Color failed with return code %d...\n", rc); for (maxcol=i=0; i<ngids; ++i) if (color[i] > maxcol) maxcol = color[i]; MPI_Reduce(&maxcol, &ncolors, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); if (uz->myRank==0) { struct rusage usage; printf("%s setup Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[1]-times[0], gtimes[1]-gtimes[0]); printf("%s gen rand edges Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[2]-times[1], gtimes[2]-gtimes[1]); printf("%s set gids Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[3]-times[2], gtimes[3]-gtimes[2]); printf("%s Zoltan_Color call Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[4]-times[3], gtimes[4]-gtimes[3]); printf("%s Coloring Time : %.2lf # Colors used : %d\n", argv[0], gtimes[4]-gtimes[0], ncolors); getrusage(RUSAGE_SELF, &usage); printf("%s maxrss=%ld minflt=%ld majflt=%ld nswap=%ld\n", argv[0], usage.ru_maxrss, usage.ru_minflt, usage.ru_majflt, usage.ru_nswap); } #ifdef _DEBUG saveColor(argv[0], uz, (int *) gid_list, color, ngids); #endif /****************************************************************** ** Clean up ******************************************************************/ if (gid_list) free(gid_list); if (color) free(color); if (xadj) free(xadj); if (adj) free(adj); if (adjTemp) free(adjTemp); if (uz->redgeto) free(uz->redgeto); End: Zoltan_Destroy(&zz); MPI_Finalize(); return 0; }
int Zoltan_Invert_Lists( ZZ *zz, /* Zoltan structure. */ int num_in, /* Number of objects in the input lists. */ ZOLTAN_ID_PTR in_global_ids, /* Array of input global IDs. */ ZOLTAN_ID_PTR in_local_ids, /* Array of input local IDs. */ int *in_procs, /* Array of processor IDs of processors owning the input objects. */ int *in_to_part, /* Optional: Array of partition numbers to which input objects should be assigned. */ int *num_out, /* Returned value: Number of output objs. */ ZOLTAN_ID_PTR *out_global_ids,/* Returned value: Array of global IDs of output objects. */ ZOLTAN_ID_PTR *out_local_ids, /* Returned value: Array of local IDs of output objects. */ int **out_procs, /* Returned value: Array of processor IDs to which output objects are assigned. */ int **out_to_part /* Optional: Returned value: Array of partition numbers to which output objects should be assigned. */ ) { /* * Routine to compute the inverse map. Can be used in two ways: * 1. Given, for each processor, a list of objects to be received by the * processor, compute the list of objects that the processor needs to send * to other processors to satisfy their needs. * 2. Given, for each processor, a list of objects to be sent to other * processors, compute the list of objects that the processor needs to receive * to satisfy its needs. */ char *yo = "Zoltan_Invert_Lists"; char msg[256]; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned communication routines */ int msgtag, msgtag2; /* Message tags for communication routines */ int num_gid_entries, num_lid_entries; /* Length of global and local ids */ int include_parts; /* Flag indicating whether to compute inverse list for partitions. */ int ierr, ret_ierr = ZOLTAN_OK; ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Check that all procs use the same id types. */ ierr = check_invert_input(zz, num_in, in_procs, in_to_part, &num_gid_entries, &num_lid_entries, &include_parts); if (ierr != ZOLTAN_OK) { ZOLTAN_TRACE_EXIT(zz, yo); return ierr; } /* Initialize returned arrays. */ *out_global_ids = NULL; *out_local_ids = NULL; *out_procs = NULL; if (include_parts) *out_to_part = NULL; /* * Compute communication map and num_out, the number of objs this * processor has to out to establish the new decomposition. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator, msgtag, num_out); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Create.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; goto End; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create"); /* * Allocate space for the object tags that need to be outed. Communicate * to get the list of objects to be outed. */ if (*num_out > 0) { if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out, ZOLTAN_SPECIAL_MALLOC_GID)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out, ZOLTAN_SPECIAL_MALLOC_LID)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out, ZOLTAN_SPECIAL_MALLOC_INT)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (include_parts) { if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out, ZOLTAN_SPECIAL_MALLOC_INT)) { ret_ierr = ZOLTAN_MEMERR; goto End; } } } /* * Use the communication plan to send global IDs, local IDs, and processor * numbers. Do in separate communications to avoid a memory copy and to * simplify implementation when a data type is added to the comm. package * (to support heterogeneous computing). */ msgtag2 = 32766; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) *out_global_ids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) *out_local_ids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } } Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, *out_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part, (int) sizeof(int), (char *) *out_to_part); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do"); End: Zoltan_Comm_Destroy(&comm_plan); if (ret_ierr == ZOLTAN_MEMERR) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID); Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID); Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT); if (include_parts) Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT); } ZOLTAN_TRACE_EXIT(zz, yo); return (ret_ierr); }
static int read_comm_map_info(int pexoid, int Proc, PROB_INFO_PTR prob, MESH_INFO_PTR mesh) { /* Local declarations. */ char *yo = "read_comm_map_info"; int ielem, imap, loc_elem, iblk, max_len, offset, index; int nnodei, nnodeb, nnodee, nelemi, nelemb, nncmap; int *int_elem, *bor_elem; int *proc_ids; int *gids, *my_procs, *recv_procs; int ierr, nrecv; int msg = 200; int sid; ELEM_INFO_PTR elements = mesh->elements; ZOLTAN_COMM_OBJ *comm_obj; E_Type etype; /***************************** BEGIN EXECUTION ******************************/ DEBUG_TRACE_START(Proc, yo); if (ne_get_loadbal_param(pexoid, &nnodei, &nnodeb, &nnodee, &nelemi, &nelemb, &nncmap, &(mesh->necmap), Proc) < 0) { Gen_Error(0, "fatal: Error returned from ne_get_loadbal_param"); return 0; } /* * get the list of the border elements in order to set * the border flag in the element structures */ int_elem = (int *) malloc ((nelemi + nelemb) * sizeof(int)); if (!int_elem) { Gen_Error(0, "fatal: insufficient memory"); return 0; } bor_elem = int_elem + nelemi; if (ne_get_elem_map(pexoid, int_elem, bor_elem, Proc) < 0) { Gen_Error(0, "fatal: Error returned from ne_get_elem_map"); return 0; } for (ielem = 0; ielem < nelemb; ielem++) { elements[bor_elem[ielem]-1].border = 1; } free(int_elem); /* * For now, only get the elemental communication maps, * since, in the driver, elements are only considered * adjacent if they share a face (same definition used * in element communication maps). Eventually, the ability * to consider elements that are connected by any nodes * adjacent will have to be added. When that happens, * the nodal communication maps will be needed. */ mesh->ecmap_cnt = (int *) malloc (mesh->necmap * sizeof(int)); mesh->ecmap_id = (int *) malloc(mesh->necmap * sizeof(int)); if (!mesh->ecmap_cnt || !mesh->ecmap_id) { Gen_Error(0, "fatal: insufficient memory"); return 0; } if (ne_get_cmap_params(pexoid, NULL, NULL, mesh->ecmap_id, mesh->ecmap_cnt, Proc) < 0) { Gen_Error(0, "fatal: Error returned from ne_get_cmap_params"); return 0; } max_len = 0; for (imap = 0; imap < mesh->necmap; imap++) max_len += mesh->ecmap_cnt[imap]; proc_ids = (int *) malloc(4 * max_len * sizeof(int)); gids = proc_ids + max_len; my_procs = gids + max_len; recv_procs = my_procs + max_len; mesh->ecmap_elemids = (int *) malloc(max_len * sizeof(int)); mesh->ecmap_sideids = (int *) malloc(max_len * sizeof(int)); mesh->ecmap_neighids = (int *) malloc(max_len * sizeof(int)); if (!mesh->ecmap_elemids || !mesh->ecmap_sideids || !mesh->ecmap_neighids) { Gen_Error(0, "fatal: insufficient memory"); return 0; } offset = 0; for (imap = 0; imap < mesh->necmap; imap++) { if(ne_get_elem_cmap(pexoid, mesh->ecmap_id[imap], &(mesh->ecmap_elemids[offset]), &(mesh->ecmap_sideids[offset]), &(proc_ids[offset]), Proc) < 0) { Gen_Error(0, "fatal: Error returned from ne_get_elem_cmap"); return 0; } offset += mesh->ecmap_cnt[imap]; } /* End: "for (imap = 0; imap < mesh->necmap; imap++)" */ /* * Decrement the ecmap_elemids by one for zero-based local numbering. * Convert the element ids to global ids to send to * the neighboring processor. */ for (ielem = 0; ielem < max_len; ielem++) { mesh->ecmap_elemids[ielem]--; gids[ielem] = elements[mesh->ecmap_elemids[ielem]].globalID; my_procs[ielem] = Proc; } /* * Now communicate with other processor to get global IDs * for the adjacent elements in this communication map. */ ierr = Zoltan_Comm_Create(&comm_obj, max_len, proc_ids, MPI_COMM_WORLD, msg, &nrecv); if (ierr != ZOLTAN_OK) { Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Create"); return 0; } if (nrecv != max_len) { /* Sanity check; this should never happen. */ Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Create"); return 0; } /* Exchange ids to neighbors. * Assuming messages will be stored in order of processor number in * ecmap_neighids. */ ierr = Zoltan_Comm_Do(comm_obj, msg+1, (char *) gids, sizeof(int), (char *) (mesh->ecmap_neighids)); /* Exchange sanity check information. * Allows to check assumption that messages are stored in order of * processor number. */ if (ierr == ZOLTAN_OK) ierr = Zoltan_Comm_Do(comm_obj, msg+2, (char *) my_procs, sizeof(int), (char *) recv_procs); if (ierr != ZOLTAN_OK) { Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Do"); return 0; } ierr = Zoltan_Comm_Destroy(&comm_obj); /* Sanity check: messages stored in order of processor number. */ for (ielem = 0; ielem < max_len; ielem++) { if (proc_ids[ielem] != recv_procs[ielem]) { Gen_Error(0, "fatal: Sanity check failed; assumption wrong"); return 0; } } /* now process all of the element ids that have been received */ offset = 0; for (imap = 0; imap < mesh->necmap; imap++) { for (ielem = 0; ielem < mesh->ecmap_cnt[imap]; ielem++) { index = ielem + offset; /* translate from element id in the communication map to local elem id */ loc_elem = mesh->ecmap_elemids[index]; iblk = elements[loc_elem].elem_blk; etype = (E_Type) (mesh->eb_etypes[iblk]); (elements[loc_elem].nadj)++; if(elements[loc_elem].nadj > elements[loc_elem].adj_len) { /* Shouldn't happen as long as only side adjacencies are used. */ /* adj_len == number of sides. */ /* Space should already be allocated for the adjacencies read */ /* from the communication maps. */ Gen_Error(0, "fatal: Number of adj greater than adj_len"); return 0; } /* Store adjacency info in the adj entry corresponding to this side. */ sid = mesh->ecmap_sideids[index] - 1; elements[loc_elem].adj[sid] = mesh->ecmap_neighids[index]; elements[loc_elem].adj_proc[sid] = mesh->ecmap_id[imap]; elements[loc_elem].edge_wgt[sid] = (float) get_elem_info(NSNODES, etype, mesh->ecmap_sideids[index]); } /* End: "for (ielem = 0; ielem < mesh->ecmap_cnt[imap]; ielem++)" */ offset += mesh->ecmap_cnt[imap]; } /* End: "for for (imap = 0; imap < mesh->necmap; imap++)" */ free (proc_ids); DEBUG_TRACE_END(Proc, yo); return 1; }
static int Zoltan_Preprocess_Scatter_Graph (ZZ *zz, ZOLTAN_Third_Graph *gr, ZOLTAN_Third_Part *prt, ZOLTAN_Third_Geom *geo, ZOLTAN_Third_Vsize *vsp) { int ierr = ZOLTAN_OK; int tmp; if ((gr->scatter>0) && (gr->scatter<3)){ int j; /* Decide if the data imbalance is so bad that we should scatter the graph. */ /* scatter==1: Scatter if all the objects are on a single processor. */ /* scatter==2: Scatter if any processor has no objects. */ gr->num_obj_orig = gr->num_obj; /* Save value for later. */ if (gr->num_obj==0) j = 1; else j = 0; MPI_Allreduce(&j, &tmp, 1, MPI_INT, MPI_SUM, zz->Communicator); if (gr->scatter == 1){ if (tmp < zz->Num_Proc-1) gr->scatter = 0; } else if (gr->scatter==2){ if (tmp == 0) gr->scatter = 0; } } /* We need to make sure we don't scatter the graph * if graph_type = LOCAL_GRAPH, i.e. METIS is used. */ if (gr->scatter && (gr->graph_type == LOCAL_GRAPH)){ gr->scatter = 0; ZOLTAN_PRINT_WARN(zz->Proc, __func__, "Setting scatter_graph=0 since the graph" " is local on each proc"); ierr = ZOLTAN_WARN; } if (gr->scatter){ if (geo) ierr = Zoltan_Scatter_Graph(&gr->vtxdist, &gr->xadj, &gr->adjncy, &gr->vwgt, &vsp->vsize, &gr->ewgts, &geo->xyz, geo->ndims, gr->obj_wgt_dim, zz, &gr->comm_plan); else { float* xyz = NULL; ierr = Zoltan_Scatter_Graph(&gr->vtxdist, &gr->xadj, &gr->adjncy, &gr->vwgt, &vsp->vsize, &gr->ewgts, &xyz, 0, gr->obj_wgt_dim, zz, &gr->comm_plan); } if ((ierr == ZOLTAN_FATAL) || (ierr == ZOLTAN_MEMERR)){ ZOLTAN_THIRD_ERROR(ierr, "Zoltan_Scatter_Graph returned error."); } gr->num_obj = gr->vtxdist[zz->Proc+1]-gr->vtxdist[zz->Proc]; if (prt) { prt->part_orig = prt->part; prt->part = (indextype *) ZOLTAN_MALLOC((gr->num_obj+1) * sizeof(indextype)); Zoltan_Comm_Do(gr->comm_plan, TAG1, (char *) prt->part_orig, sizeof(indextype), (char *) prt->part); } } return ierr; }
int Zoltan_Order_Get_GID_Order( struct Zoltan_Struct *zz, ZOLTAN_ID_PTR global_ids, ZOLTAN_ID_PTR order_ids ) { int * proctab; ZOLTAN_ID_PTR sendgidtab; int * sendtab, *recvtab; int i; int nrecv; struct Zoltan_Comm_Obj *plan; int ierr = ZOLTAN_OK; int *vtxdist; int tag = 24061986; int offset; zz->Order.gidrank = order_ids; if (!strcmp(zz->Order.order_type, "LOCAL")) { for (i = 0 ; i < zz->Order.nbr_objects ; ++i) { order_ids[i] = global_ids[zz->Order.rank[i] - zz->Order.start_index]; } return (ZOLTAN_OK); } ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr != ZOLTAN_OK) return (ierr); proctab = (int*) ZOLTAN_MALLOC(3*zz->Order.nbr_objects*sizeof(int)); sendgidtab = ZOLTAN_MALLOC_GID_ARRAY(zz, 2*zz->Order.nbr_objects); if (proctab == NULL) { ZOLTAN_FREE(&vtxdist); return (ZOLTAN_MEMERR); } if (sendgidtab == NULL) { ZOLTAN_FREE(&proctab); ZOLTAN_FREE(&vtxdist); return (ZOLTAN_MEMERR); } sendtab = proctab + zz->Order.nbr_objects; recvtab = sendtab + zz->Order.nbr_objects; for (i = 0 ; i < zz->Order.nbr_objects ; ++i) { proctab[i] = Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, zz->Order.rank[i] - zz->Order.start_index); sendtab[i] = zz->Order.rank[i] - zz->Order.start_index; } ierr = Zoltan_Comm_Create(&plan, zz->Order.nbr_objects, proctab, zz->Communicator, tag++, &nrecv); if (ierr != ZOLTAN_OK) { ZOLTAN_FREE(&sendgidtab); ZOLTAN_FREE(&proctab); ZOLTAN_FREE(&vtxdist); return (ierr); } ierr = Zoltan_Comm_Do(plan, tag++, (char *) sendtab, sizeof(int), (char *) recvtab); if (ierr != ZOLTAN_OK) { ZOLTAN_FREE(&sendgidtab); ZOLTAN_FREE(&proctab); ZOLTAN_FREE(&vtxdist); return (ierr); } offset = vtxdist[zz->Proc]; for (i = 0 ; i < zz->Order.nbr_objects ; ++i) { int position; position = recvtab[i]-offset; ZOLTAN_SET_GID(zz, &sendgidtab[i], &global_ids[position]); } ierr = Zoltan_Comm_Do_Reverse(plan, tag++, (char *) sendgidtab, zz->Num_GID*sizeof(int), NULL, (char *) order_ids); Zoltan_Comm_Destroy(&plan); ZOLTAN_FREE(&sendgidtab); ZOLTAN_FREE(&proctab); ZOLTAN_FREE(&vtxdist); return (ierr); }
/* if !copy, inmat is not usable after this call */ int Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */ Zoltan_matrix_2d *outmat, int copy) { static char *yo = "Zoltan_Matrix2d_Distribute"; int ierr = ZOLTAN_OK; int nProc_x, nProc_y; int myProc_x, myProc_y; int i, j, cnt; int *proclist = NULL; Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL; ZOLTAN_GNO_TYPE *perm_y = NULL; float *wgtarray = NULL; float *tmpwgtarray = NULL; int msg_tag = 1021982; ZOLTAN_COMM_OBJ *plan; MPI_Comm communicator = MPI_COMM_NULL; int nProc; ZOLTAN_GNO_TYPE *yGNO = NULL; ZOLTAN_GNO_TYPE *pinGNO = NULL; ZOLTAN_GNO_TYPE tmp_gno; void *partdata = NULL; MPI_Datatype zoltan_gno_mpi_type; ZOLTAN_TRACE_ENTER(zz, yo); zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix)); if(copy) { /* TODO: We need to copy the arrays also */ Zoltan_Matrix_Reset (&outmat->mtx); /* Copy also directories */ outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX); if (inmat.ddY == inmat.ddX) outmat->mtx.ddY = outmat->mtx.ddX; else outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY); } communicator = outmat->comm->Communicator; nProc = outmat->comm->nProc; nProc_x = outmat->comm->nProc_x; nProc_y = outmat->comm->nProc_y; myProc_x = outmat->comm->myProc_x; myProc_y = outmat->comm->myProc_y; KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_Duplicates"); ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx); /* KDDKDDKDD FIX INDENTATION OF THIS BLOCK */ if (inmat.opts.speed != MATRIX_NO_REDIST) { if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) { /* I need to know the original distribution */ if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */ outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear; } else { int *cmember = NULL; cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int)); if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR; Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL, outmat->mtx.nY, NULL); KDDKDDKDD(zz->Proc, " Zoltan_Distribute_Partition_Register"); partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO, cmember, zz->Num_Proc, zz->Num_Proc); ZOLTAN_FREE(&cmember); Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata); } } /* * Build comm plan for sending non-zeros to their target processors in * 2D data distribution. */ /* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */ proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int)); sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc)); if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR; wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float)); if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR; yGNO = outmat->mtx.yGNO; pinGNO = outmat->mtx.pinGNO; KDDKDDKDD(zz->Proc, " CommPlan Hash"); cnt = 0; for (i = 0; i < outmat->mtx.nY; i++) { ZOLTAN_GNO_TYPE edge_gno=-1; /* processor row for the edge */ edge_gno = yGNO[i]; for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) { ZOLTAN_GNO_TYPE vtx_gno=-1; /* processor column for the vertex */ vtx_gno = pinGNO[j]; proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData, &sendbuf[cnt].part_y); if (proclist[cnt] < 0) /* Discard this nnz */ continue; sendbuf[cnt].GNO[0] = edge_gno; sendbuf[cnt].GNO[1] = vtx_gno; memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim, outmat->mtx.pinwgtdim*sizeof(float)); cnt++; } if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) { proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData, &sendbuf[cnt].part_y); if (proclist[cnt] < 0) /* Discard this nnz */ continue; sendbuf[cnt].GNO[0] = edge_gno; sendbuf[cnt].GNO[1] = -1; memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float)); cnt++; } } if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) Zoltan_Distribute_Partition_Free(&outmat->hashDistData); if (outmat->mtx.yend != outmat->mtx.ystart + 1) ZOLTAN_FREE(&outmat->mtx.yend); outmat->mtx.yend = NULL; ZOLTAN_FREE(&outmat->mtx.ystart); ZOLTAN_FREE(&outmat->mtx.yGNO); ZOLTAN_FREE(&outmat->mtx.pinGNO); ZOLTAN_FREE(&outmat->mtx.pinwgt); ZOLTAN_FREE(&outmat->mtx.yGID); /* * Send pins to their target processors. * They become non-zeros in the 2D data distribution. */ KDDKDDKDD(zz->Proc, " CommPlan Create"); msg_tag--; ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins); ZOLTAN_FREE(&proclist); nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc)); if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR; msg_tag--; Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros); ZOLTAN_FREE(&sendbuf); if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */ tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float)); if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR; msg_tag--; Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float), (char *) tmpwgtarray); ZOLTAN_FREE(&wgtarray); } Zoltan_Comm_Destroy(&plan); /* Unpack the non-zeros received. */ KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_DupArcs"); /* TODO: do take care about singletons */ Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray, &outmat->mtx); } /* Now we just have to change numbering */ outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE)); outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE)); if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR; /* FIXME: Work only in 1D */ tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY; MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator); for (i = 1 ; i <= nProc_y ; i ++) { outmat->dist_y[i] += outmat->dist_y[i-1]; } outmat->dist_x[1] = outmat->mtx.globalX; perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE)); if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR; for (i = 0 ; i < outmat->mtx.nY ; ++i){ perm_y[i] = i + outmat->dist_y[myProc_y]; } KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute"); Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y); KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute done"); End: ZOLTAN_FREE(&perm_y); ZOLTAN_FREE(&proclist); ZOLTAN_FREE(&sendbuf); ZOLTAN_FREE(&nonzeros); ZOLTAN_FREE(&tmpwgtarray); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_DD_Find ( Zoltan_DD_Directory *dd, /* contains directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */ ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */ ZOLTAN_ID_PTR data, /* Outgoing optional corresponding user data */ int *partition, /* Outgoing optional partition information */ int count, /* Count of GIDs in above list (in) */ int *owner) /* Outgoing corresponding list of data locations */ { ZOLTAN_COMM_OBJ *plan = NULL ; /* efficient MPI communication */ char *rbuff = NULL ; /* receive buffer */ char *sbuff = NULL ; /* send buffer */ int *procs = NULL ; /* list of processors to contact */ DD_Find_Msg *ptr = NULL ; int i ; int nrec ; /* number of messages to receive */ int err ; /* return error condition */ int errcount ; /* count of GIDs not found */ char *yo = "Zoltan_DD_Find" ; if (dd != NULL && dd->debug_level > 1) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* input sanity check */ if (dd == NULL || count < 0 || ((owner == NULL || gid == NULL) && count > 0)) { ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, "Invalid input argument.") ; if (dd != NULL && dd->debug_level > 1) ZOLTAN_TRACE_OUT((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, NULL); return ZOLTAN_DD_INPUT_ERROR ; } /* allocate memory for processors to contact for directory info */ if (count > 0) { procs = (int *) ZOLTAN_MALLOC (sizeof (int) * count) ; if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list.") ; if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_DD_MEMORY_ERROR ; } } /* allocate memory for DD_Find_Msg send buffer */ if (count > 0) { sbuff = (char *) ZOLTAN_MALLOC (dd->find_msg_size * count) ; if (sbuff == NULL) { ZOLTAN_FREE (&procs) ; ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer.") ; if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_DD_MEMORY_ERROR ; } } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs."); /* for each GID, fill DD_Find_Msg buffer and contact list */ for (i = 0 ; i < count ; i++) { procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc) ; ptr = (DD_Find_Msg *) (sbuff + i * dd->find_msg_size) ; ptr->index = i ; ptr->proc = procs[i] ; ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length) ; } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill."); /* create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_FIND_MSG_TAG, &nrec) ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create."); if (err != ZOLTAN_OK) goto fini ; /* allocate receive buffer */ if (nrec > 0) { rbuff = (char *) ZOLTAN_MALLOC (nrec * dd->find_msg_size) ; if (rbuff == NULL) { err = ZOLTAN_DD_MEMORY_ERROR ; goto fini ; } } /* send out find messages across entire system */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff, dd->find_msg_size, rbuff) ; if (err != ZOLTAN_OK) goto fini ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do."); /* get find messages directed to me, fill in return information */ errcount = 0 ; for (i = 0 ; i < nrec ; i++) { ptr = (DD_Find_Msg *) (rbuff + i*dd->find_msg_size) ; err = DD_Find_Local (dd, ptr->id, ptr->id, ptr->id + dd->max_id_length, (partition) ? (&ptr->partition) : NULL, &ptr->proc) ; if (err == ZOLTAN_DD_GID_NOT_FOUND_ERROR) errcount++ ; } if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info."); /* send return information back to requester */ err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff, dd->find_msg_size, NULL, sbuff) ; if (err != ZOLTAN_OK) goto fini ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse."); /* fill in user supplied lists with returned information */ for (i = 0 ; i < count ; i++) { ptr = (DD_Find_Msg *) (sbuff + i*dd->find_msg_size) ; owner[ptr->index] = ptr->proc ; if (partition != NULL) partition[ptr->index] = ptr->partition ; if (lid != NULL) ZOLTAN_SET_ID (dd->lid_length, lid + ptr->index * dd->lid_length, ptr->id) ; if (data != NULL) ZOLTAN_SET_ID (dd->user_data_length, data + ptr->index * dd->user_data_length, ptr->id + dd->max_id_length) ; } /* if at least one GID was not found, notify caller of error */ err = (errcount == 0) ? ZOLTAN_DD_NORMAL_RETURN : ZOLTAN_DD_GID_NOT_FOUND_ERROR ; if (dd->debug_level > 2) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists."); fini: ZOLTAN_FREE (&sbuff) ; ZOLTAN_FREE (&rbuff) ; ZOLTAN_FREE (&procs) ; Zoltan_Comm_Destroy (&plan) ; if (err != ZOLTAN_DD_NORMAL_RETURN) ZOLTAN_PRINT_WARN (dd->my_proc, yo, "Return is not normal.") ; if (dd->debug_level > 0) { char str[100] ; /* diagnostic message string */ sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount) ; ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ; } if (dd->debug_level > 1) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err ; }
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int *npids, int nregions, int *c2) { char *yo = "Zoltan_Oct_migreg_migrate_regions"; int i; /* index counter */ int ierr = ZOLTAN_OK; int n_import; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ Region *import_objs = NULL; /* Array of import objects used to request the objs from other processors. */ ZOLTAN_ID_PTR import_gids = NULL; /* Array of global IDs of import_objs. */ ZOLTAN_ID_PTR import_lids = NULL; /* Array of local IDs of import_objs. */ int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator, MIGMIGREGCommCreate, &n_import); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create"); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } *c2 = n_import; if (n_import > 0) { import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region)); import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import); import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import); if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region), (char *) import_objs); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids, sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } if (num_lid_entries > 0) { ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids, sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } } for (i=0; i<n_import; i++) { import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]); import_objs[i].Local_ID = (num_lid_entries ? &(import_lids[i*num_lid_entries]) : NULL); Zoltan_Oct_insert_orphan(zz, import_objs[i]); } ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } return ierr; }
int Zoltan_Oct_migrate_octants(ZZ *zz, int *newpids, pOctant *octs, int nocts, int *nrecocts) { int i,j = 0; int nsends = 0; int nreceives = 0; int *despid = NULL; OCTNEW_msg *snd_reply = NULL; OCTNEW_msg *rcv_reply = NULL; int ierr = ZOLTAN_OK; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ char *yo = "Zoltan_Oct_migrate_octants"; pOctant *newocts = NULL; /* New foreign octant pointers */ if((newocts = (pOctant *) ZOLTAN_MALLOC(sizeof(pOctant)*(nocts+10))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } for (i=0; i<nocts; i++) newocts[i]=NULL; /* count number of sends */ nsends=0; for (i=0; i<nocts; i++) if (newpids[i]!=zz->Proc) nsends++; /* build message array */ /* if(nsends > 0) { */ if((snd_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nsends + 1) * sizeof(OCTNEW_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((despid = (int *) ZOLTAN_MALLOC((nsends+10) * sizeof(int))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); return ZOLTAN_MEMERR; } /* } */ /* else { */ /* snd_reply = NULL; */ /* despid = NULL; */ /* } */ j = 0; for (i=0; i<nocts; i++) if (newpids[i]!=zz->Proc) { snd_reply[j].num = i; despid[j++] = newpids[i]; } /* send messages */ ierr = Zoltan_Comm_Create(&comm_plan, nsends, despid, zz->Communicator, MigOctCommCreate, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); return (ierr); } if((rcv_reply = (OCTNEW_msg *) ZOLTAN_MALLOC((nreceives + 1) * sizeof(OCTNEW_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); return ZOLTAN_MEMERR; } ierr = Zoltan_Comm_Do(comm_plan, MigOctCommDo, (char *) snd_reply, sizeof(OCTNEW_msg), (char *) rcv_reply); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); return (ierr); } /* Reply to malloc requests and Receive malloc replies */ for (i=0; i< nreceives; i++) { rcv_reply[i].ptr = Zoltan_Oct_POct_new((OCT_Global_Info *) (zz->LB.Data_Structure)); } ; ierr = Zoltan_Comm_Do_Reverse(comm_plan, MigOctCommReverse, (char *) rcv_reply, sizeof(OCTNEW_msg), NULL, (char *) snd_reply); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); return (ierr); } /* store remote pointers locally for future migration */ for (i=0; i<nsends; i++) { newocts[snd_reply[i].num] = snd_reply[i].ptr; } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); return (ierr); } ZOLTAN_FREE(&snd_reply); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_reply); /* set return value */ *nrecocts = nreceives; ierr = Zoltan_Oct_Update_Connections(zz, octs, newpids, newocts, nocts); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); abort(); return (ierr); } ierr = Zoltan_Oct_Final_Migration(zz, octs,newpids,newocts,nocts, *nrecocts); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); abort(); return (ierr); } Zoltan_Oct_Update_Map(zz); ZOLTAN_FREE(&newocts); return ierr; }
static int Zoltan_Oct_build_global_rootlist(ZZ *zz,Migrate_msg **ret_rmsg, int *size) { int j, k = 0; int *despid = NULL; int nroots, nreceives; pRList RootList; /* list of the local roots */ pOctant RootOct; Migrate_msg *snd_rmsg = NULL; Migrate_msg *rcv_rmsg = NULL; OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); /*Map *array = OCT_info->map;*/ ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ int ierr = ZOLTAN_OK; char *yo = "Zoltan_Oct_build_global_rootlist"; nroots = RL_numRootOctants(Zoltan_Oct_POct_localroots(OCT_info)); if (nroots > 0) { /* KDDKDD -- Added test to prevent departure before comm */ if((despid = (int *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(int))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((snd_rmsg = (Migrate_msg *) ZOLTAN_MALLOC((zz->Num_Proc)*nroots * sizeof(Migrate_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); return ZOLTAN_MEMERR; } k = 0; for (j=0; j<zz->Num_Proc; j++) { RootList = Zoltan_Oct_POct_localroots(OCT_info); while((RootOct = RL_nextRootOctant(&RootList))) { /* if(array[Zoltan_Oct_mapidx(RootOct)].npid > 0) { */ FILLMIGRATEMSG(RootOct, RootOct, snd_rmsg[k], zz->Proc); despid[k] = j; k++; /* } */ } } } /* KDDKDD */ ierr = Zoltan_Comm_Create(&comm_plan, k, despid, zz->Communicator, RootListCommCreate, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&snd_rmsg); return (ierr); } if (nreceives > 0) { if((rcv_rmsg = (Migrate_msg *) ZOLTAN_MALLOC(nreceives * sizeof(Migrate_msg))) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&snd_rmsg); return ZOLTAN_MEMERR; } } ierr = Zoltan_Comm_Do(comm_plan, RootListCommDo, (char *) snd_rmsg, sizeof(Migrate_msg), (char *) rcv_rmsg); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&snd_rmsg); ZOLTAN_FREE(&rcv_rmsg); return (ierr); } ZOLTAN_FREE(&despid); ZOLTAN_FREE(&snd_rmsg); ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } *ret_rmsg = rcv_rmsg; *size = nreceives; return ierr; }
int Zoltan_Scatter_Graph( indextype **vtxdist, indextype **xadj, indextype **adjncy, weighttype **vwgt, indextype **vsize, weighttype **adjwgt, realtype **xyz, int ndims, /* # dimensions of xyz geometry data */ int vwgt_dim, ZZ *zz, ZOLTAN_COMM_OBJ **plan ) { static char *yo = "Zoltan_Scatter_Graph"; char msg[256]; indextype *old_vtxdist, *old_adjncy; indextype *old_xadj; indextype *old_vsize; weighttype *old_vwgt, *old_adjwgt; realtype *old_xyz; int *ptr, *proclist = NULL, *proclist2 = NULL; int i, j, num_obj, old_num_obj, num_edges, nrecv; int use_graph; /* do we use graph data, or only the geometry? */ int use_vsize; /* do we use the vsize array? */ int ewgt_dim= zz->Edge_Weight_Dim; ZOLTAN_COMM_OBJ *plan2; ZOLTAN_TRACE_ENTER(zz, yo); /* Save pointers to "old" data distribution */ old_adjncy = NULL; old_xadj = NULL; old_vwgt = old_adjwgt = NULL; old_vsize = NULL; old_xyz = NULL; old_vtxdist = *vtxdist; if (xadj) old_xadj = *xadj; if (adjncy) old_adjncy = *adjncy; if (vwgt) old_vwgt = *vwgt; if (vsize) old_vsize = *vsize; if (adjwgt) old_adjwgt = *adjwgt; if (xyz) old_xyz = *xyz; old_num_obj = (int)(old_vtxdist[zz->Proc+1] - old_vtxdist[zz->Proc]); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Old number of objects = %d\n", zz->Proc, old_num_obj); /* Compute new distribution, *vtxdist */ (*vtxdist) = (indextype *)ZOLTAN_MALLOC((zz->Num_Proc+1)* sizeof(indextype)); for (i=0; i<=zz->Num_Proc; i++){ (*vtxdist)[i] = (i*old_vtxdist[zz->Num_Proc])/zz->Num_Proc; } /* Check if any proc has graph data */ i = (old_xadj != NULL); MPI_Allreduce(&i, &use_graph, 1, MPI_INT, MPI_LOR, zz->Communicator); j = (old_vsize != NULL); MPI_Allreduce(&j, &use_vsize, 1, MPI_INT, MPI_LOR, zz->Communicator); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: use_graph = %1d, use_vsize = %1d\n", zz->Proc, use_graph, use_vsize); /* Reset all data pointers to NULL for now */ *xadj = NULL; *adjncy = NULL; *vwgt = *adjwgt = NULL; *xyz = NULL; if (use_vsize) *vsize = NULL; /* Convert the xdj array so that it contains the degree of each vertex */ if (use_graph){ for (i=0; i<old_num_obj; i++){ old_xadj[i] = old_xadj[i+1] - old_xadj[i]; } } /* Allocate new space for vertex data */ num_obj = (int)((*vtxdist)[zz->Proc+1] - (*vtxdist)[zz->Proc]); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: New number of objects = %d\n", zz->Proc, num_obj); if (use_graph) *xadj = (indextype *) ZOLTAN_MALLOC((num_obj+1)*sizeof(indextype)); if (vwgt_dim) *vwgt = (weighttype *) ZOLTAN_MALLOC(vwgt_dim*num_obj*sizeof(weighttype)); if (use_vsize) *vsize = (indextype *) ZOLTAN_MALLOC(num_obj*sizeof(indextype)); if (ndims) *xyz = (realtype *) ZOLTAN_MALLOC(ndims*num_obj*sizeof(realtype)); if (old_num_obj > 0) { /* Set up the communication plan for the vertex data */ proclist = (int *) ZOLTAN_MALLOC(old_num_obj * sizeof(int)); /* Let j be the new owner of vertex old_vtxdist[zz->Proc]+i */ j = 0; while (old_vtxdist[zz->Proc] >= (*vtxdist)[j+1]) j++; for (i=0; i<old_num_obj; i++){ if (old_vtxdist[zz->Proc]+i >= (*vtxdist)[j+1]) j++; proclist[i] = j; } } Zoltan_Comm_Create(plan, old_num_obj, proclist, zz->Communicator, TAG1, &nrecv); if (nrecv != num_obj){ sprintf(msg,"Proc %d received %d object but expected %d.", zz->Proc, nrecv, num_obj); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); /* Free data */ ZOLTAN_FREE(&proclist); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_FATAL; } /* Do the communication. To save memory, we do not pack all the data into * a buffer, but send directly from the old arrays to the new arrays. * We use the vertex communication plan for all the vertex-based arrays * and the edge communication plan for all the edge-based arrays. */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Starting vertex-based communication.\n", zz->Proc); if (use_graph){ Zoltan_Comm_Do( *plan, TAG2, (char *) old_xadj, sizeof(indextype), (char *) *xadj); } if (vwgt_dim){ Zoltan_Comm_Do( *plan, TAG3, (char *) old_vwgt, vwgt_dim*sizeof(weighttype), (char *) *vwgt); } if (use_vsize){ Zoltan_Comm_Do( *plan, TAG4, (char *) old_vsize, sizeof(indextype), (char *) *vsize); } if (ndims){ Zoltan_Comm_Do( *plan, TAG5, (char *) old_xyz, ndims*sizeof(realtype), (char *) *xyz); } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Finished vertex-based communication.\n", zz->Proc); if (use_graph){ /* Rebuild xadj from degrees */ for (i=1; i<num_obj; i++) (*xadj)[i] += (*xadj)[i-1]; for (i=num_obj; i>0; i--) (*xadj)[i] = (*xadj)[i-1]; (*xadj)[0] = 0; /* Allocate space for new edge data structures */ num_edges = (*xadj)[num_obj]; *adjncy = (indextype *) ZOLTAN_MALLOC(num_edges*sizeof(indextype)); if (ewgt_dim) *adjwgt = (weighttype *) ZOLTAN_MALLOC(ewgt_dim*num_edges*sizeof(weighttype)); /* Set up the communication plan for the edge data. */ ptr = proclist2 = (int *) ZOLTAN_MALLOC(old_xadj[old_num_obj] * sizeof(int)); for (i=0; i<old_num_obj; i++) for (j=0; j<old_xadj[i]; j++) *ptr++ = proclist[i]; if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] Debug: Allocated proclist of length " TPL_IDX_SPEC " for edges.\n", zz->Proc, old_xadj[old_num_obj]); } Zoltan_Comm_Create(&plan2, (int)old_xadj[old_num_obj], proclist2, zz->Communicator, TAG1, &nrecv); if (nrecv != num_edges){ sprintf(msg,"Proc %d received %d edges but expected %d.", zz->Proc, nrecv, num_edges); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); /* Free data */ ZOLTAN_FREE(&proclist); ZOLTAN_FREE(&proclist2); ZOLTAN_FREE(&old_vtxdist); ZOLTAN_FREE(&old_xadj); ZOLTAN_FREE(&old_adjncy); ZOLTAN_FREE(&old_vwgt); ZOLTAN_FREE(&old_vsize); ZOLTAN_FREE(&old_adjwgt); ZOLTAN_FREE(&old_xyz); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_FATAL; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Starting edge-based communication.\n", zz->Proc); /* Do the communication. */ Zoltan_Comm_Do( plan2, TAG2, (char *) old_adjncy, sizeof(indextype), (char *) *adjncy); if (ewgt_dim){ Zoltan_Comm_Do( plan2, TAG3, (char *) old_adjwgt, ewgt_dim*sizeof(weighttype), (char *) *adjwgt); } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Finished edge-based communication.\n", zz->Proc); /* Free the comm. plan for edge data */ Zoltan_Comm_Destroy(&plan2); } /* end of use_graph */ /* Free data structures */ ZOLTAN_FREE(&proclist); ZOLTAN_FREE(&proclist2); ZOLTAN_FREE(&old_vtxdist); ZOLTAN_FREE(&old_xadj); ZOLTAN_FREE(&old_adjncy); ZOLTAN_FREE(&old_vwgt); ZOLTAN_FREE(&old_vsize); ZOLTAN_FREE(&old_adjwgt); ZOLTAN_FREE(&old_xyz); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_OK; }
int Zoltan_DD_Find ( Zoltan_DD_Directory *dd, /* contains directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */ ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */ char *data, /* Outgoing optional corresponding user data */ int *partition, /* Outgoing optional partition information */ int count, /* Count of GIDs in above list (in) */ int *owner) /* Outgoing optional list of data owners */ { ZOLTAN_COMM_OBJ *plan = NULL; /* efficient MPI communication */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL; /* pointer into receive buffer */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL; /* pointer into send buffer */ int *procs = NULL; /* list of processors to contact */ DD_Find_Msg *ptr = NULL; int i; int nrec; /* number of messages to receive */ int err = ZOLTAN_OK; /* return error condition */ int errcount; /* count of GIDs not found */ char *yo = "Zoltan_DD_Find"; /* input sanity check */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR (dd ? dd->my_proc : ZOLTAN_DD_NO_PROC, yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* allocate memory for processors to contact for directory info */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } /* allocate memory for DD_Find_Msg send buffer */ if (count) { sbuff = (char*) ZOLTAN_CALLOC (count, dd->find_msg_size); if (sbuff == NULL) { ZOLTAN_FREE (&procs); ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return ZOLTAN_MEMERR; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs"); /* for each GID, fill DD_Find_Msg buffer and contact list */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Find_Msg*) sbufftmp; sbufftmp += dd->find_msg_size; ptr->index = i; ptr->proc = procs[i]; ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length); } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill"); /* create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_FIND_MSG_TAG, &nrec); if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create"); if (err != ZOLTAN_OK) goto fini; /* allocate receive buffer */ if (nrec) { rbuff = (char*) ZOLTAN_MALLOC ((size_t)nrec*(size_t)(dd->find_msg_size)); if (rbuff == NULL) { err = ZOLTAN_MEMERR; goto fini; } } /* send out find messages across entire system */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff, dd->find_msg_size, rbuff); if (err != ZOLTAN_OK) goto fini; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do"); /* get find messages directed to me, fill in return information */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Find_Msg*) rbufftmp; rbufftmp += dd->find_msg_size; err = DD_Find_Local (dd, ptr->id, ptr->id, (char *)(ptr->id + dd->max_id_length), &ptr->partition, &ptr->proc); if (err == ZOLTAN_WARN) ++errcount; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info"); /* send return information back to requester */ err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff, dd->find_msg_size, NULL, sbuff); if (err != ZOLTAN_OK) goto fini; if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse"); /* fill in user supplied lists with returned information */ sbufftmp = sbuff; for (i = 0; i < count; i++) { ptr = (DD_Find_Msg*) sbufftmp; sbufftmp += dd->find_msg_size; if (owner) owner[ptr->index] = ptr->proc; if (partition) partition[ptr->index] = ptr->partition ; if (lid) ZOLTAN_SET_ID(dd->lid_length,lid+ptr->index*dd->lid_length,ptr->id); if (data) memcpy(data + (size_t)(ptr->index) * (size_t)(dd->user_data_length), ptr->id + dd->max_id_length, dd->user_data_length); } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists"); /* err = ZOLTAN_OK; */ MPI_Allreduce(&errcount, &err, 1, MPI_INT, MPI_SUM, dd->comm); err = (err) ? ZOLTAN_WARN : ZOLTAN_OK; /* if at least one GID was not found, potentially notify caller of error */ if (dd->debug_level > 0) { char str[100]; /* diagnostic message string */ sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); } fini: ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); ZOLTAN_FREE (&procs) ; Zoltan_Comm_Destroy (&plan); if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err; }
/* Main partitioning function for hypergraph partitioning. */ int Zoltan_PHG_Partition ( ZZ *zz, /* Zoltan data structure */ HGraph *hg, /* Input hypergraph to be partitioned */ int p, /* Input: number partitions to be generated */ float *part_sizes, /* Input: array of length p containing percentages of work to be assigned to each partition */ Partition parts, /* Input: initial partition #s; aligned with vtx arrays. Output: computed partition #s */ PHGPartParams *hgp) /* Input: parameters for hgraph partitioning. */ { PHGComm *hgc = hg->comm; VCycle *vcycle=NULL, *del=NULL; int i, err = ZOLTAN_OK, middle; ZOLTAN_GNO_TYPE origVpincnt; /* for processor reduction test */ ZOLTAN_GNO_TYPE prevVcnt = 2*hg->dist_x[hgc->nProc_x]; /* initialized so that the */ ZOLTAN_GNO_TYPE prevVedgecnt = 2*hg->dist_y[hgc->nProc_y]; /* while loop will be entered before any coarsening */ ZOLTAN_GNO_TYPE tot_nPins, local_nPins; MPI_Datatype zoltan_gno_mpi_type; char *yo = "Zoltan_PHG_Partition"; int do_timing = (hgp->use_timers > 1); int fine_timing = (hgp->use_timers > 2); int vcycle_timing = (hgp->use_timers > 4 && hgp->ProRedL == 0); short refine = 0; struct phg_timer_indices *timer = Zoltan_PHG_LB_Data_timers(zz); int reset_geometric_matching = 0; char reset_geometric_string[4]; ZOLTAN_TRACE_ENTER(zz, yo); zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); if (do_timing) { if (timer->vcycle < 0) timer->vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle"); if (timer->procred < 0) timer->procred = Zoltan_Timer_Init(zz->ZTime, 0, "Processor Reduction"); if (timer->match < 0) timer->match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching"); if (timer->coarse < 0) timer->coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening"); if (timer->coarsepart < 0) timer->coarsepart = Zoltan_Timer_Init(zz->ZTime, 1, "Coarse_Partition"); if (timer->refine < 0) timer->refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement"); if (timer->project < 0) timer->project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up"); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins; MPI_Allreduce(&local_nPins,&tot_nPins,1,zoltan_gno_mpi_type,MPI_SUM,hgc->Communicator); origVpincnt = tot_nPins; if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } /* For geometric coarsening, hgp->matching pointer and string are reset * after geometric_levels of coarsening. Will need to reset them after * this vcycle is completed. Capture that fact now! */ if (!strcasecmp(hgp->redm_str, "rcb") || !strcasecmp(hgp->redm_str, "rib")) { reset_geometric_matching = 1; strcpy(reset_geometric_string, hgp->redm_str); } /****** Coarsening ******/ #define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */ while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > (ZOLTAN_GNO_TYPE)hg->redl) && ((hg->dist_x[hgc->nProc_x] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5)) /* prevVcnt initialized to 2*hg->dist_x[hgc->nProc_x] */ || (hg->dist_y[hgc->nProc_y] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5))) /* prevVedgecnt initialized to 2*hg->dist_y[hgc->nProc_y] */ && hg->dist_y[hgc->nProc_y] && hgp->matching) { ZOLTAN_GNO_TYPE *match = NULL; VCycle *coarser=NULL, *redistributed=NULL; prevVcnt = hg->dist_x[hgc->nProc_x]; prevVedgecnt = hg->dist_y[hgc->nProc_y]; #ifdef _DEBUG /* UVC: load balance stats */ Zoltan_PHG_LoadBalStat(zz, hg); #endif if (hgp->output_level >= PHG_DEBUG_LIST) { uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n", hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p); if (hgp->output_level > PHG_DEBUG_LIST) { err = Zoltan_HG_Info(zz, hg); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) goto End; } } if (hgp->output_level >= PHG_DEBUG_PLOT) Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL, "coarsening plot"); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->match, hgc->Communicator); } if (vcycle_timing) { if (vcycle->timer_match < 0) { char str[80]; sprintf(str, "VC Matching %d", hg->info); vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str); } ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match, hgc->Communicator); } /* Allocate and initialize Matching Array */ if (hg->nVtx && !(match = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC (hg->nVtx*sizeof(ZOLTAN_GNO_TYPE)))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array"); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } for (i = 0; i < hg->nVtx; i++) match[i] = i; /* Calculate matching (packing or grouping) */ err = Zoltan_PHG_Matching (zz, hg, match, hgp); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) { ZOLTAN_FREE (&match); goto End; } if (vcycle_timing) ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match, hgc->Communicator); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->match, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->coarse, hgc->Communicator); } if (vcycle_timing) { if (vcycle->timer_coarse < 0) { char str[80]; sprintf(str, "VC Coarsening %d", hg->info); vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str); } ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse, hgc->Communicator); } if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) { ZOLTAN_FREE (&match); ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL."); goto End; } /* Construct coarse hypergraph and LevelMap */ err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap, &vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData, &vcycle->comm_plan, hgp); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) goto End; if (vcycle_timing) ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse, hgc->Communicator); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarse, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } ZOLTAN_FREE (&match); if ((err=allocVCycle(coarser))!= ZOLTAN_OK) goto End; vcycle = coarser; hg = vcycle->hg; if (hgc->nProc > 1 && hgp->ProRedL > 0) { local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins; MPI_Allreduce(&local_nPins, &tot_nPins, 1, zoltan_gno_mpi_type, MPI_SUM, hgc->Communicator); if (tot_nPins < (ZOLTAN_GNO_TYPE)(hgp->ProRedL * origVpincnt + 0.5)) { if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->procred, hgc->Communicator); } /* redistribute to half the processors */ origVpincnt = tot_nPins; /* update for processor reduction test */ if(hg->nVtx&&!(hg->vmap=(int*)ZOLTAN_MALLOC(hg->nVtx*sizeof(int)))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: hg->vmap"); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } for (i = 0; i < hg->nVtx; i++) hg->vmap[i] = i; middle = (int)((float) (hgc->nProc-1) * hgp->ProRedL); if (hgp->nProc_x_req!=1&&hgp->nProc_y_req!=1) { /* Want 2D decomp */ if ((middle+1) > SMALL_PRIME && Zoltan_PHG_isPrime(middle+1)) --middle; /* if it was prime just use one less #procs (since it should be bigger than SMALL_PRIME it is safe to decrement) */ } if (!(hgc = (PHGComm*) ZOLTAN_MALLOC (sizeof(PHGComm)))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: PHGComm"); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if (!(redistributed=newVCycle(zz,NULL,NULL,vcycle,vcycle_timing))) { ZOLTAN_FREE (&hgc); ZOLTAN_PRINT_ERROR (zz->Proc, yo, "redistributed is NULL."); goto End; } Zoltan_PHG_Redistribute(zz,hgp,hg,0,middle,hgc, redistributed->hg, &vcycle->vlno,&vcycle->vdest); if (hgp->UseFixedVtx || hgp->UsePrefPart) redistributed->hg->bisec_split = hg->bisec_split; if ((err=allocVCycle(redistributed))!= ZOLTAN_OK) goto End; vcycle = redistributed; if (hgc->myProc < 0) /* I'm not in the redistributed part so I should go to uncoarsening refinement and wait */ { if (fine_timing) { if (timer->cpgather < 0) timer->cpgather = Zoltan_Timer_Init(zz->ZTime, 1, "CP Gather"); if (timer->cprefine < 0) timer->cprefine =Zoltan_Timer_Init(zz->ZTime, 0, "CP Refine"); if (timer->cpart < 0) timer->cpart = Zoltan_Timer_Init(zz->ZTime, 0, "CP Part"); } if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } goto Refine; } hg = vcycle->hg; hg->redl = hgp->redl; /* not set with hg creation */ if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } } } } if (hgp->output_level >= PHG_DEBUG_LIST) { uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n", hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p); if (hgp->output_level > PHG_DEBUG_LIST) { err = Zoltan_HG_Info(zz, hg); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) goto End; } } if (hgp->output_level >= PHG_DEBUG_PLOT) Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL, "coarsening plot"); /* free array that may have been allocated in matching */ if (hgp->vtx_scal) { hgp->vtx_scal_size = 0; ZOLTAN_FREE(&(hgp->vtx_scal)); } if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->coarsepart, hgc->Communicator); } /****** Coarse Partitioning ******/ err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) goto End; if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarsepart, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } Refine: del = vcycle; refine = 1; /****** Uncoarsening/Refinement ******/ while (vcycle) { VCycle *finer = vcycle->finer; hg = vcycle->hg; if (refine && hgc->myProc >= 0) { if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->refine, hgc->Communicator); } if (vcycle_timing) { if (vcycle->timer_refine < 0) { char str[80]; sprintf(str, "VC Refinement %d", hg->info); vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str); } ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine, hgc->Communicator); } err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->refine, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } if (vcycle_timing) ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine, hgc->Communicator); if (hgp->output_level >= PHG_DEBUG_LIST) uprintf(hgc, "FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n", hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p, Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, 0, p, vcycle->Part), Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err)); if (hgp->output_level >= PHG_DEBUG_PLOT) Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part, "partitioned plot"); } if (finer) { int *rbuffer; /* Project coarse partition to fine partition */ if (finer->comm_plan) { refine = 1; if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->project, hgc->Communicator); } if (vcycle_timing) { if (vcycle->timer_project < 0) { char str[80]; sprintf(str, "VC Project Up %d", hg->info); vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str); } ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project, hgc->Communicator); } /* easy to assign partitions to internal matches */ for (i = 0; i < finer->hg->nVtx; i++) if (finer->LevelMap[i] >= 0) /* if considers only the local vertices */ finer->Part[i] = vcycle->Part[finer->LevelMap[i]]; /* now that the course partition assignments have been propagated */ /* upward to the finer level for the local vertices, we need to */ /* fill the LevelData (matched pairs of a local vertex with a */ /* off processor vertex) with the partition assignment of the */ /* local vertex - can be done totally in the finer level! */ for (i = 0; i < finer->LevelCnt; i++) { ++i; /* skip over off processor lno */ finer->LevelData[i] = finer->Part[finer->LevelData[i]]; } /* allocate rec buffer to exchange LevelData information */ rbuffer = NULL; if (finer->LevelSndCnt > 0) { rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int)); if (!rbuffer) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } /* get partition assignments from owners of externally matched vtxs */ Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i); Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1, (char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer); /* process data to assign partitions to expernal matches */ for (i = 0; i < 2 * finer->LevelSndCnt;) { int lno, partition; lno = rbuffer[i++]; partition = rbuffer[i++]; finer->Part[lno] = partition; } ZOLTAN_FREE (&rbuffer); Zoltan_Comm_Destroy (&finer->comm_plan); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->project, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } if (vcycle_timing) ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project, hgc->Communicator); } else { int *sendbuf = NULL, size; refine = 0; /* ints local and partition numbers */ if (finer->vlno) { sendbuf = (int*) ZOLTAN_MALLOC (2 * hg->nVtx * sizeof(int)); if (!sendbuf) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } for (i = 0; i < hg->nVtx; ++i) { sendbuf[2 * i] = finer->vlno[i]; /* assign local numbers */ sendbuf[2 * i + 1] = vcycle->Part[i];/* assign partition numbers */ } } ZOLTAN_FREE (&hgc); hgc = finer->hg->comm; /* updating hgc is required when the processors change */ /* Create comm plan to unredistributed processors */ err = Zoltan_Comm_Create(&finer->comm_plan, finer->vlno ? hg->nVtx : 0, finer->vdest, hgc->Communicator, COMM_TAG+2, &size); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(hgc->myProc, yo, "Zoltan_Comm_Create failed."); goto End; } /* allocate rec buffer to exchange sendbuf information */ rbuffer = NULL; if (finer->hg->nVtx) { rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->hg->nVtx * sizeof(int)); if (!rbuffer) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } /* Use plan to send partitions to the unredistributed processors */ Zoltan_Comm_Do(finer->comm_plan, COMM_TAG+3, (char *) sendbuf, 2*sizeof(int), (char *) rbuffer); MPI_Bcast(rbuffer, 2*finer->hg->nVtx, MPI_INT, 0, hgc->col_comm); /* process data to assign partitions to unredistributed processors */ for (i = 0; i < 2 * finer->hg->nVtx;) { int lno, partition; lno = rbuffer[i++]; partition = rbuffer[i++]; finer->Part[lno] = partition; } if (finer->vlno) ZOLTAN_FREE (&sendbuf); ZOLTAN_FREE (&rbuffer); Zoltan_Comm_Destroy (&finer->comm_plan); } } vcycle = finer; } /* while (vcycle) */ End: vcycle = del; while (vcycle) { if (vcycle_timing) { Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout); Zoltan_Timer_Destroy(&vcycle->timer); } if (vcycle->finer) { /* cleanup by level */ Zoltan_HG_HGraph_Free (vcycle->hg); if (vcycle->LevelData) Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part, &vcycle->LevelMap, &vcycle->LevelData, &vcycle->hg); else if (vcycle->vlno) Zoltan_Multifree (__FILE__, __LINE__, 5, &vcycle->Part, &vcycle->vdest, &vcycle->vlno, &vcycle->LevelMap, &vcycle->hg); else Zoltan_Multifree (__FILE__, __LINE__, 3, &vcycle->Part, &vcycle->LevelMap, &vcycle->hg); } else /* cleanup top level */ Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap, &vcycle->LevelData); del = vcycle; vcycle = vcycle->finer; ZOLTAN_FREE(&del); } if (reset_geometric_matching) { strcpy(hgp->redm_str, reset_geometric_string); Zoltan_PHG_Set_Matching_Fn(hgp); } if (do_timing) ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TRACE_EXIT(zz, yo) ; return err; }
int Zoltan_Verify_Graph(MPI_Comm comm, indextype *vtxdist, indextype *xadj, indextype *adjncy, weighttype *vwgt, weighttype *adjwgt, int vwgt_dim, int ewgt_dim, int graph_type, int check_graph, int output_level) { int ierr, flag, cross_edges = 0, mesg_size, sum; int nprocs, proc, *proclist, errors, global_errors; int *perm=NULL; int free_adjncy_sort=0; ZOLTAN_COMM_OBJ *comm_plan; static char *yo = "Zoltan_Verify_Graph"; char msg[256]; ZOLTAN_GNO_TYPE num_obj; int nrecv; indextype *ptr, *ptr1, *ptr2; indextype global_i, global_j; indextype *sendgno=NULL, *recvgno=NULL, *adjncy_sort=NULL; weighttype *sendwgt, *recvwgt; ZOLTAN_GNO_TYPE num_duplicates, num_singletons; ZOLTAN_GNO_TYPE num_selfs, nedges, global_sum, num_zeros; ZOLTAN_GNO_TYPE i, j, ii, k; MPI_Datatype zoltan_gno_mpi_type; ierr = ZOLTAN_OK; zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); /* Make sure all procs have same value of check_graph. */ MPI_Allreduce(&check_graph, &i, 1, MPI_INT, MPI_MAX, comm); check_graph = i; if (check_graph == 0) /* perform no error checking at all */ return ierr; /* Get number of procs and my rank */ MPI_Comm_size(comm, &nprocs); MPI_Comm_rank(comm, &proc); /* Check number of vertices (objects) */ num_obj = (ZOLTAN_GNO_TYPE)(vtxdist[proc+1] - vtxdist[proc]); MPI_Reduce(&num_obj, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm); if ((proc==0) && (global_sum==0)){ if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN; if (output_level>0) ZOLTAN_PRINT_WARN(proc, yo, "No vertices in graph."); } /* Verify that vertex weights are non-negative */ num_zeros = 0; if (vwgt_dim){ for (i=0; i<num_obj; i++){ sum = 0; for (k=0; k<vwgt_dim; k++){ if (vwgt[i*vwgt_dim+k] < 0) { sprintf(msg, "Negative object weight of " TPL_WGT_SPEC " for object " ZOLTAN_GNO_SPEC ".", vwgt[i*vwgt_dim+k], i); ZOLTAN_PRINT_ERROR(proc, yo, msg); ierr = ZOLTAN_FATAL; } sum += vwgt[i*vwgt_dim+k]; } if (sum == 0){ num_zeros++; if (output_level>1) { sprintf(msg, "Zero vertex (object) weights for object " ZOLTAN_GNO_SPEC ".", i); ZOLTAN_PRINT_WARN(proc, yo, msg); } if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN; } } MPI_Reduce(&num_zeros, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm); if ((proc==0) && (global_sum>0)){ if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN; if (output_level>0){ sprintf(msg, ZOLTAN_GNO_SPEC " objects have zero weights.", global_sum); ZOLTAN_PRINT_WARN(proc, yo, msg); } } } /* Check number of edges */ nedges = (ZOLTAN_GNO_TYPE)xadj[num_obj]; MPI_Reduce(&nedges, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm); if ((proc==0) && (global_sum==0)){ if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN; if (output_level>0) ZOLTAN_PRINT_WARN(proc, yo, "No edges in graph."); } /* Verify that edge weights are non-negative */ num_zeros = 0; if (ewgt_dim){ for (j=0; j<nedges; j++){ sum = 0; for (k=0; k<ewgt_dim; k++){ if (adjwgt[j*ewgt_dim+k] < 0) { sprintf(msg, "Negative edge weight of " TPL_WGT_SPEC " in edge " ZOLTAN_GNO_SPEC ".", adjwgt[j*ewgt_dim+k], j); ZOLTAN_PRINT_ERROR(proc, yo, msg); ierr = ZOLTAN_FATAL; } sum += adjwgt[j*ewgt_dim+k]; } if (sum == 0){ num_zeros++; if (output_level>1) { sprintf(msg, "Zero edge (communication) weights for edge " ZOLTAN_GNO_SPEC ".", j); ZOLTAN_PRINT_WARN(proc, yo, msg); } if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN; } } MPI_Reduce(&num_zeros, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm); if ((proc==0) && (global_sum>0)){ if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN; if (output_level>0){ sprintf(msg, ZOLTAN_GNO_SPEC " edges have zero weights.", global_sum); ZOLTAN_PRINT_WARN(proc, yo, msg); } } } /* Verify that the graph is symmetric (edge weights, too) */ /* Also check for self-edges and multi-edges */ /* Pre-processing: Check if edge lists are sorted. If not, */ /* make a copy and sort so we can save time in the lookups. */ flag = 0; /* Assume sorted. */ for (i=0; (i<num_obj) && (flag==0); i++){ for (ii=xadj[i]; ii<xadj[i+1]-1; ii++){ if (adjncy[ii] > adjncy[ii+1]){ flag = 1; /* Not sorted. */ break; } } } if (flag){ /* Need to sort. */ adjncy_sort = (indextype *) ZOLTAN_MALLOC(nedges*sizeof(indextype)); perm = (int *) ZOLTAN_MALLOC(nedges*sizeof(int)); free_adjncy_sort = 1; if (nedges && (!adjncy_sort || !perm)){ /* Out of memory. */ ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory."); ierr = ZOLTAN_MEMERR; } for (k=0; k<nedges; k++){ adjncy_sort[k] = adjncy[k]; perm[k] = k; } if (sizeof(indextype) == sizeof(short)){ for (i=0; i<num_obj; i++) Zoltan_quicksort_list_inc_short((short *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1); } else if (sizeof(indextype) == sizeof(int)){ for (i=0; i<num_obj; i++) Zoltan_quicksort_list_inc_int((int *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1); } else if (sizeof(indextype) == sizeof(long)){ for (i=0; i<num_obj; i++) Zoltan_quicksort_list_inc_long((long *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1); } else if (sizeof(indextype) == sizeof(int64_t)){ for (i=0; i<num_obj; i++) Zoltan_quicksort_list_inc_long_long((int64_t*)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1); } else{ ZOLTAN_PRINT_ERROR(proc, yo, "Error in third party library data type support."); ierr = ZOLTAN_MEMERR; } } else { /* Already sorted. */ adjncy_sort = adjncy; } /* First pass: Check on-processor edges and count # off-proc edges */ cross_edges = 0; num_selfs = 0; num_duplicates = 0; num_singletons = 0; for (i=0; i<num_obj; i++){ if (IS_GLOBAL_GRAPH(graph_type)){ global_i = vtxdist[proc]+i; } else{ /* graph_type == LOCAL_GRAPH */ global_i = i; /* A bit confusingly, global_i = i for local graphs */ } /* Singleton? */ if (xadj[i] == xadj[i+1]){ num_singletons++; if (output_level>1){ sprintf(msg, "Vertex " TPL_IDX_SPEC " has no edges.", global_i); ZOLTAN_PRINT_WARN(proc, yo, msg); } } for (ii=xadj[i]; ii<xadj[i+1]; ii++){ global_j = adjncy_sort[ii]; /* Valid vertex number? */ if ((IS_GLOBAL_GRAPH(graph_type) && ((global_j < vtxdist[0]) || (global_j >= vtxdist[nprocs]))) || (IS_LOCAL_GRAPH(graph_type) && ((global_j < 0) || (global_j >= num_obj)))){ sprintf(msg, "Edge to invalid vertex " TPL_IDX_SPEC " detected.", global_j); ZOLTAN_PRINT_ERROR(proc, yo, msg); ierr = ZOLTAN_FATAL; } /* Self edge? */ if (global_j == global_i){ num_selfs++; if (output_level>1){ sprintf(msg, "Self edge for vertex " TPL_IDX_SPEC " detected.", global_i); ZOLTAN_PRINT_WARN(proc, yo, msg); } } /* Duplicate edge? */ if ((ii+1<xadj[i+1]) && (adjncy_sort[ii]==adjncy_sort[ii+1])){ num_duplicates++; if (output_level>1){ sprintf(msg, "Duplicate edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") detected.", global_i, global_j); ZOLTAN_PRINT_WARN(proc, yo, msg); } } /* Is global_j a local vertex? */ if (IS_LOCAL_GRAPH(graph_type) || (IS_GLOBAL_GRAPH(graph_type) && (global_j >= vtxdist[proc]) && (global_j < vtxdist[proc+1]))){ /* Check if (global_j, global_i) is an edge */ if (IS_GLOBAL_GRAPH(graph_type)) j = global_j - vtxdist[proc]; else /* graph_type == LOCAL_GRAPH */ j = global_j; /* Binary search for edge (global_j, global_i) */ ptr = (indextype *)bsearch(&global_i, &adjncy_sort[xadj[j]], (int)(xadj[j+1]-xadj[j]), sizeof(indextype), Zoltan_Compare_Indextypes); if (ptr){ /* OK, found edge (global_j, global_i) */ if ((adjncy_sort==adjncy) && ewgt_dim){ /* Compare weights */ /* EBEB For now, don't compare weights if we sorted edge lists. */ flag = 0; for (k=0; k<ewgt_dim; k++){ if (adjwgt[(ptr-adjncy_sort)*ewgt_dim+k] != adjwgt[ii*ewgt_dim+k]){ /* Numerically nonsymmetric */ flag = -1; if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN; } } if (flag<0 && output_level>0){ sprintf(msg, "Graph is numerically nonsymmetric " "in edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ")", global_i, global_j); ZOLTAN_PRINT_WARN(proc, yo, msg); } } } else { /* bsearch failed */ sprintf(msg, "Graph is not symmetric. " "Edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") exists, but no edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ").", global_i, global_j, global_j, global_i); ZOLTAN_PRINT_ERROR(proc, yo, msg); ierr = ZOLTAN_FATAL; } } else { cross_edges++; } } } /* Sum up warnings so far. */ MPI_Reduce(&num_selfs, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm); if ((proc==0) && (global_sum>0)){ ierr = ZOLTAN_WARN; if (output_level>0){ sprintf(msg, ZOLTAN_GNO_SPEC " self-edges in graph.", global_sum); ZOLTAN_PRINT_WARN(proc, yo, msg); } } MPI_Reduce(&num_duplicates, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm); if ((proc==0) && (global_sum>0)){ ierr = ZOLTAN_WARN; if (output_level>0){ sprintf(msg, ZOLTAN_GNO_SPEC " duplicate edges in graph.", global_sum); ZOLTAN_PRINT_WARN(proc, yo, msg); } } MPI_Reduce(&num_singletons, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm); if ((proc==0) && (global_sum>0)){ ierr = ZOLTAN_WARN; if (output_level>0){ sprintf(msg, ZOLTAN_GNO_SPEC " vertices in the graph are singletons (have no edges).", global_sum); ZOLTAN_PRINT_WARN(proc, yo, msg); } } /* Check if any processor has encountered an error so far */ errors = 0; if (ierr == ZOLTAN_WARN) errors |= 1; if (ierr == ZOLTAN_MEMERR) errors |= 2; if (ierr == ZOLTAN_FATAL) errors |= 4; MPI_Allreduce(&errors, &global_errors, 1, MPI_INT, MPI_BOR, comm); if (global_errors & 4){ /* Fatal error: return now */ if (free_adjncy_sort) ZOLTAN_FREE(&adjncy_sort); if (free_adjncy_sort) ZOLTAN_FREE(&perm); return ZOLTAN_FATAL; } if ((IS_GLOBAL_GRAPH(graph_type)) && (check_graph >= 2)) { /* Test for consistency across processors. */ /* Allocate space for off-proc data */ mesg_size = (2*sizeof(indextype)) + (ewgt_dim * sizeof(weighttype)); sendgno = (indextype *) ZOLTAN_MALLOC(cross_edges*mesg_size); recvgno = (indextype *) ZOLTAN_MALLOC(cross_edges*mesg_size); proclist = (int *) ZOLTAN_MALLOC(cross_edges*sizeof(int)); if (cross_edges && !(sendgno && recvgno && proclist)){ ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory."); ierr = ZOLTAN_MEMERR; } /* Second pass: Copy data to send buffer */ nedges = 0; ptr1 = (indextype *) sendgno; for (i=0; i<num_obj; i++){ global_i = vtxdist[proc]+i; for (ii=xadj[i]; ii<xadj[i+1]; ii++){ global_j = adjncy[ii]; /* Is global_j off-proc? */ if ((global_j < vtxdist[proc]) || (global_j >= vtxdist[proc+1])){ /* Add to list */ k=0; while (global_j >= vtxdist[k+1]) k++; proclist[nedges++] = k; /* Copy (global_i, global_j) and corresponding weights to sendgno */ *ptr1++ = global_i; *ptr1++ = global_j; for (k=0; k<ewgt_dim; k++){ *ptr1++ = adjwgt[ii*ewgt_dim+k]; } } } } /* Do the irregular communication */ ierr = Zoltan_Comm_Create(&comm_plan, cross_edges, proclist, comm, TAG1, &nrecv); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Create.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(proc, yo, msg); } else { if (nrecv != cross_edges){ sprintf(msg, "Incorrect number of edges to/from proc %d.", proc); ZOLTAN_PRINT_ERROR(proc, yo, msg); ierr = ZOLTAN_FATAL; } ierr = Zoltan_Comm_Do(comm_plan, TAG2, (char *)sendgno, mesg_size, (char *)recvgno); Zoltan_Comm_Destroy(&comm_plan); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(proc, yo, msg); } else { /* Third pass: Compare on-proc data to the off-proc data we received */ /* sendgno and recvgno should contain the same data except (i,j) is */ /* (j,i) */ for (i=0, ptr1=sendgno; i<cross_edges; i++){ flag = 0; sendwgt = (weighttype *)(ptr1 + 2); for (j=0, ptr2=recvgno; j<cross_edges; j++){ recvwgt = (weighttype *)(ptr2 + 2); if ((ptr2[0] == ptr1[1]) && (ptr2[1] == ptr1[0])){ /* Found matching edge */ flag = 1; /* Check weights */ for (k=0; k<ewgt_dim; k++){ if (sendwgt[k] != recvwgt[k]){ flag = -1; ierr = ZOLTAN_WARN; } } if (flag<0 && output_level>0){ sprintf(msg, "Edge weight (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") is not symmetric", ptr1[0], ptr1[1]); ZOLTAN_PRINT_WARN(proc, yo, msg); } } ptr2 = (indextype *)(recvwgt + ewgt_dim); } if (!flag){ sprintf(msg, "Graph is not symmetric. " "Edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") exists, but not (" TPL_IDX_SPEC "," TPL_IDX_SPEC ").", ptr1[0], ptr1[1], ptr1[1], ptr1[0]); ZOLTAN_PRINT_ERROR(proc, yo, msg); ierr = ZOLTAN_FATAL; } ptr1 = (indextype *)(sendwgt + ewgt_dim); } } } /* Free memory */ ZOLTAN_FREE(&sendgno); ZOLTAN_FREE(&recvgno); ZOLTAN_FREE(&proclist); } if (free_adjncy_sort) ZOLTAN_FREE(&adjncy_sort); if (free_adjncy_sort) ZOLTAN_FREE(&perm); /* Compute global error code */ errors = 0; if (ierr == ZOLTAN_WARN) errors |= 1; if (ierr == ZOLTAN_MEMERR) errors |= 2; if (ierr == ZOLTAN_FATAL) errors |= 4; MPI_Allreduce(&errors, &global_errors, 1, MPI_INT, MPI_BOR, comm); if (global_errors & 4){ return ZOLTAN_FATAL; } else if (global_errors & 2){ return ZOLTAN_MEMERR; } else if (global_errors & 1){ return ZOLTAN_WARN; } else { if (proc==0 && output_level>0){ printf("ZOLTAN %s: The graph is valid with check_graph = %1d\n", yo, check_graph); } return ZOLTAN_OK; } }
int main(int argc, char *argv[]) { FILE *in_file = NULL; /* file with data for problems */ ZOLTAN_COMM_OBJ *plan; /* communication data structure pointer */ struct Params params; /* parameters describing a problem */ struct Data data; /* data describing a problem instance */ struct Data my_send_data; /* data I initially own */ struct Answer true_answer; /* expected outcome of exchange */ int nvals_recv; /* number of vals I own after exchange */ float *recv_data; /* values I own after exchange */ float *reverse_data; /* values I own after reversing communication */ char file_name[100]; /* name of input file */ int nbytes; /* size of objects */ int my_proc; /* my processor ID */ int nprocs; /* total number of processors */ int flag; /* return code from comm operations */ int more_problems; /* are there more problems to do? */ int i, j; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &my_proc); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); Zoltan_Memory_Debug(2); if (argc > 1) strcpy(file_name, argv[1]); else strcpy(file_name,"comm_input.dat"); if (my_proc == 0) { in_file = fopen(file_name, "r"); if (in_file == NULL) { printf("No input file `%s' found.\n", file_name); } } /* Read some problem descriptors from a file */ more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc); while (more_problems) { /* Generate full data at random on each proc */ gen_comm_data(¶ms, &data, nprocs); if (out_level > 2) if (my_proc == 0) print_data("DATA", &data); /* Figure out from the data what what to expect to receive */ extract_comm_answer(&data, &true_answer, my_proc, nprocs); /* Extract what to send */ set_up_comm_from_send(&data, &my_send_data, params.blocked, my_proc, nprocs); if (out_level > 2) print_data("MY_DATA", &my_send_data); if (out_level > 1) printf("%d: About to call comm_create\n", my_proc); /* Call comm routines */ Zoltan_Comm_Create(&plan, my_send_data.nvals, my_send_data.proc_dest, MPI_COMM_WORLD, 1, &nvals_recv); if (out_level > 1) printf("%d: About to call comm_info\n", my_proc); check_comm_info(plan, &my_send_data, nvals_recv, my_proc); if (out_level > 2) print_plan("BEFORE RESIZE", plan, my_proc); /* "4" reflects the max_sizes value in gen_comm_data */ recv_data = (float *) ZOLTAN_MALLOC(nvals_recv * 4 * sizeof(float)); reverse_data = (float *) ZOLTAN_MALLOC(my_send_data.nvals * 4 * sizeof(float)); if (my_send_data.sizes != NULL) { if (out_level > 1) printf("%d: About to call comm_resize\n", my_proc); Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL); Zoltan_Comm_Resize(plan, NULL, 43, NULL); Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL); if (out_level > 2) print_plan("AFTER RESIZE", plan, my_proc); } if (my_send_data.sizes == NULL) nbytes = my_send_data.same_size * sizeof(float); else nbytes = sizeof(float); if (out_level > 1) printf("%d: About to call comm_do\n", my_proc); flag = Zoltan_Comm_Do(plan, 2, (char *) my_send_data.vals, nbytes, (char *) recv_data); if (flag == ZOLTAN_OK) { if (out_level > 1) printf("%d: About to call check_answer\n", my_proc); /* Check answers */ check_comm_answer(&true_answer, recv_data, my_proc); } else { printf("%d: Comm_Do returned error code %d\n", my_proc, flag); } if (out_level > 1) printf("%d: About to call comm_do_reverse\n", my_proc); i = (true_answer.sizes != NULL && plan->indices_to != NULL); MPI_Allreduce(&i, &j, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (j == 0) flag = Zoltan_Comm_Do_Reverse(plan, 2, (char *) recv_data, nbytes, true_answer.sizes, (char *) reverse_data); else { if (my_proc == 0) printf(">> Non-blocked, variable-sized recvs not supported\n"); flag = ZOLTAN_FATAL; } if (flag == ZOLTAN_OK) { if (out_level > 1) printf("%d: About to call check_answer_reverse\n", my_proc); /* Check answers */ check_comm_answer_reverse(&my_send_data, reverse_data, my_proc); } else { if (out_level > 1) printf("%d: Comm_Do_Reverse returned error code %d\n", my_proc, flag); } /* Free up data structures */ ZOLTAN_FREE(&reverse_data); ZOLTAN_FREE(&recv_data); free_comm_data(&data, &my_send_data, &true_answer); Zoltan_Comm_Destroy(&plan); /* Read some problem descriptors from a file */ more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc); } Zoltan_Memory_Stats(); MPI_Finalize(); return(0); }
static int local_HEs_from_export_lists( ZZ *zz, int remap_type, /* type of remapping to do: parts, procs, or none. */ int nobj, /* # objs the processor knows about (keep + exports) */ int *new_proc, /* On input, new processor assignment for each obj; Upon return, remapped new proc assignment for each obj. */ int *old_part, /* old partition assignments for each objs */ int *new_part, /* On input, new partition assignments for each objs. Upon return, remapped new partition assignments */ int *HEcnt, /* # of HEs allocated. */ int **HEinfo /* Array of HE info; for each HE, two pins and one edge weight. Stored as a single vector to minimize communication calls. */ ) { /* Routine to remap partitions (to new processors or new partition numbers) * to reduce data movement. * This routine assumes the load-balancing algorithm built export lists. * Objects described are those that STARTED on zz->Proc due to load balancing. * For all these objects, old_proc == zz->Proc. */ char *yo = "local_HEs_from_export_lists"; int ierr = ZOLTAN_OK; int i, cnt, tmp; int *tmp_HEinfo; int my_proc = zz->Proc; /* This processor's rank. */ int nimp = 0; int *imp_proc = NULL, /* Temporary arrays if inversion of export to */ *imp_old_part = NULL, /* import lists is needed. */ *imp_new_part = NULL; int HEwgt_size; /* # of HE weights allocated. */ int *HEwgt = NULL; /* Array of HE weights. Initially includes zero weights; later zero-weights are removed.*/ if (remap_type == ZOLTAN_LB_REMAP_PROCESSORS) { /* Build HEs based on processor assignment. * We know the old processor for all objects we are keeping and all * export objects -- it is my_proc! * We also know the new processor number for all objects initially on * my_proc (since we built export lists.) * This case is a special case of partition remapping; it is easy to * build the hyperedges in this special case. */ HEwgt_size = zz->Num_Proc; HEwgt = (int *) ZOLTAN_CALLOC(HEwgt_size, sizeof(int)); if (!HEwgt) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } for (i = 0; i < nobj; i++) HEwgt[new_proc[i]]++; *HEcnt = 0; for (i = 0; i < HEwgt_size; i++) if (HEwgt[i] != 0) (*HEcnt)++; ierr = malloc_HEinfo(zz, *HEcnt, HEinfo); if (ierr < 0) goto End; tmp_HEinfo = *HEinfo; cnt = 0; for (i = 0; i < HEwgt_size; i++) { if (HEwgt[i] != 0) { tmp = cnt * HEINFO_ENTRIES; tmp_HEinfo[tmp] = my_proc; /* Old processor number */ tmp_HEinfo[tmp+1] = i; /* New processor number */ tmp_HEinfo[tmp+2] = HEwgt[i]; /* shift non-zero weights down. */ cnt++; } } } else { /* ZOLTAN_LB_REMAP_PARTS */ /* Cannot renumber partitions given export lists without summing HE weights * across processors. This summation is not straightforward. Also, a * potentially large number of HEs may exist * (max_old_partition_number * zz->Num_Global_Parts). Rather than build * this large matrix, just compute import lists from the export lists * and run the import-list algorithm. */ ZOLTAN_COMM_OBJ *plan; int msg_tag = 22345; ierr = Zoltan_Comm_Create(&plan, nobj, new_proc, zz->Communicator, msg_tag, &nimp); if (nimp > 0) { imp_proc = (int *) ZOLTAN_MALLOC(3 * nimp * sizeof(int)); imp_old_part = imp_proc + nimp; imp_new_part = imp_old_part + nimp; if (!imp_proc) { ierr = ZOLTAN_MEMERR; ZOLTAN_PRINT_ERROR(my_proc, yo, "Memory error."); goto End; } } ierr = Zoltan_Comm_Info(plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, imp_proc, NULL); msg_tag++; ierr = Zoltan_Comm_Do(plan, msg_tag, (char *) old_part, sizeof(int), (char *) imp_old_part); msg_tag++; ierr = Zoltan_Comm_Do(plan, msg_tag, (char *) new_part, sizeof(int), (char *) imp_new_part); Zoltan_Comm_Destroy(&plan); ierr = local_HEs_from_import_lists(zz, remap_type, nimp, imp_proc, imp_old_part, imp_new_part, HEcnt, HEinfo); } End: if (HEwgt) ZOLTAN_FREE(&HEwgt); if (imp_proc) ZOLTAN_FREE(&imp_proc); return ierr; }
void check_comm_info( ZOLTAN_COMM_OBJ *plan, struct Data *my_send_data, int nvals_recv, int my_proc ) { int *info_tmp = NULL, /* Temp bufs for verifying Zoltan_Comm_Info */ *info_tmp_send = NULL, *info_tmp_recv = NULL; int *info_send_list = NULL, *info_send_procs = NULL, *info_send_lengths = NULL; int info_nsend = 0, info_send_nvals = 0, info_send_size = 0, info_max_send_size = 0; int *info_recv_list = NULL, *info_recv_procs = NULL, *info_recv_lengths = NULL; int info_nrecv = 0, info_recv_nvals = 0, info_recv_size = 0, info_total_recv_size = 0; int info_self_msg = 0; int i; Zoltan_Comm_Info(plan, &info_nsend, NULL, NULL, &info_send_nvals, NULL, NULL, &info_nrecv, NULL, NULL, &info_recv_nvals, NULL, NULL, &info_self_msg); if (info_send_nvals != my_send_data->nvals) printf("%d Error in Zoltan_Comm_Info info_send_nvals %d != %d\n", my_proc, info_send_nvals, my_send_data->nvals); if (info_recv_nvals != nvals_recv) printf("%d Error in Zoltan_Comm_Info info_recv_nvals %d != %d\n", my_proc, info_recv_nvals, nvals_recv); info_send_size = 2 * (info_nsend + info_self_msg) + info_send_nvals; info_send_procs = (int *) ZOLTAN_MALLOC(info_send_size * sizeof(int)); info_send_lengths = info_send_procs + (info_nsend + info_self_msg); info_send_list = info_send_lengths + (info_nsend + info_self_msg); info_recv_size = 2 * (info_nrecv + info_self_msg) + info_recv_nvals; info_recv_procs = (int *) ZOLTAN_MALLOC(info_recv_size * sizeof(int)); info_recv_lengths = info_recv_procs + (info_nrecv + info_self_msg); info_recv_list = info_recv_lengths + (info_nrecv + info_self_msg); Zoltan_Comm_Info(plan, &info_nsend, info_send_procs, info_send_lengths, &info_send_nvals, &info_max_send_size, info_send_list, &info_nrecv, info_recv_procs, info_recv_lengths, &info_recv_nvals, &info_total_recv_size, info_recv_list, &info_self_msg); for (i = 0; i < info_send_nvals; i++) if (info_send_list[i] != my_send_data->proc_dest[i]) printf("%d Error in Zoltan_Comm_Info send_list[%d]: %d != %d\n", my_proc, i, info_send_list[i], my_send_data->proc_dest[i]); info_tmp = (int *) ZOLTAN_MALLOC((info_send_nvals + info_recv_nvals) * sizeof(int)); info_tmp_send = info_tmp; info_tmp_recv = info_tmp_send + info_send_nvals; for (i = 0; i < info_send_nvals; i++) info_tmp_send[i] = my_proc; Zoltan_Comm_Do(plan, 12, (char *) info_tmp_send, sizeof(int), (char *) info_tmp_recv); for (i = 0; i < info_recv_nvals; i++) if (info_recv_list[i] != info_tmp_recv[i]) printf("%d Error in Zoltan_Comm_Info recv_list[%d]: %d != %d\n", my_proc, i, info_recv_list[i], info_tmp_recv[i]); ZOLTAN_FREE(&info_tmp); ZOLTAN_FREE(&info_recv_procs); ZOLTAN_FREE(&info_send_procs); }
int Zoltan_DD_Update ( Zoltan_DD_Directory *dd, /* directory state information */ ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to update */ ZOLTAN_ID_PTR lid, /* Incoming corresponding LIDs (optional) */ char *user, /* Incoming list of user data (optional) */ int *partition, /* Optional, grouping of GIDs to partitions */ int count) /* Number of GIDs in update list */ { int *procs = NULL; /* list of processors to contact */ DD_Update_Msg *ptr = NULL; ZOLTAN_COMM_OBJ *plan = NULL; /* for efficient MPI communication */ char *sbuff = NULL; /* send buffer */ char *sbufftmp = NULL;/* pointer into send buffer */ char *rbuff = NULL; /* receive buffer */ char *rbufftmp = NULL;/* pointer into receive buffer */ int nrec = 0; /* number of receives to expect */ int i; int err; int errcount = 0; /* count of GIDs not found, added */ char str[100]; /* build error message string */ char *yo = "Zoltan_DD_Update"; /* input sanity checking */ if (dd == NULL || count < 0 || (gid == NULL && count > 0)) { ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc), yo, "Invalid input argument"); return ZOLTAN_FATAL; } if (dd->debug_level > 4) ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL); /* part of initializing the error checking process */ /* for each linked list head, walk its list resetting errcheck */ if (dd->debug_level) for (i = 0; i < dd->table_length; i++) { DD_NodeIdx nodeidx; for (nodeidx = dd->table[i]; nodeidx != -1; nodeidx = dd->nodelist[nodeidx].next) dd->nodelist[nodeidx].errcheck = ZOLTAN_DD_NO_PROC; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After reset errcheck"); /* allocate memory for list of processors to contact */ if (count) { procs = (int*) ZOLTAN_MALLOC (sizeof(int) * count); if (procs == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list"); err = ZOLTAN_MEMERR; goto fini; } } /* allocate memory for DD_Update_Msg send buffer */ if (count) { sbuff = (char*) ZOLTAN_CALLOC (count, dd->update_msg_size); if (sbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer"); err = ZOLTAN_MEMERR; goto fini; } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs"); /* for each GID given, fill in contact list and then message structure */ sbufftmp = sbuff; for (i = 0; i < count; i++) { procs[i] = dd->hash(gid + i*dd->gid_length, dd->gid_length, dd->nproc, dd->hashdata, dd->hashfn); ptr = (DD_Update_Msg*) sbufftmp; sbufftmp += dd->update_msg_size; ptr->lid_flag = (lid) ? 1 : 0; ptr->user_flag = (user) ? 1 : 0; ptr->partition_flag = (partition) ? 1 : 0; ptr->partition = (partition) ? *(partition + i) : -1; ptr->owner = dd->my_proc; ZOLTAN_SET_ID (dd->gid_length, ptr->gid, gid + i * dd->gid_length); if (lid) { ZOLTAN_SET_ID(dd->lid_length, ptr->gid + dd->gid_length, lid + i * dd->lid_length); } else { memset(ptr->gid + dd->gid_length, 0, dd->lid_length); } if (user) { memcpy(ptr->gid + (dd->gid_length + dd->lid_length), user + (size_t)i * (size_t)(dd->user_data_length), dd->user_data_length); } else { memset(ptr->gid + (dd->gid_length + dd->lid_length), 0, dd->user_data_length); } } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill contact list"); /* now create efficient communication plan */ err = Zoltan_Comm_Create (&plan, count, procs, dd->comm, ZOLTAN_DD_UPDATE_MSG_TAG, &nrec); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Create error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create"); /* If dd has no nodes allocated (e.g., first call to DD_Update; * create the nodelist and freelist */ if (nrec && dd->nodelistlen == 0) { DD_Memory_Alloc_Nodelist(dd, (DD_NodeIdx) nrec, 0.); /* TODO Add overalloc parameter */ } /* allocate receive buffer for nrec DD_Update_Msg structures */ if (nrec) { rbuff = (char*)ZOLTAN_MALLOC((size_t)nrec*(size_t)(dd->update_msg_size)); if (rbuff == NULL) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Receive buffer malloc failed"); err = ZOLTAN_MEMERR; goto fini; } } /* send my update messages & receive updates directed to me */ err = Zoltan_Comm_Do (plan, ZOLTAN_DD_UPDATE_MSG_TAG+1, sbuff, dd->update_msg_size, rbuff); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Comm_Do error"); goto fini; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do"); /* for each message rec'd, update local directory information */ errcount = 0; rbufftmp = rbuff; for (i = 0; i < nrec; i++) { ptr = (DD_Update_Msg *) rbufftmp; rbufftmp += dd->update_msg_size; err = DD_Update_Local (dd, ptr->gid, (ptr->lid_flag) ? (ptr->gid + dd->gid_length) : NULL, (char*)((ptr->user_flag)?(ptr->gid+(dd->gid_length+dd->lid_length)):NULL), (ptr->partition_flag) ? (ptr->partition) : -1, /* illegal partition */ ptr->owner); if (err != ZOLTAN_OK) ++errcount; } if (dd->debug_level > 6) ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Local update"); err = ZOLTAN_OK; if (dd->debug_level) /* overwrite error return if extra checking is on */ err = (errcount) ? ZOLTAN_WARN : ZOLTAN_OK; fini: ZOLTAN_FREE (&procs); ZOLTAN_FREE (&sbuff); ZOLTAN_FREE (&rbuff); Zoltan_Comm_Destroy (&plan); if (dd->debug_level) { sprintf (str, "Processed %d GIDs (%d local), %d GID errors", count, nrec, errcount); ZOLTAN_PRINT_INFO (dd->my_proc, yo, str); } if (dd->debug_level > 4) ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL); return err; }
int Zoltan_Postprocess_FinalOutput (ZZ* zz, ZOLTAN_Third_Graph *gr, ZOLTAN_Third_Part *prt, ZOLTAN_Third_Vsize *vsp, int use_timers, realtype itr) { #define FOMAXDIM 10 static char * yo = "Zoltan_Postprocess_FinalOutput"; static int nRuns=0; static double balsum[FOMAXDIM], cutesum[FOMAXDIM]; static double balmax[FOMAXDIM], cutemax[FOMAXDIM]; static double balmin[FOMAXDIM], cutemin[FOMAXDIM]; /* following variables are defined double to avoid overflow */ static double cutlsum = 0.0, cutnsum = 0.0, movesum = 0.0, repartsum = 0.0; static float cutlmax = 0, cutnmax = 0; static double movemax = 0, repartmax = 0; static float cutlmin = INT_MAX, cutnmin = INT_MAX; static double movemin = 1e100, repartmin = 1e100; static int timer_final_output = -1; double bal[FOMAXDIM]; /* Balance: max / avg */ double cute[FOMAXDIM]; /* Traditional weighted graph edge cuts */ float cutl; /* Connnectivity cuts: sum_over_edges((npart-1)) */ float cutn; /* Net cuts: sum_over_edges((nparts>1)) */ double move = 0.0, gmove =0.0; /* migration cost */ double repart; /* total repartitioning cost; cutl x multiplier + move */ int *adjpart = NULL; int vdim; int edim; indextype *vsizeBACKUP = NULL; indextype *input_part; int i,rc; /* #define UVC_DORUK_COMP_OBJSIZE */ #ifdef UVC_DORUK_COMP_OBJSIZE double minD, maxD, gminD, gmaxD; #endif if (use_timers) { if (timer_final_output < 0) timer_final_output = Zoltan_Timer_Init(zz->ZTime, 1, "Final_Output"); ZOLTAN_TIMER_START(zz->ZTime, timer_final_output, zz->Communicator); } if (nRuns == 0) { for (i = 0; i < FOMAXDIM; i++) { /* Initialize first time */ balsum[i] = cutesum[i] = 0.0; balmax[i] = cutemax[i] = 0.0; balmin[i] = cutemin[i] = 1e100; } } if (vsp && vsp->vsizeBACKUP) vsizeBACKUP = vsp->vsizeBACKUP; vdim = MAX(gr->obj_wgt_dim,1); edim = MAX(zz->Edge_Weight_Dim,1); if (gr->obj_wgt_dim < FOMAXDIM && zz->Edge_Weight_Dim < FOMAXDIM) { if (gr->xadj[gr->num_obj]){ adjpart = (int *) ZOLTAN_MALLOC(gr->xadj[gr->num_obj] * sizeof(int)); if (!adjpart){ ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Error 1 returned from Zoltan_Postprocess_FinalOutput"); } } Compute_Bal(zz, gr->num_obj, gr->vwgt, gr->obj_wgt_dim, prt->part, bal); rc = Compute_Adjpart(zz, gr->num_obj, gr->vtxdist, gr->xadj, gr->adjncy, gr->adjproc, prt->part, adjpart); if (rc != ZOLTAN_OK){ ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Error 2 returned from Zoltan_Postprocess_FinalOutput"); } Compute_EdgeCut(zz, gr->num_obj, gr->xadj, gr->float_ewgts, prt->part, adjpart, cute); cutl = Compute_ConCut(zz, gr->num_obj, gr->xadj, gr->float_ewgts, prt->part, adjpart); cutn = Compute_NetCut(zz, gr->num_obj, gr->xadj, gr->float_ewgts, prt->part, adjpart); #ifdef UVC_DORUK_COMP_OBJSIZE if (vsizeBACKUP) { minD = vsizeBACKUP[0]; maxD = vsizeBACKUP[0]; } #endif if (gr->scatter != 0) { /* Project input part in the scatter graph */ input_part = (indextype*) ZOLTAN_MALLOC(gr->num_obj * sizeof(int)); Zoltan_Comm_Do(gr->comm_plan, TAG1, (char *) prt->input_part, sizeof(indextype), (char *) input_part); } else input_part = prt->input_part; for (i=0; i<gr->num_obj; i++) { /*printf("obj[%d] = %d\n", i, vsize[i]);*/ if (prt->part[i] != input_part[i]) { move += (double) ((vsizeBACKUP) ? vsizeBACKUP[i] : 1.0); } #ifdef UVC_DORUK_COMP_OBJSIZE if (vsizeBACKUP) { minD = minD < vsizeBACKUP[i] ? minD : vsizeBACKUP[i]; maxD = maxD > vsizeBACKUP[i] ? maxD : vsizeBACKUP[i]; } #endif } if (gr->scatter != 0) ZOLTAN_FREE(&input_part); #ifdef UVC_DORUK_COMP_OBJSIZE if (gr->showMoveVol) { MPI_Allreduce(&minD, &gminD, 1, MPI_DOUBLE, MPI_MIN, zz->Communicator); MPI_Allreduce(&maxD, &gmaxD, 1, MPI_DOUBLE, MPI_MAX, zz->Communicator); if (zz->Proc == 0) printf("minD: %f, maxD: %f, gminD: %f, gmaxD: %f\n", minD, maxD, gminD, gmaxD); } #endif MPI_Allreduce(&move, &gmove, 1, MPI_DOUBLE, MPI_SUM, zz->Communicator); repart = (itr) * (double) cutl + gmove; repartsum += repart; if (repart > repartmax) repartmax = repart; if (repart < repartmin) repartmin = repart; movesum += gmove; if (gmove > movemax) movemax = gmove; if (gmove < movemin) movemin = gmove; cutlsum += cutl; if (cutl > cutlmax) cutlmax = cutl; if (cutl < cutlmin) cutlmin = cutl; cutnsum += cutn; if (cutn > cutnmax) cutnmax = cutn; if (cutn < cutnmin) cutnmin = cutn; for (i = 0; i < vdim; i++) { balsum[i] += bal[i]; if (bal[i] > balmax[i]) balmax[i] = bal[i]; if (bal[i] < balmin[i]) balmin[i] = bal[i]; } for (i = 0; i < edim; i++) { cutesum[i] += cute[i]; if (cute[i] > cutemax[i]) cutemax[i] = cute[i]; if (cute[i] < cutemin[i]) cutemin[i] = cute[i]; } nRuns++; if (zz->Proc == 0) { for (i = 0; i < vdim; i++) printf("STATS Runs %d bal[%d] CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, i, bal[i], balmax[i], balmin[i], balsum[i]/nRuns); printf("STATS Runs %d cutl CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, cutl, cutlmax, cutlmin, cutlsum/nRuns); printf("STATS Runs %d cutn CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, cutn, cutnmax, cutnmin, cutnsum/nRuns); for (i = 0; i < edim; i++) printf("STATS Runs %d cute[%d] CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, i, cute[i], cutemax[i], cutemin[i], cutesum[i]/nRuns); printf("STATS Runs %d %s CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, gr->showMoveVol ? "moveVol" : "moveCnt", gmove, movemax, movemin, movesum/nRuns); if (gr->showMoveVol) printf("STATS Runs %d repart CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, repart, repartmax, repartmin, repartsum/nRuns); } ZOLTAN_FREE(&adjpart); } #undef FOMAXDIM if (use_timers) ZOLTAN_TIMER_STOP(zz->ZTime, timer_final_output, zz->Communicator); return (ZOLTAN_OK); }
int Zoltan_Migrate( ZZ *zz, /* Zoltan structure. */ int num_import, /* Number of non-local objects assigned to the processor in the new decomposition. */ ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects assigned to this processor in the new decomposition; this field can be NULL if the application doesn't provide import IDs.*/ ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects assigned to the processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_procs, /* Array of processor IDs of processors owning the non-local objects that are assigned to this processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_to_part, /* Array of partition numbers to which imported objects should be assigned. */ int num_export, /* Number of objs to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of objects to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of objects to be exported to other processors to establish the new decomposition. */ int *export_procs, /* Array of processor IDs to which objects will be exported to establish the new decomposition. */ int *export_to_part /* Array of partition numbers to which exported objects should be assigned. */ ) { /* * Routine to help perform migration. If migration pre-processing routine * (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn. * It then calls a function to obtain the size of the migrating objects * (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified * object packing routine (ZOLTAN_PACK_OBJ_FN) for each object * to be exported. It develops the needed communication map to move the * objects to other processors. It performs the communication according * to the map, and then calls an application-specified object unpacking * routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported. */ char *yo = "Zoltan_Migrate"; int num_gid_entries, num_lid_entries; /* lengths of global & local ids */ int *sizes = NULL; /* sizes (in bytes) of the object data for export. */ int id_size; /* size (in bytes) of ZOLTAN_GID + padding for alignment */ int tag_size; /* size (in bytes) of ZOLTAN_GID + one int (for message size) */ char *export_buf = NULL; /* buffer for packing export data. */ char *import_buf = NULL; /* buffer for receiving imported data. */ char *tmp; /* temporary pointer into buffers. */ int i; /* loop counter. */ int tmp_size; /* size of a single object's data. */ int *idx = NULL; /* index used for multi-fn packs and unpacks. */ int idx_cnt = 0; /* index counter for idx array. */ ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm buf */ ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass NULL to query functions when NUM_LID_ENTRIES=0. */ ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */ ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */ int msgtag, msgtag2; /* Tags for communication routines */ int total_send_size; /* Total size of outcoming message (in #items) */ int total_recv_size; /* Total size of incoming message (in #items) */ int aligned_int; /* size of an int padded for alignment */ int dest; /* temporary destination partition. */ int include_parts = 0; /* flag indicating whether partition info is provided */ int ierr = ZOLTAN_OK; int actual_num_exp = 0; int actual_exp_allocated = 0; ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */ int *actual_exp_procs = NULL; /* are changing partition but not */ int *actual_exp_to_part = NULL; /* processor may not be included. */ int actual_num_imp = 0; int actual_imp_allocated = 0; ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */ int *actual_imp_procs = NULL; /* are changing partition but not */ int *actual_imp_to_part = NULL; /* processor may not be included. */ ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { goto End; } /* * Check that all procs use the same id types. */ ierr = check_input(zz, ((num_export >= 0 && export_to_part) || (num_import >= 0 && import_to_part)), &include_parts); if (ierr != ZOLTAN_OK) goto End; num_gid_entries = zz->Num_GID; num_lid_entries = zz->Num_LID; /* * Check that all necessary query functions are available. */ if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (num_export >= 0) { /* Build the actual export arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &actual_num_exp, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part, &actual_exp_allocated); if (ierr < 0) goto End; /* Compute communication map based on actual exports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs, zz->Communicator, msgtag, &actual_num_imp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } } else if (num_import >= 0) { /* Build the actual import arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, &actual_num_imp, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part, &actual_imp_allocated); if (ierr < 0) goto End; /* Compute communication map based on imports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs, zz->Communicator, msgtag, &actual_num_exp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } /* Compute actual export lists for packing objects */ if (actual_num_exp > 0) { actual_exp_allocated = 1; actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp); actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp); actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp); if (include_parts) actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp); if (actual_exp_gids == NULL || (num_lid_entries && actual_exp_lids == NULL) || actual_exp_procs == NULL || (import_to_part != NULL && actual_exp_to_part == NULL)) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); ierr = ZOLTAN_MEMERR; goto End; } } msgtag2 = 32766; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) actual_exp_gids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) actual_exp_lids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part, (int) sizeof(int), (char *) actual_exp_to_part); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } /* Create inverse plan (i.e., plan based on exports) so can set * variable sizes. * (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable * but does not tell how large to allocate receive buffer. */ ierr = Zoltan_Comm_Invert_Plan(&imp_plan); exp_plan = imp_plan; imp_plan = NULL; } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Migrate.Pre_Migrate_PP != NULL) { zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Pre_Migrate != NULL) { zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_FN function."); goto End; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing"); id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE)); /* Note that alignment is not strictly necessary when ZOLTAN_ID_TYPE is int or unsigned int. */ aligned_int = Zoltan_Align(sizeof(int)); tag_size = id_size + aligned_int; /* * For each object, allow space for its global ID and its data plus * one int (for the object data size). * Zoltan will pack the global IDs; the application must pack the data * through the pack routine. Zoltan needs the global IDs for unpacking, * as the order of the data received during communication is not * necessarily the same order as import_global_ids[]. * Zoltan also needs to communicate the sizes of the objects because * only the sender knows the size of each object. */ if (actual_num_exp > 0) { sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!sizes) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Get_Obj_Size_Multi != NULL) { zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, sizes, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE_MULTI function."); goto End; } } else { for (i = 0; i < actual_num_exp; i++) { lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE function."); goto End; } } } total_send_size = 0; for (i = 0; i < actual_num_exp; i++) { sizes[i] = Zoltan_Align(sizes[i]); total_send_size += sizes[i] + tag_size; } export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char)); if (!export_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Pack_Obj_Multi != NULL) { /* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */ idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Pack the objects for export. */ idx_cnt = 0; tmp = export_buf; for (i = 0; i < actual_num_exp; i++) { /* Pack the object's global ID */ tmp_id = (ZOLTAN_ID_PTR) tmp; ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries])); tmp += id_size; /* Pack the object's size */ *((int *)tmp) = sizes[i]; tmp += aligned_int; /* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Pack_Obj_Multi != NULL) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n", zz->Proc, yo); } zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, (actual_exp_to_part!=NULL ? actual_exp_to_part : actual_exp_procs), sizes, idx, export_buf, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ_MULTI function."); goto End; } } else { tmp = export_buf + tag_size; for (i = 0; i < actual_num_exp; i++) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries])); printf("size = %d bytes\n", sizes[i]); } /* Pack the object's data */ lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i] : actual_exp_procs[i]); zz->Pack_Obj(zz->Pack_Obj_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, dest, sizes[i], tmp, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ function."); goto End; } tmp += sizes[i] + tag_size; } } ZOLTAN_FREE(&idx); tmp_id = NULL; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects"); /* Modify sizes[] to contain message sizes, not object sizes */ for (i=0; i<actual_num_exp; i++) { sizes[i] += tag_size; } msgtag--; ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize."); goto End; } if (actual_num_imp > 0) { import_buf = (char *) ZOLTAN_MALLOC(total_recv_size); if (!import_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Send the export data using the communication plan. */ msgtag2 = 32765; ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } /* * Free whatever memory we can. */ Zoltan_Comm_Destroy(&exp_plan); ZOLTAN_FREE(&export_buf); ZOLTAN_FREE(&sizes); ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication"); /* * Perform application-specified processing before unpacking the data. */ if (zz->Migrate.Mid_Migrate_PP != NULL) { zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Mid_Migrate != NULL) { zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_FN function."); goto End; } } /* * Unpack the object data. */ if (actual_num_imp > 0) { if (zz->Unpack_Obj_Multi != NULL) { /* Allocate and fill input arrays for Unpack_Obj_Multi. */ sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp); idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); if (!sizes || !tmp_id || !idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } tmp = import_buf; idx_cnt = 0; for (i = 0; i < actual_num_imp; i++) { /* Unpack the object's global ID */ ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp); tmp += id_size; /* Unpack the object's size */ sizes[i] = *((int *)tmp); tmp += aligned_int; /* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n", zz->Proc,yo); } zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries, actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr); ZOLTAN_FREE(&import_buf); ZOLTAN_FREE(&sizes); ZOLTAN_FREE(&tmp_id); ZOLTAN_FREE(&idx); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_MULTI_FN."); goto End; } } else { tmp = import_buf; for (i = 0; i < actual_num_imp; i++) { tmp_size = *((int *)(tmp + id_size)); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp); printf("size = %d bytes\n", tmp_size); } /* Unpack the object's data */ zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries, (ZOLTAN_ID_PTR) tmp, tmp_size, tmp + tag_size, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_FN."); goto End; } tmp += (tmp_size + tag_size); } ZOLTAN_FREE(&import_buf); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects"); if (zz->Migrate.Post_Migrate_PP != NULL) { zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Post_Migrate != NULL) { zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_FN function."); goto End; } } End: if (actual_exp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); } if (actual_imp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part); } if (ierr < 0) { if (exp_plan) Zoltan_Comm_Destroy(&exp_plan); Zoltan_Multifree(__FILE__, __LINE__, 5, &import_buf, &tmp_id, &sizes, &idx, &export_buf); } ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Inverse_Perm( ZZ *zz, /* Input: Zoltan struct */ int *perm, /* Input: Permutation to invert. */ int *inv_perm, /* Output: Inverse permutation of perm. */ int *vtxdist, /* Input: Distribution of the vectors. */ char *order_type, /* Input: Local or global ordering? */ int start_index /* Input: Do permutations start with 0 or 1? */ ) { int i, ierr, num_obj, nrecv, offset; int *proclist, *sendlist, *recvlist; ZOLTAN_COMM_OBJ *comm_plan; char msg[256]; char *yo = "Zoltan_Inverse_Perm"; ierr = ZOLTAN_OK; proclist = sendlist = recvlist = NULL; comm_plan = NULL; /* num_obj = local number of objects (elements in the perm vectors) */ num_obj = vtxdist[(zz->Proc)+1] - vtxdist[zz->Proc]; /* Verify that input permutation is really a permutation */ /* Also check that start_index is correct. */ /* Convert permutation vector to 0-base if necessary */ if (start_index>0){ for (i=0; i<num_obj; i++) perm[i] -= start_index; } if (strcmp(order_type, "LOCAL")==0){ /* Local inverse */ for (i=0; i<num_obj; i++) inv_perm[perm[i]] = i; } else if (strcmp(order_type, "GLOBAL")==0){ /* Global inverse; use Zoltan Comm package */ proclist = (int *) ZOLTAN_MALLOC (5*num_obj*sizeof(int)); sendlist = &proclist[num_obj]; recvlist = &proclist[3*num_obj]; /* Set up comm plan. We know where to send. */ /* Send pairs of (i, perm[i]) to other procs */ offset = vtxdist[zz->Proc]; for (i=0; i<num_obj; i++){ sendlist[2*i] = offset+i; sendlist[2*i+1] = perm[i]; proclist[i] = Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, perm[i]); } ierr = Zoltan_Comm_Create(&comm_plan, num_obj, proclist, zz->Communicator, TAG1, &nrecv); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in Zoltan_Comm_Create"); goto error; } if (nrecv != num_obj){ /* This should never happen. */ sprintf(msg, "Internal error: nrecv (%3d) != num_obj (%3d). Invalid permutation.\n", nrecv, num_obj); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ierr = ZOLTAN_FATAL; goto error; } /* Do the communication. */ ierr = Zoltan_Comm_Do(comm_plan, TAG2, (char *)sendlist, 2*sizeof(int), (char *) recvlist); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in Zoltan_Comm_Do"); goto error; } /* Permute data locally. */ for (i=0; i<num_obj; i++){ /* inv_perm[perm[i]] = i; */ inv_perm[recvlist[2*i+1]-offset] = recvlist[2*i]; } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown order_type."); ierr = ZOLTAN_FATAL; goto error; } /* Convert permutation vectors back to their right start_index */ if (start_index>0){ for (i=0; i<num_obj; i++){ perm[i] += start_index; inv_perm[i] += start_index; } } error: /* Free the comm_plan, proclist, sendlist, recvlist. */ if (comm_plan) Zoltan_Comm_Destroy( &comm_plan); if (proclist ) ZOLTAN_FREE(&proclist); return (ierr); }
static int Zoltan_PHG_Redistribute_Hypergraph( ZZ *zz, PHGPartParams *hgp, /* Input: parameters; used only for UseFixedVtx */ HGraph *ohg, /* Input: Local part of distributed hypergraph */ int firstproc, /* Input: rank (in ocomm) of the first proc of the ncomm*/ int *v2Col, /* Input: Vertex to processor Column Mapping */ int *n2Row, /* Input: Net to processor Row Mapping */ PHGComm *ncomm, /* Input: communicators of new distribution */ HGraph *nhg, /* Output: Newly redistributed hypergraph */ int **vmap, /* Output: allocated with the size nhg->nVtx and vertex map from nhg to ohg's local vertex number*/ int **vdest /* Output: allocated with the size nhg->nVtx and stores dest proc in ocomm */ ) { char * yo = "Zoltan_PHG_Redistribute_Hypergraph"; PHGComm *ocomm = ohg->comm; int ierr=ZOLTAN_OK; int i, v, n, nPins, nsend, elemsz, nVtx, nEdge; int msg_tag = 9999; int *proclist=NULL, *sendbuf=NULL; int *vno=NULL, *nno=NULL, *dist_x=NULL, *dist_y=NULL, *vsn=NULL, *nsn=NULL, *pins=NULL, *cnt=NULL; ZOLTAN_COMM_OBJ *plan; Zoltan_HG_HGraph_Init (nhg); nhg->comm = ncomm; nhg->dist_x = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int)); nhg->dist_y = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int)); dist_x = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int)); dist_y = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int)); vsn = (int *) ZOLTAN_CALLOC(ncomm->nProc_x+1, sizeof(int)); nsn = (int *) ZOLTAN_CALLOC(ncomm->nProc_y+1, sizeof(int)); vno = (int *) ZOLTAN_MALLOC(ohg->nVtx * sizeof(int)); nno = (int *) ZOLTAN_MALLOC(ohg->nEdge * sizeof(int)); if (!nhg->dist_x || !nhg->dist_y || !dist_x || !dist_y || !vsn || !nsn || (ohg->nVtx && !vno) || (ohg->nEdge && !nno) ) { uprintf(ocomm, " new comm nProcx=%d nProcy=%d nvtx=%d nedge=%d", ncomm->nProc_x, ncomm->nProc_y, ohg->nVtx, ohg->nEdge); MEMORY_ERROR; } for (v = 0; v < ohg->nVtx; ++v) ++dist_x[v2Col[v]]; for (n = 0; n < ohg->nEdge; ++n) ++dist_y[n2Row[n]]; /* UVCUVC: CHECK ASSUMPTION This code assumes that the objects in the receive buffer of Zoltan_Comm_Do function are 1- in the increasing processor order, 2- order of the items send by a processor is preserved. */ /* compute prefix sum to find new vertex start numbers; for each processor */ MPI_Scan(dist_x, vsn, ncomm->nProc_x, MPI_INT, MPI_SUM, ocomm->row_comm); /* All reduce to compute how many each processor will have */ MPI_Allreduce(dist_x, &(nhg->dist_x[1]), ncomm->nProc_x, MPI_INT, MPI_SUM, ocomm->row_comm); nhg->dist_x[0] = 0; for (i=1; i<=ncomm->nProc_x; ++i) nhg->dist_x[i] += nhg->dist_x[i-1]; MPI_Scan(dist_y, nsn, ncomm->nProc_y, MPI_INT, MPI_SUM, ocomm->col_comm); MPI_Allreduce(dist_y, &(nhg->dist_y[1]), ncomm->nProc_y, MPI_INT, MPI_SUM, ocomm->col_comm); nhg->dist_y[0] = 0; for (i=1; i<=ncomm->nProc_y; ++i) nhg->dist_y[i] += nhg->dist_y[i-1]; #ifdef _DEBUG1 PrintArr(ocomm, "vsn", vsn, ncomm->nProc_x); PrintArr(ocomm, "nsn", nsn, ncomm->nProc_y); #endif /* find mapping of current LOCAL vertex no (in my node) to "new" vertex no LOCAL to dest node*/ for (v = ohg->nVtx-1; v>=0; --v) vno[v] = --vsn[v2Col[v]]; for (n = ohg->nEdge-1; n>=0; --n) nno[n] = --nsn[n2Row[n]]; nsend = MAX(MAX(ohg->nPins, ohg->nVtx), ohg->nEdge); elemsz = MAX(MAX(2, ohg->VtxWeightDim), ohg->EdgeWeightDim); elemsz = (sizeof(float)>sizeof(int)) ? sizeof(float)*elemsz : sizeof(int)*elemsz; proclist = (int *) ZOLTAN_MALLOC(nsend * sizeof(int)); sendbuf = (int *) ZOLTAN_MALLOC(nsend * elemsz); /* first communicate pins */ nPins = 0; for (v = 0; v < ohg->nVtx; ++v) { for (i = ohg->vindex[v]; i < ohg->vindex[v+1]; ++i) { #ifdef _DEBUG1 if ((n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v])<0 || (n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v])>=ocomm->nProc) errexit("vertex %d vedge[%d]=%d n2Row=%d #Proc_x=%d v2Col=%d", i, ohg->vedge[i], n2Row[ohg->vedge[i]], ncomm->nProc_x , v2Col[v]); #endif proclist[nPins] = firstproc + n2Row[ohg->vedge[i]] * ncomm->nProc_x + v2Col[v]; sendbuf[2*nPins] = vno[v]; sendbuf[2*nPins+1]= nno[ohg->vedge[i]]; ++nPins; } } #ifdef _DEBUG1 if (nPins!=ohg->nPins) { uprintf(ocomm, "sanity check failed nPins(%d)!=hg->nPins(%d)\n", nPins, ohg->nPins); errexit("terminating"); } #endif --msg_tag; ierr |= Zoltan_Comm_Create(&plan, ohg->nPins, proclist, ocomm->Communicator, msg_tag, &nPins); #ifdef _DEBUG1 if (ncomm->myProc==-1 && nPins>1) { /* this processor is not in new comm but receiving data?*/ uprintf(ocomm, "Something wrong; why I'm receiving data nPins=%d\n", nPins); errexit("terminating"); } #endif if (nPins && (pins = (int *) ZOLTAN_MALLOC(nPins * 2 * sizeof(int)))==NULL) MEMORY_ERROR; --msg_tag; Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, 2*sizeof(int), (char *) pins); Zoltan_Comm_Destroy(&plan); /* now communicate vertex map */ nsend = 0; if (!ocomm->myProc_y) { /* only first row sends to the first row of ncomm */ for (v = 0; v < ohg->nVtx; ++v) { proclist[nsend] = firstproc+v2Col[v]; sendbuf[nsend++] = ohg->vmap[v]; } } --msg_tag; ierr |= Zoltan_Comm_Create(&plan, nsend, proclist, ocomm->Communicator, msg_tag, &nVtx); #ifdef _DEBUG1 if (ncomm->myProc==-1 && nVtx>1) { /* this processor is not in new comm but receiving data?*/ uprintf(ocomm, "Something wrong; why I'm receiving data nVtx=%d\n", nVtx); errexit("terminating"); } #endif /* those are only needed in the first row of ncomm */ *vmap = *vdest = NULL; if (!ncomm->myProc_y && nVtx && (!(*vmap = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int))) || !(*vdest = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int))))) MEMORY_ERROR; --msg_tag; Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(int), (char *) *vmap); if (!ocomm->myProc_y) { /* only first row sends to the first row of ncomm */ for (v = 0; v < ohg->nVtx; ++v) sendbuf[v] = ocomm->myProc; } --msg_tag; Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(int), (char *) *vdest); if (ncomm->myProc!=-1) { /* I'm in the new comm */ /* ncomm's first row now bcast to other rows */ MPI_Bcast(&nVtx, 1, MPI_INT, 0, ncomm->col_comm); #ifdef _DEBUG1 if (nVtx!=(nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x])) errexit("nVtx(%d)!= nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x](%d)", nVtx, nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x]); #endif if (nVtx && (nhg->vmap = (int *) ZOLTAN_MALLOC(nVtx * sizeof(int)))==NULL) MEMORY_ERROR; for (i=0; i<nVtx; ++i) nhg->vmap[i] = i; } /* now communicate vertex weights */ if (ohg->VtxWeightDim) { if (nVtx) nhg->vwgt = (float*) ZOLTAN_MALLOC(nVtx*ohg->VtxWeightDim*sizeof(float)); --msg_tag; Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->vwgt, ohg->VtxWeightDim*sizeof(float), (char *) nhg->vwgt); if (ncomm->myProc!=-1) /* ncomm's first row now bcast to other rows */ MPI_Bcast(nhg->vwgt, nVtx*ohg->VtxWeightDim, MPI_FLOAT, 0, ncomm->col_comm); } /* communicate fixed vertices, if any */ if (hgp->UseFixedVtx) { if (nVtx) nhg->fixed_part = (int *) ZOLTAN_MALLOC(nVtx*sizeof(int)); --msg_tag; Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->fixed_part, sizeof(int), (char *) nhg->fixed_part); if (ncomm->myProc!=-1) /* ncomm's first row now bcast to other rows */ MPI_Bcast(nhg->fixed_part, nVtx, MPI_INT, 0, ncomm->col_comm); } /* communicate pref parts, if any */ if (hgp->UsePrefPart) { if (nVtx) nhg->pref_part = (int *) ZOLTAN_MALLOC(nVtx*sizeof(int)); --msg_tag; Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->pref_part, sizeof(int), (char *) nhg->pref_part); if (ncomm->myProc!=-1) /* ncomm's first row now bcast to other rows */ MPI_Bcast(nhg->pref_part, nVtx, MPI_INT, 0, ncomm->col_comm); } /* this comm plan is no longer needed. */ Zoltan_Comm_Destroy(&plan); if (ohg->EdgeWeightDim) { /* now communicate edge weights */ nsend = 0; if (!ocomm->myProc_x) /* only first column sends to first column of ncomm */ for (n = 0; n < ohg->nEdge; ++n) proclist[nsend++] = firstproc + n2Row[n]*ncomm->nProc_x; --msg_tag; ierr |= Zoltan_Comm_Create(&plan, nsend, proclist, ocomm->Communicator, msg_tag, &nEdge); #ifdef _DEBUG1 if (ncomm->myProc==-1 && nEdge>1) { /* this processor is not in new comm but receiving data?*/ uprintf(ocomm, "Something wrong; why I'm receiving data nEdge=%d\n", nEdge); errexit("terminating"); } #endif if (ncomm->myProc!=-1) { /* if we're in the new comm */ /* ncomm's first column now bcast to other columns */ MPI_Bcast(&nEdge, 1, MPI_INT, 0, ncomm->row_comm); #ifdef _DEBUG1 if (nEdge != (nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y])) errexit("nEdge(%d)!=nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y](%d)", nEdge, nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y]); #endif } if (nEdge) nhg->ewgt = (float*) ZOLTAN_MALLOC(nEdge*ohg->EdgeWeightDim*sizeof(float)); --msg_tag; Zoltan_Comm_Do(plan, msg_tag, (char *) ohg->ewgt, ohg->EdgeWeightDim*sizeof(float), (char *) nhg->ewgt); if (ncomm->myProc!=-1) { /* if we're in the new comm */ /* ncomm's first column now bcast to other columns */ if (nEdge) MPI_Bcast(nhg->ewgt, nEdge*ohg->EdgeWeightDim, MPI_FLOAT, 0, ncomm->row_comm); } Zoltan_Comm_Destroy(&plan); } else nEdge = (ncomm->myProc==-1) ? 0 : nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y]; if (ncomm->myProc==-1) { #ifdef _DEBUG1 if (nPins || nVtx || nEdge) errexit("I should not have any data: hey nPins=%d nVtx=%d nEdge=%d\n", nPins, nVtx, nEdge); #endif nhg->nEdge = nhg->nVtx = nhg->nPins = 0; } else { nhg->nEdge = nhg->dist_y[ncomm->myProc_y+1] - nhg->dist_y[ncomm->myProc_y]; nhg->nVtx = nhg->dist_x[ncomm->myProc_x+1] - nhg->dist_x[ncomm->myProc_x]; nhg->nPins = nPins; /* Unpack the pins received. */ cnt = (int *) ZOLTAN_CALLOC(nhg->nVtx + 1, sizeof(int)); nhg->vindex = (int *) ZOLTAN_CALLOC(nhg->nVtx + 1, sizeof(int)); nhg->vedge = (int *) ZOLTAN_MALLOC(nhg->nPins * sizeof(int)); if (!cnt || !nhg->vindex || (nPins && !nhg->vedge)) MEMORY_ERROR; /* Count the number of pins per vertex */ for (i = 0; i < nPins; ++i) ++cnt[pins[2*i]]; /* Compute prefix sum to represent hindex correctly. */ for (i = 0; i < nhg->nVtx; ++i) { nhg->vindex[i+1] = nhg->vindex[i] + cnt[i]; cnt[i] = nhg->vindex[i]; } for (i = 0; i < nPins; ++i) nhg->vedge[cnt[pins[2*i]]++] = pins[2*i+1]; nhg->info = ohg->info; nhg->VtxWeightDim = ohg->VtxWeightDim; nhg->EdgeWeightDim = ohg->EdgeWeightDim; ierr = Zoltan_HG_Create_Mirror(zz, nhg); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) MEMORY_ERROR; } End: Zoltan_Multifree(__FILE__, __LINE__, 10, &proclist, &sendbuf, &pins, &cnt, &vno, &nno, &dist_x, &dist_y, &vsn, &nsn ); return ierr; }
static int Zoltan_Oct_Update_Connections( ZZ *zz, pOctant *octs, /* octs[nocts] */ int *newpids, /* newpids[nocts] */ pOctant *newocts, /* newocts[nocts] */ int nocts) /* number of octants leaving this processor */ { int i, j; int nsends; int nreceives; pOctant parent; pOctant child; int ppid; int cpid; int childnum; int *despid = NULL; Update_msg umsg; Update_msg *localumsg = NULL; Update_msg *remoteumsg = NULL; Update_msg *rcv_umsg = NULL; int localcount; int remotecount; int ierr = ZOLTAN_OK; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ char *yo = "Zoltan_Oct_Update_Connections"; OCT_Global_Info *OCT_info = (OCT_Global_Info *) zz->LB.Data_Structure; localcount=0; remotecount=0; /* count number of sends */ nsends = 0; for (i=0; i<nocts; i++) if (newpids[i]!=zz->Proc) nsends++; if(nocts > 0) { if((remoteumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((localumsg = (Update_msg *) ZOLTAN_MALLOC((nocts+1) * sizeof(Update_msg)*9)) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); return ZOLTAN_MEMERR; } if((despid = (int *) ZOLTAN_MALLOC((nocts+1) * sizeof(int)*9)) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); return ZOLTAN_MEMERR; } } else { remoteumsg = NULL; localumsg = NULL; despid = NULL; } localcount = 0; remotecount = 0; for (i=0; i<nocts; i++) /* Send connection updates */ if (newpids[i]!=zz->Proc) { parent = Zoltan_Oct_parent(octs[i]); ppid = Zoltan_Oct_Ppid(octs[i]); childnum = Zoltan_Oct_childnum(octs[i]); if (parent) { /* Let parent of oct[i] know that it's moving */ if (ppid==zz->Proc) { FILLUPDATEMSG(localumsg[localcount], parent, childnum, newocts[i], newpids[i]); localcount++; } else { FILLUPDATEMSG(remoteumsg[remotecount], parent, childnum, newocts[i], newpids[i]); despid[remotecount++] = ppid; } } for (j=0; j<8; j++) { child = Zoltan_Oct_child(octs[i],j); cpid = octs[i]->cpid[j]; /* Tell child of oct[i] that it is moving */ if (child) { if (cpid==zz->Proc) { /* NOTE: -1 signals PARENT */ FILLUPDATEMSG(localumsg[localcount], child, -1, newocts[i], newpids[i]); localcount++; } else { /* NOTE: -1 signals PARENT */ FILLUPDATEMSG(remoteumsg[remotecount], child, -1, newocts[i], newpids[i]); despid[remotecount++] = cpid; } } } } ierr = Zoltan_Comm_Create(&comm_plan, remotecount, despid, zz->Communicator, MigUpdCommCreate, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&despid); return (ierr); } /* if(nreceives > 0) { */ if((rcv_umsg = (Update_msg *) ZOLTAN_MALLOC((nreceives +1) * sizeof(Update_msg)*9)) == NULL) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&despid); return ZOLTAN_MEMERR; } ierr = Zoltan_Comm_Do(comm_plan, MigUpdCommDo, (char *) remoteumsg, sizeof(Update_msg), (char *) rcv_umsg); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_umsg); return (ierr); } /* } */ /* else { */ /* rcv_umsg = NULL; */ /* } */ /* update new octants */ for (i=0; i< (localcount+nreceives); i++) { if (i<localcount) umsg=localumsg[i]; else umsg=rcv_umsg[i-localcount]; if (umsg.childnum>=0) { Zoltan_Oct_setchild(umsg.oct,umsg.childnum,umsg.newptr); Zoltan_Oct_setCpid(umsg.oct,umsg.childnum,umsg.newpid); } else { if((Zoltan_Oct_data_newpid(umsg.oct) == OCT_info->OCT_localpid) || ((Zoltan_Oct_data_newpid(umsg.oct) != OCT_info->OCT_localpid) && (umsg.newpid == OCT_info->OCT_localpid))) Zoltan_Oct_POct_setparent(OCT_info, umsg.oct,umsg.newptr,umsg.newpid); else { umsg.oct->ppid = umsg.newpid; umsg.oct->parent = umsg.newptr; } } } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&despid); ZOLTAN_FREE(&rcv_umsg); return (ierr); } ZOLTAN_FREE(&remoteumsg); ZOLTAN_FREE(&localumsg); ZOLTAN_FREE(&rcv_umsg); ZOLTAN_FREE(&despid); return ierr; }