/***************************************************************************** * This function computes a partitioning. *****************************************************************************/ void ParMETIS_PartKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part, MPI_Comm *comm) { int i; int ncon = 1; float *tpwgts, ubvec[MAXNCON]; int myoptions[10]; tpwgts = fmalloc(*nparts*ncon, "tpwgts"); for (i=0; i<*nparts*ncon; i++) tpwgts[i] = 1.0/(float)(*nparts); for (i=0; i<ncon; i++) ubvec[i] = UNBALANCE_FRACTION; if (options[0] == 0) { myoptions[0] = 0; } else { myoptions[0] = 1; myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL]; myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED; } ParMETIS_V3_PartKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, &ncon, nparts, tpwgts, ubvec, myoptions, edgecut, part, comm); GKfree((void **)&tpwgts, LTERM); }
void FC_FUNC_(oct_parmetis_v3_partkway, OCT_PARMETIS_PARTKWAY) (idx_t *vtxdist, idx_t *xadj, idx_t *adjncy, idx_t *ncon, idx_t *nparts, real_t *tpwgts, real_t *ubvec, idx_t *options, idx_t *edgecut, idx_t *part, MPI_Fint *fcomm) { idx_t wgtflag = 0, numflag = 1; MPI_Comm comm; #ifdef HAVE_MPI2 comm = MPI_Comm_f2c(*fcomm); #else comm = *fcomm; #endif ParMETIS_V3_PartKway(vtxdist, xadj, adjncy, NULL, NULL, &wgtflag, &numflag, ncon, nparts, tpwgts, ubvec, options, edgecut, part, &comm); }
std::tuple<ptrdiff_t, ptrdiff_t> partition(const distributed_matrix<B> &A, idx_t npart, std::vector<ptrdiff_t> &perm) const { communicator comm = A.comm(); idx_t n = A.loc_rows(); int active = (n > 0); std::vector<idx_t> ptr; std::vector<idx_t> col; symm_graph(A, ptr, col); idx_t wgtflag = 0; idx_t numflag = 0; idx_t options = 0; idx_t edgecut = 0; idx_t ncon = 1; std::vector<real_t> tpwgts(npart, 1.0 / npart); std::vector<real_t> ubvec(ncon, 1.05); std::vector<idx_t> part(n); if (!n) part.reserve(1); // So that part.data() is not NULL MPI_Comm scomm; MPI_Comm_split(comm, active ? 0 : MPI_UNDEFINED, comm.rank, &scomm); if (active) { communicator sc(scomm); std::vector<idx_t> vtxdist = sc.exclusive_sum(n); sc.check( METIS_OK == ParMETIS_V3_PartKway( &vtxdist[0], &ptr[0], &col[0], NULL, NULL, &wgtflag, &numflag, &ncon, &npart, &tpwgts[0], &ubvec[0], &options, &edgecut, &part[0], &scomm), "Error in ParMETIS" ); MPI_Comm_free(&scomm); } return graph_perm_index(comm, npart, part, perm); }
/************************************************************************* * Let the game begin **************************************************************************/ int main(int argc, char *argv[]) { idx_t i, j, npes, mype, optype, nparts, adptf, options[10]; idx_t *part=NULL, *sizes=NULL; graph_t graph; real_t ipc2redist, *xyz=NULL, *tpwgts=NULL, ubvec[MAXNCON]; MPI_Comm comm; idx_t numflag=0, wgtflag=0, ndims, edgecut; char xyzfilename[8192]; MPI_Init(&argc, &argv); MPI_Comm_dup(MPI_COMM_WORLD, &comm); gkMPI_Comm_size(comm, &npes); gkMPI_Comm_rank(comm, &mype); if (argc != 8) { if (mype == 0) printf("Usage: %s <graph-file> <op-type> <nparts> <adapth-factor> <ipc2redist> <dbglvl> <seed>\n", argv[0]); MPI_Finalize(); exit(0); } optype = atoi(argv[2]); nparts = atoi(argv[3]); adptf = atoi(argv[4]); ipc2redist = atof(argv[5]); options[0] = 1; options[PMV3_OPTION_DBGLVL] = atoi(argv[6]); options[PMV3_OPTION_SEED] = atoi(argv[7]); if (mype == 0) printf("reading file: %s\n", argv[1]); ParallelReadGraph(&graph, argv[1], comm); /* Remove the edges for testing */ /*iset(graph.vtxdist[mype+1]-graph.vtxdist[mype]+1, 0, graph.xadj); */ rset(graph.ncon, 1.05, ubvec); tpwgts = rmalloc(nparts*graph.ncon, "tpwgts"); rset(nparts*graph.ncon, 1.0/(real_t)nparts, tpwgts); /* ChangeToFortranNumbering(graph.vtxdist, graph.xadj, graph.adjncy, mype, npes); numflag = 1; nvtxs = graph.vtxdist[mype+1]-graph.vtxdist[mype]; nedges = graph.xadj[nvtxs]; printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1)); */ if (optype >= 20) { sprintf(xyzfilename, "%s.xyz", argv[1]); xyz = ReadTestCoordinates(&graph, xyzfilename, &ndims, comm); } if (mype == 0) printf("finished reading file: %s\n", argv[1]); part = ismalloc(graph.nvtxs, mype%nparts, "main: part"); sizes = imalloc(2*npes, "main: sizes"); switch (optype) { case 1: wgtflag = 3; ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD); break; case 2: wgtflag = 3; options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED; ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD); break; case 3: options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED; graph.vwgt = ismalloc(graph.nvtxs, 1, "main: vwgt"); if (npes > 1) { AdaptGraph(&graph, adptf, comm); } else { wgtflag = 3; ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); printf("Initial partitioning with edgecut of %"PRIDX"\n", edgecut); for (i=0; i<graph.ncon; i++) { for (j=0; j<graph.nvtxs; j++) { if (part[j] == i) graph.vwgt[j*graph.ncon+i] = adptf; else graph.vwgt[j*graph.ncon+i] = 1; } } } wgtflag = 3; ParMETIS_V3_AdaptiveRepart(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec, &ipc2redist, options, &edgecut, part, &comm); break; case 4: ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options, part, sizes, &comm); /* WriteOVector(argv[1], graph.vtxdist, part, comm); */ break; case 5: ParMETIS_SerialNodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options, part, sizes, &comm); /* WriteOVector(argv[1], graph.vtxdist, part, comm); */ printf("%"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX"\n", sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], sizes[5], sizes[6]); break; case 11: /* TestAdaptiveMETIS(graph.vtxdist, graph.xadj, graph.adjncy, part, options, adptf, comm); */ break; case 20: wgtflag = 3; ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &ndims, xyz, &graph.ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); break; case 21: ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm); break; } /* printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1)); */ gk_free((void **)&part, &sizes, &tpwgts, &graph.vtxdist, &graph.xadj, &graph.adjncy, &graph.vwgt, &graph.adjwgt, &xyz, LTERM); MPI_Comm_free(&comm); MPI_Finalize(); return 0; }
void AlgParMETIS<Adapter>::partition( const RCP<PartitioningSolution<Adapter> > &solution ) { HELLO; size_t numGlobalParts = solution->getTargetGlobalNumberOfParts(); int np = problemComm->getSize(); // Get vertex info ArrayView<const gno_t> vtxgnos; ArrayView<StridedData<lno_t, scalar_t> > vwgts; int nVwgt = model->getNumWeightsPerVertex(); size_t nVtx = model->getVertexList(vtxgnos, vwgts); pm_idx_t pm_nVtx; TPL_Traits<pm_idx_t,size_t>::ASSIGN_TPL_T(pm_nVtx, nVtx); pm_idx_t *pm_vwgts = NULL; if (nVwgt) { pm_vwgts = new pm_idx_t[nVtx*nVwgt]; scale_weights(nVtx, vwgts, pm_vwgts); } // Get edge info ArrayView<const gno_t> adjgnos; ArrayView<const lno_t> offsets; ArrayView<StridedData<lno_t, scalar_t> > ewgts; int nEwgt = model->getNumWeightsPerEdge(); size_t nEdge = model->getEdgeList(adjgnos, offsets, ewgts); pm_idx_t *pm_ewgts = NULL; if (nEwgt) { pm_ewgts = new pm_idx_t[nEdge*nEwgt]; scale_weights(nEdge, ewgts, pm_ewgts); } // Convert index types for edges, if needed pm_idx_t *pm_offsets; TPL_Traits<pm_idx_t,const lno_t>::ASSIGN_TPL_T_ARRAY(&pm_offsets, offsets); pm_idx_t *pm_adjs; pm_idx_t pm_dummy_adj; if (nEdge) TPL_Traits<pm_idx_t,const gno_t>::ASSIGN_TPL_T_ARRAY(&pm_adjs, adjgnos); else pm_adjs = &pm_dummy_adj; // ParMETIS does not like NULL pm_adjs; // Build vtxdist pm_idx_t *pm_vtxdist; ArrayView<size_t> vtxdist; model->getVertexDist(vtxdist); TPL_Traits<pm_idx_t,size_t>::ASSIGN_TPL_T_ARRAY(&pm_vtxdist, vtxdist); // ParMETIS does not like processors having no vertices. // Inspect vtxdist and remove from communicator procs that have no vertices RCP<Comm<int> > subcomm; MPI_Comm mpicomm; // Note: mpicomm is valid only while subcomm is in scope if (np > 1) { int nKeep = 0; Array<int> keepRanks(np); for (int i = 0; i < np; i++) { if ((pm_vtxdist[i+1] - pm_vtxdist[i]) > 0) { keepRanks[nKeep] = i; pm_vtxdist[nKeep] = pm_vtxdist[i]; nKeep++; } } pm_vtxdist[nKeep] = pm_vtxdist[np]; if (nKeep < np) { subcomm = problemComm->createSubcommunicator(keepRanks.view(0,nKeep)); if (subcomm != Teuchos::null) mpicomm = Teuchos::getRawMpiComm(*subcomm); else mpicomm = MPI_COMM_NULL; } else { mpicomm = Teuchos::getRawMpiComm(*problemComm); } } else { mpicomm = Teuchos::getRawMpiComm(*problemComm); } // Create array for ParMETIS to return results in. pm_idx_t *pm_partList = NULL; if (nVtx) pm_partList = new pm_idx_t[nVtx]; if (mpicomm != MPI_COMM_NULL) { // If in ParMETIS' communicator (i.e., have vertices), call ParMETIS // Get target part sizes pm_idx_t pm_nCon = (nVwgt == 0 ? 1 : pm_idx_t(nVwgt)); pm_real_t *pm_partsizes = new pm_real_t[numGlobalParts*pm_nCon]; for (pm_idx_t dim = 0; dim < pm_nCon; dim++) { if (!solution->criteriaHasUniformPartSizes(dim)) for (size_t i=0; i<numGlobalParts; i++) pm_partsizes[i*pm_nCon+dim] = pm_real_t(solution->getCriteriaPartSize(dim,i)); else for (size_t i=0; i<numGlobalParts; i++) pm_partsizes[i*pm_nCon+dim] = pm_real_t(1.)/pm_real_t(numGlobalParts); } // Get imbalance tolerances double tolerance = 1.1; const Teuchos::ParameterList &pl = env->getParameters(); const Teuchos::ParameterEntry *pe = pl.getEntryPtr("imbalance_tolerance"); if (pe) tolerance = pe->getValue<double>(&tolerance); pm_real_t *pm_imbTols = new pm_real_t[pm_nCon]; for (pm_idx_t dim = 0; dim < pm_nCon; dim++) pm_imbTols[dim] = pm_real_t(tolerance); std::string parmetis_method("PARTKWAY"); pe = pl.getEntryPtr("partitioning_approach"); if (pe){ std::string approach; approach = pe->getValue<std::string>(&approach); if ((approach == "repartition") || (approach == "maximize_overlap")) parmetis_method = "REFINE_KWAY"; // TODO: AdaptiveRepart } // Other ParMETIS parameters? pm_idx_t pm_wgtflag = 2*(nVwgt > 0) + (nEwgt > 0); pm_idx_t pm_numflag = 0; pm_idx_t pm_edgecut = -1; pm_idx_t pm_options[METIS_NOPTIONS]; pm_options[0] = 1; // Use non-default options for some ParMETIS options for (int i = 0; i < METIS_NOPTIONS; i++) pm_options[i] = 0; // Default options pm_options[2] = 15; // Matches default value used in Zoltan pm_idx_t pm_nPart; TPL_Traits<pm_idx_t,size_t>::ASSIGN_TPL_T(pm_nPart, numGlobalParts); if (parmetis_method == "PARTKWAY") { ParMETIS_V3_PartKway(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts, pm_ewgts, &pm_wgtflag, &pm_numflag, &pm_nCon, &pm_nPart, pm_partsizes, pm_imbTols, pm_options, &pm_edgecut, pm_partList, &mpicomm); } else if (parmetis_method == "ADAPTIVE_REPART") { // Get object sizes: pm_vsize std::cout << "NOT READY FOR ADAPTIVE_REPART YET; NEED VSIZE" << std::endl; exit(-1); //pm_real_t itr = 100.; // Same default as in Zoltan //ParMETIS_V3_AdaptiveRepart(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts, // pm_vsize, pm_ewgts, &pm_wgtflag, // &pm_numflag, &pm_nCon, &pm_nPart, // pm_partsizes, pm_imbTols, // &itr, pm_options, // &pm_edgecut, pm_partList, &mpicomm); } else if (parmetis_method == "REFINE_KWAY") { ParMETIS_V3_RefineKway(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts, pm_ewgts, &pm_wgtflag, &pm_numflag, &pm_nCon, &pm_nPart, pm_partsizes, pm_imbTols, pm_options, &pm_edgecut, pm_partList, &mpicomm); } // Clean up delete [] pm_partsizes; delete [] pm_imbTols; } // Load answer into the solution. ArrayRCP<part_t> partList; if (nVtx) { if (TPL_Traits<pm_idx_t, part_t>::OK_TO_CAST_TPL_T()) { partList = ArrayRCP<part_t>((part_t *)pm_partList, 0, nVtx, true); } else { // TODO Probably should have a TPL_Traits function to do the following partList = ArrayRCP<part_t>(new part_t[nVtx], 0, nVtx, true); for (size_t i = 0; i < nVtx; i++) { partList[i] = part_t(pm_partList[i]); } delete [] pm_partList; } } solution->setParts(partList); env->memory("Zoltan2-ParMETIS: After creating solution"); // Clean up copies made due to differing data sizes. TPL_Traits<pm_idx_t,size_t>::DELETE_TPL_T_ARRAY(&pm_vtxdist); TPL_Traits<pm_idx_t,const lno_t>::DELETE_TPL_T_ARRAY(&pm_offsets); if (nEdge) TPL_Traits<pm_idx_t,const gno_t>::DELETE_TPL_T_ARRAY(&pm_adjs); if (nVwgt) delete [] pm_vwgts; if (nEwgt) delete [] pm_ewgts; }
int main(int argc, char *argv[]) { int mpi_rank, mpi_size; MPI_Init(&argc,&argv); MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); idx_t vtxdist[4] = {0,4,8,12}; idx_t *xadj; idx_t eptr0[5] = {0,1,3,5,7}; idx_t eptr1[5] = {0,3,6,9,12}; idx_t eptr2[5] = {0,2,4,6,7}; if (mpi_rank==0) xadj = eptr0; if (mpi_rank==1) xadj = eptr1; if (mpi_rank==2) xadj = eptr2; idx_t *adjncy; idx_t eind0[7] = {6, 6,7, 7,8, 6,9}; idx_t eind1[12] = {7,9,10, 8,10,11, 0,1,3, 1,2,4}; idx_t eind2[7] = {2,5, 3,4, 4,5, 5}; if (mpi_rank==0) adjncy = eind0; if (mpi_rank==1) adjncy = eind1; if (mpi_rank==2) adjncy = eind2; idx_t *vwgt = NULL; idx_t *adjwgt = NULL; idx_t wgtflag = 0; idx_t numflag = 0; idx_t ncon = 1; idx_t ncommonnodes = 2; idx_t nparts = 3; real_t tpwgts[3] = {1./3., 1./3., 1./3.}; real_t ubvec[1] = {1.05}; idx_t options[1] = {0}; idx_t edgecut; idx_t *part; part = malloc(sizeof(real_t)*4); part = malloc(sizeof(real_t)*4); part = malloc(sizeof(real_t)*4); MPI_Comm comm = MPI_COMM_WORLD; int err = ParMETIS_V3_PartKway(vtxdist, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); if (err==METIS_OK) { printf("ok\n"); } else if (err==METIS_ERROR) { printf("error\n"); } int iind; printf("rank: %d\n", mpi_rank); printf("part: "); for (iind = 0 ; iind < 4 ; iind++) { printf("%d ",part[iind]); } printf("\n"); printf("edgecut: %d\n", edgecut); printf("\n"); MPI_Finalize(); return 0; }
void TestParMetis_GPart(char *filename, char *xyzfile, MPI_Comm comm) { idx_t ncon, nparts, npes, mype, opt2, realcut; graph_t graph, mgraph; idx_t *part, *mpart, *savepart, *order, *sizes; idx_t numflag=0, wgtflag=0, options[10], edgecut, ndims; real_t ipc2redist, *xyz=NULL, *tpwgts = NULL, ubvec[MAXNCON]; gkMPI_Comm_size(comm, &npes); gkMPI_Comm_rank(comm, &mype); ParallelReadGraph(&graph, filename, comm); if (xyzfile) xyz = ReadTestCoordinates(&graph, xyzfile, &ndims, comm); gkMPI_Barrier(comm); part = imalloc(graph.nvtxs, "TestParMetis_V3: part"); tpwgts = rmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts"); rset(MAXNCON, 1.05, ubvec); graph.vwgt = ismalloc(graph.nvtxs*5, 1, "TestParMetis_GPart: vwgt"); /*====================================================================== / ParMETIS_V3_PartKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; wgtflag = 2; numflag = 0; edgecut = 0; for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) { for (ncon=1; ncon<=NCON; ncon++) { if (ncon > 1 && nparts > 1) Mc_AdaptGraph(&graph, part, ncon, nparts, comm); else iset(graph.nvtxs, 1, graph.vwgt); if (mype == 0) printf("\nTesting ParMETIS_V3_PartKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts); rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) { printf("ParMETIS_V3_PartKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } if (mype == 0) printf("\nTesting ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts); options[3] = PARMETIS_PSR_UNCOUPLED; ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) { printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } } } /*====================================================================== / ParMETIS_V3_PartGeomKway /=======================================================================*/ if (xyzfile != NULL) { options[0] = 1; options[1] = 3; options[2] = 1; wgtflag = 2; numflag = 0; for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) { for (ncon=1; ncon<=NCON; ncon++) { if (ncon > 1) Mc_AdaptGraph(&graph, part, ncon, nparts, comm); else iset(graph.nvtxs, 1, graph.vwgt); if (mype == 0) printf("\nTesting ParMETIS_V3_PartGeomKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts); rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) printf("ParMETIS_V3_PartGeomKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } } } /*====================================================================== / ParMETIS_V3_PartGeom /=======================================================================*/ if (xyz != NULL) { wgtflag = 0; numflag = 0; if (mype == 0) printf("\nTesting ParMETIS_V3_PartGeom\n"); ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm); realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) printf("ParMETIS_V3_PartGeom reported a cut of %"PRIDX"\n", realcut); } /*====================================================================== / Coupled ParMETIS_V3_RefineKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; options[3] = PARMETIS_PSR_COUPLED; nparts = npes; wgtflag = 0; numflag = 0; ncon = 1; rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); if (mype == 0) printf("\nTesting coupled ParMETIS_V3_RefineKway with default options (before move)\n"); ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); /* Compute a good partition and move the graph. Do so quietly! */ options[0] = 0; nparts = npes; wgtflag = 0; numflag = 0; ncon = 1; rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut, part, &comm); TestMoveGraph(&graph, &mgraph, part, comm); gk_free((void **)&(graph.vwgt), LTERM); mpart = ismalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart"); savepart = imalloc(mgraph.nvtxs, "TestParMetis_V3: savepart"); /*====================================================================== / Coupled ParMETIS_V3_RefineKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; options[3] = PARMETIS_PSR_COUPLED; nparts = npes; wgtflag = 0; numflag = 0; for (ncon=1; ncon<=NCON; ncon++) { if (mype == 0) printf("\nTesting coupled ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts); rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, mpart, &comm); realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart, filename, comm); if (mype == 0) printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } /*ADAPTIVE:*/ /*====================================================================== / ParMETIS_V3_AdaptiveRepart /=======================================================================*/ mgraph.vwgt = ismalloc(mgraph.nvtxs*NCON, 1, "TestParMetis_V3: mgraph.vwgt"); mgraph.vsize = ismalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize"); AdaptGraph(&mgraph, 4, comm); options[0] = 1; options[1] = 7; options[2] = 1; options[3] = PARMETIS_PSR_COUPLED; wgtflag = 2; numflag = 0; for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) { options[0] = 0; ncon = 1; wgtflag = 0; rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, savepart, &comm); options[0] = 1; wgtflag = 2; for (ncon=1; ncon<=NCON; ncon++) { rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); if (ncon > 1) Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm); else AdaptGraph(&mgraph, 4, comm); for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) { icopy(mgraph.nvtxs, savepart, mpart); if (mype == 0) printf("\nTesting ParMETIS_V3_AdaptiveRepart with ipc2redist: %.3"PRREAL", ncon: %"PRIDX", nparts: %"PRIDX"\n", ipc2redist, ncon, nparts); ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, mgraph.vwgt, mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, &ipc2redist, options, &edgecut, mpart, &comm); realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart, filename, comm); if (mype == 0) printf("ParMETIS_V3_AdaptiveRepart reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } } } gk_free((void **)&tpwgts, &part, &mpart, &savepart, &xyz, &mgraph.vwgt, &mgraph.vsize, LTERM); }
int Zoltan_ParMetis( ZZ *zz, /* Zoltan structure */ float *part_sizes, /* Input: Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ int *num_imp, /* number of objects to be imported */ ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */ ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */ int **imp_procs, /* list of processors to import from */ int **imp_to_part, /* list of partitions to which imported objects are assigned. */ int *num_exp, /* number of objects to be exported */ ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */ ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */ int **exp_procs, /* list of processors to export to */ int **exp_to_part /* list of partitions to which exported objects are assigned. */ ) { char *yo = "Zoltan_ParMetis"; int ierr; ZOLTAN_Third_Graph gr; ZOLTAN_Third_Geom *geo = NULL; ZOLTAN_Third_Vsize vsp; ZOLTAN_Third_Part prt; ZOLTAN_Output_Part part; ZOLTAN_ID_PTR global_ids = NULL; ZOLTAN_ID_PTR local_ids = NULL; int use_timers = 0; int timer_p = -1; int get_times = 0; double times[5]; double pmv3_itr = 0.0; realtype itr = 0.0; indextype options[MAX_PARMETIS_OPTIONS]; char alg[MAX_PARAM_STRING_LEN]; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */ /* change our zz struct. */ #endif indextype i; realtype *imb_tols; indextype ncon; indextype edgecut; indextype wgtflag; indextype numflag = 0; indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */ ZOLTAN_TRACE_ENTER(zz, yo); Zoltan_Third_Init(&gr, &prt, &vsp, &part, imp_gids, imp_lids, imp_procs, imp_to_part, exp_gids, exp_lids, exp_procs, exp_to_part); if (sizeof(realtype) != sizeof(float)) { int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1); prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype)); for (i = 0; i < tmp; i++) prt.input_part_sizes[i] = (realtype) part_sizes[i]; /* KDD 2/2014: removed re-scaling part sizes so they sum to one. * part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes. * plus, the code here was wrong for multiple object weights. * similar scaling code did not exist in the Scotch interface. */ prt.part_sizes = prt.input_part_sizes; } else prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes; ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } gr.graph_type = 0; #ifdef ZOLTAN_PARMETIS SET_GLOBAL_GRAPH(&gr.graph_type); /* Select type of graph, negative because we impose them */ /* TODO: add a parameter to select the type, shared with Scotch */ /* if (strcmp (graph_type, "GLOBAL") != 0) { */ /* gr.graph_type = - LOCAL_GRAPH; */ /* if (zz->Num_Proc > 1) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */ /* gr.graph_type = - GLOBAL_GRAPH; */ /* retval = ZOLTAN_WARN; */ /* } */ /* } */ #else /* graph is local */ SET_LOCAL_GRAPH(&gr.graph_type); #endif /* ZOLTAN_PARMETIS */ /* Some algorithms use geometry data */ if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */ geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom)); memset (geo, 0, sizeof(ZOLTAN_Third_Geom)); /* ParMETIS will crash if geometric method and some procs have no nodes. */ /* Avoid fatal crash by setting scatter to level 2 or higher. */ gr.scatter_min = 2; if (geo == NULL) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory."); return (ZOLTAN_MEMERR); } if (strcmp(alg, "PARTGEOM") == 0) { gr.get_data = 0; } } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } vsp.vsize_malloc = 0; #ifdef PARMETIS31_ALWAYS_FREES_VSIZE if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) { /* ParMETIS will free this memory; use malloc to allocate so ZOLTAN_MALLOC counters don't show an error. */ vsp.vsize_malloc = 1 ; } #endif /* PARMETIS31_ALWAYS_FREES_VSIZE */ ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr, geo, &prt, &vsp); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } /* Get object sizes if requested */ if (options[PMV3_OPT_USE_OBJ_SIZE] && (zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) && (!strcmp(alg, "ADAPTIVEREPART") || gr.final_output)) gr.showMoveVol = 1; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); /* Get ready to call ParMETIS */ edgecut = -1; wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0); numflag = 0; ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1); if (!prt.part_sizes){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL."); } if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) { for (i=0; i<num_part; i++){ indextype j; printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i); for (j=0; j<ncon; j++) printf("%f ", prt.part_sizes[i*ncon+j]); printf("\n"); } } /* if (strcmp(alg, "ADAPTIVEREPART") == 0) */ for (i = 0; i < num_part*ncon; i++) if (prt.part_sizes[i] == 0) ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! " "ParMETIS 3.x will likely fail. Please use a " "different method, or remove the zero-sized " "parts from the problem."); /* Set Imbalance Tolerance for each weight component. */ imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype)); if (!imb_tols){ /* Not enough memory */ ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } for (i=0; i<ncon; i++) imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]); /* Now we can call ParMetis */ /* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */ #ifdef ZOLTAN_PARMETIS if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */ /* First check for ParMetis 3 routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartKway"); ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOMKWAY") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeomKway"); ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts, &wgtflag, &numflag, &ndims, geo->xyz, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOM") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeom"); ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "ADAPTIVEREPART") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_AdaptiveRepart"); ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, &itr, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "REFINEKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_RefineKway"); ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown ParMetis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_METIS /* TODO: I don't know how to set balance ! */ if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Check for Metis routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library "); /* Use default options for METIS */ #if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5 options[0] = 0; METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &num_part, prt.part_sizes, options, &edgecut, prt.part); #else METIS_SetDefaultOptions(options); METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown Metis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (gr.final_output) { /* Do final output now because after the data will not be coherent: unscatter only unscatter part data, not graph */ ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr); } /* Ignore the timings of Final Ouput */ if (get_times) times[3] = Zoltan_Time(zz->Timer); ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr, geo, &prt, &vsp, NULL, &part); Zoltan_Third_Export_User(&part, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part); /* Get a time here */ if (get_times) times[4] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers && timer_p >= 0) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL); if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols); if (geo != NULL) ZOLTAN_FREE(&geo); ZOLTAN_FREE(&global_ids); ZOLTAN_FREE(&local_ids); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
void ParMETISGraphPartitionerImpl::p_partition(void) { int me(this->processor_rank()); std::vector<idx_t> vtxdist; std::vector<idx_t> xadj; std::vector<idx_t> adjncy; ParMETISGraphWrapper wrap(p_adjacency_list); wrap.get_csr_local(vtxdist, xadj, adjncy); int nnodes(vtxdist[me+1] - vtxdist[me]); #if 0 for (int p = 0; p < this->processor_size(); ++p) { if (this->processor_rank() == p) { std::cout << "Processor " << p << ": nodes: "; for (Index n = 0; n < nnodes; ++n) { std::cout << p_adjacency_list.node_index(n) << ","; } std::cout << std::endl; std::cout << "Processor " << p << ": vtxdist: "; std::copy(vtxdist.begin(), vtxdist.end(), std::ostream_iterator<idx_t>(std::cout, ",")); std::cout << std::endl; std::cout << "Processor " << p << ": xadj: "; std::copy(xadj.begin(), xadj.end(), std::ostream_iterator<idx_t>(std::cout, ",")); std::cout << std::endl; std::cout << "Processor " << p << ": adjncy: "; std::copy(adjncy.begin(), adjncy.end(), std::ostream_iterator<idx_t>(std::cout, ",")); std::cout << std::endl; } this->communicator().barrier(); } #endif // Call the partitioner (try to use variable names that match the documentation) int status; idx_t ncon(1); idx_t wgtflag(3), numflag(0); idx_t nparts(this->processor_size()); std::vector<idx_t> vwgt(nnodes, 1); std::vector<idx_t> adjwgt(adjncy.size(), 2); std::vector<real_t> tpwgts(nparts*ncon, 1.0/static_cast<real_t>(nparts)); real_t ubvec(1.05); std::vector<idx_t> options(3); options[0] = 1; options[1] = 127; options[2] = 14; MPI_Comm comm(this->communicator()); idx_t edgecut; std::vector<idx_t> part(nnodes); status = ParMETIS_V3_PartKway(&vtxdist[0], &xadj[0], &adjncy[0], &vwgt[0], &adjwgt[0], &wgtflag, &numflag, &ncon, &nparts, &tpwgts[0], &ubvec, &options[0], &edgecut, &part[0], &comm); if (status != 0) { // FIXME: throw an exception } // "part" contains the destination processors; transfer this to the // local array wrap.set_partition(vtxdist, part); wrap.get_partition(p_node_destinations); }
void AlgParMETIS<Adapter>::partition( const RCP<PartitioningSolution<Adapter> > &solution ) { HELLO; size_t numGlobalParts = solution->getTargetGlobalNumberOfParts(); int me = problemComm->getRank(); int np = problemComm->getSize(); // Get vertex info ArrayView<const gno_t> vtxgnos; ArrayView<StridedData<lno_t, scalar_t> > vwgts; int nVwgt = model->getNumWeightsPerVertex(); size_t nVtx = model->getVertexList(vtxgnos, vwgts); pm_idx_t pm_nVtx; TPL_Traits<pm_idx_t,size_t>::ASSIGN(pm_nVtx, nVtx); pm_idx_t *pm_vwgts = NULL; if (nVwgt) { pm_vwgts = new pm_idx_t[nVtx*nVwgt]; scale_weights(nVtx, vwgts, pm_vwgts); } // Get edge info ArrayView<const gno_t> adjgnos; ArrayView<const lno_t> offsets; ArrayView<StridedData<lno_t, scalar_t> > ewgts; int nEwgt = model->getNumWeightsPerEdge(); size_t nEdge = model->getEdgeList(adjgnos, offsets, ewgts); pm_idx_t *pm_ewgts = NULL; if (nEwgt) { pm_ewgts = new pm_idx_t[nEdge*nEwgt]; scale_weights(nEdge, ewgts, pm_ewgts); } // Convert index types for edges, if needed pm_idx_t *pm_offsets; TPL_Traits<pm_idx_t,const lno_t>::ASSIGN_ARRAY(&pm_offsets, offsets); pm_idx_t *pm_adjs; pm_idx_t pm_dummy_adj; if (nEdge) TPL_Traits<pm_idx_t,const gno_t>::ASSIGN_ARRAY(&pm_adjs, adjgnos); else pm_adjs = &pm_dummy_adj; // ParMETIS does not like NULL pm_adjs; // Build vtxdist pm_idx_t *pm_vtxdist; ArrayView<size_t> vtxdist; model->getVertexDist(vtxdist); TPL_Traits<pm_idx_t,size_t>::ASSIGN_ARRAY(&pm_vtxdist, vtxdist); // ParMETIS does not like processors having no vertices. // Inspect vtxdist and remove from communicator procs that have no vertices RCP<Comm<int> > subcomm; MPI_Comm mpicomm; // Note: mpicomm is valid only while subcomm is in scope if (np > 1) { int nKeep = 0; Array<int> keepRanks(np); for (int i = 0; i < np; i++) { if ((pm_vtxdist[i+1] - pm_vtxdist[i]) > 0) { keepRanks[nKeep] = i; pm_vtxdist[nKeep] = pm_vtxdist[i]; nKeep++; } } pm_vtxdist[nKeep] = pm_vtxdist[np]; if (nKeep < np) { subcomm = problemComm->createSubcommunicator(keepRanks.view(0,nKeep)); if (subcomm != Teuchos::null) mpicomm = Teuchos::getRawMpiComm(*subcomm); else mpicomm = MPI_COMM_NULL; } else { mpicomm = Teuchos::getRawMpiComm(*problemComm); } } else { mpicomm = Teuchos::getRawMpiComm(*problemComm); } // Create array for ParMETIS to return results in. pm_idx_t *pm_partList = NULL; if (nVtx) pm_partList = new pm_idx_t[nVtx]; for (size_t i = 0; i < nVtx; i++) pm_partList[i] = 0; int pm_return = METIS_OK; if (mpicomm != MPI_COMM_NULL) { // If in ParMETIS' communicator (i.e., have vertices), call ParMETIS // Get target part sizes pm_idx_t pm_nCon = (nVwgt == 0 ? 1 : pm_idx_t(nVwgt)); pm_real_t *pm_partsizes = new pm_real_t[numGlobalParts*pm_nCon]; for (pm_idx_t dim = 0; dim < pm_nCon; dim++) { if (!solution->criteriaHasUniformPartSizes(dim)) for (size_t i=0; i<numGlobalParts; i++) pm_partsizes[i*pm_nCon+dim] = pm_real_t(solution->getCriteriaPartSize(dim,i)); else for (size_t i=0; i<numGlobalParts; i++) pm_partsizes[i*pm_nCon+dim] = pm_real_t(1.)/pm_real_t(numGlobalParts); } // Get imbalance tolerances double tolerance = 1.1; const Teuchos::ParameterList &pl = env->getParameters(); const Teuchos::ParameterEntry *pe = pl.getEntryPtr("imbalance_tolerance"); if (pe) tolerance = pe->getValue<double>(&tolerance); // ParMETIS requires tolerance to be greater than 1.0; // fudge it if condition is not met if (tolerance <= 1.0) { if (me == 0) std::cerr << "Warning: ParMETIS requires imbalance_tolerance > 1.0; " << "to comply, Zoltan2 reset imbalance_tolerance to 1.01." << std::endl; tolerance = 1.01; } pm_real_t *pm_imbTols = new pm_real_t[pm_nCon]; for (pm_idx_t dim = 0; dim < pm_nCon; dim++) pm_imbTols[dim] = pm_real_t(tolerance); std::string parmetis_method("PARTKWAY"); pe = pl.getEntryPtr("partitioning_approach"); if (pe) { std::string approach; approach = pe->getValue<std::string>(&approach); if ((approach == "repartition") || (approach == "maximize_overlap")) { if (np > 1) // ParMETIS_V3_AdaptiveRepart requires two or more processors parmetis_method = "ADAPTIVE_REPART"; else parmetis_method = "REFINE_KWAY"; } } // Other ParMETIS parameters? pm_idx_t pm_wgtflag = 2*(nVwgt > 0) + (nEwgt > 0); pm_idx_t pm_numflag = 0; pm_idx_t pm_edgecut = -1; pm_idx_t pm_options[METIS_NOPTIONS]; pm_options[0] = 1; // Use non-default options for some ParMETIS options for (int i = 0; i < METIS_NOPTIONS; i++) pm_options[i] = 0; // Default options pm_options[2] = 15; // Matches default value used in Zoltan pm_idx_t pm_nPart; TPL_Traits<pm_idx_t,size_t>::ASSIGN(pm_nPart, numGlobalParts); if (parmetis_method == "PARTKWAY") { pm_return = ParMETIS_V3_PartKway(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts, pm_ewgts, &pm_wgtflag, &pm_numflag, &pm_nCon, &pm_nPart, pm_partsizes, pm_imbTols, pm_options, &pm_edgecut, pm_partList, &mpicomm); } else if (parmetis_method == "ADAPTIVE_REPART") { // Get object sizes: pm_vsize // TODO: get pm_vsize info from input adapter or graph model // TODO: This is just a placeholder pm_idx_t *pm_vsize = new pm_idx_t[nVtx]; for (size_t i = 0; i < nVtx; i++) pm_vsize[i] = 1; pm_real_t itr = 100.; // Same default as in Zoltan pm_return = ParMETIS_V3_AdaptiveRepart(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts, pm_vsize, pm_ewgts, &pm_wgtflag, &pm_numflag, &pm_nCon, &pm_nPart, pm_partsizes, pm_imbTols, &itr, pm_options, &pm_edgecut, pm_partList, &mpicomm); delete [] pm_vsize; } else if (parmetis_method == "REFINE_KWAY") { pm_return = ParMETIS_V3_RefineKway(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts, pm_ewgts, &pm_wgtflag, &pm_numflag, &pm_nCon, &pm_nPart, pm_partsizes, pm_imbTols, pm_options, &pm_edgecut, pm_partList, &mpicomm); } // Clean up delete [] pm_partsizes; delete [] pm_imbTols; } // Load answer into the solution. ArrayRCP<part_t> partList; if (nVtx) TPL_Traits<part_t, pm_idx_t>::SAVE_ARRAYRCP(&partList, pm_partList, nVtx); TPL_Traits<pm_idx_t, part_t>::DELETE_ARRAY(&pm_partList); solution->setParts(partList); env->memory("Zoltan2-ParMETIS: After creating solution"); // Clean up copies made due to differing data sizes. TPL_Traits<pm_idx_t,size_t>::DELETE_ARRAY(&pm_vtxdist); TPL_Traits<pm_idx_t,const lno_t>::DELETE_ARRAY(&pm_offsets); if (nEdge) TPL_Traits<pm_idx_t,const gno_t>::DELETE_ARRAY(&pm_adjs); if (nVwgt) delete [] pm_vwgts; if (nEwgt) delete [] pm_ewgts; if (pm_return != METIS_OK) { throw std::runtime_error( "\nParMETIS returned an error; no valid partition generated.\n" "Look for 'PARMETIS ERROR' in your output for more details.\n"); } }
static PetscErrorCode MatPartitioningApply_Parmetis(MatPartitioning part,IS *partitioning) { MatPartitioning_Parmetis *parmetis = (MatPartitioning_Parmetis*)part->data; PetscErrorCode ierr; PetscInt *locals = PETSC_NULL; Mat mat = part->adj,amat,pmat; PetscBool flg; PetscInt bs = 1; PetscFunctionBegin; ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr); if (flg) { amat = mat; PetscObjectReference((PetscObject)amat);CHKERRQ(ierr); } else { /* bs indicates if the converted matrix is "reduced" from the original and hence the resulting partition results need to be stretched to match the original matrix */ ierr = MatConvert(mat,MATMPIADJ,MAT_INITIAL_MATRIX,&amat);CHKERRQ(ierr); if (amat->rmap->n > 0) bs = mat->rmap->n/amat->rmap->n; } ierr = MatMPIAdjCreateNonemptySubcommMat(amat,&pmat);CHKERRQ(ierr); ierr = MPI_Barrier(((PetscObject)part)->comm);CHKERRQ(ierr); if (pmat) { MPI_Comm pcomm = ((PetscObject)pmat)->comm,comm_pmetis; Mat_MPIAdj *adj = (Mat_MPIAdj*)pmat->data; PetscInt *vtxdist = pmat->rmap->range; PetscInt *xadj = adj->i; PetscInt *adjncy = adj->j; PetscInt itmp = 0,wgtflag=0, numflag=0, ncon=1, nparts=part->n, options[24], i, j; real_t *tpwgts,*ubvec; int status; #if defined(PETSC_USE_DEBUG) /* check that matrix has no diagonal entries */ { PetscInt rstart; ierr = MatGetOwnershipRange(pmat,&rstart,PETSC_NULL);CHKERRQ(ierr); for (i=0; i<pmat->rmap->n; i++) { for (j=xadj[i]; j<xadj[i+1]; j++) { if (adjncy[j] == i+rstart) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Row %d has diagonal entry; Parmetis forbids diagonal entry",i+rstart); } } } #endif ierr = PetscMalloc(amat->rmap->n*sizeof(PetscInt),&locals);CHKERRQ(ierr); if (PetscLogPrintInfo) {itmp = parmetis->printout; parmetis->printout = 127;} ierr = PetscMalloc(ncon*nparts*sizeof(real_t),&tpwgts);CHKERRQ(ierr); for (i=0; i<ncon; i++) { for (j=0; j<nparts; j++) { if (part->part_weights) { tpwgts[i*nparts+j] = part->part_weights[i*nparts+j]; } else { tpwgts[i*nparts+j] = 1./nparts; } } } ierr = PetscMalloc(ncon*sizeof(real_t),&ubvec);CHKERRQ(ierr); for (i=0; i<ncon; i++) { ubvec[i] = 1.05; } /* This sets the defaults */ options[0] = 0; for (i=1; i<24; i++) { options[i] = -1; } /* Duplicate the communicator to be sure that ParMETIS attribute caching does not interfere with PETSc. */ ierr = MPI_Comm_dup(pcomm,&comm_pmetis);CHKERRQ(ierr); status = ParMETIS_V3_PartKway(vtxdist,xadj,adjncy,part->vertex_weights,adj->values,&wgtflag,&numflag,&ncon,&nparts,tpwgts,ubvec,options,&parmetis->cuts,locals,&comm_pmetis);CHKERRQPARMETIS(status); ierr = MPI_Comm_free(&comm_pmetis);CHKERRQ(ierr); ierr = PetscFree(tpwgts);CHKERRQ(ierr); ierr = PetscFree(ubvec);CHKERRQ(ierr); if (PetscLogPrintInfo) {parmetis->printout = itmp;} } if (bs > 1) { PetscInt i,j,*newlocals; ierr = PetscMalloc(bs*amat->rmap->n*sizeof(PetscInt),&newlocals);CHKERRQ(ierr); for (i=0; i<amat->rmap->n; i++) { for (j=0; j<bs; j++) { newlocals[bs*i + j] = locals[i]; } } ierr = PetscFree(locals);CHKERRQ(ierr); ierr = ISCreateGeneral(((PetscObject)part)->comm,bs*amat->rmap->n,newlocals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr); } else { ierr = ISCreateGeneral(((PetscObject)part)->comm,amat->rmap->n,locals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr); } ierr = MatDestroy(&pmat);CHKERRQ(ierr); ierr = MatDestroy(&amat);CHKERRQ(ierr); PetscFunctionReturn(0); }
void AlgParMETIS<Adapter>::partition( const RCP<PartitioningSolution<Adapter> > &solution ) { HELLO; size_t numGlobalParts = solution->getTargetGlobalNumberOfParts(); int np = problemComm->getSize(); // Get vertex info ArrayView<const gno_t> vtxgnos; ArrayView<StridedData<lno_t, scalar_t> > xyz; ArrayView<StridedData<lno_t, scalar_t> > vwgts; int nVwgt = model->getNumWeightsPerVertex(); size_t nVtx = model->getVertexList(vtxgnos, xyz, vwgts); pm_idx_t pm_nVtx; TPL_Traits<pm_idx_t,size_t>::ASSIGN_TPL_T(pm_nVtx, nVtx, env); pm_idx_t *pm_vwgts = NULL; if (nVwgt) { pm_vwgts = new pm_idx_t[nVtx*nVwgt]; scale_weights(nVtx, vwgts, pm_vwgts); } // Get edge info ArrayView<const gno_t> adjgnos; ArrayView<const int> procs; ArrayView<const lno_t> offsets; ArrayView<StridedData<lno_t, scalar_t> > ewgts; int nEwgt = model->getNumWeightsPerEdge(); size_t nEdge = model->getEdgeList(adjgnos, procs, offsets, ewgts); pm_idx_t *pm_ewgts = NULL; if (nEwgt) { pm_ewgts = new pm_idx_t[nEdge*nEwgt]; scale_weights(nEdge, ewgts, pm_ewgts); } // Convert index types for edges, if needed pm_idx_t *pm_offsets; TPL_Traits<pm_idx_t,lno_t>::ASSIGN_TPL_T_ARRAY(&pm_offsets, offsets, env); pm_idx_t *pm_adjs; TPL_Traits<pm_idx_t,gno_t>::ASSIGN_TPL_T_ARRAY(&pm_adjs, adjgnos, env); // Build vtxdist pm_idx_t *pm_vtxdist = new pm_idx_t[np+1]; pm_vtxdist[0] = 0; Teuchos::gatherAll(*problemComm, 1, &pm_nVtx, np, &(pm_vtxdist[1])); for (int i = 2; i <= np; i++) pm_vtxdist[i] += pm_vtxdist[i-1]; // Create array for ParMETIS to return results in. // Note: ParMETIS does not like NULL arrays, // so add 1 to always have non-null. // See Zoltan bug 4299. pm_idx_t *pm_partList = new pm_idx_t[nVtx+1]; // Get target part sizes and imbalance tolerances pm_idx_t pm_nCon = (nVwgt == 0 ? 1 : pm_idx_t(nVwgt)); pm_real_t *pm_partsizes = new pm_real_t[numGlobalParts*pm_nCon]; for (pm_idx_t dim = 0; dim < pm_nCon; dim++) { if (!solution->criteriaHasUniformPartSizes(dim)) for (size_t i=0; i<numGlobalParts; i++) pm_partsizes[i*pm_nCon+dim] = pm_real_t(solution->getCriteriaPartSize(dim,i)); else for (size_t i=0; i<numGlobalParts; i++) pm_partsizes[i*pm_nCon+dim] = pm_real_t(1.) / pm_real_t(numGlobalParts); } pm_real_t *pm_imbTols = new pm_real_t[pm_nCon]; for (pm_idx_t dim = 0; dim < pm_nCon; dim++) pm_imbTols[dim] = 1.05; // TODO: GET THE PARAMETER std::string parmetis_method("PARTKWAY"); pm_idx_t pm_wgtflag = 2*(nVwgt > 0) + (nEwgt > 0); pm_idx_t pm_numflag = 0; pm_idx_t pm_nPart; TPL_Traits<pm_idx_t,size_t>::ASSIGN_TPL_T(pm_nPart, numGlobalParts, env); if (parmetis_method == "PARTKWAY") { pm_idx_t pm_edgecut = -1; pm_idx_t pm_options[3]; pm_options[0] = 0; // Use default options pm_options[1] = 0; // Debug level (ignored if pm_options[0] == 0) pm_options[2] = 0; // Seed (ignored if pm_options[0] == 0) ParMETIS_V3_PartKway(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts, pm_ewgts, &pm_wgtflag, &pm_numflag, &pm_nCon, &pm_nPart, pm_partsizes, pm_imbTols, pm_options, &pm_edgecut, pm_partList, &mpicomm); } else if (parmetis_method == "ADAPTIVE_REPART") { // Get object sizes std::cout << "NOT READY FOR ADAPTIVE_REPART YET" << std::endl; exit(-1); } else if (parmetis_method == "PART_GEOM") { // Get coordinate info, too. std::cout << "NOT READY FOR PART_GEOM YET" << std::endl; exit(-1); } // Clean up delete [] pm_vtxdist; delete [] pm_partsizes; delete [] pm_imbTols; // Load answer into the solution. ArrayRCP<part_t> partList; if (TPL_Traits<pm_idx_t, part_t>::OK_TO_CAST_TPL_T()) { partList = ArrayRCP<part_t>((part_t *)pm_partList, 0, nVtx, true); } else { // TODO Probably should have a TPL_Traits function to do the following partList = ArrayRCP<part_t>(new part_t[nVtx], 0, nVtx, true); for (size_t i = 0; i < nVtx; i++) { partList[i] = part_t(pm_partList[i]); } delete [] pm_partList; } solution->setParts(partList); env->memory("Zoltan2-ParMETIS: After creating solution"); // Clean up copies made due to differing data sizes. TPL_Traits<pm_idx_t,lno_t>::DELETE_TPL_T_ARRAY(&pm_offsets); TPL_Traits<pm_idx_t,gno_t>::DELETE_TPL_T_ARRAY(&pm_adjs); if (nVwgt) delete [] pm_vwgts; if (nEwgt) delete [] pm_ewgts; }
AnyType ParMETIS_Op<Type, Mesh>::operator()(Stack stack) const { KN<Type>* ptKN = GetAny<KN<Type>*>((*part)(stack)); idx_t nparts = GetAny<long>((*lparts)(stack)); Type* pt = *ptKN; long n = ptKN->n; idx_t* ptInt = reinterpret_cast<idx_t*>(pt); std::fill_n(ptInt, n, 0); MPI_Comm comm = nargs[0] ? *((MPI_Comm*)GetAny<pcommworld>((*nargs[0])(stack))) : MPI_COMM_WORLD; int worker = nargs[1] ? GetAny<long>((*nargs[1])(stack)) : 0; MPI_Comm workComm = comm; if(worker == 0) MPI_Comm_size(comm, &worker); else { int size; MPI_Comm_size(comm, &size); worker = std::min(size, worker); MPI_Group worldGroup, workGroup; MPI_Comm_group(workComm, &worldGroup); int ranges[1][3]; ranges[0][0] = 0; ranges[0][1] = worker - 1; ranges[0][2] = 1; MPI_Group_range_incl(worldGroup, 1, ranges, &workGroup); MPI_Comm_create(comm, workGroup, &workComm); MPI_Group_free(&worldGroup); } int rank; MPI_Comm_rank(comm, &rank); if(rank < worker) { idx_t* vtxdist = new idx_t[worker + 1]; vtxdist[0] = 0; for(int i = 1; i < worker; ++i) vtxdist[i] = vtxdist[i - 1] + n / worker; vtxdist[worker] = n; int loc = vtxdist[rank + 1] - vtxdist[rank]; idx_t* xadg = new idx_t[loc + 1]; const Mesh& Th(*GetAny<const Mesh*>((*pTh)(stack))); idx_t nv = Th.nv; idx_t nve = Mesh::Rd::d + 1; std::vector<idx_t> adjncy; adjncy.reserve(loc * nve); xadg[0] = 0; for(idx_t k = vtxdist[rank]; k < vtxdist[rank + 1]; ++k) { for(idx_t j = 0; j < nve; ++j) { idx_t l = j; idx_t m = Th.ElementAdj(k, l); if(k != m && m > 0) adjncy.push_back(m); } xadg[k + 1 - vtxdist[rank]] = adjncy.size(); } #if 0 for(int i = 0; i < worker; ++i) { MPI_Barrier(workComm); if(i == rank) { for(int j = 0; j < worker + 1; ++j) { std::cout << vtxdist[j] << " "; } std::cout << std::endl; for(int j = 0; j < loc + 1; ++j) { std::cout << xadg[j] << " "; } std::cout << std::endl; for(int j = 0; j < adjncy.size(); ++j) { std::cout << adjncy[j] << " "; } std::cout << std::endl; } MPI_Barrier(workComm); } #endif idx_t wgtflag = 0; idx_t ncon = 1; idx_t edgecut; real_t* tpwgts = new real_t[nparts]; for(int i = 0; i < nparts; ++i) tpwgts[i] = 1.0 / static_cast<real_t>(nparts); real_t ubvec = 1.05; idx_t* part = ptInt + vtxdist[rank]; ParMETIS_V3_PartKway(vtxdist, xadg, adjncy.data(), NULL, NULL, &wgtflag, &wgtflag, &ncon, &nparts, tpwgts, &ubvec, &wgtflag, &edgecut, part, &workComm); delete [] tpwgts; delete [] xadg; delete [] vtxdist; } MPI_Allreduce(MPI_IN_PLACE, ptInt, n, MPI_INT, MPI_SUM, comm); for(int i = n; i-- > 0; ) pt[i] = ptInt[i]; if(nargs[1] && workComm != MPI_COMM_NULL) MPI_Comm_free(&workComm); return 0L; }
bool ParMetisPartitioner::partition(map<int, double>& elemWeights, PartitionMode mode) { FUNCNAME("ParMetisPartitioner::partition()"); int mpiSize = mpiComm->Get_size(); // === Create parmetis mesh === if (parMetisMesh) delete parMetisMesh; TEST_EXIT_DBG(elementInRank.size() != 0)("Should not happen!\n"); parMetisMesh = new ParMetisMesh(mesh, mpiComm, elementInRank, mapLocalGlobal); int nElements = parMetisMesh->getNumElements(); // === Create weight array === vector<int> wgts(nElements); vector<float> floatWgts(nElements); unsigned int floatWgtsPos = 0; float maxWgt = 0.0; TraverseStack stack; ElInfo* elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL); while (elInfo) { int index = elInfo->getElement()->getIndex(); if (elementInRank[index]) { // get weight float wgt = static_cast<float>(elemWeights[index]); maxWgt = std::max(wgt, maxWgt); // write float weight TEST_EXIT_DBG(floatWgtsPos < floatWgts.size())("Should not happen!\n"); floatWgts[floatWgtsPos++] = wgt; } elInfo = stack.traverseNext(elInfo); } TEST_EXIT_DBG(floatWgtsPos == floatWgts.size())("Should not happen!\n"); float tmp; mpiComm->Allreduce(&maxWgt, &tmp, 1, MPI_FLOAT, MPI_MAX); maxWgt = tmp; // === Create dual graph === ParMetisGraph parMetisGraph(parMetisMesh, mpiComm); // === Partitioning of dual graph === int wgtflag = 2; // weights at vertices only! int numflag = 0; // c numbering style! int ncon = 1; // one weight at each vertex! int nparts = mpiSize; // number of partitions vector<double> tpwgts(mpiSize); double ubvec = 1.05; int options[4] = {0, 0, 15, PARMETIS_PSR_COUPLED}; // default options int edgecut = -1; vector<int> part(nElements); // set tpwgts for (int i = 0; i < mpiSize; i++) tpwgts[i] = 1.0 / static_cast<double>(nparts); // float scale = 10000.0 / maxWgt; for (int i = 0; i < nElements; i++) wgts[i] = floatWgts[i]; // wgts[i] = static_cast<int>(floatWgts[i] * scale); // === Start ParMETIS. === MPI_Comm tmpComm = MPI_Comm(*mpiComm); switch (mode) { case INITIAL: ParMETIS_V3_PartKway(parMetisMesh->getElementDist(), parMetisGraph.getXAdj(), parMetisGraph.getAdjncy(), &(wgts[0]), NULL, &wgtflag, &numflag, &ncon, &nparts, &(tpwgts[0]), &ubvec, options, &edgecut, &(part[0]), &tmpComm); break; case ADAPTIVE_REPART: { vector<int> vsize(nElements); for (int i = 0; i < nElements; i++) vsize[i] = static_cast<int>(floatWgts[i]); ParMETIS_V3_AdaptiveRepart(parMetisMesh->getElementDist(), parMetisGraph.getXAdj(), parMetisGraph.getAdjncy(), &(wgts[0]), NULL, &(vsize[0]), &wgtflag, &numflag, &ncon, &nparts, &(tpwgts[0]), &ubvec, &itr, options, &edgecut, &(part[0]), &tmpComm); } break; case REFINE_PART: ParMETIS_V3_RefineKway(parMetisMesh->getElementDist(), parMetisGraph.getXAdj(), parMetisGraph.getAdjncy(), &(wgts[0]), NULL, &wgtflag, &numflag, &ncon, &nparts, &(tpwgts[0]), &ubvec, options, &edgecut, &(part[0]), &tmpComm); break; default: ERROR_EXIT("unknown partitioning mode\n"); } // === Distribute new partition data. === return distributePartitioning(&(part[0])); }
/*********************************************************************************** * This function is the testing routine for the adaptive multilevel partitioning code. * It computes a partition from scratch, it then moves the graph and changes some * of the vertex weights and then call the adaptive code. ************************************************************************************/ void TestParMetis_V3(char *filename, MPI_Comm comm) { int ncon, nparts, npes, mype, opt2, realcut; GraphType graph, mgraph; idxtype *part, *mpart, *savepart, *order, *sizes; int numflag=0, wgtflag=0, options[10], edgecut, ndims; float ipc2redist, *xyz, *tpwgts = NULL, ubvec[MAXNCON]; MPI_Comm_size(comm, &npes); MPI_Comm_rank(comm, &mype); ndims = 2; ParallelReadGraph(&graph, filename, comm); xyz = ReadTestCoordinates(&graph, filename, 2, comm); MPI_Barrier(comm); part = idxmalloc(graph.nvtxs, "TestParMetis_V3: part"); tpwgts = fmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts"); sset(MAXNCON, 1.05, ubvec); graph.vwgt = idxsmalloc(graph.nvtxs*5, 1, "TestParMetis_V3: vwgt"); /*====================================================================== / ParMETIS_V3_PartKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; wgtflag = 2; numflag = 0; edgecut = 0; for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) { for (ncon=1; ncon<=5; ncon+=2) { if (ncon > 1 && nparts > 1) Mc_AdaptGraph(&graph, part, ncon, nparts, comm); else idxset(graph.nvtxs, 1, graph.vwgt); for (opt2=1; opt2<=2; opt2++) { options[2] = opt2; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (mype == 0) printf("\nTesting ParMETIS_V3_PartKway with options[1-2] = {%d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], ncon, nparts); ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); if (mype == 0) { printf("ParMETIS_V3_PartKway reported a cut of %d\n", edgecut); } } } } /*====================================================================== / ParMETIS_V3_PartGeomKway /=======================================================================*/ options[0] = 1; options[1] = 3; wgtflag = 2; numflag = 0; for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) { for (ncon=1; ncon<=5; ncon+=2) { if (ncon > 1) Mc_AdaptGraph(&graph, part, ncon, nparts, comm); else idxset(graph.nvtxs, 1, graph.vwgt); for (opt2=1; opt2<=2; opt2++) { options[2] = opt2; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (mype == 0) printf("\nTesting ParMETIS_V3_PartGeomKway with options[1-2] = {%d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], ncon, nparts); ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); if (mype == 0) { printf("ParMETIS_V3_PartGeomKway reported a cut of %d\n", edgecut); } } } } /*====================================================================== / ParMETIS_V3_PartGeom /=======================================================================*/ wgtflag = 0; numflag = 0; if (mype == 0) printf("\nTesting ParMETIS_V3_PartGeom\n"); /* ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm); */ if (mype == 0) printf("ParMETIS_V3_PartGeom partition complete\n"); /* realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) printf("ParMETIS_V3_PartGeom reported a cut of %d\n", realcut); */ /*====================================================================== / ParMETIS_V3_RefineKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; options[3] = COUPLED; nparts = npes; wgtflag = 0; numflag = 0; ncon = 1; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (mype == 0) printf("\nTesting ParMETIS_V3_RefineKway with default options (before move)\n"); ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); MALLOC_CHECK(NULL); if (mype == 0) { printf("ParMETIS_V3_RefineKway reported a cut of %d\n", edgecut); } MALLOC_CHECK(NULL); /* Compute a good partition and move the graph. Do so quietly! */ options[0] = 0; nparts = npes; wgtflag = 0; numflag = 0; ncon = 1; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut, part, &comm); TestMoveGraph(&graph, &mgraph, part, comm); GKfree((void *)&(graph.vwgt), LTERM); mpart = idxsmalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart"); savepart = idxmalloc(mgraph.nvtxs, "TestParMetis_V3: savepart"); MALLOC_CHECK(NULL); /*====================================================================== / ParMETIS_V3_RefineKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[3] = COUPLED; nparts = npes; wgtflag = 0; numflag = 0; for (ncon=1; ncon<=5; ncon+=2) { for (opt2=1; opt2<=2; opt2++) { options[2] = opt2; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (mype == 0) printf("\nTesting ParMETIS_V3_RefineKway with options[1-3] = {%d %d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], options[3], ncon, nparts); ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, mpart, &comm); if (mype == 0) { printf("ParMETIS_V3_RefineKway reported a cut of %d\n", edgecut); } } } /*====================================================================== / ParMETIS_V3_AdaptiveRepart /=======================================================================*/ mgraph.vwgt = idxsmalloc(mgraph.nvtxs*5, 1, "TestParMetis_V3: mgraph.vwgt"); mgraph.vsize = idxsmalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize"); AdaptGraph(&mgraph, 4, comm); options[0] = 1; options[1] = 7; options[3] = COUPLED; wgtflag = 2; numflag = 0; for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) { ncon = 1; wgtflag = 0; options[0] = 0; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, savepart, &comm); options[0] = 1; wgtflag = 2; for (ncon=1; ncon<=3; ncon+=2) { sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (ncon > 1) Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm); else AdaptGraph(&mgraph, 4, comm); /* idxset(mgraph.nvtxs, 1, mgraph.vwgt); */ for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) { for (opt2=1; opt2<=2; opt2++) { idxcopy(mgraph.nvtxs, savepart, mpart); options[2] = opt2; if (mype == 0) printf("\nTesting ParMETIS_V3_AdaptiveRepart with options[1-3] = {%d %d %d}, ipc2redist: %.3f, Ncon: %d, Nparts: %d\n", options[1], options[2], options[3], ipc2redist, ncon, nparts); ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, mgraph.vwgt, mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, &ipc2redist, options, &edgecut, mpart, &comm); if (mype == 0) { printf("ParMETIS_V3_AdaptiveRepart reported a cut of %d\n", edgecut); } } } } } free(mgraph.vwgt); free(mgraph.vsize); /*====================================================================== / ParMETIS_V3_NodeND /=======================================================================*/ sizes = idxmalloc(2*npes, "TestParMetis_V3: sizes"); order = idxmalloc(graph.nvtxs, "TestParMetis_V3: sizes"); options[0] = 1; options[PMV3_OPTION_DBGLVL] = 3; options[PMV3_OPTION_SEED] = 1; numflag = 0; for (opt2=1; opt2<=2; opt2++) { options[PMV3_OPTION_IPART] = opt2; if (mype == 0) printf("\nTesting ParMETIS_V3_NodeND with options[1-3] = {%d %d %d}\n", options[1], options[2], options[3]); ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options, order, sizes, &comm); } GKfree(&tpwgts, &part, &mpart, &savepart, &order, &sizes, LTERM); }