/*****************************************************************************
* This function performs refinement on a partitioning.
*****************************************************************************/
void ParMETIS_RefineKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *options,
int *edgecut, idxtype *part, MPI_Comm *comm)
{
int i;
int nparts;
int ncon = 1;
float *tpwgts, ubvec[MAXNCON];
int myoptions[10];
MPI_Comm_size(*comm, &nparts);
tpwgts = fmalloc(nparts*ncon, "tpwgts");
for (i=0; i<nparts*ncon; i++)
tpwgts[i] = 1.0/(float)(nparts);
for (i=0; i<ncon; i++)
ubvec[i] = UNBALANCE_FRACTION;
if (options[0] == 0) {
myoptions[0] = 0;
}
else {
myoptions[0] = 1;
myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
myoptions[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED;
}
ParMETIS_V3_RefineKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag,
&ncon, &nparts, tpwgts, ubvec, myoptions, edgecut, part, comm);
GKfree((void **)&tpwgts, LTERM);
}
/*************************************************************************
* Let the game begin
**************************************************************************/
int main(int argc, char *argv[])
{
idx_t i, j, npes, mype, optype, nparts, adptf, options[10];
idx_t *part=NULL, *sizes=NULL;
graph_t graph;
real_t ipc2redist, *xyz=NULL, *tpwgts=NULL, ubvec[MAXNCON];
MPI_Comm comm;
idx_t numflag=0, wgtflag=0, ndims, edgecut;
char xyzfilename[8192];
MPI_Init(&argc, &argv);
MPI_Comm_dup(MPI_COMM_WORLD, &comm);
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
if (argc != 8) {
if (mype == 0)
printf("Usage: %s <graph-file> <op-type> <nparts> <adapth-factor> <ipc2redist> <dbglvl> <seed>\n", argv[0]);
MPI_Finalize();
exit(0);
}
optype = atoi(argv[2]);
nparts = atoi(argv[3]);
adptf = atoi(argv[4]);
ipc2redist = atof(argv[5]);
options[0] = 1;
options[PMV3_OPTION_DBGLVL] = atoi(argv[6]);
options[PMV3_OPTION_SEED] = atoi(argv[7]);
if (mype == 0)
printf("reading file: %s\n", argv[1]);
ParallelReadGraph(&graph, argv[1], comm);
/* Remove the edges for testing */
/*iset(graph.vtxdist[mype+1]-graph.vtxdist[mype]+1, 0, graph.xadj); */
rset(graph.ncon, 1.05, ubvec);
tpwgts = rmalloc(nparts*graph.ncon, "tpwgts");
rset(nparts*graph.ncon, 1.0/(real_t)nparts, tpwgts);
/*
ChangeToFortranNumbering(graph.vtxdist, graph.xadj, graph.adjncy, mype, npes);
numflag = 1;
nvtxs = graph.vtxdist[mype+1]-graph.vtxdist[mype];
nedges = graph.xadj[nvtxs];
printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1));
*/
if (optype >= 20) {
sprintf(xyzfilename, "%s.xyz", argv[1]);
xyz = ReadTestCoordinates(&graph, xyzfilename, &ndims, comm);
}
if (mype == 0)
printf("finished reading file: %s\n", argv[1]);
part = ismalloc(graph.nvtxs, mype%nparts, "main: part");
sizes = imalloc(2*npes, "main: sizes");
switch (optype) {
case 1:
wgtflag = 3;
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD);
break;
case 2:
wgtflag = 3;
options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED;
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD);
break;
case 3:
options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED;
graph.vwgt = ismalloc(graph.nvtxs, 1, "main: vwgt");
if (npes > 1) {
AdaptGraph(&graph, adptf, comm);
}
else {
wgtflag = 3;
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts,
ubvec, options, &edgecut, part, &comm);
printf("Initial partitioning with edgecut of %"PRIDX"\n", edgecut);
for (i=0; i<graph.ncon; i++) {
for (j=0; j<graph.nvtxs; j++) {
if (part[j] == i)
graph.vwgt[j*graph.ncon+i] = adptf;
else
graph.vwgt[j*graph.ncon+i] = 1;
}
}
}
wgtflag = 3;
ParMETIS_V3_AdaptiveRepart(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
&ipc2redist, options, &edgecut, part, &comm);
break;
case 4:
ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options,
part, sizes, &comm);
/* WriteOVector(argv[1], graph.vtxdist, part, comm); */
break;
case 5:
ParMETIS_SerialNodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options,
part, sizes, &comm);
/* WriteOVector(argv[1], graph.vtxdist, part, comm); */
printf("%"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX"\n", sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], sizes[5], sizes[6]);
break;
case 11:
/* TestAdaptiveMETIS(graph.vtxdist, graph.xadj, graph.adjncy, part, options, adptf, comm); */
break;
case 20:
wgtflag = 3;
ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &ndims, xyz, &graph.ncon, &nparts,
tpwgts, ubvec, options, &edgecut, part, &comm);
break;
case 21:
ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm);
break;
}
/* printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1)); */
gk_free((void **)&part, &sizes, &tpwgts, &graph.vtxdist, &graph.xadj, &graph.adjncy,
&graph.vwgt, &graph.adjwgt, &xyz, LTERM);
MPI_Comm_free(&comm);
MPI_Finalize();
return 0;
}
void AlgParMETIS<Adapter>::partition(
const RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
int np = problemComm->getSize();
// Get vertex info
ArrayView<const gno_t> vtxgnos;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
int nVwgt = model->getNumWeightsPerVertex();
size_t nVtx = model->getVertexList(vtxgnos, vwgts);
pm_idx_t pm_nVtx;
TPL_Traits<pm_idx_t,size_t>::ASSIGN_TPL_T(pm_nVtx, nVtx);
pm_idx_t *pm_vwgts = NULL;
if (nVwgt) {
pm_vwgts = new pm_idx_t[nVtx*nVwgt];
scale_weights(nVtx, vwgts, pm_vwgts);
}
// Get edge info
ArrayView<const gno_t> adjgnos;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
int nEwgt = model->getNumWeightsPerEdge();
size_t nEdge = model->getEdgeList(adjgnos, offsets, ewgts);
pm_idx_t *pm_ewgts = NULL;
if (nEwgt) {
pm_ewgts = new pm_idx_t[nEdge*nEwgt];
scale_weights(nEdge, ewgts, pm_ewgts);
}
// Convert index types for edges, if needed
pm_idx_t *pm_offsets;
TPL_Traits<pm_idx_t,const lno_t>::ASSIGN_TPL_T_ARRAY(&pm_offsets, offsets);
pm_idx_t *pm_adjs;
pm_idx_t pm_dummy_adj;
if (nEdge)
TPL_Traits<pm_idx_t,const gno_t>::ASSIGN_TPL_T_ARRAY(&pm_adjs, adjgnos);
else
pm_adjs = &pm_dummy_adj; // ParMETIS does not like NULL pm_adjs;
// Build vtxdist
pm_idx_t *pm_vtxdist;
ArrayView<size_t> vtxdist;
model->getVertexDist(vtxdist);
TPL_Traits<pm_idx_t,size_t>::ASSIGN_TPL_T_ARRAY(&pm_vtxdist, vtxdist);
// ParMETIS does not like processors having no vertices.
// Inspect vtxdist and remove from communicator procs that have no vertices
RCP<Comm<int> > subcomm;
MPI_Comm mpicomm; // Note: mpicomm is valid only while subcomm is in scope
if (np > 1) {
int nKeep = 0;
Array<int> keepRanks(np);
for (int i = 0; i < np; i++) {
if ((pm_vtxdist[i+1] - pm_vtxdist[i]) > 0) {
keepRanks[nKeep] = i;
pm_vtxdist[nKeep] = pm_vtxdist[i];
nKeep++;
}
}
pm_vtxdist[nKeep] = pm_vtxdist[np];
if (nKeep < np) {
subcomm = problemComm->createSubcommunicator(keepRanks.view(0,nKeep));
if (subcomm != Teuchos::null)
mpicomm = Teuchos::getRawMpiComm(*subcomm);
else
mpicomm = MPI_COMM_NULL;
}
else {
mpicomm = Teuchos::getRawMpiComm(*problemComm);
}
}
else {
mpicomm = Teuchos::getRawMpiComm(*problemComm);
}
// Create array for ParMETIS to return results in.
pm_idx_t *pm_partList = NULL;
if (nVtx) pm_partList = new pm_idx_t[nVtx];
if (mpicomm != MPI_COMM_NULL) {
// If in ParMETIS' communicator (i.e., have vertices), call ParMETIS
// Get target part sizes
pm_idx_t pm_nCon = (nVwgt == 0 ? 1 : pm_idx_t(nVwgt));
pm_real_t *pm_partsizes = new pm_real_t[numGlobalParts*pm_nCon];
for (pm_idx_t dim = 0; dim < pm_nCon; dim++) {
if (!solution->criteriaHasUniformPartSizes(dim))
for (size_t i=0; i<numGlobalParts; i++)
pm_partsizes[i*pm_nCon+dim] =
pm_real_t(solution->getCriteriaPartSize(dim,i));
else
for (size_t i=0; i<numGlobalParts; i++)
pm_partsizes[i*pm_nCon+dim] = pm_real_t(1.)/pm_real_t(numGlobalParts);
}
// Get imbalance tolerances
double tolerance = 1.1;
const Teuchos::ParameterList &pl = env->getParameters();
const Teuchos::ParameterEntry *pe = pl.getEntryPtr("imbalance_tolerance");
if (pe) tolerance = pe->getValue<double>(&tolerance);
pm_real_t *pm_imbTols = new pm_real_t[pm_nCon];
for (pm_idx_t dim = 0; dim < pm_nCon; dim++)
pm_imbTols[dim] = pm_real_t(tolerance);
std::string parmetis_method("PARTKWAY");
pe = pl.getEntryPtr("partitioning_approach");
if (pe){
std::string approach;
approach = pe->getValue<std::string>(&approach);
if ((approach == "repartition") || (approach == "maximize_overlap"))
parmetis_method = "REFINE_KWAY";
// TODO: AdaptiveRepart
}
// Other ParMETIS parameters?
pm_idx_t pm_wgtflag = 2*(nVwgt > 0) + (nEwgt > 0);
pm_idx_t pm_numflag = 0;
pm_idx_t pm_edgecut = -1;
pm_idx_t pm_options[METIS_NOPTIONS];
pm_options[0] = 1; // Use non-default options for some ParMETIS options
for (int i = 0; i < METIS_NOPTIONS; i++)
pm_options[i] = 0; // Default options
pm_options[2] = 15; // Matches default value used in Zoltan
pm_idx_t pm_nPart;
TPL_Traits<pm_idx_t,size_t>::ASSIGN_TPL_T(pm_nPart, numGlobalParts);
if (parmetis_method == "PARTKWAY") {
ParMETIS_V3_PartKway(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts, pm_ewgts,
&pm_wgtflag, &pm_numflag, &pm_nCon, &pm_nPart,
pm_partsizes, pm_imbTols, pm_options,
&pm_edgecut, pm_partList, &mpicomm);
}
else if (parmetis_method == "ADAPTIVE_REPART") {
// Get object sizes: pm_vsize
std::cout << "NOT READY FOR ADAPTIVE_REPART YET; NEED VSIZE" << std::endl;
exit(-1);
//pm_real_t itr = 100.; // Same default as in Zoltan
//ParMETIS_V3_AdaptiveRepart(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts,
// pm_vsize, pm_ewgts, &pm_wgtflag,
// &pm_numflag, &pm_nCon, &pm_nPart,
// pm_partsizes, pm_imbTols,
// &itr, pm_options,
// &pm_edgecut, pm_partList, &mpicomm);
}
else if (parmetis_method == "REFINE_KWAY") {
ParMETIS_V3_RefineKway(pm_vtxdist, pm_offsets, pm_adjs, pm_vwgts,
pm_ewgts,
&pm_wgtflag, &pm_numflag, &pm_nCon, &pm_nPart,
pm_partsizes, pm_imbTols,
pm_options, &pm_edgecut, pm_partList, &mpicomm);
}
// Clean up
delete [] pm_partsizes;
delete [] pm_imbTols;
}
// Load answer into the solution.
ArrayRCP<part_t> partList;
if (nVtx) {
if (TPL_Traits<pm_idx_t, part_t>::OK_TO_CAST_TPL_T()) {
partList = ArrayRCP<part_t>((part_t *)pm_partList, 0, nVtx, true);
}
else {
// TODO Probably should have a TPL_Traits function to do the following
partList = ArrayRCP<part_t>(new part_t[nVtx], 0, nVtx, true);
for (size_t i = 0; i < nVtx; i++) {
partList[i] = part_t(pm_partList[i]);
}
delete [] pm_partList;
}
}
solution->setParts(partList);
env->memory("Zoltan2-ParMETIS: After creating solution");
// Clean up copies made due to differing data sizes.
TPL_Traits<pm_idx_t,size_t>::DELETE_TPL_T_ARRAY(&pm_vtxdist);
TPL_Traits<pm_idx_t,const lno_t>::DELETE_TPL_T_ARRAY(&pm_offsets);
if (nEdge)
TPL_Traits<pm_idx_t,const gno_t>::DELETE_TPL_T_ARRAY(&pm_adjs);
if (nVwgt) delete [] pm_vwgts;
if (nEwgt) delete [] pm_ewgts;
}
void TestParMetis_GPart(char *filename, char *xyzfile, MPI_Comm comm)
{
idx_t ncon, nparts, npes, mype, opt2, realcut;
graph_t graph, mgraph;
idx_t *part, *mpart, *savepart, *order, *sizes;
idx_t numflag=0, wgtflag=0, options[10], edgecut, ndims;
real_t ipc2redist, *xyz=NULL, *tpwgts = NULL, ubvec[MAXNCON];
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
ParallelReadGraph(&graph, filename, comm);
if (xyzfile)
xyz = ReadTestCoordinates(&graph, xyzfile, &ndims, comm);
gkMPI_Barrier(comm);
part = imalloc(graph.nvtxs, "TestParMetis_V3: part");
tpwgts = rmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts");
rset(MAXNCON, 1.05, ubvec);
graph.vwgt = ismalloc(graph.nvtxs*5, 1, "TestParMetis_GPart: vwgt");
/*======================================================================
/ ParMETIS_V3_PartKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
wgtflag = 2;
numflag = 0;
edgecut = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=NCON; ncon++) {
if (ncon > 1 && nparts > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
iset(graph.nvtxs, 1, graph.vwgt);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options,
&edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0) {
printf("ParMETIS_V3_PartKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
if (mype == 0)
printf("\nTesting ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
options[3] = PARMETIS_PSR_UNCOUPLED;
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options,
&edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0) {
printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeomKway
/=======================================================================*/
if (xyzfile != NULL) {
options[0] = 1;
options[1] = 3;
options[2] = 1;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=NCON; ncon++) {
if (ncon > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
iset(graph.nvtxs, 1, graph.vwgt);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeomKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0)
printf("ParMETIS_V3_PartGeomKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeom
/=======================================================================*/
if (xyz != NULL) {
wgtflag = 0;
numflag = 0;
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeom\n");
ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0)
printf("ParMETIS_V3_PartGeom reported a cut of %"PRIDX"\n", realcut);
}
/*======================================================================
/ Coupled ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
if (mype == 0)
printf("\nTesting coupled ParMETIS_V3_RefineKway with default options (before move)\n");
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
part, &comm);
/* Compute a good partition and move the graph. Do so quietly! */
options[0] = 0;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut,
part, &comm);
TestMoveGraph(&graph, &mgraph, part, comm);
gk_free((void **)&(graph.vwgt), LTERM);
mpart = ismalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart");
savepart = imalloc(mgraph.nvtxs, "TestParMetis_V3: savepart");
/*======================================================================
/ Coupled ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
for (ncon=1; ncon<=NCON; ncon++) {
if (mype == 0)
printf("\nTesting coupled ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
mpart, &comm);
realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart,
filename, comm);
if (mype == 0)
printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
/*ADAPTIVE:*/
/*======================================================================
/ ParMETIS_V3_AdaptiveRepart
/=======================================================================*/
mgraph.vwgt = ismalloc(mgraph.nvtxs*NCON, 1, "TestParMetis_V3: mgraph.vwgt");
mgraph.vsize = ismalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize");
AdaptGraph(&mgraph, 4, comm);
options[0] = 1;
options[1] = 7;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) {
options[0] = 0;
ncon = 1;
wgtflag = 0;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
savepart, &comm);
options[0] = 1;
wgtflag = 2;
for (ncon=1; ncon<=NCON; ncon++) {
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
if (ncon > 1)
Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm);
else
AdaptGraph(&mgraph, 4, comm);
for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) {
icopy(mgraph.nvtxs, savepart, mpart);
if (mype == 0)
printf("\nTesting ParMETIS_V3_AdaptiveRepart with ipc2redist: %.3"PRREAL", ncon: %"PRIDX", nparts: %"PRIDX"\n",
ipc2redist, ncon, nparts);
ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy,
mgraph.vwgt, mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts,
tpwgts, ubvec, &ipc2redist, options, &edgecut, mpart, &comm);
realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart,
filename, comm);
if (mype == 0)
printf("ParMETIS_V3_AdaptiveRepart reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
gk_free((void **)&tpwgts, &part, &mpart, &savepart, &xyz, &mgraph.vwgt, &mgraph.vsize, LTERM);
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
realtype itr = 0.0;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
/* change our zz struct. */
#endif
indextype i;
realtype *imb_tols;
indextype ncon;
indextype edgecut;
indextype wgtflag;
indextype numflag = 0;
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
ZOLTAN_TRACE_ENTER(zz, yo);
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
if (sizeof(realtype) != sizeof(float)) {
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));
for (i = 0; i < tmp; i++)
prt.input_part_sizes[i] = (realtype) part_sizes[i];
/* KDD 2/2014: removed re-scaling part sizes so they sum to one.
* part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
* plus, the code here was wrong for multiple object weights.
* similar scaling code did not exist in the Scotch interface.
*/
prt.part_sizes = prt.input_part_sizes;
}
else
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
indextype j;
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
/* Now we can call ParMetis */
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
/* Use default options for METIS */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0;
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
#else
METIS_SetDefaultOptions(options);
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
prt.part_sizes, imb_tols, options,
&edgecut, prt.part);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) {
/* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
void AlgParMETIS<Adapter>::partition(
const RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
int me = problemComm->getRank();
int np = problemComm->getSize();
// Get vertex info
ArrayView<const gno_t> vtxgnos;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
int nVwgt = model->getNumWeightsPerVertex();
size_t nVtx = model->getVertexList(vtxgnos, vwgts);
pm_idx_t pm_nVtx;
TPL_Traits<pm_idx_t,size_t>::ASSIGN(pm_nVtx, nVtx);
pm_idx_t *pm_vwgts = NULL;
if (nVwgt) {
pm_vwgts = new pm_idx_t[nVtx*nVwgt];
scale_weights(nVtx, vwgts, pm_vwgts);
}
// Get edge info
ArrayView<const gno_t> adjgnos;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
int nEwgt = model->getNumWeightsPerEdge();
size_t nEdge = model->getEdgeList(adjgnos, offsets, ewgts);
pm_idx_t *pm_ewgts = NULL;
if (nEwgt) {
pm_ewgts = new pm_idx_t[nEdge*nEwgt];
scale_weights(nEdge, ewgts, pm_ewgts);
}
// Convert index types for edges, if needed
pm_idx_t *pm_offsets;
TPL_Traits<pm_idx_t,const lno_t>::ASSIGN_ARRAY(&pm_offsets, offsets);
pm_idx_t *pm_adjs;
pm_idx_t pm_dummy_adj;
if (nEdge)
TPL_Traits<pm_idx_t,const gno_t>::ASSIGN_ARRAY(&pm_adjs, adjgnos);
else
pm_adjs = &pm_dummy_adj; // ParMETIS does not like NULL pm_adjs;
// Build vtxdist
pm_idx_t *pm_vtxdist;
ArrayView<size_t> vtxdist;
model->getVertexDist(vtxdist);
TPL_Traits<pm_idx_t,size_t>::ASSIGN_ARRAY(&pm_vtxdist, vtxdist);
// ParMETIS does not like processors having no vertices.
// Inspect vtxdist and remove from communicator procs that have no vertices
RCP<Comm<int> > subcomm;
MPI_Comm mpicomm; // Note: mpicomm is valid only while subcomm is in scope
if (np > 1) {
int nKeep = 0;
Array<int> keepRanks(np);
for (int i = 0; i < np; i++) {
if ((pm_vtxdist[i+1] - pm_vtxdist[i]) > 0) {
keepRanks[nKeep] = i;
pm_vtxdist[nKeep] = pm_vtxdist[i];
nKeep++;
}
}
pm_vtxdist[nKeep] = pm_vtxdist[np];
if (nKeep < np) {
subcomm = problemComm->createSubcommunicator(keepRanks.view(0,nKeep));
if (subcomm != Teuchos::null)
mpicomm = Teuchos::getRawMpiComm(*subcomm);
else
mpicomm = MPI_COMM_NULL;
}
else {
mpicomm = Teuchos::getRawMpiComm(*problemComm);
}
}
else {
mpicomm = Teuchos::getRawMpiComm(*problemComm);
}
// Create array for ParMETIS to return results in.
pm_idx_t *pm_partList = NULL;
if (nVtx) pm_partList = new pm_idx_t[nVtx];
for (size_t i = 0; i < nVtx; i++) pm_partList[i] = 0;
int pm_return = METIS_OK;
if (mpicomm != MPI_COMM_NULL) {
// If in ParMETIS' communicator (i.e., have vertices), call ParMETIS
// Get target part sizes
pm_idx_t pm_nCon = (nVwgt == 0 ? 1 : pm_idx_t(nVwgt));
pm_real_t *pm_partsizes = new pm_real_t[numGlobalParts*pm_nCon];
for (pm_idx_t dim = 0; dim < pm_nCon; dim++) {
if (!solution->criteriaHasUniformPartSizes(dim))
for (size_t i=0; i<numGlobalParts; i++)
pm_partsizes[i*pm_nCon+dim] =
pm_real_t(solution->getCriteriaPartSize(dim,i));
else
for (size_t i=0; i<numGlobalParts; i++)
pm_partsizes[i*pm_nCon+dim] = pm_real_t(1.)/pm_real_t(numGlobalParts);
}
// Get imbalance tolerances
double tolerance = 1.1;
const Teuchos::ParameterList &pl = env->getParameters();
const Teuchos::ParameterEntry *pe = pl.getEntryPtr("imbalance_tolerance");
if (pe) tolerance = pe->getValue<double>(&tolerance);
// ParMETIS requires tolerance to be greater than 1.0;
// fudge it if condition is not met
if (tolerance <= 1.0) {
if (me == 0)
std::cerr << "Warning: ParMETIS requires imbalance_tolerance > 1.0; "
<< "to comply, Zoltan2 reset imbalance_tolerance to 1.01."
<< std::endl;
tolerance = 1.01;
}
pm_real_t *pm_imbTols = new pm_real_t[pm_nCon];
for (pm_idx_t dim = 0; dim < pm_nCon; dim++)
pm_imbTols[dim] = pm_real_t(tolerance);
std::string parmetis_method("PARTKWAY");
pe = pl.getEntryPtr("partitioning_approach");
if (pe) {
std::string approach;
approach = pe->getValue<std::string>(&approach);
if ((approach == "repartition") || (approach == "maximize_overlap")) {
if (np > 1)
// ParMETIS_V3_AdaptiveRepart requires two or more processors
parmetis_method = "ADAPTIVE_REPART";
else
parmetis_method = "REFINE_KWAY";
}
}
// Other ParMETIS parameters?
pm_idx_t pm_wgtflag = 2*(nVwgt > 0) + (nEwgt > 0);
pm_idx_t pm_numflag = 0;
pm_idx_t pm_edgecut = -1;
pm_idx_t pm_options[METIS_NOPTIONS];
pm_options[0] = 1; // Use non-default options for some ParMETIS options
for (int i = 0; i < METIS_NOPTIONS; i++)
pm_options[i] = 0; // Default options
pm_options[2] = 15; // Matches default value used in Zoltan
pm_idx_t pm_nPart;
TPL_Traits<pm_idx_t,size_t>::ASSIGN(pm_nPart, numGlobalParts);
if (parmetis_method == "PARTKWAY") {
pm_return = ParMETIS_V3_PartKway(pm_vtxdist, pm_offsets, pm_adjs,
pm_vwgts, pm_ewgts, &pm_wgtflag,
&pm_numflag, &pm_nCon, &pm_nPart,
pm_partsizes, pm_imbTols, pm_options,
&pm_edgecut, pm_partList, &mpicomm);
}
else if (parmetis_method == "ADAPTIVE_REPART") {
// Get object sizes: pm_vsize
// TODO: get pm_vsize info from input adapter or graph model
// TODO: This is just a placeholder
pm_idx_t *pm_vsize = new pm_idx_t[nVtx];
for (size_t i = 0; i < nVtx; i++) pm_vsize[i] = 1;
pm_real_t itr = 100.; // Same default as in Zoltan
pm_return = ParMETIS_V3_AdaptiveRepart(pm_vtxdist, pm_offsets, pm_adjs,
pm_vwgts,
pm_vsize, pm_ewgts, &pm_wgtflag,
&pm_numflag, &pm_nCon, &pm_nPart,
pm_partsizes, pm_imbTols,
&itr, pm_options, &pm_edgecut,
pm_partList, &mpicomm);
delete [] pm_vsize;
}
else if (parmetis_method == "REFINE_KWAY") {
pm_return = ParMETIS_V3_RefineKway(pm_vtxdist, pm_offsets, pm_adjs,
pm_vwgts, pm_ewgts, &pm_wgtflag,
&pm_numflag, &pm_nCon, &pm_nPart,
pm_partsizes, pm_imbTols, pm_options,
&pm_edgecut, pm_partList, &mpicomm);
}
// Clean up
delete [] pm_partsizes;
delete [] pm_imbTols;
}
// Load answer into the solution.
ArrayRCP<part_t> partList;
if (nVtx)
TPL_Traits<part_t, pm_idx_t>::SAVE_ARRAYRCP(&partList, pm_partList, nVtx);
TPL_Traits<pm_idx_t, part_t>::DELETE_ARRAY(&pm_partList);
solution->setParts(partList);
env->memory("Zoltan2-ParMETIS: After creating solution");
// Clean up copies made due to differing data sizes.
TPL_Traits<pm_idx_t,size_t>::DELETE_ARRAY(&pm_vtxdist);
TPL_Traits<pm_idx_t,const lno_t>::DELETE_ARRAY(&pm_offsets);
if (nEdge)
TPL_Traits<pm_idx_t,const gno_t>::DELETE_ARRAY(&pm_adjs);
if (nVwgt) delete [] pm_vwgts;
if (nEwgt) delete [] pm_ewgts;
if (pm_return != METIS_OK) {
throw std::runtime_error(
"\nParMETIS returned an error; no valid partition generated.\n"
"Look for 'PARMETIS ERROR' in your output for more details.\n");
}
}
bool ParMetisPartitioner::partition(map<int, double>& elemWeights,
PartitionMode mode)
{
FUNCNAME("ParMetisPartitioner::partition()");
int mpiSize = mpiComm->Get_size();
// === Create parmetis mesh ===
if (parMetisMesh)
delete parMetisMesh;
TEST_EXIT_DBG(elementInRank.size() != 0)("Should not happen!\n");
parMetisMesh = new ParMetisMesh(mesh, mpiComm, elementInRank, mapLocalGlobal);
int nElements = parMetisMesh->getNumElements();
// === Create weight array ===
vector<int> wgts(nElements);
vector<float> floatWgts(nElements);
unsigned int floatWgtsPos = 0;
float maxWgt = 0.0;
TraverseStack stack;
ElInfo* elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo)
{
int index = elInfo->getElement()->getIndex();
if (elementInRank[index])
{
// get weight
float wgt = static_cast<float>(elemWeights[index]);
maxWgt = std::max(wgt, maxWgt);
// write float weight
TEST_EXIT_DBG(floatWgtsPos < floatWgts.size())("Should not happen!\n");
floatWgts[floatWgtsPos++] = wgt;
}
elInfo = stack.traverseNext(elInfo);
}
TEST_EXIT_DBG(floatWgtsPos == floatWgts.size())("Should not happen!\n");
float tmp;
mpiComm->Allreduce(&maxWgt, &tmp, 1, MPI_FLOAT, MPI_MAX);
maxWgt = tmp;
// === Create dual graph ===
ParMetisGraph parMetisGraph(parMetisMesh, mpiComm);
// === Partitioning of dual graph ===
int wgtflag = 2; // weights at vertices only!
int numflag = 0; // c numbering style!
int ncon = 1; // one weight at each vertex!
int nparts = mpiSize; // number of partitions
vector<double> tpwgts(mpiSize);
double ubvec = 1.05;
int options[4] = {0, 0, 15, PARMETIS_PSR_COUPLED}; // default options
int edgecut = -1;
vector<int> part(nElements);
// set tpwgts
for (int i = 0; i < mpiSize; i++)
tpwgts[i] = 1.0 / static_cast<double>(nparts);
// float scale = 10000.0 / maxWgt;
for (int i = 0; i < nElements; i++)
wgts[i] = floatWgts[i];
// wgts[i] = static_cast<int>(floatWgts[i] * scale);
// === Start ParMETIS. ===
MPI_Comm tmpComm = MPI_Comm(*mpiComm);
switch (mode)
{
case INITIAL:
ParMETIS_V3_PartKway(parMetisMesh->getElementDist(),
parMetisGraph.getXAdj(),
parMetisGraph.getAdjncy(),
&(wgts[0]),
NULL,
&wgtflag,
&numflag,
&ncon,
&nparts,
&(tpwgts[0]),
&ubvec,
options,
&edgecut,
&(part[0]),
&tmpComm);
break;
case ADAPTIVE_REPART:
{
vector<int> vsize(nElements);
for (int i = 0; i < nElements; i++)
vsize[i] = static_cast<int>(floatWgts[i]);
ParMETIS_V3_AdaptiveRepart(parMetisMesh->getElementDist(),
parMetisGraph.getXAdj(),
parMetisGraph.getAdjncy(),
&(wgts[0]),
NULL,
&(vsize[0]),
&wgtflag,
&numflag,
&ncon,
&nparts,
&(tpwgts[0]),
&ubvec,
&itr,
options,
&edgecut,
&(part[0]),
&tmpComm);
}
break;
case REFINE_PART:
ParMETIS_V3_RefineKway(parMetisMesh->getElementDist(),
parMetisGraph.getXAdj(),
parMetisGraph.getAdjncy(),
&(wgts[0]),
NULL,
&wgtflag,
&numflag,
&ncon,
&nparts,
&(tpwgts[0]),
&ubvec,
options,
&edgecut,
&(part[0]),
&tmpComm);
break;
default:
ERROR_EXIT("unknown partitioning mode\n");
}
// === Distribute new partition data. ===
return distributePartitioning(&(part[0]));
}
/***********************************************************************************
* This function is the testing routine for the adaptive multilevel partitioning code.
* It computes a partition from scratch, it then moves the graph and changes some
* of the vertex weights and then call the adaptive code.
************************************************************************************/
void TestParMetis_V3(char *filename, MPI_Comm comm)
{
int ncon, nparts, npes, mype, opt2, realcut;
GraphType graph, mgraph;
idxtype *part, *mpart, *savepart, *order, *sizes;
int numflag=0, wgtflag=0, options[10], edgecut, ndims;
float ipc2redist, *xyz, *tpwgts = NULL, ubvec[MAXNCON];
MPI_Comm_size(comm, &npes);
MPI_Comm_rank(comm, &mype);
ndims = 2;
ParallelReadGraph(&graph, filename, comm);
xyz = ReadTestCoordinates(&graph, filename, 2, comm);
MPI_Barrier(comm);
part = idxmalloc(graph.nvtxs, "TestParMetis_V3: part");
tpwgts = fmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts");
sset(MAXNCON, 1.05, ubvec);
graph.vwgt = idxsmalloc(graph.nvtxs*5, 1, "TestParMetis_V3: vwgt");
/*======================================================================
/ ParMETIS_V3_PartKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
wgtflag = 2;
numflag = 0;
edgecut = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=5; ncon+=2) {
if (ncon > 1 && nparts > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
idxset(graph.nvtxs, 1, graph.vwgt);
for (opt2=1; opt2<=2; opt2++) {
options[2] = opt2;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartKway with options[1-2] = {%d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], ncon, nparts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag,
&numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm);
if (mype == 0) {
printf("ParMETIS_V3_PartKway reported a cut of %d\n", edgecut);
}
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeomKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=5; ncon+=2) {
if (ncon > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
idxset(graph.nvtxs, 1, graph.vwgt);
for (opt2=1; opt2<=2; opt2++) {
options[2] = opt2;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeomKway with options[1-2] = {%d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], ncon, nparts);
ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag,
&numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm);
if (mype == 0) {
printf("ParMETIS_V3_PartGeomKway reported a cut of %d\n", edgecut);
}
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeom
/=======================================================================*/
wgtflag = 0;
numflag = 0;
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeom\n");
/* ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm); */
if (mype == 0)
printf("ParMETIS_V3_PartGeom partition complete\n");
/*
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0)
printf("ParMETIS_V3_PartGeom reported a cut of %d\n", realcut);
*/
/*======================================================================
/ ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
options[3] = COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (mype == 0)
printf("\nTesting ParMETIS_V3_RefineKway with default options (before move)\n");
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag,
&numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm);
MALLOC_CHECK(NULL);
if (mype == 0) {
printf("ParMETIS_V3_RefineKway reported a cut of %d\n", edgecut);
}
MALLOC_CHECK(NULL);
/* Compute a good partition and move the graph. Do so quietly! */
options[0] = 0;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag,
&numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut, part, &comm);
TestMoveGraph(&graph, &mgraph, part, comm);
GKfree((void *)&(graph.vwgt), LTERM);
mpart = idxsmalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart");
savepart = idxmalloc(mgraph.nvtxs, "TestParMetis_V3: savepart");
MALLOC_CHECK(NULL);
/*======================================================================
/ ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[3] = COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
for (ncon=1; ncon<=5; ncon+=2) {
for (opt2=1; opt2<=2; opt2++) {
options[2] = opt2;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (mype == 0)
printf("\nTesting ParMETIS_V3_RefineKway with options[1-3] = {%d %d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], options[3], ncon, nparts);
ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag,
&numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, mpart, &comm);
if (mype == 0) {
printf("ParMETIS_V3_RefineKway reported a cut of %d\n", edgecut);
}
}
}
/*======================================================================
/ ParMETIS_V3_AdaptiveRepart
/=======================================================================*/
mgraph.vwgt = idxsmalloc(mgraph.nvtxs*5, 1, "TestParMetis_V3: mgraph.vwgt");
mgraph.vsize = idxsmalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize");
AdaptGraph(&mgraph, 4, comm);
options[0] = 1;
options[1] = 7;
options[3] = COUPLED;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) {
ncon = 1;
wgtflag = 0;
options[0] = 0;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, savepart, &comm);
options[0] = 1;
wgtflag = 2;
for (ncon=1; ncon<=3; ncon+=2) {
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (ncon > 1)
Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm);
else
AdaptGraph(&mgraph, 4, comm);
/* idxset(mgraph.nvtxs, 1, mgraph.vwgt); */
for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) {
for (opt2=1; opt2<=2; opt2++) {
idxcopy(mgraph.nvtxs, savepart, mpart);
options[2] = opt2;
if (mype == 0)
printf("\nTesting ParMETIS_V3_AdaptiveRepart with options[1-3] = {%d %d %d}, ipc2redist: %.3f, Ncon: %d, Nparts: %d\n", options[1], options[2], options[3], ipc2redist, ncon, nparts);
ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, mgraph.vwgt,
mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, &ipc2redist,
options, &edgecut, mpart, &comm);
if (mype == 0) {
printf("ParMETIS_V3_AdaptiveRepart reported a cut of %d\n", edgecut);
}
}
}
}
}
free(mgraph.vwgt);
free(mgraph.vsize);
/*======================================================================
/ ParMETIS_V3_NodeND
/=======================================================================*/
sizes = idxmalloc(2*npes, "TestParMetis_V3: sizes");
order = idxmalloc(graph.nvtxs, "TestParMetis_V3: sizes");
options[0] = 1;
options[PMV3_OPTION_DBGLVL] = 3;
options[PMV3_OPTION_SEED] = 1;
numflag = 0;
for (opt2=1; opt2<=2; opt2++) {
options[PMV3_OPTION_IPART] = opt2;
if (mype == 0)
printf("\nTesting ParMETIS_V3_NodeND with options[1-3] = {%d %d %d}\n", options[1], options[2], options[3]);
ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options,
order, sizes, &comm);
}
GKfree(&tpwgts, &part, &mpart, &savepart, &order, &sizes, LTERM);
}
