int SCOTCH_dgraphPart ( SCOTCH_Dgraph * const grafptr, /*+ Graph to map +*/ const SCOTCH_Num partnbr, /*+ Number of parts +*/ SCOTCH_Strat * const stratptr, /*+ Mapping strategy +*/ SCOTCH_Num * const termloctab) /*+ Mapping array +*/ { SCOTCH_Arch archdat; int o; SCOTCH_archInit (&archdat); SCOTCH_archCmplt (&archdat, partnbr); o = SCOTCH_dgraphMap (grafptr, &archdat, stratptr, termloctab); SCOTCH_archExit (&archdat); return (o); }
void AlgPTScotch<Adapter>::partition( const RCP<PartitioningSolution<Adapter> > &solution ) { HELLO; size_t numGlobalParts = solution->getTargetGlobalNumberOfParts(); SCOTCH_Num partnbr=0; TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(partnbr, numGlobalParts, env); #ifdef HAVE_ZOLTAN2_MPI int ierr = 0; int me = problemComm->getRank(); const SCOTCH_Num baseval = 0; // Base value for array indexing. // GraphModel returns GNOs from base 0. SCOTCH_Strat stratstr; // Strategy string // TODO: Set from parameters SCOTCH_stratInit(&stratstr); // Allocate and initialize PTScotch Graph data structure. SCOTCH_Dgraph *gr = SCOTCH_dgraphAlloc(); // Scotch distributed graph ierr = SCOTCH_dgraphInit(gr, mpicomm); env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphInit", !ierr, BASIC_ASSERTION, problemComm); // Get vertex info ArrayView<const gno_t> vtxID; ArrayView<StridedData<lno_t, scalar_t> > xyz; ArrayView<StridedData<lno_t, scalar_t> > vwgts; size_t nVtx = model->getVertexList(vtxID, xyz, vwgts); SCOTCH_Num vertlocnbr=0; TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(vertlocnbr, nVtx, env); SCOTCH_Num vertlocmax = vertlocnbr; // Assumes no holes in global nums. // Get edge info ArrayView<const gno_t> edgeIds; ArrayView<const int> procIds; ArrayView<const lno_t> offsets; ArrayView<StridedData<lno_t, scalar_t> > ewgts; size_t nEdge = model->getEdgeList(edgeIds, procIds, offsets, ewgts); SCOTCH_Num edgelocnbr=0; TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(edgelocnbr, nEdge, env); const SCOTCH_Num edgelocsize = edgelocnbr; // Assumes adj array is compact. SCOTCH_Num *vertloctab; // starting adj/vtx TPL_Traits<SCOTCH_Num, lno_t>::ASSIGN_TPL_T_ARRAY(&vertloctab, offsets, env); SCOTCH_Num *edgeloctab; // adjacencies TPL_Traits<SCOTCH_Num, gno_t>::ASSIGN_TPL_T_ARRAY(&edgeloctab, edgeIds, env); // We don't use these arrays, but we need them as arguments to Scotch. SCOTCH_Num *vendloctab = NULL; // Assume consecutive storage for adj SCOTCH_Num *vlblloctab = NULL; // Vertex label array SCOTCH_Num *edgegsttab = NULL; // Array for ghost vertices // Get weight info. SCOTCH_Num *velotab = NULL; // Vertex weights SCOTCH_Num *edlotab = NULL; // Edge weights int nVwgts = model->getNumWeightsPerVertex(); int nEwgts = model->getNumWeightsPerEdge(); if (nVwgts > 1 && me == 0) { std::cerr << "Warning: NumWeightsPerVertex is " << nVwgts << " but Scotch allows only one weight. " << " Zoltan2 will use only the first weight per vertex." << std::endl; } if (nEwgts > 1 && me == 0) { std::cerr << "Warning: NumWeightsPerEdge is " << nEwgts << " but Scotch allows only one weight. " << " Zoltan2 will use only the first weight per edge." << std::endl; } if (nVwgts) { velotab = new SCOTCH_Num[nVtx+1]; // +1 since Scotch wants all procs // to have non-NULL arrays scale_weights(nVtx, vwgts[0], velotab); } if (nEwgts) { edlotab = new SCOTCH_Num[nEdge+1]; // +1 since Scotch wants all procs // to have non-NULL arrays scale_weights(nEdge, ewgts[0], edlotab); } // Build PTScotch distributed data structure ierr = SCOTCH_dgraphBuild(gr, baseval, vertlocnbr, vertlocmax, vertloctab, vendloctab, velotab, vlblloctab, edgelocnbr, edgelocsize, edgeloctab, edgegsttab, edlotab); env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphBuild", !ierr, BASIC_ASSERTION, problemComm); // Create array for Scotch to return results in. ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx,true); SCOTCH_Num *partloctab = NULL; if (nVtx && (sizeof(SCOTCH_Num) == sizeof(part_t))) { // Can write directly into the solution's memory partloctab = (SCOTCH_Num *) partList.getRawPtr(); } else { // Can't use solution memory directly; will have to copy later. // Note: Scotch does not like NULL arrays, so add 1 to always have non-null. // ParMETIS has this same "feature." See Zoltan bug 4299. partloctab = new SCOTCH_Num[nVtx+1]; } // Get target part sizes float *partsizes = new float[numGlobalParts]; if (!solution->criteriaHasUniformPartSizes(0)) for (size_t i=0; i<numGlobalParts; i++) partsizes[i] = solution->getCriteriaPartSize(0, i); else for (size_t i=0; i<numGlobalParts; i++) partsizes[i] = 1.0 / float(numGlobalParts); // Allocate and initialize PTScotch target architecture data structure SCOTCH_Arch archdat; SCOTCH_archInit(&archdat); SCOTCH_Num velosum = 0; SCOTCH_dgraphSize (gr, &velosum, NULL, NULL, NULL); SCOTCH_Num *goalsizes = new SCOTCH_Num[partnbr]; // TODO: The goalsizes are set as in Zoltan; not sure it is correct there // or here. // It appears velosum is global NUMBER of vertices, not global total // vertex weight. I think we should use the latter. // Fix this when we add vertex weights. for (SCOTCH_Num i = 0; i < partnbr; i++) goalsizes[i] = SCOTCH_Num(ceil(velosum * partsizes[i])); delete [] partsizes; SCOTCH_archCmpltw(&archdat, partnbr, goalsizes); // Call partitioning; result returned in partloctab. ierr = SCOTCH_dgraphMap(gr, &archdat, &stratstr, partloctab); env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphMap", !ierr, BASIC_ASSERTION, problemComm); SCOTCH_archExit(&archdat); delete [] goalsizes; // TODO - metrics #ifdef SHOW_ZOLTAN2_SCOTCH_MEMORY int me = env->comm_->getRank(); #endif #ifdef HAVE_SCOTCH_ZOLTAN2_GETMEMORYMAX if (me == 0){ size_t scotchBytes = SCOTCH_getMemoryMax(); std::cout << "Rank " << me << ": Maximum bytes used by Scotch: "; std::cout << scotchBytes << std::endl; } #endif // Clean up PTScotch SCOTCH_dgraphExit(gr); free(gr); SCOTCH_stratExit(&stratstr); // Load answer into the solution. if ((sizeof(SCOTCH_Num) != sizeof(part_t)) || (nVtx == 0)) { for (size_t i = 0; i < nVtx; i++) partList[i] = partloctab[i]; delete [] partloctab; } solution->setParts(partList); env->memory("Zoltan2-Scotch: After creating solution"); // Clean up copies made due to differing data sizes. TPL_Traits<SCOTCH_Num, lno_t>::DELETE_TPL_T_ARRAY(&vertloctab); TPL_Traits<SCOTCH_Num, gno_t>::DELETE_TPL_T_ARRAY(&edgeloctab); if (nVwgts) delete [] velotab; if (nEwgts) delete [] edlotab; #else // DO NOT HAVE_MPI // TODO: Handle serial case with calls to Scotch. // TODO: For now, assign everything to rank 0 and assume only one part. // TODO: Can probably use the code above for loading solution, // TODO: instead of duplicating it here. // TODO // TODO: Actual logic should call Scotch when number of processes == 1. ArrayView<const gno_t> vtxID; ArrayView<StridedData<lno_t, scalar_t> > xyz; ArrayView<StridedData<lno_t, scalar_t> > vwgts; size_t nVtx = model->getVertexList(vtxID, xyz, vwgts); ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx, true); for (size_t i = 0; i < nVtx; i++) partList[i] = 0; solution->setParts(partList); #endif // DO NOT HAVE_MPI }
// Call scotch with options from dictionary. Foam::label Foam::ptscotchDecomp::decompose ( const fileName& meshPath, const List<int>& adjncy, const List<int>& xadj, const scalarField& cWeights, List<int>& finalDecomp ) const { if (debug) { Pout<< "ptscotchDecomp : entering with xadj:" << xadj.size() << endl; } // Dump graph if (decompositionDict_.found("ptscotchCoeffs")) { const dictionary& scotchCoeffs = decompositionDict_.subDict("ptscotchCoeffs"); if (scotchCoeffs.lookupOrDefault("writeGraph", false)) { OFstream str ( meshPath + "_" + Foam::name(Pstream::myProcNo()) + ".dgr" ); Pout<< "Dumping Scotch graph file to " << str.name() << endl << "Use this in combination with dgpart." << endl; globalIndex globalCells(xadj.size()-1); // Distributed graph file (.grf) label version = 2; str << version << nl; // Number of files (procglbnbr) str << Pstream::nProcs(); // My file number (procloc) str << ' ' << Pstream::myProcNo() << nl; // Total number of vertices (vertglbnbr) str << globalCells.size(); // Total number of connections (edgeglbnbr) str << ' ' << returnReduce(xadj[xadj.size()-1], sumOp<label>()) << nl; // Local number of vertices (vertlocnbr) str << xadj.size()-1; // Local number of connections (edgelocnbr) str << ' ' << xadj[xadj.size()-1] << nl; // Numbering starts from 0 label baseval = 0; // 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs str << baseval << ' ' << "000" << nl; for (label cellI = 0; cellI < xadj.size()-1; cellI++) { label start = xadj[cellI]; label end = xadj[cellI+1]; str << end-start; for (label i = start; i < end; i++) { str << ' ' << adjncy[i]; } str << nl; } } } // Strategy // ~~~~~~~~ // Default. SCOTCH_Strat stradat; check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit"); if (decompositionDict_.found("scotchCoeffs")) { const dictionary& scotchCoeffs = decompositionDict_.subDict("scotchCoeffs"); string strategy; if (scotchCoeffs.readIfPresent("strategy", strategy)) { if (debug) { Info<< "ptscotchDecomp : Using strategy " << strategy << endl; } SCOTCH_stratDgraphMap(&stradat, strategy.c_str()); //fprintf(stdout, "S\tStrat="); //SCOTCH_stratSave(&stradat, stdout); //fprintf(stdout, "\n"); } } // Graph // ~~~~~ List<int> velotab; // Check for externally provided cellweights and if so initialise weights scalar minWeights = gMin(cWeights); if (cWeights.size() > 0) { if (minWeights <= 0) { WarningIn ( "ptscotchDecomp::decompose(..)" ) << "Illegal minimum weight " << minWeights << endl; } if (cWeights.size() != xadj.size()-1) { FatalErrorIn ( "ptscotchDecomp::decompose(..)" ) << "Number of cell weights " << cWeights.size() << " does not equal number of cells " << xadj.size()-1 << exit(FatalError); } // Convert to integers. velotab.setSize(cWeights.size()); forAll(velotab, i) { velotab[i] = int(cWeights[i]/minWeights); } } if (debug) { Pout<< "SCOTCH_dgraphInit" << endl; } SCOTCH_Dgraph grafdat; check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit"); if (debug) { Pout<< "SCOTCH_dgraphBuild with:" << nl << "xadj.size()-1 : " << xadj.size()-1 << nl << "xadj : " << long(xadj.begin()) << nl << "velotab : " << long(velotab.begin()) << nl << "adjncy.size() : " << adjncy.size() << nl << "adjncy : " << long(adjncy.begin()) << nl << endl; } check ( SCOTCH_dgraphBuild ( &grafdat, // grafdat 0, // baseval, c-style numbering xadj.size()-1, // vertlocnbr, nCells xadj.size()-1, // vertlocmax const_cast<SCOTCH_Num*>(xadj.begin()), // vertloctab, start index per cell into // adjncy const_cast<SCOTCH_Num*>(&xadj[1]),// vendloctab, end index ,, const_cast<SCOTCH_Num*>(velotab.begin()),// veloloctab, vtx weights NULL, // vlblloctab adjncy.size(), // edgelocnbr, number of arcs adjncy.size(), // edgelocsiz const_cast<SCOTCH_Num*>(adjncy.begin()), // edgeloctab NULL, // edgegsttab NULL // edlotab, edge weights ), "SCOTCH_dgraphBuild" ); if (debug) { Pout<< "SCOTCH_dgraphCheck" << endl; } check(SCOTCH_dgraphCheck(&grafdat), "SCOTCH_dgraphCheck"); // Architecture // ~~~~~~~~~~~~ // (fully connected network topology since using switch) if (debug) { Pout<< "SCOTCH_archInit" << endl; } SCOTCH_Arch archdat; check(SCOTCH_archInit(&archdat), "SCOTCH_archInit"); List<label> processorWeights; if (decompositionDict_.found("scotchCoeffs")) { const dictionary& scotchCoeffs = decompositionDict_.subDict("scotchCoeffs"); scotchCoeffs.readIfPresent("processorWeights", processorWeights); } if (processorWeights.size()) { if (debug) { Info<< "ptscotchDecomp : Using procesor weights " << processorWeights << endl; } check ( SCOTCH_archCmpltw(&archdat, nProcessors_, processorWeights.begin()), "SCOTCH_archCmpltw" ); } else { if (debug) { Pout<< "SCOTCH_archCmplt" << endl; } check ( SCOTCH_archCmplt(&archdat, nProcessors_), "SCOTCH_archCmplt" ); } //SCOTCH_Mapping mapdat; //SCOTCH_dgraphMapInit(&grafdat, &mapdat, &archdat, NULL); //SCOTCH_dgraphMapCompute(&grafdat, &mapdat, &stradat); /*Perform mapping*/ //SCOTCHdgraphMapExit(&grafdat, &mapdat); // Hack:switch off fpu error trapping # ifdef LINUX_GNUC int oldExcepts = fedisableexcept ( FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW ); # endif if (debug) { Pout<< "SCOTCH_dgraphMap" << endl; } finalDecomp.setSize(xadj.size()-1); finalDecomp = 0; check ( SCOTCH_dgraphMap ( &grafdat, &archdat, &stradat, // const SCOTCH_Strat * finalDecomp.begin() // parttab ), "SCOTCH_graphMap" ); # ifdef LINUX_GNUC feenableexcept(oldExcepts); # endif //finalDecomp.setSize(xadj.size()-1); //check //( // SCOTCH_dgraphPart // ( // &grafdat, // nProcessors_, // partnbr // &stradat, // const SCOTCH_Strat * // finalDecomp.begin() // parttab // ), // "SCOTCH_graphPart" //); if (debug) { Pout<< "SCOTCH_dgraphExit" << endl; } // Release storage for graph SCOTCH_dgraphExit(&grafdat); // Release storage for strategy SCOTCH_stratExit(&stradat); // Release storage for network topology SCOTCH_archExit(&archdat); return 0; }