int SCOTCH_graphPart ( SCOTCH_Graph * const grafptr, /*+ Graph to map +*/ const SCOTCH_Num partnbr, /*+ Number of parts +*/ SCOTCH_Strat * const stratptr, /*+ Mapping strategy +*/ SCOTCH_Num * const maptab) /*+ Mapping array +*/ { SCOTCH_Arch archdat; int o; SCOTCH_archInit (&archdat); SCOTCH_archCmplt (&archdat, partnbr); o = SCOTCH_graphMap (grafptr, &archdat, stratptr, maptab); SCOTCH_archExit (&archdat); return (o); }
// Call scotch with options from dictionary. Foam::label Foam::scotchDecomp::decompose ( const List<int>& adjncy, const List<int>& xadj, const scalarField& cWeights, List<int>& finalDecomp ) { // Dump graph if (decompositionDict_.found("scotchCoeffs")) { const dictionary& scotchCoeffs = decompositionDict_.subDict("scotchCoeffs"); if (scotchCoeffs.found("writeGraph")) { Switch writeGraph(scotchCoeffs.lookup("writeGraph")); if (writeGraph) { OFstream str(mesh_.time().path() / mesh_.name() + ".grf"); Info<< "Dumping Scotch graph file to " << str.name() << endl << "Use this in combination with gpart." << endl; label version = 0; str << version << nl; // Numer of vertices str << xadj.size()-1 << ' ' << adjncy.size() << nl; // Numbering starts from 0 label baseval = 0; // Has weights? label hasEdgeWeights = 0; label hasVertexWeights = 0; label numericflag = 10*hasEdgeWeights+hasVertexWeights; str << baseval << ' ' << numericflag << nl; for (label cellI = 0; cellI < xadj.size()-1; cellI++) { label start = xadj[cellI]; label end = xadj[cellI+1]; str << end-start; for (label i = start; i < end; i++) { str << ' ' << adjncy[i]; } str << nl; } } } } // Strategy // ~~~~~~~~ // Default. SCOTCH_Strat stradat; check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit"); if (decompositionDict_.found("scotchCoeffs")) { const dictionary& scotchCoeffs = decompositionDict_.subDict("scotchCoeffs"); string strategy; if (scotchCoeffs.readIfPresent("strategy", strategy)) { if (debug) { Info<< "scotchDecomp : Using strategy " << strategy << endl; } SCOTCH_stratGraphMap(&stradat, strategy.c_str()); //fprintf(stdout, "S\tStrat="); //SCOTCH_stratSave(&stradat, stdout); //fprintf(stdout, "\n"); } } // Graph // ~~~~~ List<int> velotab; // Check for externally provided cellweights and if so initialise weights scalar minWeights = gMin(cWeights); if (cWeights.size() > 0) { if (minWeights <= 0) { WarningIn ( "scotchDecomp::decompose" "(const pointField&, const scalarField&)" ) << "Illegal minimum weight " << minWeights << endl; } if (cWeights.size() != xadj.size()-1) { FatalErrorIn ( "scotchDecomp::decompose" "(const pointField&, const scalarField&)" ) << "Number of cell weights " << cWeights.size() << " does not equal number of cells " << xadj.size()-1 << exit(FatalError); } // Convert to integers. velotab.setSize(cWeights.size()); forAll(velotab, i) { velotab[i] = int(cWeights[i]/minWeights); } } SCOTCH_Graph grafdat; check(SCOTCH_graphInit(&grafdat), "SCOTCH_graphInit"); check ( SCOTCH_graphBuild ( &grafdat, 0, // baseval, c-style numbering xadj.size()-1, // vertnbr, nCells xadj.begin(), // verttab, start index per cell into adjncy &xadj[1], // vendtab, end index ,, velotab.begin(), // velotab, vertex weights NULL, // vlbltab adjncy.size(), // edgenbr, number of arcs adjncy.begin(), // edgetab NULL // edlotab, edge weights ), "SCOTCH_graphBuild" ); check(SCOTCH_graphCheck(&grafdat), "SCOTCH_graphCheck"); // Architecture // ~~~~~~~~~~~~ // (fully connected network topology since using switch) SCOTCH_Arch archdat; check(SCOTCH_archInit(&archdat), "SCOTCH_archInit"); List<label> processorWeights; if (decompositionDict_.found("scotchCoeffs")) { const dictionary& scotchCoeffs = decompositionDict_.subDict("scotchCoeffs"); scotchCoeffs.readIfPresent("processorWeights", processorWeights); } if (processorWeights.size()) { if (debug) { Info<< "scotchDecomp : Using procesor weights " << processorWeights << endl; } check ( SCOTCH_archCmpltw(&archdat, nProcessors_, processorWeights.begin()), "SCOTCH_archCmpltw" ); } else { check ( SCOTCH_archCmplt(&archdat, nProcessors_), "SCOTCH_archCmplt" ); } //SCOTCH_Mapping mapdat; //SCOTCH_graphMapInit(&grafdat, &mapdat, &archdat, NULL); //SCOTCH_graphMapCompute(&grafdat, &mapdat, &stradat); /* Perform mapping */ //SCOTCH_graphMapExit(&grafdat, &mapdat); // Hack:switch off fpu error trapping # ifdef LINUX_GNUC int oldExcepts = fedisableexcept ( FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW ); # endif finalDecomp.setSize(xadj.size()-1); finalDecomp = 0; check ( SCOTCH_graphMap ( &grafdat, &archdat, &stradat, // const SCOTCH_Strat * finalDecomp.begin() // parttab ), "SCOTCH_graphMap" ); # ifdef LINUX_GNUC feenableexcept(oldExcepts); # endif //finalDecomp.setSize(xadj.size()-1); //check //( // SCOTCH_graphPart // ( // &grafdat, // nProcessors_, // partnbr // &stradat, // const SCOTCH_Strat * // finalDecomp.begin() // parttab // ), // "SCOTCH_graphPart" //); // Release storage for graph SCOTCH_graphExit(&grafdat); // Release storage for strategy SCOTCH_stratExit(&stradat); // Release storage for network topology SCOTCH_archExit(&archdat); return 0; }
static void run_scotch( Container &c, MapContainer &mapping, const std::size_t cores, weight_function_t weight_func, void *weight ) { #if 0 static_assert( std::is_signed< std::remove_reference< decltype( c.end() ) >::type >::value, "Container must have signed types so that -1 may signify no mapping" ); #endif raftgraph_t raft_graph; get_graph_info( c, raft_graph, weight_func, nullptr ); SCOTCH_Graph graph; if( SCOTCH_graphInit( &graph ) != 0 ) { /** TODO, add RaftLib Exception **/ std::cerr << "Failed to initialize graph!!\n"; exit( EXIT_FAILURE ); } auto table( raft_graph.getScotchTables() ); if( SCOTCH_graphBuild( &graph /** graph ptr **/, 0 /** base value **/, table.num_vertices /** vertex nmbr (zero indexed) **/, table.vtable /** vertex tab **/, &table.vtable[ 1 ] /** vendtab **/, nullptr /** velotab **/, nullptr /** vlbltab **/, table.num_edges /** edge number **/, table.etable /** edge tab **/, table.eweight /** edlotab **/ ) != 0 ) { /** TODO, add RaftLib Exception **/ std::cerr << "Failed to build graph\n"; exit( EXIT_FAILURE ); } if( SCOTCH_graphCheck( &graph ) != 0 ) { /** TODO, add RaftLib Exception **/ std::cerr << "Graph is inconsistent\n"; std::remove_reference< decltype( table ) >::type::print( std::cerr, table ); std::cerr << "\n"; raft_graph.print( std::cerr ); exit( EXIT_FAILURE ); } /** TODO, we can do much more with this arch file **/ SCOTCH_Arch archdat; if( SCOTCH_archInit( &archdat ) != 0 ) { /** TODO, add RaftLib Exception **/ std::cerr << "Architecture initialization failed\n"; exit( EXIT_FAILURE ); } /** core are equal **/ if( SCOTCH_archCmplt( &archdat, cores /** num cores **/) != 0 ) { /** TODO, add RaftLib Exception **/ std::cerr << "Failed to create architecture file\n"; exit( EXIT_FAILURE ); } /** strategy **/ SCOTCH_Strat stradat; if( SCOTCH_stratInit( &stradat ) != 0 ) { /** TODO, add RaftLib Exception **/ std::cerr << "Failed to init strategy!!\n"; exit( EXIT_FAILURE ); } /** build recursive strategy **/ if( SCOTCH_stratGraphClusterBuild( &stradat, SCOTCH_STRATSPEED, cores, .75, .01) != 0 ) { /** TODO, add RaftLib Exception **/ std::cerr << "Failed to map strategy graph!!\n"; exit( EXIT_FAILURE ); } if( SCOTCH_graphMap( &graph /** graph ptr **/, &archdat, &stradat, table.partition /** parttab **/ ) != 0 ) { /** TODO, add RaftLib Exception **/ std::cerr << "Failed to map!!\n"; exit( EXIT_FAILURE ); } /** * first case is for if we've mapped all vertices, * second is for when some of the kernels are innactive * in which case the number of vertices in the * table will be less than the size of c in which case * we need to get which vertices (the actual number id * from the application) are mapped and to where, the * returned table in mapping must include even the * vertices that aren't active (indicated by a -1) so * that the returning loop can be as simple as possible */ if( c.size() == table.num_vertices ) { /** copy mapping **/ for( auto i( 0 ); i < table.num_vertices; i++ ) { mapping.emplace_back( table.partition[ i ] ); } } else { const auto &vmapping( raft_graph.getVertexNumbersAtIndicies() ); auto it_map_index( vmapping.cbegin() ); auto table_index( 0 ); const auto size( c.size() ); for( auto i( 0 ); i < size; i++ ) { if( i == (*it_map_index) && it_map_index != vmapping.cend() ) { mapping.emplace_back( table.partition[ table_index++ ] ); ++it_map_index; } else { mapping.emplace_back( -1 ); } } } /** call exit graph **/ SCOTCH_graphExit( &graph ); SCOTCH_stratExit( &stradat ); SCOTCH_archExit ( &archdat ); return; }
/* Internal function : kPartBoxCompute * it computes a new numbering of graph vertices, using a k-partitioning. * Assuming that baseval of the graph is 1 * * - graf : the input graph * - vertNbr : the number of vertices * - boxVertNbr : the number of vertices of each box * - permVrtTab : the new numbering * * returning 0 if OK, 1 else */ int kPartBoxCompute(SCOTCH_Graph graf, int vertNbr, int boxVertNbr, SCOTCH_Num *permVrtTab) { int boxNbr, vertIdx; SCOTCH_Num logMaxVal, SupMaxVal, InfMaxVal, maxVal; char s[200]; SCOTCH_Num *sortPartTb; SCOTCH_Strat strat ; SCOTCH_Arch arch; /* Computing the number of boxes */ boxNbr = vertNbr / boxVertNbr; if (boxNbr * boxVertNbr != vertNbr) { boxNbr = boxNbr + 1; } /* Initializing SCOTCH functions */ CHECK_SCOTCH(SCOTCH_stratInit(&strat), "scotch_stratInit", 0) ; CHECK_SCOTCH(SCOTCH_archVcmplt(&arch), "scotch_archVcmplt", 0) ; sprintf(s, "m{vert=%d,low=r{job=t,map=t,poli=S,sep=m{type=h,vert=80,low=h{pass=10}f{bal=0.0005,move=80},asc=f{bal=0.005,move=80}}}}", vertNbr / boxVertNbr); CHECK_SCOTCH(SCOTCH_stratGraphMap(&strat, s), "scotch_stratGraphMap", 0) ; sortPartTb= (SCOTCH_Num *)M_calloc(2*vertNbr, sizeof(SCOTCH_Num), "boxCompute"); /* Partionning the graph */ CHECK_SCOTCH(SCOTCH_graphMap(&graf, &arch, &strat, sortPartTb), "scotch_graphMap", 0); // Looking for the max value in sortPartTb and computing sortPartTb as // followed : // - sortPartTb[2i] is the box value // - sortPartTb[2i+1] is the vertex number maxVal = sortPartTb[0]; for (vertIdx = vertNbr - 1 ; vertIdx >= 0 ; vertIdx--) { sortPartTb[2*vertIdx] = sortPartTb[vertIdx]; sortPartTb[2*vertIdx+1] = vertIdx + 1; if (sortPartTb[vertIdx] > maxVal) maxVal = sortPartTb[vertIdx]; } // Determining the log of MaxVal logMaxVal = 0; while ( maxVal > 0) { logMaxVal++; maxVal >>= 1; } // Infering the interval in which box values will be InfMaxVal = logMaxVal << logMaxVal; SupMaxVal = (logMaxVal << (logMaxVal + 1)) - 1; // Increasing box values until they are in the previous interval for (vertIdx = 0 ; vertIdx < vertNbr ; vertIdx++) { while (!(sortPartTb[2*vertIdx] >= InfMaxVal && sortPartTb[2*vertIdx] <= SupMaxVal)) { sortPartTb[2*vertIdx] <<= 1; } } // Sorting the tabular, which contains box values and vertex numbers _SCOTCHintSort2asc1(sortPartTb, vertNbr); /* Infering the new numbering */ for (vertIdx = 0; vertIdx < vertNbr ; vertIdx++) { permVrtTab[sortPartTb[2*vertIdx + 1]] = vertIdx + 1; } SCOTCH_stratExit(&strat) ; SCOTCH_archExit(&arch) ; M_free(sortPartTb); return 0; }