void computeOrdering(const ordinal_type treecut = 0) { int ierr = 0; // pointers for global graph ordering ordinal_type *perm = _perm.ptr_on_device(); ordinal_type *peri = _peri.ptr_on_device(); ordinal_type *range = _range.ptr_on_device(); ordinal_type *tree = _tree.ptr_on_device(); { // set desired tree level if (_strat & SCOTCH_STRATLEVELMAX || _strat & SCOTCH_STRATLEVELMIN) { TACHO_TEST_FOR_ABORT(_level == 0, "SCOTCH_STRATLEVEL(MIN/MAX) is used but level is not specified"); } const int level = Util::max(1, _level-treecut); SCOTCH_Strat stradat; SCOTCH_Num straval = _strat; ierr = SCOTCH_stratInit(&stradat); TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_stratInit"); // if both are zero, do not build strategy if (_strat || _level) { if (_verbose) std::cout << "GraphTools_Scotch:: User provide a strategy and/or level" << std::endl << " strategy = " << _strat << ", level = " << _level << ", treecut = " << treecut << std::endl << " strategy & SCOTCH_STRATLEVELMAX = " << (_strat & SCOTCH_STRATLEVELMAX) << std::endl << " strategy & SCOTCH_STRATLEVELMIN = " << (_strat & SCOTCH_STRATLEVELMIN) << std::endl << " strategy & SCOTCH_STRATLEAFSIMPLE = " << (_strat & SCOTCH_STRATLEAFSIMPLE) << std::endl << " strategy & SCOTCH_STRATSEPASIMPLE = " << (_strat & SCOTCH_STRATSEPASIMPLE) << std::endl << std::endl; ierr = SCOTCH_stratGraphOrderBuild(&stradat, straval, level, 0.2); TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_stratGraphOrderBuild"); } ierr = SCOTCH_graphOrder(&_graph, &stradat, perm, peri, &_cblk, range, tree); TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphOrder"); SCOTCH_stratExit(&stradat); } { ordinal_type nroot = 0; for (ordinal_type i=0; i<_cblk; ++i) nroot += (_tree[i] == -1); if (nroot > 1) { if (_verbose) std::cout << "GraphTools_Scotch:: # of roots " << nroot << std::endl << " a fake root is created to complete the tree" << std::endl << std::endl; _tree [_cblk] = -1; // dummy root _range[_cblk+1] = _range[_cblk]; // zero range for the dummy root for (ordinal_type i=0; i<_cblk; ++i) if (_tree[i] == -1) // multiple roots becomes children of the dummy root _tree[i] = (_cblk+1); ++_cblk; // include the dummy root } } _is_ordered = true; //std::cout << "SCOTCH level = " << level << std::endl; //std::cout << "Range Tree " << std::endl; //for (int i=0;i<_cblk;++i) // std::cout << _range[i] << " :: " << i << " " << _tree[i] << std::endl; }
bool ScotchSplitter(unsigned dim, const int* ptRows, const int* indCols, unsigned& nbMaxLevels, unsigned minSize, int* loc2glob, int* glob2loc, int& nbDoms, int*& ptOnDomains, int*& sizeOfDomains, bool checkData, const bool verbose, FILE *fp) { int ierr; // check consistency between Scotch and Dissection library CHECK(sizeof(int) == sizeof(SCOTCH_Num), "Incompatible integer representation between Scotch and Dissection"); if (verbose) { int vers, rela, patc; SCOTCH_version(&vers, &rela, &patc); diss_printf(verbose, fp, "%s %d : Soctch version : %d.%d.%d\n", __FILE__, __LINE__, vers, rela, patc); } // Allocate and initialize the Scotch graph SCOTCH_Graph ptGraph; ierr = SCOTCH_graphInit(&ptGraph); CHECK(ierr==0,"Fail initializing Scotch graph"); // TRACE("Building Scotch graph\n"); ierr = SCOTCH_graphBuild(&ptGraph, (SCOTCH_Num)0, // offset, (SCOTCH_Num)dim, (const SCOTCH_Num*)ptRows, (const SCOTCH_Num*)ptRows+1, NULL, NULL, (SCOTCH_Num)ptRows[dim], (const SCOTCH_Num*)indCols, NULL); // if (verbose) { // fprintf(fp, "%s %d : 1 : ptRows = %p indCols = %p\n", // __FILE__, __LINE__, (void *)ptRows, (void *)indCols); // } CHECK(ierr==0,"Scotch graph building failed !"); if (checkData) { //TRACE("Check Scotch graph\n"); ierr = SCOTCH_graphCheck(&ptGraph); if (ierr) { diss_printf(verbose, fp, "Failed the checking of the graph : bad data ?\n"); SCOTCH_graphFree(&ptGraph); return false; } } // Allocate and initialize the strategy wanted for Scotch // TRACE("Initialize strategy for splitting\n"); SCOTCH_Strat ptStrat; ierr = SCOTCH_stratInit(&ptStrat); CHECK(ierr==0, "Failed initializing Scotch strategy structure"); char *str_Strat = new char[1024]; int nbLvls = std::min(unsigned(nbMaxLevels), highestbit(unsigned(dim/minSize))); diss_printf(verbose, fp, "nbLevels = %d\n", nbLvls); sprintf(str_Strat,"c{rat=0.7,cpr=n{sep=/((levl<%d)|(vert>%d))?m{type=h,rat=0.7,vert=100,low=h{pass=10},asc=b{width=3,bnd=f{bal=0.2},org=h{pass=10}f{bal=0.2}}}|m{type=h,rat=0.7,vert=100,low=h{pass=10},asc=b{width=3,bnd=f{bal=0.2},org=h{pass=10}f{bal=0.2}}};,ole=f{cmin=%d,cmax=%d,frat=0.05},ose=s},unc=n{sep=/(levl<%d)?(m{type=h,rat=0.7,vert=100,low=h{pass=10},asc=b{width=3,bnd=f{bal=0.2},org=h{pass=10}f{bal=0.2}}})|m{type=h,rat=0.7,vert=100,low=h{pass=10},asc=b{width=3,bnd=f{bal=0.2},org=h{pass=10}f{bal=0.2}}};,ole=f{cmin=%d,cmax=%d,frat=0.05},ose=s}}", nbLvls-1,2*minSize-1,minSize,dim,nbLvls-1,minSize,dim); // DBG_PRINT("Strategy string : %s\n", str_Strat); ierr = SCOTCH_stratGraphOrder(&ptStrat, str_Strat); delete [] str_Strat; CHECK(ierr==0, "Failed build graph ordering strategy for Scotch"); // Ordering with nested bisection : // TRACE("Split the graph\n"); int* rangtab = new int[dim+1]; int* treetab = new int[dim]; int nbSplitDoms; bool repeat = true; int lastCompleteLevel; int *levels, *nbDomsPerLevels; SCOTCH_randomReset(); while (repeat) { ierr = SCOTCH_graphOrder(&ptGraph, &ptStrat, (SCOTCH_Num*)loc2glob, (SCOTCH_Num*)glob2loc, (SCOTCH_Num*)&nbSplitDoms, (SCOTCH_Num*)rangtab, (SCOTCH_Num*)treetab); if (ierr) { diss_printf(verbose, fp, "Failed reordering sparse matrix graph !\n"); SCOTCH_stratExit(&ptStrat); SCOTCH_graphFree(&ptGraph); return false; } levels = new int[nbSplitDoms]; // int *nbDomsPerLevels;// = new int[nbLvls]; unsigned nbLvlsScotch= compLevelOfDoms(nbSplitDoms, nbLvls, treetab, levels, nbDomsPerLevels); /** Search last level where number of domains is a power of two */ lastCompleteLevel = 0; while ((lastCompleteLevel<nbLvlsScotch) && ((1<<lastCompleteLevel) == nbDomsPerLevels[lastCompleteLevel]) ) { lastCompleteLevel ++; } lastCompleteLevel = std::min(lastCompleteLevel, nbLvls); nbDoms = (1<<lastCompleteLevel)-1; // Search where start each domain per bisection level // and compute the size of each subdomain : // int indDom = 0; bool flag_size_check = false; for (int i = 0; i < nbSplitDoms; i++) { int sz = 0; while (levels[i]>=lastCompleteLevel) { sz += rangtab[i+1]-rangtab[i]; i++; } if (sz+rangtab[i+1]-rangtab[i] <= TOO_SMALL) { flag_size_check = true; break; } //DBG_PRINT("Domain %d begin at %d\n",indDom+1,begDom); } // loop : i if (!flag_size_check) { repeat = false; break; } else { delete [] levels; delete [] nbDomsPerLevels; } } // while (repeat) ptOnDomains = new int[nbDoms+1]; sizeOfDomains = new int[nbDoms]; memset(sizeOfDomains, 0, nbDoms*sizeof(int)); int* indDomPerLevel = new int[lastCompleteLevel+1]; memset(indDomPerLevel,0,(lastCompleteLevel+1)*sizeof(int)); int begDom = 0; for (int i = 0; i < nbSplitDoms; i++) { int sz = 0; while (levels[i]>=lastCompleteLevel) { sz += rangtab[i+1]-rangtab[i]; i++; } int indDom = (1<<levels[i])-1+indDomPerLevel[levels[i]]; // DBG_PRINT("level %d : current dom = %d\n", levels[i],indDom+1); ptOnDomains[indDom] = begDom; //DBG_PRINT("Domain %d begin at %d\n",indDom+1,begDom); sizeOfDomains[indDom] = sz+rangtab[i+1]-rangtab[i]; //DBG_PRINT("Domain %d size of %d\n",indDom+1,sizeOfDomains[indDom]); begDom += sizeOfDomains[indDom]; indDomPerLevel[levels[i]] += 1; } diss_printf(verbose, fp, "%s %d : indDomPerlevel[lastCompleteLevel] = %d\n", __FILE__, __LINE__, indDomPerLevel[lastCompleteLevel]); ptOnDomains[nbDoms] = dim; nbMaxLevels = lastCompleteLevel; delete [] indDomPerLevel; delete [] nbDomsPerLevels; delete [] levels; delete [] treetab; delete [] rangtab; // Cleaning all Scotch structures : SCOTCH_stratExit(&ptStrat); SCOTCH_graphFree(&ptGraph); return true; }