GraphHelper_Scotch(const CrsMatrixType& A,
const int seed = GraphHelper::DefaultRandomSeed) {
_label = "GraphHelper_Scotch::" + A.Label();
_is_ordered = false;
_cblk = 0;
// scotch does not allow self-contribution (diagonal term in sparse matrix)
_base = 0; //A.BaseVal();
_m = A.NumRows();
_nnz = A.NumNonZeros();
_rptr = size_type_array(_label+"::RowPtrArray", _m+1);
_cidx = ordinal_type_array(_label+"::ColIndexArray", _nnz);
_perm = ordinal_type_array(_label+"::PermutationArray", _m);
_peri = ordinal_type_array(_label+"::InvPermutationArray", _m);
_range = ordinal_type_array(_label+"::RangeArray", _m);
_tree = ordinal_type_array(_label+"::TreeArray", _m);
// create a graph structure without diagonals
A.convertGraph(_nnz, _rptr, _cidx);
int ierr = 0;
ordinal_type *rptr = reinterpret_cast<ordinal_type*>(_rptr.ptr_on_device());
ordinal_type *cidx = reinterpret_cast<ordinal_type*>(_cidx.ptr_on_device());
if (seed != GraphHelper::DefaultRandomSeed) {
SCOTCH_randomSeed(seed);
SCOTCH_randomReset();
}
ierr = SCOTCH_graphInit(&_graph);
CHKERR(ierr);
ierr = SCOTCH_graphBuild(&_graph, // scotch graph
_base, // base value
_m, // # of vertices
rptr, // column index array pointer begin
rptr+1, // column index array pointer end
NULL, // weights on vertices (optional)
NULL, // label array on vertices (optional)
_nnz, // # of nonzeros
cidx, // column index array
NULL);
CHKERR(ierr); // edge load array (optional)
ierr = SCOTCH_graphCheck(&_graph);
CHKERR(ierr);
}
void setGraph(const ordinal_type m,
const size_type_array rptr,
const ordinal_type_array cidx) {
_is_ordered = false;
_cblk = 0;
/// Scotch graph spec
/// - no diagonals, symmetric
_base = 0;
_m = m;
_nnz = rptr[m];
_rptr = rptr;
_cidx = cidx;
_perm = ordinal_type_array("Scotch::PermutationArray", _m);
_peri = ordinal_type_array("Scotch::InvPermutationArray", _m);
_range = ordinal_type_array("Scotch::RangeArray", _m);
_tree = ordinal_type_array("Scotch::TreeArray", _m);
_strat = 0;
_level = 0;
int ierr = 0;
ordinal_type *rptr_ptr = reinterpret_cast<ordinal_type*>(_rptr.ptr_on_device());
ordinal_type *cidx_ptr = reinterpret_cast<ordinal_type*>(_cidx.ptr_on_device());
ierr = SCOTCH_graphInit(&_graph);
TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphInit");
ierr = SCOTCH_graphBuild(&_graph, // scotch graph
_base, // base value
_m, // # of vertices
rptr_ptr, // column index array pointer begin
rptr_ptr+1, // column index array pointer end
NULL, // weights on vertices (optional)
NULL, // label array on vertices (optional)
_nnz, // # of nonzeros
cidx_ptr, // column index array
NULL); // edge load array (optional)
TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphBuild");
ierr = SCOTCH_graphCheck(&_graph);
TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphCheck");
}
void computeOrdering(const ordinal_type treecut = 0) {
int ierr = 0;
// pointers for global graph ordering
ordinal_type *perm = _perm.ptr_on_device();
ordinal_type *peri = _peri.ptr_on_device();
ordinal_type *range = _range.ptr_on_device();
ordinal_type *tree = _tree.ptr_on_device();
{
// set desired tree level
if (_strat & SCOTCH_STRATLEVELMAX ||
_strat & SCOTCH_STRATLEVELMIN) {
TACHO_TEST_FOR_ABORT(_level == 0, "SCOTCH_STRATLEVEL(MIN/MAX) is used but level is not specified");
}
const int level = Util::max(1, _level-treecut);
SCOTCH_Strat stradat;
SCOTCH_Num straval = _strat;
ierr = SCOTCH_stratInit(&stradat);
TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_stratInit");
// if both are zero, do not build strategy
if (_strat || _level) {
if (_verbose)
std::cout << "GraphTools_Scotch:: User provide a strategy and/or level" << std::endl
<< " strategy = " << _strat << ", level = " << _level << ", treecut = " << treecut << std::endl
<< " strategy & SCOTCH_STRATLEVELMAX = " << (_strat & SCOTCH_STRATLEVELMAX) << std::endl
<< " strategy & SCOTCH_STRATLEVELMIN = " << (_strat & SCOTCH_STRATLEVELMIN) << std::endl
<< " strategy & SCOTCH_STRATLEAFSIMPLE = " << (_strat & SCOTCH_STRATLEAFSIMPLE) << std::endl
<< " strategy & SCOTCH_STRATSEPASIMPLE = " << (_strat & SCOTCH_STRATSEPASIMPLE) << std::endl
<< std::endl;
ierr = SCOTCH_stratGraphOrderBuild(&stradat, straval, level, 0.2);
TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_stratGraphOrderBuild");
}
ierr = SCOTCH_graphOrder(&_graph,
&stradat,
perm,
peri,
&_cblk,
range,
tree);
TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphOrder");
SCOTCH_stratExit(&stradat);
}
{
ordinal_type nroot = 0;
for (ordinal_type i=0; i<_cblk; ++i)
nroot += (_tree[i] == -1);
if (nroot > 1) {
if (_verbose)
std::cout << "GraphTools_Scotch:: # of roots " << nroot << std::endl
<< " a fake root is created to complete the tree" << std::endl
<< std::endl;
_tree [_cblk] = -1; // dummy root
_range[_cblk+1] = _range[_cblk]; // zero range for the dummy root
for (ordinal_type i=0; i<_cblk; ++i)
if (_tree[i] == -1) // multiple roots becomes children of the dummy root
_tree[i] = (_cblk+1);
++_cblk; // include the dummy root
}
}
_is_ordered = true;
//std::cout << "SCOTCH level = " << level << std::endl;
//std::cout << "Range Tree " << std::endl;
//for (int i=0;i<_cblk;++i)
// std::cout << _range[i] << " :: " << i << " " << _tree[i] << std::endl;
}
int computeOrdering() {
int r_val = 0;
camd_defaults(_control);
camd_control(_control);
ordinal_type *rptr = reinterpret_cast<ordinal_type*>(_rptr.ptr_on_device());
ordinal_type *cidx = reinterpret_cast<ordinal_type*>(_cidx.ptr_on_device());
ordinal_type *cnst = reinterpret_cast<ordinal_type*>(_cnst.ptr_on_device());
ordinal_type *next = reinterpret_cast<ordinal_type*>(_next.ptr_on_device());
ordinal_type *perm = reinterpret_cast<ordinal_type*>(_perm.ptr_on_device());
// length array
// assume there always is diagonal and the given matrix is symmetry
ordinal_type_array lwork(_label+"::LWorkArray", _m);
ordinal_type *lwork_ptr = reinterpret_cast<ordinal_type*>(lwork.ptr_on_device());
for (ordinal_type i=0;i<_m;++i)
lwork_ptr[i] = rptr[i+1] - rptr[i];
// workspace
const size_type swlen = _nnz + _nnz/5 + 5*(_m+1);;
ordinal_type_array swork(_label+"::SWorkArray", swlen);
ordinal_type *swork_ptr = reinterpret_cast<ordinal_type*>(swork.ptr_on_device());
ordinal_type *pe_ptr = reinterpret_cast<ordinal_type*>(_pe.ptr_on_device()); // 1) Pe
size_type pfree = 0;
for (ordinal_type i=0;i<_m;++i) {
pe_ptr[i] = pfree;
pfree += lwork_ptr[i];
}
if (_nnz != pfree)
ERROR(">> nnz in the graph does not match to nnz count (pfree)");
ordinal_type *nv_ptr = reinterpret_cast<ordinal_type*>(_nv.ptr_on_device()); // 2) Nv
ordinal_type *hd_ptr = swork_ptr; swork_ptr += (_m+1); // 3) Head
ordinal_type *el_ptr = reinterpret_cast<ordinal_type*>(_el.ptr_on_device()); // 4) Elen
ordinal_type *dg_ptr = swork_ptr; swork_ptr += _m; // 5) Degree
ordinal_type *wk_ptr = swork_ptr; swork_ptr += (_m+1); // 6) W
ordinal_type *bk_ptr = swork_ptr; swork_ptr += _m; // 7) BucketSet
const size_type iwlen = swlen - (4*_m+2);
ordinal_type *iw_ptr = swork_ptr; swork_ptr += iwlen; // Iw
for (ordinal_type i=0;i<pfree;++i)
iw_ptr[i] = cidx[i];
CAMD::run(_m, pe_ptr, iw_ptr, lwork_ptr, iwlen, pfree,
// output
nv_ptr, next, perm, hd_ptr, el_ptr, dg_ptr, wk_ptr,
_control, _info, cnst, bk_ptr);
r_val = (_info[CAMD_STATUS] != CAMD_OK);
for (ordinal_type i=0;i<_m;++i)
_peri[_perm[i]] = i;
_is_ordered = true;
return r_val;
}
int computeOrdering(const ordinal_type treecut = 15,
const ordinal_type minblksize = 0) {
int ierr = 0;
// pointers for global graph ordering
ordinal_type *perm = _perm.ptr_on_device();
ordinal_type *peri = _peri.ptr_on_device();
ordinal_type *range = _range.ptr_on_device();
ordinal_type *tree = _tree.ptr_on_device();
{
const int level = max(1, int(log2(_m)-treecut)); // level = log2(_nnz)+10;
SCOTCH_Strat stradat;
SCOTCH_Num straval = (SCOTCH_STRATLEVELMAX |
SCOTCH_STRATLEVELMIN |
SCOTCH_STRATLEAFSIMPLE |
SCOTCH_STRATSEPASIMPLE);
ierr = SCOTCH_stratInit(&stradat);CHKERR(ierr);
ierr = SCOTCH_stratGraphOrderBuild (&stradat, straval, level, 0.2);CHKERR(ierr);
ierr = SCOTCH_graphOrder(&_graph,
&stradat,
perm,
peri,
&_cblk,
range,
tree);CHKERR(ierr);
SCOTCH_stratExit(&stradat);
}
// provided blksize is greater than 0, reorder internally
// if (treecut > 0 && minblksize > 0) {
// // graph array
// ordinal_type *rptr = reinterpret_cast<ordinal_type*>(_rptr.ptr_on_device());
// ordinal_type *cidx = reinterpret_cast<ordinal_type*>(_cidx.ptr_on_device());
// // create workspace in
// size_type_array rptr_work = size_type_array(_label+"::Block::RowPtrArray", _m+1);
// ordinal_type_array cidx_work = ordinal_type_array(_label+"::Block::ColIndexArray", _nnz);
// // create workspace output
// ordinal_type_array perm_work = ordinal_type_array(_label+"::Block::PermutationArray", _m);
// ordinal_type_array peri_work = ordinal_type_array(_label+"::Block::InvPermutationArray", _m);
// ordinal_type_array range_work = ordinal_type_array(_label+"::Block::RangeArray", _m);
// ordinal_type_array tree_work = ordinal_type_array(_label+"::Block::TreeArray", _m);
// // scotch input
// ordinal_type *rptr_blk = reinterpret_cast<ordinal_type*>(rptr_work.ptr_on_device());
// ordinal_type *cidx_blk = reinterpret_cast<ordinal_type*>(cidx_work.ptr_on_device());
// size_type nnz = 0;
// rptr_blk[0] = nnz;
// for (ordinal_type iblk=0;iblk<_cblk;++iblk) {
// // allocate graph
// SCOTCH_Graph graph;
// ierr = SCOTCH_graphInit(&graph);CHKERR(ierr);
// SCOTCH_Strat stradat;
// SCOTCH_Num straval = (/*SCOTCH_STRATLEVELMAX |
// SCOTCH_STRATLEVELMIN |*/
// SCOTCH_STRATLEAFSIMPLE |
// SCOTCH_STRATSEPASIMPLE);
// ierr = SCOTCH_stratInit(&stradat);CHKERR(ierr);
// ierr = SCOTCH_stratGraphOrderBuild(&stradat, straval, 0, 0.2);CHKERR(ierr);
// const ordinal_type ibegin = range[iblk], iend = range[iblk+1], m = iend - ibegin;
// // scotch output
// ordinal_type cblk_blk = 0;
// ordinal_type *perm_blk = perm_work.ptr_on_device() + ibegin;
// ordinal_type *peri_blk = peri_work.ptr_on_device() + ibegin;
// ordinal_type *range_blk = range_work.ptr_on_device() + ibegin;
// ordinal_type *tree_blk = tree_work.ptr_on_device() + ibegin;
// // if each blk is greater than the given minblksize, reorder internally
// if (m > minblksize) {
// for (int i=ibegin;i<iend;++i) {
// const ordinal_type ii = peri[i];
// const ordinal_type jbegin = rptr[ii];
// const ordinal_type jend = rptr[ii+1];
// for (int j=jbegin;j<jend;++j) {
// const ordinal_type jj = perm[cidx[j]];
// if (ibegin <= jj && jj < iend)
// cidx_blk[nnz++] = (jj - ibegin);
// }
// rptr_blk[i+1] = nnz;
// }
// const size_type nnz_blk = nnz - rptr_blk[ibegin];
// ierr = SCOTCH_graphBuild(&graph, // scotch graph
// 0, // base value
// m, // # of vertices
// &rptr_blk[ibegin], // column index array pointer begin
// &rptr_blk[ibegin]+1,// column index array pointer end
// NULL, // weights on vertices (optional)
// NULL, // label array on vertices (optional)
// nnz_blk, // # of nonzeros
// cidx_blk, // column index array
// NULL);CHKERR(ierr); // edge load array (optional)
// ierr = SCOTCH_graphCheck(&graph);CHKERR(ierr);
// ierr = SCOTCH_graphOrder(&graph,
// &stradat,
// perm_blk,
// peri_blk,
// &cblk_blk,
// range_blk,
// tree_blk);CHKERR(ierr);
// } else {
// for (ordinal_type i=0;i<m;++i) {
// perm_blk[i] = i;
// peri_blk[i] = i;
// }
// range_blk[1] = m;
// tree_blk[0] = -1;
// }
// SCOTCH_stratExit(&stradat);
// SCOTCH_graphFree(&graph);
// for (ordinal_type i=0;i<m;++i) {
// const ordinal_type ii = peri_blk[i] + ibegin;
// peri_blk[i] = peri[ii];
// }
// for (ordinal_type i=0;i<m;++i) {
// const ordinal_type ii = i + ibegin;
// peri[ii] = peri_blk[i];
// }
// }
// for (ordinal_type i=0;i<_m;++i)
// perm[peri[i]] = i;
// }
_is_ordered = true;
//cout << "SCOTCH level = " << level << endl;
//cout << "Range Tree " << endl;
//for (int i=0;i<_cblk;++i)
// cout << _range[i] << " :: " << i << " " << _tree[i] << endl;
return 0;
}
