/*! \brief Copy data to host memory from a parallel buffer.
@param[in] size The number of entries to copy from \c hostSrc to \c buffDest.
@param[in] hostSrc The location in host memory from where the data is copied.
@param[out] buffDest The parallel buffer to which the data is copied.
\pre \c size is non-negative.
\pre \c hostSrc has length equal to \c size.
\pre \c buffSrc has length at least <tt>size</tt>.
\post On return, entries in the range <tt>[0 , size)</tt> of \c hostSrc are allowed to be written to. The data is guaranteed to be present in \c buffDest before it is used in a parallel computation.
*/
template <class T> inline
void copyToBuffer(size_t size, const ArrayView<const T> &hostSrc, const ArrayRCP<T> &buffDest) {
if (isHostNode == false) {
CHECK_COMPUTE_BUFFER(buffDest);
}
ArrayRCP<const T> buffSrc = arcpFromArrayView(hostSrc);
copyBuffers<T>(size,buffSrc,buffDest);
}
void AlgPTScotch(
const RCP<const Environment> &env, // parameters & app comm
const RCP<const Comm<int> > &problemComm, // problem comm
#ifdef HAVE_ZOLTAN2_MPI
MPI_Comm mpicomm,
#endif
const RCP<GraphModel<typename Adapter::base_adapter_t> > &model, // the graph
RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
typedef typename Adapter::lno_t lno_t;
typedef typename Adapter::gno_t gno_t;
typedef typename Adapter::scalar_t scalar_t;
int ierr = 0;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
SCOTCH_Num partnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(partnbr, numGlobalParts, env);
#ifdef HAVE_ZOLTAN2_MPI
const SCOTCH_Num baseval = 0; // Base value for array indexing.
// GraphModel returns GNOs from base 0.
SCOTCH_Strat stratstr; // Strategy string
// TODO: Set from parameters
SCOTCH_stratInit(&stratstr);
// Allocate & initialize PTScotch data structure.
SCOTCH_Dgraph *gr = SCOTCH_dgraphAlloc(); // Scotch distributed graph
ierr = SCOTCH_dgraphInit(gr, mpicomm);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphInit",
!ierr, BASIC_ASSERTION, problemComm);
// Get vertex info
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vtxWt;
size_t nVtx = model->getVertexList(vtxID, xyz, vtxWt);
SCOTCH_Num vertlocnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(vertlocnbr, nVtx, env);
SCOTCH_Num vertlocmax = vertlocnbr; // Assumes no holes in global nums.
// Get edge info
ArrayView<const gno_t> edgeIds;
ArrayView<const int> procIds;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
size_t nEdges = model->getEdgeList(edgeIds, procIds, offsets, ewgts);
SCOTCH_Num edgelocnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(edgelocnbr, nEdges, env);
const SCOTCH_Num edgelocsize = edgelocnbr; // Assumes adj array is compact.
SCOTCH_Num *vertloctab; // starting adj/vtx
SCOTCH_Num_Traits<lno_t>::ASSIGN_SCOTCH_NUM_ARRAY(&vertloctab, offsets, env);
SCOTCH_Num *edgeloctab; // adjacencies
SCOTCH_Num_Traits<gno_t>::ASSIGN_SCOTCH_NUM_ARRAY(&edgeloctab, edgeIds, env);
// We don't use these arrays, but we need them as arguments to Scotch.
SCOTCH_Num *vendloctab = NULL; // Assume consecutive storage for adj
SCOTCH_Num *vlblloctab = NULL; // Vertex label array
SCOTCH_Num *edgegsttab = NULL; // Array for ghost vertices
// Get weight info.
// TODO: Actually get the weights; for now, not using weights.
SCOTCH_Num *veloloctab = NULL; // Vertex weights
SCOTCH_Num *edloloctab = NULL; // Edge weights
//TODO int vwtdim = model->getVertexWeightDim();
//TODO int ewtdim = model->getEdgeWeightDim();
//TODO if (vwtdim) veloloctab = new SCOTCH_Num[nVtx];
//TODO if (ewtdim) edloloctab = new SCOTCH_Num[nEdges];
//TODO scale weights to SCOTCH_Nums.
// Build PTScotch distributed data structure
ierr = SCOTCH_dgraphBuild(gr, baseval, vertlocnbr, vertlocmax,
vertloctab, vendloctab, veloloctab, vlblloctab,
edgelocnbr, edgelocsize,
edgeloctab, edgegsttab, edloloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphBuild",
!ierr, BASIC_ASSERTION, problemComm);
// Create array for Scotch to return results in.
ArrayRCP<partId_t> partList(new partId_t [nVtx], 0, nVtx,true);
SCOTCH_Num *partloctab = NULL;
if (nVtx && (sizeof(SCOTCH_Num) == sizeof(partId_t))) {
// Can write directly into the solution's memory
partloctab = (SCOTCH_Num *) partList.getRawPtr();
}
else {
// Can't use solution memory directly; will have to copy later.
// Note: Scotch does not like NULL arrays, so add 1 to always have non-null.
// ParMETIS has this same "feature." See Zoltan bug 4299.
partloctab = new SCOTCH_Num[nVtx+1];
}
// Call partitioning; result returned in partloctab.
// TODO: Use SCOTCH_dgraphMap so can include a machine model in partitioning
ierr = SCOTCH_dgraphPart(gr, partnbr, &stratstr, partloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphPart",
!ierr, BASIC_ASSERTION, problemComm);
// TODO - metrics
#ifdef SHOW_ZOLTAN2_SCOTCH_MEMORY
int me = env->comm_->getRank();
#endif
#ifdef HAVE_SCOTCH_ZOLTAN2_GETMEMORYMAX
if (me == 0){
size_t scotchBytes = SCOTCH_getMemoryMax();
std::cout << "Rank " << me << ": Maximum bytes used by Scotch: ";
std::cout << scotchBytes << std::endl;
}
#endif
// Clean up PTScotch
SCOTCH_dgraphExit(gr);
delete gr;
SCOTCH_stratExit(&stratstr);
// Load answer into the solution.
if ((sizeof(SCOTCH_Num) != sizeof(partId_t)) || (nVtx == 0)) {
for (size_t i = 0; i < nVtx; i++) partList[i] = partloctab[i];
delete [] partloctab;
}
ArrayRCP<const gno_t> gnos = arcpFromArrayView(vtxID);
solution->setParts(gnos, partList, true);
env->memory("Zoltan2-Scotch: After creating solution");
//if (me == 0) cout << " done." << endl;
// Clean up Zoltan2
//TODO if (vwtdim) delete [] velotab;
//TODO if (ewtdim) delete [] edlotab;
// Clean up copies made due to differing data sizes.
SCOTCH_Num_Traits<lno_t>::DELETE_SCOTCH_NUM_ARRAY(&vertloctab);
SCOTCH_Num_Traits<gno_t>::DELETE_SCOTCH_NUM_ARRAY(&edgeloctab);
#else // DO NOT HAVE_MPI
// TODO: Handle serial case with calls to Scotch.
// TODO: For now, assign everything to rank 0 and assume only one part.
// TODO: Can probably use the code above for loading solution,
// TODO: instead of duplicating it here.
// TODO
// TODO: Actual logic should call Scotch when number of processes == 1.
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vtxWt;
size_t nVtx = model->getVertexList(vtxID, xyz, vtxWt);
for (size_t i = 0; i < nVtx; i++) partList[i] = 0;
#endif // DO NOT HAVE_MPI
}