void globalWeightedCutsMessagesByPart(
const RCP<const Environment> &env,
const RCP<const Comm<int> > &comm,
const RCP<const GraphModel<typename Adapter::base_adapter_t> > &graph,
const ArrayView<const typename Adapter::part_t> &parts,
typename Adapter::part_t &numParts,
ArrayRCP<RCP<BaseClassMetrics<typename Adapter::scalar_t> > > &metrics,
ArrayRCP<typename Adapter::scalar_t> &globalSums)
{
env->debug(DETAILED_STATUS, "Entering globalWeightedCutsMessagesByPart");
//////////////////////////////////////////////////////////
// Initialize return values
typedef typename Adapter::lno_t t_lno_t;
typedef typename Adapter::gno_t t_gno_t;
typedef typename Adapter::scalar_t t_scalar_t;
typedef typename Adapter::part_t part_t;
typedef typename Adapter::node_t t_node_t;
typedef typename Zoltan2::GraphModel<typename Adapter::base_adapter_t>::input_t t_input_t;
t_lno_t localNumVertices = graph->getLocalNumVertices();
t_gno_t globalNumVertices = graph->getGlobalNumVertices();
t_lno_t localNumEdges = graph->getLocalNumEdges();
ArrayView<const t_gno_t> Ids;
ArrayView<t_input_t> v_wghts;
graph->getVertexList(Ids, v_wghts);
typedef GraphMetrics<t_scalar_t> mv_t;
//get the edge ids, and weights
ArrayView<const t_gno_t> edgeIds;
ArrayView<const t_lno_t> offsets;
ArrayView<t_input_t> e_wgts;
graph->getEdgeList(edgeIds, offsets, e_wgts);
std::vector <t_scalar_t> edge_weights;
int numWeightPerEdge = graph->getNumWeightsPerEdge();
int numMetrics = 2; // "edge cuts", messages
if (numWeightPerEdge) numMetrics += numWeightPerEdge; // "weight n"
// add some more metrics to the array
typedef typename ArrayRCP<RCP<BaseClassMetrics<typename Adapter::scalar_t> > >::size_type array_size_type;
metrics.resize( metrics.size() + numMetrics );
for( array_size_type n = metrics.size() - numMetrics; n < metrics.size(); ++n ){
mv_t * newMetric = new mv_t; // allocate the new memory
env->localMemoryAssertion(__FILE__,__LINE__,1,newMetric); // check errors
metrics[n] = rcp( newMetric); // create the new members
}
array_size_type next = metrics.size() - numMetrics; // MDM - this is most likely temporary to preserve the format here - we are now filling a larger array so we may not have started at 0
std::vector <part_t> e_parts (localNumEdges);
#ifdef HAVE_ZOLTAN2_MPI
if (comm->getSize() > 1)
{
Zoltan_DD_Struct *dd = NULL;
MPI_Comm mpicomm = Teuchos::getRawMpiComm(*comm);
int size_gnot = Zoltan2::TPL_Traits<ZOLTAN_ID_PTR, t_gno_t>::NUM_ID;
int debug_level = 0;
Zoltan_DD_Create(&dd, mpicomm,
size_gnot, 0,
sizeof(part_t), localNumVertices, debug_level);
ZOLTAN_ID_PTR ddnotneeded = NULL; // Local IDs not needed
Zoltan_DD_Update(
dd,
(ZOLTAN_ID_PTR) Ids.getRawPtr(),
ddnotneeded,
(char *) &(parts[0]),
NULL,
int(localNumVertices));
Zoltan_DD_Find(
dd,
(ZOLTAN_ID_PTR) edgeIds.getRawPtr(),
ddnotneeded,
(char *)&(e_parts[0]),
NULL,
localNumEdges,
NULL
);
Zoltan_DD_Destroy(&dd);
} else
#endif
{
std::map<t_gno_t,t_lno_t> global_id_to_local_index;
//else everything is local.
//we need a globalid to local index conversion.
//this does not exists till this point, so we need to create one.
for (t_lno_t i = 0; i < localNumVertices; ++i){
//at the local index i, we have the global index Ids[i].
//so write i, to Ids[i] index of the vector.
global_id_to_local_index[Ids[i]] = i;
}
for (t_lno_t i = 0; i < localNumEdges; ++i){
t_gno_t ei = edgeIds[i];
//ei is the global index of the neighbor one.
part_t p = parts[global_id_to_local_index[ei]];
e_parts[i] = p;
}
}
RCP<const Teuchos::Comm<int> > tcomm = comm;
env->timerStart(MACRO_TIMERS, "Communication Graph Create");
{
//get the vertices in each part in my part.
std::vector <t_lno_t> part_begins(numParts, -1);
std::vector <t_lno_t> part_nexts(localNumVertices, -1);
//cluster vertices according to their parts.
//create local part graph.
for (t_lno_t i = 0; i < localNumVertices; ++i){
part_t ap = parts[i];
part_nexts[i] = part_begins[ap];
part_begins[ap] = i;
}
for (int weight_index = -1; weight_index < numWeightPerEdge ; ++weight_index){
//MD: these two should be part_t.
//but we dont want to compile tpetra from the beginning.
//This can be changed when directory is updated.
typedef t_lno_t local_part_type;
typedef t_gno_t global_part_type;
typedef Tpetra::Map<local_part_type, global_part_type, t_node_t> map_t;
Teuchos::RCP<const map_t> map = Teuchos::rcp (new map_t (numParts, 0, tcomm));
typedef Tpetra::CrsMatrix<t_scalar_t, local_part_type, global_part_type, t_node_t> tcrsMatrix_t;
Teuchos::RCP<tcrsMatrix_t> tMatrix(new tcrsMatrix_t (map, 0));
std::vector <global_part_type> part_neighbors (numParts);
std::vector <t_scalar_t> part_neighbor_weights(numParts, 0);
std::vector <t_scalar_t> part_neighbor_weights_ordered(numParts);
//coarsen for all vertices in my part in order with parts.
for (global_part_type i = 0; i < (global_part_type) numParts; ++i){
part_t num_neighbor_parts = 0;
t_lno_t v = part_begins[i];
//get part i, and first vertex in this part v.
while (v != -1){
//now get the neightbors of v.
for (t_lno_t j = offsets[v]; j < offsets[v+1]; ++j){
//get the part of the second vertex.
part_t ep = e_parts[j];
t_scalar_t ew = 1;
if (weight_index > -1){
ew = e_wgts[weight_index][j];
}
//add it to my local part neighbors for part i.
if (part_neighbor_weights[ep] < 0.00001){
part_neighbors[num_neighbor_parts++] = ep;
}
part_neighbor_weights[ep] += ew;
}
v = part_nexts[v];
}
//now get the part list.
for (t_lno_t j = 0; j < num_neighbor_parts; ++j){
part_t neighbor_part = part_neighbors[j];
part_neighbor_weights_ordered[j] = part_neighbor_weights[neighbor_part];
part_neighbor_weights[neighbor_part] = 0;
}
//insert it to tpetra crsmatrix.
if (num_neighbor_parts > 0){
Teuchos::ArrayView<const global_part_type> destinations(&(part_neighbors[0]), num_neighbor_parts);
Teuchos::ArrayView<const t_scalar_t> vals(&(part_neighbor_weights_ordered[0]), num_neighbor_parts);
tMatrix->insertGlobalValues (i,destinations, vals);
}
}
tMatrix->fillComplete ();
local_part_type num_local_parts = map->getNodeNumElements();
Array<global_part_type> Indices;
Array<t_scalar_t> Values;
t_scalar_t max_edge_cut = 0;
t_scalar_t total_edge_cut = 0;
global_part_type max_message = 0;
global_part_type total_message = 0;
for (local_part_type i=0; i < num_local_parts; i++) {
const global_part_type globalRow = map->getGlobalElement(i);
size_t NumEntries = tMatrix->getNumEntriesInGlobalRow (globalRow);
Indices.resize (NumEntries);
Values.resize (NumEntries);
tMatrix->getGlobalRowCopy (globalRow,Indices(),Values(),NumEntries);
t_scalar_t part_edge_cut = 0;
global_part_type part_messages = 0;
for (size_t j=0; j < NumEntries; j++){
if (Indices[j] != globalRow){
part_edge_cut += Values[j];
part_messages += 1;
}
}
if (part_edge_cut > max_edge_cut){
max_edge_cut = part_edge_cut;
}
total_edge_cut += part_edge_cut;
if (part_messages > max_message){
max_message = part_messages;
}
total_message += part_messages;
}
t_scalar_t g_max_edge_cut = 0;
t_scalar_t g_total_edge_cut = 0;
global_part_type g_max_message = 0;
global_part_type g_total_message = 0;
try{
Teuchos::reduceAll<int, t_scalar_t>(*comm,Teuchos::REDUCE_MAX,1,&max_edge_cut,&g_max_edge_cut);
Teuchos::reduceAll<int, global_part_type>(*comm,Teuchos::REDUCE_MAX,1,&max_message,&g_max_message);
Teuchos::reduceAll<int, t_scalar_t>(*comm,Teuchos::REDUCE_SUM,1,&total_edge_cut,&g_total_edge_cut);
Teuchos::reduceAll<int, global_part_type>(*comm,Teuchos::REDUCE_SUM,1,&total_message,&g_total_message);
}
Z2_THROW_OUTSIDE_ERROR(*env);
if (weight_index == -1){
metrics[next]->setName("md edge cuts");
}
else {
std::ostringstream oss;
oss << "md weight " << weight_index;
metrics[next]->setName( oss.str());
}
metrics[next]->setMetricValue("global maximum", g_max_edge_cut);
metrics[next]->setMetricValue("global sum", g_total_edge_cut);
next++;
if (weight_index == -1){
metrics[next]->setName("md message");
metrics[next]->setMetricValue("global maximum", g_max_message);
metrics[next]->setMetricValue("global sum", g_total_message);
next++;
}
}
}
env->timerStop(MACRO_TIMERS, "Communication Graph Create");
env->debug(DETAILED_STATUS, "Exiting globalWeightedCutsMessagesByPart");
}
/* this file is automatically generated - do not edit (see script/tpetra.py) */
#include "Xpetra_TpetraConfigDefs.hpp"
#include "Tpetra_CrsMatrix.hpp"
#include "Xpetra_CrsMatrix.hpp"
#include "Xpetra_TpetraMap.hpp"
#include "Xpetra_TpetraMultiVector.hpp"
#include "Xpetra_TpetraVector.hpp"
#include "Xpetra_TpetraCrsGraph.hpp"
//#include "Xpetra_TpetraRowMatrix.hpp"
#include "Xpetra_Exceptions.hpp"
namespace Xpetra {
// TODO: move that elsewhere
// template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node, class LocalMatOps>
// const Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> toTpetraCrsMatrix(const Xpetra::DistObject<char, LocalOrdinal, GlobalOrdinal, Node> &);
//
template <class Scalar, class LocalOrdinal = int, class GlobalOrdinal = LocalOrdinal, class Node = Kokkos::DefaultNode::DefaultNodeType, class LocalMatOps = typename Kokkos::DefaultKernels<Scalar,LocalOrdinal,Node>::SparseOps>
class TpetraCrsMatrix
: public CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps>//, public TpetraRowMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>
{
// The following typedef are used by the XPETRA_DYNAMIC_CAST() macro.
typedef TpetraCrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> TpetraCrsMatrixClass;
typedef TpetraVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> TpetraVectorClass;
typedef TpetraImport<LocalOrdinal,GlobalOrdinal,Node> TpetraImportClass;
typedef TpetraExport<LocalOrdinal,GlobalOrdinal,Node> TpetraExportClass;
public:
//! @name Constructor/Destructor Methods
//@{
//! Constructor specifying fixed number of entries for each row.
TpetraCrsMatrix(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rowMap, size_t maxNumEntriesPerRow, ProfileType pftype=DynamicProfile, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null)
: mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(rowMap), maxNumEntriesPerRow, toTpetra(pftype), params))) { }
//! Constructor specifying (possibly different) number of entries in each row.
TpetraCrsMatrix(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rowMap, const ArrayRCP< const size_t > &NumEntriesPerRowToAlloc, ProfileType pftype=DynamicProfile, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null)
: mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(rowMap), NumEntriesPerRowToAlloc, toTpetra(pftype), params))) { }
//! Constructor specifying column Map and fixed number of entries for each row.
TpetraCrsMatrix(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rowMap, const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &colMap, size_t maxNumEntriesPerRow, ProfileType pftype=DynamicProfile, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null)
: mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(rowMap), toTpetra(colMap), maxNumEntriesPerRow, toTpetra(pftype), params))) { }
//! Constructor specifying column Map and number of entries in each row.
TpetraCrsMatrix(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rowMap, const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &colMap, const ArrayRCP< const size_t > &NumEntriesPerRowToAlloc, ProfileType pftype=DynamicProfile, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null)
: mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(rowMap), toTpetra(colMap), NumEntriesPerRowToAlloc, toTpetra(pftype), params))) { }
//! Constructor specifying a previously constructed graph.
TpetraCrsMatrix(const Teuchos::RCP< const CrsGraph< LocalOrdinal, GlobalOrdinal, Node, LocalMatOps > > &graph, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null)
: mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(graph), params))) { }
//! Constructor for a fused import
TpetraCrsMatrix(const Teuchos::RCP<const CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> >& sourceMatrix,
const Import<LocalOrdinal,GlobalOrdinal,Node> & importer,
const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& domainMap = Teuchos::null,
const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& rangeMap = Teuchos::null,
const Teuchos::RCP<Teuchos::ParameterList>& params = Teuchos::null)
{
typedef Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> MyTpetraCrsMatrix;
XPETRA_DYNAMIC_CAST(const TpetraCrsMatrixClass, *sourceMatrix, tSourceMatrix, "Xpetra::TpetraCrsMatrix constructor only accepts Xpetra::TpetraCrsMatrix as the input argument.");//TODO: remove and use toTpetra()
RCP< const Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > v = tSourceMatrix.getTpetra_CrsMatrix();
RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > myDomainMap = domainMap!=Teuchos::null ? toTpetra(domainMap) : Teuchos::null;
RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > myRangeMap = rangeMap!=Teuchos::null ? toTpetra(rangeMap) : Teuchos::null;
mtx_=Tpetra::importAndFillCompleteCrsMatrix<MyTpetraCrsMatrix>(tSourceMatrix.getTpetra_CrsMatrix(),toTpetra(importer),myDomainMap,myRangeMap,params);
bool restrictComm=false;
if(!params.is_null()) restrictComm = params->get("Restrict Communicator",restrictComm);
if(restrictComm && mtx_->getRowMap().is_null()) mtx_=Teuchos::null;
}
//! Constructor for a fused export
TpetraCrsMatrix(const Teuchos::RCP<const CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> >& sourceMatrix,
const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter,
const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& domainMap = Teuchos::null,
const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& rangeMap = Teuchos::null,
const Teuchos::RCP<Teuchos::ParameterList>& params = Teuchos::null)
{
typedef Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> MyTpetraCrsMatrix;
XPETRA_DYNAMIC_CAST(const TpetraCrsMatrixClass, *sourceMatrix, tSourceMatrix, "Xpetra::TpetraCrsMatrix constructor only accepts Xpetra::TpetraCrsMatrix as the input argument.");//TODO: remove and use toTpetra()
RCP< const Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > v = tSourceMatrix.getTpetra_CrsMatrix();
RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > myDomainMap = domainMap!=Teuchos::null ? toTpetra(domainMap) : Teuchos::null;
RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > myRangeMap = rangeMap!=Teuchos::null ? toTpetra(rangeMap) : Teuchos::null;
mtx_=Tpetra::exportAndFillCompleteCrsMatrix<MyTpetraCrsMatrix>(tSourceMatrix.getTpetra_CrsMatrix(),toTpetra(exporter),myDomainMap,myRangeMap,params);
}
//! Destructor.
virtual ~TpetraCrsMatrix() { }
//@}
//! @name Insertion/Removal Methods
//@{
//! Insert matrix entries, using global IDs.
void insertGlobalValues(GlobalOrdinal globalRow, const ArrayView< const GlobalOrdinal > &cols, const ArrayView< const Scalar > &vals) { XPETRA_MONITOR("TpetraCrsMatrix::insertGlobalValues"); mtx_->insertGlobalValues(globalRow, cols, vals); }
//! Insert matrix entries, using local IDs.
void insertLocalValues(LocalOrdinal localRow, const ArrayView< const LocalOrdinal > &cols, const ArrayView< const Scalar > &vals) { XPETRA_MONITOR("TpetraCrsMatrix::insertLocalValues"); mtx_->insertLocalValues(localRow, cols, vals); }
//! Replace matrix entries, using global IDs.
void replaceGlobalValues(GlobalOrdinal globalRow, const ArrayView< const GlobalOrdinal > &cols, const ArrayView< const Scalar > &vals) { XPETRA_MONITOR("TpetraCrsMatrix::replaceGlobalValues"); mtx_->replaceGlobalValues(globalRow, cols, vals); }
//! Replace matrix entries, using local IDs.
void replaceLocalValues(LocalOrdinal localRow, const ArrayView< const LocalOrdinal > &cols, const ArrayView< const Scalar > &vals) { XPETRA_MONITOR("TpetraCrsMatrix::replaceLocalValues"); mtx_->replaceLocalValues(localRow, cols, vals); }
//! Set all matrix entries equal to scalarThis.
void setAllToScalar(const Scalar &alpha) { XPETRA_MONITOR("TpetraCrsMatrix::setAllToScalar"); mtx_->setAllToScalar(alpha); }
//! Scale the current values of a matrix, this = alpha*this.
void scale(const Scalar &alpha) { XPETRA_MONITOR("TpetraCrsMatrix::scale"); mtx_->scale(alpha); }
//! Allocates and returns ArrayRCPs of the Crs arrays --- This is an Xpetra-only routine.
//** \warning This is an expert-only routine and should not be called from user code. */
void allocateAllValues(size_t numNonZeros,ArrayRCP<size_t> & rowptr, ArrayRCP<LocalOrdinal> & colind, ArrayRCP<Scalar> & values)
{ XPETRA_MONITOR("TpetraCrsMatrix::allocateAllValues"); rowptr.resize(getNodeNumRows()+1); colind.resize(numNonZeros); values.resize(numNonZeros);}
