void globalWeightedCutsMessagesByPart( const RCP<const Environment> &env, const RCP<const Comm<int> > &comm, const RCP<const GraphModel<typename Adapter::base_adapter_t> > &graph, const ArrayView<const typename Adapter::part_t> &parts, typename Adapter::part_t &numParts, ArrayRCP<RCP<BaseClassMetrics<typename Adapter::scalar_t> > > &metrics, ArrayRCP<typename Adapter::scalar_t> &globalSums) { env->debug(DETAILED_STATUS, "Entering globalWeightedCutsMessagesByPart"); ////////////////////////////////////////////////////////// // Initialize return values typedef typename Adapter::lno_t t_lno_t; typedef typename Adapter::gno_t t_gno_t; typedef typename Adapter::scalar_t t_scalar_t; typedef typename Adapter::part_t part_t; typedef typename Adapter::node_t t_node_t; typedef typename Zoltan2::GraphModel<typename Adapter::base_adapter_t>::input_t t_input_t; t_lno_t localNumVertices = graph->getLocalNumVertices(); t_gno_t globalNumVertices = graph->getGlobalNumVertices(); t_lno_t localNumEdges = graph->getLocalNumEdges(); ArrayView<const t_gno_t> Ids; ArrayView<t_input_t> v_wghts; graph->getVertexList(Ids, v_wghts); typedef GraphMetrics<t_scalar_t> mv_t; //get the edge ids, and weights ArrayView<const t_gno_t> edgeIds; ArrayView<const t_lno_t> offsets; ArrayView<t_input_t> e_wgts; graph->getEdgeList(edgeIds, offsets, e_wgts); std::vector <t_scalar_t> edge_weights; int numWeightPerEdge = graph->getNumWeightsPerEdge(); int numMetrics = 2; // "edge cuts", messages if (numWeightPerEdge) numMetrics += numWeightPerEdge; // "weight n" // add some more metrics to the array typedef typename ArrayRCP<RCP<BaseClassMetrics<typename Adapter::scalar_t> > >::size_type array_size_type; metrics.resize( metrics.size() + numMetrics ); for( array_size_type n = metrics.size() - numMetrics; n < metrics.size(); ++n ){ mv_t * newMetric = new mv_t; // allocate the new memory env->localMemoryAssertion(__FILE__,__LINE__,1,newMetric); // check errors metrics[n] = rcp( newMetric); // create the new members } array_size_type next = metrics.size() - numMetrics; // MDM - this is most likely temporary to preserve the format here - we are now filling a larger array so we may not have started at 0 std::vector <part_t> e_parts (localNumEdges); #ifdef HAVE_ZOLTAN2_MPI if (comm->getSize() > 1) { Zoltan_DD_Struct *dd = NULL; MPI_Comm mpicomm = Teuchos::getRawMpiComm(*comm); int size_gnot = Zoltan2::TPL_Traits<ZOLTAN_ID_PTR, t_gno_t>::NUM_ID; int debug_level = 0; Zoltan_DD_Create(&dd, mpicomm, size_gnot, 0, sizeof(part_t), localNumVertices, debug_level); ZOLTAN_ID_PTR ddnotneeded = NULL; // Local IDs not needed Zoltan_DD_Update( dd, (ZOLTAN_ID_PTR) Ids.getRawPtr(), ddnotneeded, (char *) &(parts[0]), NULL, int(localNumVertices)); Zoltan_DD_Find( dd, (ZOLTAN_ID_PTR) edgeIds.getRawPtr(), ddnotneeded, (char *)&(e_parts[0]), NULL, localNumEdges, NULL ); Zoltan_DD_Destroy(&dd); } else #endif { std::map<t_gno_t,t_lno_t> global_id_to_local_index; //else everything is local. //we need a globalid to local index conversion. //this does not exists till this point, so we need to create one. for (t_lno_t i = 0; i < localNumVertices; ++i){ //at the local index i, we have the global index Ids[i]. //so write i, to Ids[i] index of the vector. global_id_to_local_index[Ids[i]] = i; } for (t_lno_t i = 0; i < localNumEdges; ++i){ t_gno_t ei = edgeIds[i]; //ei is the global index of the neighbor one. part_t p = parts[global_id_to_local_index[ei]]; e_parts[i] = p; } } RCP<const Teuchos::Comm<int> > tcomm = comm; env->timerStart(MACRO_TIMERS, "Communication Graph Create"); { //get the vertices in each part in my part. std::vector <t_lno_t> part_begins(numParts, -1); std::vector <t_lno_t> part_nexts(localNumVertices, -1); //cluster vertices according to their parts. //create local part graph. for (t_lno_t i = 0; i < localNumVertices; ++i){ part_t ap = parts[i]; part_nexts[i] = part_begins[ap]; part_begins[ap] = i; } for (int weight_index = -1; weight_index < numWeightPerEdge ; ++weight_index){ //MD: these two should be part_t. //but we dont want to compile tpetra from the beginning. //This can be changed when directory is updated. typedef t_lno_t local_part_type; typedef t_gno_t global_part_type; typedef Tpetra::Map<local_part_type, global_part_type, t_node_t> map_t; Teuchos::RCP<const map_t> map = Teuchos::rcp (new map_t (numParts, 0, tcomm)); typedef Tpetra::CrsMatrix<t_scalar_t, local_part_type, global_part_type, t_node_t> tcrsMatrix_t; Teuchos::RCP<tcrsMatrix_t> tMatrix(new tcrsMatrix_t (map, 0)); std::vector <global_part_type> part_neighbors (numParts); std::vector <t_scalar_t> part_neighbor_weights(numParts, 0); std::vector <t_scalar_t> part_neighbor_weights_ordered(numParts); //coarsen for all vertices in my part in order with parts. for (global_part_type i = 0; i < (global_part_type) numParts; ++i){ part_t num_neighbor_parts = 0; t_lno_t v = part_begins[i]; //get part i, and first vertex in this part v. while (v != -1){ //now get the neightbors of v. for (t_lno_t j = offsets[v]; j < offsets[v+1]; ++j){ //get the part of the second vertex. part_t ep = e_parts[j]; t_scalar_t ew = 1; if (weight_index > -1){ ew = e_wgts[weight_index][j]; } //add it to my local part neighbors for part i. if (part_neighbor_weights[ep] < 0.00001){ part_neighbors[num_neighbor_parts++] = ep; } part_neighbor_weights[ep] += ew; } v = part_nexts[v]; } //now get the part list. for (t_lno_t j = 0; j < num_neighbor_parts; ++j){ part_t neighbor_part = part_neighbors[j]; part_neighbor_weights_ordered[j] = part_neighbor_weights[neighbor_part]; part_neighbor_weights[neighbor_part] = 0; } //insert it to tpetra crsmatrix. if (num_neighbor_parts > 0){ Teuchos::ArrayView<const global_part_type> destinations(&(part_neighbors[0]), num_neighbor_parts); Teuchos::ArrayView<const t_scalar_t> vals(&(part_neighbor_weights_ordered[0]), num_neighbor_parts); tMatrix->insertGlobalValues (i,destinations, vals); } } tMatrix->fillComplete (); local_part_type num_local_parts = map->getNodeNumElements(); Array<global_part_type> Indices; Array<t_scalar_t> Values; t_scalar_t max_edge_cut = 0; t_scalar_t total_edge_cut = 0; global_part_type max_message = 0; global_part_type total_message = 0; for (local_part_type i=0; i < num_local_parts; i++) { const global_part_type globalRow = map->getGlobalElement(i); size_t NumEntries = tMatrix->getNumEntriesInGlobalRow (globalRow); Indices.resize (NumEntries); Values.resize (NumEntries); tMatrix->getGlobalRowCopy (globalRow,Indices(),Values(),NumEntries); t_scalar_t part_edge_cut = 0; global_part_type part_messages = 0; for (size_t j=0; j < NumEntries; j++){ if (Indices[j] != globalRow){ part_edge_cut += Values[j]; part_messages += 1; } } if (part_edge_cut > max_edge_cut){ max_edge_cut = part_edge_cut; } total_edge_cut += part_edge_cut; if (part_messages > max_message){ max_message = part_messages; } total_message += part_messages; } t_scalar_t g_max_edge_cut = 0; t_scalar_t g_total_edge_cut = 0; global_part_type g_max_message = 0; global_part_type g_total_message = 0; try{ Teuchos::reduceAll<int, t_scalar_t>(*comm,Teuchos::REDUCE_MAX,1,&max_edge_cut,&g_max_edge_cut); Teuchos::reduceAll<int, global_part_type>(*comm,Teuchos::REDUCE_MAX,1,&max_message,&g_max_message); Teuchos::reduceAll<int, t_scalar_t>(*comm,Teuchos::REDUCE_SUM,1,&total_edge_cut,&g_total_edge_cut); Teuchos::reduceAll<int, global_part_type>(*comm,Teuchos::REDUCE_SUM,1,&total_message,&g_total_message); } Z2_THROW_OUTSIDE_ERROR(*env); if (weight_index == -1){ metrics[next]->setName("md edge cuts"); } else { std::ostringstream oss; oss << "md weight " << weight_index; metrics[next]->setName( oss.str()); } metrics[next]->setMetricValue("global maximum", g_max_edge_cut); metrics[next]->setMetricValue("global sum", g_total_edge_cut); next++; if (weight_index == -1){ metrics[next]->setName("md message"); metrics[next]->setMetricValue("global maximum", g_max_message); metrics[next]->setMetricValue("global sum", g_total_message); next++; } } } env->timerStop(MACRO_TIMERS, "Communication Graph Create"); env->debug(DETAILED_STATUS, "Exiting globalWeightedCutsMessagesByPart"); }
/* this file is automatically generated - do not edit (see script/tpetra.py) */ #include "Xpetra_TpetraConfigDefs.hpp" #include "Tpetra_CrsMatrix.hpp" #include "Xpetra_CrsMatrix.hpp" #include "Xpetra_TpetraMap.hpp" #include "Xpetra_TpetraMultiVector.hpp" #include "Xpetra_TpetraVector.hpp" #include "Xpetra_TpetraCrsGraph.hpp" //#include "Xpetra_TpetraRowMatrix.hpp" #include "Xpetra_Exceptions.hpp" namespace Xpetra { // TODO: move that elsewhere // template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node, class LocalMatOps> // const Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> toTpetraCrsMatrix(const Xpetra::DistObject<char, LocalOrdinal, GlobalOrdinal, Node> &); // template <class Scalar, class LocalOrdinal = int, class GlobalOrdinal = LocalOrdinal, class Node = Kokkos::DefaultNode::DefaultNodeType, class LocalMatOps = typename Kokkos::DefaultKernels<Scalar,LocalOrdinal,Node>::SparseOps> class TpetraCrsMatrix : public CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps>//, public TpetraRowMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> { // The following typedef are used by the XPETRA_DYNAMIC_CAST() macro. typedef TpetraCrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> TpetraCrsMatrixClass; typedef TpetraVector<Scalar,LocalOrdinal,GlobalOrdinal,Node> TpetraVectorClass; typedef TpetraImport<LocalOrdinal,GlobalOrdinal,Node> TpetraImportClass; typedef TpetraExport<LocalOrdinal,GlobalOrdinal,Node> TpetraExportClass; public: //! @name Constructor/Destructor Methods //@{ //! Constructor specifying fixed number of entries for each row. TpetraCrsMatrix(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rowMap, size_t maxNumEntriesPerRow, ProfileType pftype=DynamicProfile, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null) : mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(rowMap), maxNumEntriesPerRow, toTpetra(pftype), params))) { } //! Constructor specifying (possibly different) number of entries in each row. TpetraCrsMatrix(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rowMap, const ArrayRCP< const size_t > &NumEntriesPerRowToAlloc, ProfileType pftype=DynamicProfile, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null) : mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(rowMap), NumEntriesPerRowToAlloc, toTpetra(pftype), params))) { } //! Constructor specifying column Map and fixed number of entries for each row. TpetraCrsMatrix(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rowMap, const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &colMap, size_t maxNumEntriesPerRow, ProfileType pftype=DynamicProfile, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null) : mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(rowMap), toTpetra(colMap), maxNumEntriesPerRow, toTpetra(pftype), params))) { } //! Constructor specifying column Map and number of entries in each row. TpetraCrsMatrix(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rowMap, const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &colMap, const ArrayRCP< const size_t > &NumEntriesPerRowToAlloc, ProfileType pftype=DynamicProfile, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null) : mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(rowMap), toTpetra(colMap), NumEntriesPerRowToAlloc, toTpetra(pftype), params))) { } //! Constructor specifying a previously constructed graph. TpetraCrsMatrix(const Teuchos::RCP< const CrsGraph< LocalOrdinal, GlobalOrdinal, Node, LocalMatOps > > &graph, const Teuchos::RCP< Teuchos::ParameterList > ¶ms=Teuchos::null) : mtx_(Teuchos::rcp(new Tpetra::CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps >(toTpetra(graph), params))) { } //! Constructor for a fused import TpetraCrsMatrix(const Teuchos::RCP<const CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> >& sourceMatrix, const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& domainMap = Teuchos::null, const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& rangeMap = Teuchos::null, const Teuchos::RCP<Teuchos::ParameterList>& params = Teuchos::null) { typedef Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> MyTpetraCrsMatrix; XPETRA_DYNAMIC_CAST(const TpetraCrsMatrixClass, *sourceMatrix, tSourceMatrix, "Xpetra::TpetraCrsMatrix constructor only accepts Xpetra::TpetraCrsMatrix as the input argument.");//TODO: remove and use toTpetra() RCP< const Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > v = tSourceMatrix.getTpetra_CrsMatrix(); RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > myDomainMap = domainMap!=Teuchos::null ? toTpetra(domainMap) : Teuchos::null; RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > myRangeMap = rangeMap!=Teuchos::null ? toTpetra(rangeMap) : Teuchos::null; mtx_=Tpetra::importAndFillCompleteCrsMatrix<MyTpetraCrsMatrix>(tSourceMatrix.getTpetra_CrsMatrix(),toTpetra(importer),myDomainMap,myRangeMap,params); bool restrictComm=false; if(!params.is_null()) restrictComm = params->get("Restrict Communicator",restrictComm); if(restrictComm && mtx_->getRowMap().is_null()) mtx_=Teuchos::null; } //! Constructor for a fused export TpetraCrsMatrix(const Teuchos::RCP<const CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> >& sourceMatrix, const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& domainMap = Teuchos::null, const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& rangeMap = Teuchos::null, const Teuchos::RCP<Teuchos::ParameterList>& params = Teuchos::null) { typedef Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> MyTpetraCrsMatrix; XPETRA_DYNAMIC_CAST(const TpetraCrsMatrixClass, *sourceMatrix, tSourceMatrix, "Xpetra::TpetraCrsMatrix constructor only accepts Xpetra::TpetraCrsMatrix as the input argument.");//TODO: remove and use toTpetra() RCP< const Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > v = tSourceMatrix.getTpetra_CrsMatrix(); RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > myDomainMap = domainMap!=Teuchos::null ? toTpetra(domainMap) : Teuchos::null; RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > myRangeMap = rangeMap!=Teuchos::null ? toTpetra(rangeMap) : Teuchos::null; mtx_=Tpetra::exportAndFillCompleteCrsMatrix<MyTpetraCrsMatrix>(tSourceMatrix.getTpetra_CrsMatrix(),toTpetra(exporter),myDomainMap,myRangeMap,params); } //! Destructor. virtual ~TpetraCrsMatrix() { } //@} //! @name Insertion/Removal Methods //@{ //! Insert matrix entries, using global IDs. void insertGlobalValues(GlobalOrdinal globalRow, const ArrayView< const GlobalOrdinal > &cols, const ArrayView< const Scalar > &vals) { XPETRA_MONITOR("TpetraCrsMatrix::insertGlobalValues"); mtx_->insertGlobalValues(globalRow, cols, vals); } //! Insert matrix entries, using local IDs. void insertLocalValues(LocalOrdinal localRow, const ArrayView< const LocalOrdinal > &cols, const ArrayView< const Scalar > &vals) { XPETRA_MONITOR("TpetraCrsMatrix::insertLocalValues"); mtx_->insertLocalValues(localRow, cols, vals); } //! Replace matrix entries, using global IDs. void replaceGlobalValues(GlobalOrdinal globalRow, const ArrayView< const GlobalOrdinal > &cols, const ArrayView< const Scalar > &vals) { XPETRA_MONITOR("TpetraCrsMatrix::replaceGlobalValues"); mtx_->replaceGlobalValues(globalRow, cols, vals); } //! Replace matrix entries, using local IDs. void replaceLocalValues(LocalOrdinal localRow, const ArrayView< const LocalOrdinal > &cols, const ArrayView< const Scalar > &vals) { XPETRA_MONITOR("TpetraCrsMatrix::replaceLocalValues"); mtx_->replaceLocalValues(localRow, cols, vals); } //! Set all matrix entries equal to scalarThis. void setAllToScalar(const Scalar &alpha) { XPETRA_MONITOR("TpetraCrsMatrix::setAllToScalar"); mtx_->setAllToScalar(alpha); } //! Scale the current values of a matrix, this = alpha*this. void scale(const Scalar &alpha) { XPETRA_MONITOR("TpetraCrsMatrix::scale"); mtx_->scale(alpha); } //! Allocates and returns ArrayRCPs of the Crs arrays --- This is an Xpetra-only routine. //** \warning This is an expert-only routine and should not be called from user code. */ void allocateAllValues(size_t numNonZeros,ArrayRCP<size_t> & rowptr, ArrayRCP<LocalOrdinal> & colind, ArrayRCP<Scalar> & values) { XPETRA_MONITOR("TpetraCrsMatrix::allocateAllValues"); rowptr.resize(getNodeNumRows()+1); colind.resize(numNonZeros); values.resize(numNonZeros);}