void IsorropiaInterface<LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level& level) const {
FactoryMonitor m(*this, "Build", level);
RCP<Matrix> A = Get< RCP<Matrix> >(level, "A");
GO numParts = level.Get<GO>("number of partitions");
RCP<const Map> rowMap = A->getRowMap();
RCP<const Map> colMap = A->getColMap();
if (numParts == 1) {
// Running on one processor, so decomposition is the trivial one, all zeros.
RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap, true);
//Set(level, "Partition", decomposition);
Set(level, "AmalgamatedPartition", decomposition);
return;
}
// ok, reconstruct graph information of matrix A
// Note, this is the non-rebalanced matrix A and we need the graph
// of the non-rebalanced matrix A. We cannot make use of the "Graph"
// that is/will be built for the aggregates later for several reasons
// 1) the "Graph" for the aggregates is meant to be based on the rebalanced matrix A
// 2) we cannot call a CoalesceDropFactory::Build here since this is very complicated and
// completely messes up the whole factory chain
// 3) CoalesceDropFactory is meant to provide some minimal Graph information for the aggregation
// (LWGraph), but here we need the full CrsGraph for Isorropia as input
// That is, why this code is somewhat redundant to CoalesceDropFactory
LO blockdim = 1; // block dim for fixed size blocks
GO indexBase = rowMap->getIndexBase(); // index base of maps
GO offset = 0;
LO blockid = -1; // block id in strided map
LO nStridedOffset = 0; // DOF offset for strided block id "blockid" (default = 0)
LO stridedblocksize = blockdim; // size of strided block id "blockid" (default = fullblocksize, only if blockid!=-1 stridedblocksize <= fullblocksize)
// 1) check for blocking/striding information
// fill above variables
if(A->IsView("stridedMaps") &&
Teuchos::rcp_dynamic_cast<const StridedMap>(A->getRowMap("stridedMaps")) != Teuchos::null) {
Xpetra::viewLabel_t oldView = A->SwitchToView("stridedMaps"); // note: "stridedMaps are always non-overlapping (correspond to range and domain maps!)
RCP<const StridedMap> strMap = Teuchos::rcp_dynamic_cast<const StridedMap>(A->getRowMap());
TEUCHOS_TEST_FOR_EXCEPTION(strMap == Teuchos::null,Exceptions::BadCast,"MueLu::IsorropiaInterface::Build: cast to strided row map failed.");
blockdim = strMap->getFixedBlockSize();
offset = strMap->getOffset();
blockid = strMap->getStridedBlockId();
if (blockid > -1) {
std::vector<size_t> stridingInfo = strMap->getStridingData();
for (size_t j = 0; j < Teuchos::as<size_t>(blockid); j++)
nStridedOffset += stridingInfo[j];
stridedblocksize = Teuchos::as<LocalOrdinal>(stridingInfo[blockid]);
} else {
stridedblocksize = blockdim;
}
oldView = A->SwitchToView(oldView);
GetOStream(Statistics0, -1) << "IsorropiaInterface::Build():" << " found blockdim=" << blockdim << " from strided maps (blockid=" << blockid << ", strided block size=" << stridedblocksize << "). offset=" << offset << std::endl;
} else GetOStream(Statistics0, -1) << "IsorropiaInterface::Build(): no striding information available. Use blockdim=1 with offset=0" << std::endl;
// 2) build (un)amalgamation information
// prepare generation of nodeRowMap (of amalgamated matrix)
RCP<AmalgamationInfo> amalInfo = Get< RCP<AmalgamationInfo> >(level, "UnAmalgamationInfo");
RCP<std::vector<GO> > gNodeIds = amalInfo->GetNodeGIDVector();
GO cnt_amalRows = amalInfo->GetNumberOfNodes();
// inter processor communication: sum up number of block ids
GO num_blockids = 0;
Teuchos::reduceAll<int,GO>(*(A->getRowMap()->getComm()),Teuchos::REDUCE_SUM, cnt_amalRows, Teuchos::ptr(&num_blockids) );
GetOStream(Statistics0, -1) << "IsorropiaInterface::SetupAmalgamationData()" << " # of amalgamated blocks=" << num_blockids << std::endl;
// 3) generate row map for amalgamated matrix (graph of A)
// with same distribution over all procs as row map of A
Teuchos::ArrayRCP<GO> arr_gNodeIds = Teuchos::arcp( gNodeIds );
Teuchos::RCP<Map> nodeMap = MapFactory::Build(A->getRowMap()->lib(), num_blockids, arr_gNodeIds(), indexBase, A->getRowMap()->getComm()); // note: nodeMap has same indexBase as row map of A (=dof map)
GetOStream(Statistics0, -1) << "IsorropiaInterface: nodeMap " << nodeMap->getNodeNumElements() << "/" << nodeMap->getGlobalNumElements() << " local/global elements" << std::endl;
// 4) create graph of amalgamated matrix
RCP<CrsGraph> crsGraph = CrsGraphFactory::Build(nodeMap, 10, Xpetra::DynamicProfile);
// 5) do amalgamation. generate graph of amalgamated matrix
for(LO row=0; row<Teuchos::as<LO>(A->getRowMap()->getNodeNumElements()); row++) {
// get global DOF id
GO grid = rowMap->getGlobalElement(row);
// translate grid to nodeid
GO nodeId = AmalgamationFactory::DOFGid2NodeId(grid, blockdim, offset, indexBase);
size_t nnz = A->getNumEntriesInLocalRow(row);
Teuchos::ArrayView<const LO> indices;
Teuchos::ArrayView<const SC> vals;
A->getLocalRowView(row, indices, vals);
//TEUCHOS_TEST_FOR_EXCEPTION(Teuchos::as<size_t>(indices.size()) != nnz, Exceptions::RuntimeError, "MueLu::CoalesceFactory::Amalgamate: number of nonzeros not equal to number of indices? Error.");
RCP<std::vector<GO> > cnodeIds = Teuchos::rcp(new std::vector<GO>); // global column block ids
LO realnnz = 0;
for(LO col=0; col<Teuchos::as<LO>(nnz); col++) {
//TEUCHOS_TEST_FOR_EXCEPTION(A->getColMap()->isNodeLocalElement(indices[col])==false,Exceptions::RuntimeError, "MueLu::CoalesceFactory::Amalgamate: Problem with columns. Error.");
GO gcid = colMap->getGlobalElement(indices[col]); // global column id
if(vals[col]!=0.0) {
GO cnodeId = AmalgamationFactory::DOFGid2NodeId(gcid, blockdim, offset, indexBase);
cnodeIds->push_back(cnodeId);
realnnz++; // increment number of nnz in matrix row
}
}
Teuchos::ArrayRCP<GO> arr_cnodeIds = Teuchos::arcp( cnodeIds );
//TEUCHOS_TEST_FOR_EXCEPTION(crsGraph->getRowMap()->isNodeGlobalElement(nodeId)==false,Exceptions::RuntimeError, "MueLu::CoalesceFactory::Amalgamate: global row id does not belong to current proc. Error.");
if(arr_cnodeIds.size() > 0 )
crsGraph->insertGlobalIndices(nodeId, arr_cnodeIds());
}
// fill matrix graph
crsGraph->fillComplete(nodeMap,nodeMap);
#ifdef HAVE_MPI
#ifdef HAVE_MUELU_ISORROPIA
// prepare parameter list for Isorropia
Teuchos::ParameterList paramlist;
paramlist.set("NUM PARTS", toString(numParts));
/*STRUCTURALLY SYMMETRIC [NO/yes] (is symmetrization required?)
PARTITIONING METHOD [block/cyclic/random/rcb/rib/hsfc/graph/HYPERGRAPH]
NUM PARTS [int k] (global number of parts)
IMBALANCE TOL [float tol] (1.0 is perfect balance)
BALANCE OBJECTIVE [ROWS/nonzeros]
*/
Teuchos::ParameterList& sublist = paramlist.sublist("Zoltan");
sublist.set("LB_APPROACH", "PARTITION");
#ifdef HAVE_MUELU_EPETRA
RCP< Xpetra::EpetraCrsGraph> epCrsGraph = Teuchos::rcp_dynamic_cast<Xpetra::EpetraCrsGraph>(crsGraph);
if(epCrsGraph != Teuchos::null) {
RCP< const Epetra_CrsGraph> epetraCrsGraph = epCrsGraph->getEpetra_CrsGraph();
RCP<Isorropia::Epetra::Partitioner> isoPart = Teuchos::rcp(new Isorropia::Epetra::Partitioner(epetraCrsGraph,paramlist));
int size = 0;
const int* array = NULL;
isoPart->extractPartsView(size,array);
TEUCHOS_TEST_FOR_EXCEPTION(size != Teuchos::as<int>(nodeMap->getNodeNumElements()), Exceptions::RuntimeError, "length of array returned from extractPartsView does not match local length of rowMap");
RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(nodeMap, false);
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
// fill vector with amalgamated information about partitioning
for(int i = 0; i<size; i++) {
decompEntries[i] = Teuchos::as<GO>(array[i]);
}
Set(level, "AmalgamatedPartition", decomposition);
}
#endif // ENDIF HAVE_MUELU_EPETRA
#ifdef HAVE_MUELU_TPETRA
#ifdef HAVE_MUELU_INST_DOUBLE_INT_INT
RCP< Xpetra::TpetraCrsGraph<LO, GO, Node, LocalMatOps> > tpCrsGraph = Teuchos::rcp_dynamic_cast<Xpetra::TpetraCrsGraph<LO, GO, Node, LocalMatOps> >(crsGraph);
if(tpCrsGraph != Teuchos::null) {
#ifdef HAVE_ISORROPIA_TPETRA
RCP< const Tpetra::CrsGraph<LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > tpetraCrsGraph = tpCrsGraph->getTpetra_CrsGraph();
RCP<Isorropia::Tpetra::Partitioner<Node> > isoPart = rcp(new Isorropia::Tpetra::Partitioner<Node>(tpetraCrsGraph, paramlist));
int size = 0;
const int* array = NULL;
isoPart->extractPartsView(size,array);
TEUCHOS_TEST_FOR_EXCEPTION(size != Teuchos::as<int>(nodeMap->getNodeNumElements()), Exceptions::RuntimeError, "length of array returned from extractPartsView does not match local length of rowMap");
RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(nodeMap, false);
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
// fill vector with amalgamated information about partitioning
// TODO: we assume simple block maps here
// TODO: adapt this to usage of nodegid2dofgids
for(int i = 0; i<size; i++) {
decompEntries[i] = Teuchos::as<GO>(array[i]);
}
Set(level, "AmalgamatedPartition", decomposition);
#else
TEUCHOS_TEST_FOR_EXCEPTION(false, Exceptions::RuntimeError, "Tpetra is not enabled for Isorropia. Recompile Isorropia with Tpetra support.");
#endif // ENDIF HAVE_ISORROPIA_TPETRA
}
#else
TEUCHOS_TEST_FOR_EXCEPTION(false, Exceptions::RuntimeError, "Isorropia is an interface to Zoltan which only has support for LO=GO=int and SC=double.");
#endif // ENDIF HAVE_MUELU_INST_DOUBLE_INT_INT
#endif // ENDIF HAVE_MUELU_TPETRA
#endif // HAVE_MUELU_ISORROPIA
#else // if we don't have MPI
// Running on one processor, so decomposition is the trivial one, all zeros.
RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap, true);
Set(level, "AmalgamatedPartition", decomposition);
#endif // HAVE_MPI
// throw a more helpful error message if something failed
//TEUCHOS_TEST_FOR_EXCEPTION(!level.IsAvailable("AmalgamatedPartition"), Exceptions::RuntimeError, "IsorropiaInterface::Build : no \'Partition\' vector available on level. Isorropia failed to build a partition of the non-repartitioned graph of A. Please make sure, that Isorropia is correctly compiled (Epetra/Tpetra).");
} //Build()
