void Add(
const Xpetra::Matrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>& A,
bool transposeA,
Scalar scalarA,
const Xpetra::Matrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>& B,
bool transposeB,
Scalar scalarB,
Teuchos::RCP<Xpetra::Matrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > C) {
if ( !(A.getRowMap()->isSameAs(*(B.getRowMap()))) ) {
throw(Xpetra::Exceptions::RuntimeError("Xpetra::MatrixMatrix::Add: matrix row maps are not the same."));
}
if (C==Teuchos::null)
//FIXME 5 is a complete guess as to the #nonzeros per row
C = rcp( new Xpetra::CrsMatrixWrap<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>(A.getRowMap(), 5) );
if (C->getRowMap()->lib() == Xpetra::UseEpetra) {
#ifdef HAVE_XPETRA_EPETRAEXT
const Epetra_CrsMatrix& epA = Xpetra::MatrixMatrix::Op2EpetraCrs(A);
const Epetra_CrsMatrix& epB = Xpetra::MatrixMatrix::Op2EpetraCrs(B);
RCP<Epetra_CrsMatrix> epC = Xpetra::MatrixMatrix::Op2NonConstEpetraCrs(C);
Epetra_CrsMatrix* ref2epC = &*epC; //to avoid a compiler error...
//FIXME is there a bug if beta=0?
int i = EpetraExt::MatrixMatrix::Add(epA,transposeA,scalarA,epB,transposeB,scalarB,ref2epC);
if (i != 0) {
std::ostringstream buf;
buf << i;
std::string msg = "EpetraExt::MatrixMatrix::Add return value of " + buf.str();
throw(Xpetra::Exceptions::RuntimeError(msg));
}
#else
throw(Xpetra::Exceptions::RuntimeError("Xpetra must be compile with EpetraExt."));
#endif
} else if(C->getRowMap()->lib() == Xpetra::UseTpetra) {
#ifdef HAVE_XPETRA_TPETRA
const Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> & tpA = Xpetra::MatrixMatrix::Op2TpetraCrs(A);
const Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> & tpB = Xpetra::MatrixMatrix::Op2TpetraCrs(B);
RCP<Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > tpC = Xpetra::MatrixMatrix::Op2NonConstTpetraCrs(C);
Tpetra::MatrixMatrix::Add(tpA, transposeA, scalarA, tpB, transposeB, scalarB, tpC);
#else
throw(Xpetra::Exceptions::RuntimeError("Xpetra must be compile with Tpetra."));
#endif
}
///////////////////////// EXPERIMENTAL
RCP<Xpetra::Matrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > rcpA = Teuchos::rcp_const_cast<Xpetra::Matrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> >(Teuchos::rcpFromRef(A));
RCP<Xpetra::Matrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > rcpB = Teuchos::rcp_const_cast<Xpetra::Matrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> >(Teuchos::rcpFromRef(B));
if(A.IsView("stridedMaps")) C->CreateView("stridedMaps", rcpA);
if(B.IsView("stridedMaps")) C->CreateView("stridedMaps", rcpB);
///////////////////////// EXPERIMENTAL
}
void SubBlockAFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level & currentLevel) const {
typedef Xpetra::Matrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> OMatrix; //TODO
typedef Xpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> CrsMatrixClass; //TODO
typedef Xpetra::CrsMatrixWrap<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> CrsMatrixWrapClass; //TODO
typedef Xpetra::BlockedCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> BlockedCrsOMatrix; //TODO
typedef Xpetra::MapExtractor<Scalar, LocalOrdinal, GlobalOrdinal, Node> MapExtractorClass;
const ParameterList & pL = GetParameterList();
size_t row = Teuchos::as<size_t>(pL.get<int>("block row"));
size_t col = Teuchos::as<size_t>(pL.get<int>("block col"));
RCP<OMatrix> Ain = Teuchos::null;
Ain = Get< RCP<OMatrix> >(currentLevel, "A");
RCP<BlockedCrsOMatrix> bA = Teuchos::rcp_dynamic_cast<BlockedCrsOMatrix>(Ain);
TEUCHOS_TEST_FOR_EXCEPTION(bA==Teuchos::null, Exceptions::BadCast, "MueLu::SubBlockAFactory::Build: input matrix A is not of type BlockedCrsMatrix! error.");
TEUCHOS_TEST_FOR_EXCEPTION(row > bA->Rows(), Exceptions::RuntimeError, "MueLu::SubBlockAFactory::Build: A.Rows() > rows_! error.");
TEUCHOS_TEST_FOR_EXCEPTION(col > bA->Cols(), Exceptions::RuntimeError, "MueLu::SubBlockAFactory::Build: A.Cols() > cols_! error.");
Teuchos::RCP<CrsMatrixClass> A = bA->getMatrix(row, col);
Teuchos::RCP<CrsMatrixWrapClass> Op = Teuchos::rcp(new CrsMatrixWrapClass(A));
//////////////// EXPERIMENTAL
// extract striding information from RangeMapExtractor
Teuchos::RCP<const MapExtractorClass> rgMapExtractor = bA->getRangeMapExtractor();
Teuchos::RCP<const MapExtractorClass> doMapExtractor = bA->getDomainMapExtractor();
Teuchos::RCP<const Map> rgMap = rgMapExtractor->getMap(row);
Teuchos::RCP<const Map> doMap = doMapExtractor->getMap(col);
Teuchos::RCP<const StridedMap> srgMap = Teuchos::rcp_dynamic_cast<const StridedMap>(rgMap);
Teuchos::RCP<const StridedMap> sdoMap = Teuchos::rcp_dynamic_cast<const StridedMap>(doMap);
if(srgMap == Teuchos::null) {
Teuchos::RCP<const Map> fullRgMap = rgMapExtractor->getFullMap();
Teuchos::RCP<const StridedMap> sFullRgMap = Teuchos::rcp_dynamic_cast<const StridedMap>(fullRgMap);
TEUCHOS_TEST_FOR_EXCEPTION(sFullRgMap==Teuchos::null, Exceptions::BadCast, "MueLu::SubBlockAFactory::Build: full rangeMap is not a strided map");
std::vector<size_t> stridedData = sFullRgMap->getStridingData();
if(stridedData.size() == 1 && row > 0) // we have block matrices. use striding block information 0
srgMap = StridedMapFactory::Build(rgMap, stridedData, 0, sFullRgMap->getOffset());
else // we have strided matrices. use striding information of the corresponding block
srgMap = StridedMapFactory::Build(rgMap, stridedData, row, sFullRgMap->getOffset());
}
if(sdoMap == Teuchos::null) {
Teuchos::RCP<const Map> fullDoMap = doMapExtractor->getFullMap();
Teuchos::RCP<const StridedMap> sFullDoMap = Teuchos::rcp_dynamic_cast<const StridedMap>(fullDoMap);
TEUCHOS_TEST_FOR_EXCEPTION(sFullDoMap==Teuchos::null, Exceptions::BadCast, "MueLu::SubBlockAFactory::Build: full domainMap is not a strided map");
std::vector<size_t> stridedData2 = sFullDoMap->getStridingData();
if(stridedData2.size() == 1 && col > 0) // we have block matrices. use striding block information 0
sdoMap = StridedMapFactory::Build(doMap, stridedData2, 0, sFullDoMap->getOffset());
else // we have strided matrices. use striding information of the corresponding block
sdoMap = StridedMapFactory::Build(doMap, stridedData2, col, sFullDoMap->getOffset());
}
TEUCHOS_TEST_FOR_EXCEPTION(srgMap==Teuchos::null, Exceptions::BadCast, "MueLu::SubBlockAFactory::Build: rangeMap " << row << " is not a strided map");
TEUCHOS_TEST_FOR_EXCEPTION(sdoMap==Teuchos::null, Exceptions::BadCast, "MueLu::SubBlockAFactory::Build: domainMap " << col << " is not a strided map");
GetOStream(Statistics1) << "A(" << row << "," << col << ") has strided maps: range map fixed block size=" << srgMap->getFixedBlockSize() << " strided block id = " << srgMap->getStridedBlockId() << ", domain map fixed block size=" << sdoMap->getFixedBlockSize() << ", strided block id=" << sdoMap->getStridedBlockId() << std::endl;
if(Op->IsView("stridedMaps") == true) Op->RemoveView("stridedMaps");
Op->CreateView("stridedMaps", srgMap, sdoMap);
TEUCHOS_TEST_FOR_EXCEPTION(Op->IsView("stridedMaps")==false, Exceptions::RuntimeError, "MueLu::SubBlockAFactory::Build: failed to set stridedMaps");
//////////////// EXPERIMENTAL
currentLevel.Set("A", Teuchos::rcp_dynamic_cast<OMatrix>(Op), this);
}
void SchurComplementFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level & currentLevel) const
{
FactoryMonitor m(*this, "SchurComplementFactory", currentLevel);
Teuchos::RCP<Matrix> A = Get<RCP<Matrix> >(currentLevel, "A");
RCP<BlockedCrsMatrix> bA = Teuchos::rcp_dynamic_cast<BlockedCrsMatrix>(A);
TEUCHOS_TEST_FOR_EXCEPTION(bA == Teuchos::null, Exceptions::BadCast, "MueLu::SchurComplementFactory::Build: input matrix A is not of type BlockedCrsMatrix! A generated by AFact_ must be a 2x2 block operator. error.");
Teuchos::RCP<CrsMatrix> A00 = bA->getMatrix(0,0);
Teuchos::RCP<CrsMatrix> A01 = bA->getMatrix(0,1);
Teuchos::RCP<CrsMatrix> A10 = bA->getMatrix(1,0);
Teuchos::RCP<CrsMatrix> A11 = bA->getMatrix(1,1);
Teuchos::RCP<CrsMatrixWrap> Op00 = Teuchos::rcp(new CrsMatrixWrap(A00));
Teuchos::RCP<CrsMatrixWrap> Op01 = Teuchos::rcp(new CrsMatrixWrap(A01));
Teuchos::RCP<CrsMatrixWrap> Op10 = Teuchos::rcp(new CrsMatrixWrap(A10));
Teuchos::RCP<CrsMatrixWrap> Op11 = Teuchos::rcp(new CrsMatrixWrap(A11));
Teuchos::RCP<Matrix> F = Teuchos::rcp_dynamic_cast<Matrix>(Op00);
Teuchos::RCP<Matrix> G = Teuchos::rcp_dynamic_cast<Matrix>(Op01);
Teuchos::RCP<Matrix> D = Teuchos::rcp_dynamic_cast<Matrix>(Op10);
Teuchos::RCP<Matrix> Z = Teuchos::rcp_dynamic_cast<Matrix>(Op11);
// TODO move this to BlockedCrsMatrix->getMatrix routine...
F->CreateView("stridedMaps", bA->getRangeMap(0), bA->getDomainMap(0));
G->CreateView("stridedMaps", bA->getRangeMap(0), bA->getDomainMap(1));
D->CreateView("stridedMaps", bA->getRangeMap(1), bA->getDomainMap(0));
Z->CreateView("stridedMaps", bA->getRangeMap(1), bA->getDomainMap(1));
const ParameterList& pL = GetParameterList();
Scalar omega = pL.get<Scalar>("omega");
// copy the value of G so we can do the left scale.
RCP<Matrix> FhatinvG = MatrixFactory::Build(G->getRowMap(), G->getGlobalMaxNumRowEntries());
RCP<Matrix> emptyMat = MatrixFactory::Build(G->getRowMap(), G->getGlobalMaxNumRowEntries());
emptyMat->fillComplete(G->getDomainMap(),G->getRowMap());
Utils2::TwoMatrixAdd(G,false,1.0,emptyMat,false,-1.0/omega,FhatinvG);
FhatinvG->fillComplete(G->getDomainMap(),G->getRowMap()); // complete the matrix. left scaling does not change the pattern of the operator.
bool lumping = pL.get<bool>("lumping");
if(!lumping) {
// extract diagonal of F. store it in ArrayRCP object
Teuchos::ArrayRCP<SC> AdiagFinv = Utils::GetMatrixDiagonal(*F);
for (size_t k = 0; k < Teuchos::as<size_t> (AdiagFinv.size ()); ++k) {
if (Teuchos::ScalarTraits<SC>::magnitude(AdiagFinv[k]) < 1e-4)
AdiagFinv[k] = Teuchos::ScalarTraits<SC>::one();
}
Utils::MyOldScaleMatrix(FhatinvG,AdiagFinv,true,false,false); // TODO check the MyOldScaleMatrix routine...
} else {
// use diagonal of lumped matrix as approximation
Teuchos::ArrayRCP<SC> AdiagFinv = Utils::GetLumpedMatrixDiagonal(*F);
Utils::MyOldScaleMatrix(FhatinvG,AdiagFinv,true,false,false); // TODO check the MyOldScaleMatrix routine...
}
// build D \hat{F}^{-1} G
RCP<Matrix> DFhatinvG = Utils::Multiply(*D,false,*FhatinvG,false);
// build full SchurComplement operator
// S = - 1/omega D \hat{F}^{-1} G + Z
RCP<Matrix> S;
Utils2::TwoMatrixAdd(Z,false,1.0,DFhatinvG,false,-1.0/omega,S);
S->fillComplete();
{
// note: variable "A" generated by this SchurComplement factory is in fact the SchurComplement matrix
// we have to use the variable name "A" since the Smoother object expects the matrix to be called "A"
Set(currentLevel, "A", S);
}
}
TEUCHOS_UNIT_TEST(CoalesceDropFactory, AmalgamationStridedOffsetDropping2LW)
{
// unit test for block size 9 = (2,3,4). wrap block 1.
// drop small entries
// lightweight wrap = true
out << "version: " << MueLu::Version() << std::endl;
RCP<const Teuchos::Comm<int> > comm = Parameters::getDefaultComm();
Xpetra::UnderlyingLib lib = TestHelpers::Parameters::getLib();
// create strided map information
std::vector<size_t> stridingInfo;
stridingInfo.push_back(as<size_t>(2));
stridingInfo.push_back(as<size_t>(3));
stridingInfo.push_back(as<size_t>(4));
LocalOrdinal stridedBlockId = 1;
GlobalOrdinal offset = 19;
RCP<const StridedMap> dofMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(lib, 9*comm->getSize(), 0,
stridingInfo, comm,
stridedBlockId, offset);
/////////////////////////////////////////////////////
Teuchos::RCP<Matrix> mtx = TestHelpers::TestFactory<SC,LO,GO,NO>::BuildTridiag(dofMap, 2.0, 1.0, 0.0001);
Level fineLevel;
TestHelpers::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(fineLevel);
RCP<const Map> stridedRangeMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(
mtx->getRangeMap(),
stridingInfo,
stridedBlockId,
offset
);
RCP<const Map> stridedDomainMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(
mtx->getDomainMap(),
stridingInfo,
stridedBlockId,
offset
);
if(mtx->IsView("stridedMaps") == true) mtx->RemoveView("stridedMaps");
mtx->CreateView("stridedMaps", stridedRangeMap, stridedDomainMap);
fineLevel.Set("A", mtx);
CoalesceDropFactory dropFact = CoalesceDropFactory();
dropFact.SetParameter("lightweight wrap",Teuchos::ParameterEntry(true));
dropFact.SetParameter("aggregation: drop tol",Teuchos::ParameterEntry(0.3));
fineLevel.Request("Graph", &dropFact);
fineLevel.Request("DofsPerNode", &dropFact);
dropFact.Build(fineLevel);
fineLevel.print(out);
RCP<GraphBase> graph = fineLevel.Get<RCP<GraphBase> >("Graph", &dropFact);
LO myDofsPerNode = fineLevel.Get<LO>("DofsPerNode", &dropFact);
TEST_EQUALITY(as<int>(graph->GetDomainMap()->getGlobalNumElements()) == comm->getSize(), true);
TEST_EQUALITY(as<int>(myDofsPerNode) == 9, true);
bool bCorrectGraph = false;
if (comm->getSize() == 1 && graph->getNeighborVertices(0).size() == 1) {
bCorrectGraph = true;
} else {
if (comm->getRank() == 0) {
if (graph->getNeighborVertices(0).size() == 1) bCorrectGraph = true;
}
else {
if (graph->getNeighborVertices(0).size() == 2) bCorrectGraph = true;
}
}
TEST_EQUALITY(bCorrectGraph, true);
const RCP<const Map> myImportMap = graph->GetImportMap(); // < note that the ImportMap is built from the column map of the matrix A WITHOUT dropping!
const RCP<const Map> myDomainMap = graph->GetDomainMap();
TEST_EQUALITY(myImportMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myImportMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myImportMap->getMinLocalIndex(),0);
TEST_EQUALITY(myImportMap->getGlobalNumElements(),as<size_t>(comm->getSize()+2*(comm->getSize()-1)));
if (comm->getSize()>1) {
size_t numLocalRowMapElts = graph->GetNodeNumVertices();
size_t numLocalImportElts = myImportMap->getNodeNumElements();
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
TEST_EQUALITY(as<bool>(numLocalImportElts==numLocalRowMapElts+1), true);
} else {
TEST_EQUALITY(as<bool>(numLocalImportElts==numLocalRowMapElts+2), true);
}
}
TEST_EQUALITY(myDomainMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myDomainMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myDomainMap->getMinLocalIndex(),0);
TEST_EQUALITY(myDomainMap->getGlobalNumElements(),as<size_t>(comm->getSize()));
TEST_EQUALITY(as<bool>(myDomainMap->getNodeNumElements()==1), true);
} // AmalgamationStridedOffsetDropping2LW
TEUCHOS_UNIT_TEST(CoalesceDropFactory, AmalgamationStrided2LW)
{
# include "MueLu_UseShortNames.hpp"
MUELU_TESTING_SET_OSTREAM;
MUELU_TESTING_LIMIT_SCOPE(Scalar,GlobalOrdinal,NO);
out << "version: " << MueLu::Version() << std::endl;
// unit test for block size 3 = (2,1). wrap block 0
// lightweight wrap = true
RCP<const Teuchos::Comm<int> > comm = Parameters::getDefaultComm();
Xpetra::UnderlyingLib lib = TestHelpers::Parameters::getLib();
// create strided map information
std::vector<size_t> stridingInfo;
stridingInfo.push_back(as<size_t>(2));
stridingInfo.push_back(as<size_t>(1));
LocalOrdinal stridedBlockId = 0;
int blockSize=3;
RCP<const StridedMap> dofMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(lib, blockSize*comm->getSize(), 0,
stridingInfo, comm,
stridedBlockId /*blockId*/, 0 /*offset*/);
/////////////////////////////////////////////////////
Teuchos::RCP<Matrix> mtx = TestHelpers::TestFactory<SC,LO,GO,NO>::BuildTridiag(dofMap, 2.0, -1.0, -1.0);
Level fineLevel;
TestHelpers::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(fineLevel);
RCP<const Xpetra::StridedMap<LocalOrdinal, GlobalOrdinal, Node> > stridedRangeMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(
mtx->getRangeMap(),
stridingInfo,
stridedBlockId,
0 /*offset*/
);
RCP<const Map> stridedDomainMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(
mtx->getDomainMap(),
stridingInfo,
stridedBlockId,
0 /*offset*/
);
if(mtx->IsView("stridedMaps") == true) mtx->RemoveView("stridedMaps");
mtx->CreateView("stridedMaps", stridedRangeMap, stridedDomainMap);
fineLevel.Set("A", mtx);
CoalesceDropFactory dropFact = CoalesceDropFactory();
dropFact.SetParameter("lightweight wrap",Teuchos::ParameterEntry(true));
fineLevel.Request("Graph", &dropFact);
fineLevel.Request("DofsPerNode", &dropFact);
dropFact.Build(fineLevel);
fineLevel.print(out);
RCP<GraphBase> graph = fineLevel.Get<RCP<GraphBase> >("Graph", &dropFact);
LO myDofsPerNode = fineLevel.Get<LO>("DofsPerNode", &dropFact);
TEST_EQUALITY(as<int>(graph->GetDomainMap()->getGlobalNumElements()) == comm->getSize(), true);
TEST_EQUALITY(as<int>(myDofsPerNode) == blockSize, true);
bool bCorrectGraph = false;
if (comm->getSize() == 1 && graph->getNeighborVertices(0).size() == 1) {
bCorrectGraph = true;
} else {
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
if (graph->getNeighborVertices(0).size() == 2) bCorrectGraph = true;
}
else {
if (graph->getNeighborVertices(0).size() == blockSize) bCorrectGraph = true;
}
}
TEST_EQUALITY(bCorrectGraph, true);
const RCP<const Map> myImportMap = graph->GetImportMap(); // < note that the ImportMap is built from the column map of the matrix A WITHOUT dropping!
const RCP<const Map> myDomainMap = graph->GetDomainMap();
TEST_EQUALITY(myImportMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myImportMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myImportMap->getMinLocalIndex(),0);
TEST_EQUALITY(myImportMap->getGlobalNumElements(),as<size_t>(comm->getSize()+2*(comm->getSize()-1)));
if (comm->getSize()>1) {
size_t numLocalRowMapElts = graph->GetNodeNumVertices();
size_t numLocalImportElts = myImportMap->getNodeNumElements();
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
TEST_EQUALITY(as<bool>(numLocalImportElts==numLocalRowMapElts+1), true);
} else {
TEST_EQUALITY(as<bool>(numLocalImportElts==numLocalRowMapElts+2), true);
}
}
if (comm->getSize()>1) {
size_t numLocalRowMapElts = graph->GetNodeNumVertices();
size_t maxLocalIndex = myImportMap->getMaxLocalIndex();
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
TEST_EQUALITY(as<bool>(maxLocalIndex==numLocalRowMapElts*blockSize-2), true);
} else {
TEST_EQUALITY(as<bool>(maxLocalIndex==numLocalRowMapElts*blockSize-1), true);
}
}
TEST_EQUALITY(myDomainMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myDomainMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myDomainMap->getMinLocalIndex(),0);
TEST_EQUALITY(myDomainMap->getMaxLocalIndex(),0);
TEST_EQUALITY(myDomainMap->getGlobalNumElements(),as<size_t>(comm->getSize()));
TEST_EQUALITY(as<bool>(myDomainMap->getNodeNumElements()==1), true);
} // AmalgamationStrided2LW
void RebalanceBlockRestrictionFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level &fineLevel, Level &coarseLevel) const {
FactoryMonitor m(*this, "Build", coarseLevel);
//const Teuchos::ParameterList & pL = GetParameterList();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
Teuchos::RCP<Matrix> originalTransferOp = Teuchos::null;
originalTransferOp = Get< RCP<Matrix> >(coarseLevel, "R");
RCP<Xpetra::BlockedCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> > bOriginalTransferOp = Teuchos::rcp_dynamic_cast<Xpetra::BlockedCrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps> >(originalTransferOp);
TEUCHOS_TEST_FOR_EXCEPTION(bOriginalTransferOp==Teuchos::null, Exceptions::BadCast, "MueLu::RebalanceBlockTransferFactory::Build: input matrix P or R is not of type BlockedCrsMatrix! error.");
// plausibility check
TEUCHOS_TEST_FOR_EXCEPTION(bOriginalTransferOp->Rows() != 2,Exceptions::RuntimeError, "MueLu::RebalanceBlockTransferFactory::Build: number of block rows of transfer operator is not equal 2. error.");
TEUCHOS_TEST_FOR_EXCEPTION(bOriginalTransferOp->Cols() != 2,Exceptions::RuntimeError, "MueLu::RebalanceBlockTransferFactory::Build: number of block columns of transfer operator is not equal 2. error.");
// rebuild rebalanced blocked P operator
std::vector<GO> fullRangeMapVector;
std::vector<GO> fullDomainMapVector;
std::vector<RCP<const Map> > subBlockRRangeMaps;
std::vector<RCP<const Map> > subBlockRDomainMaps;
subBlockRRangeMaps.reserve(bOriginalTransferOp->Rows()); // reserve size for block P operators
subBlockRDomainMaps.reserve(bOriginalTransferOp->Cols()); // reserve size for block P operators
std::vector<Teuchos::RCP<Matrix> > subBlockRebR;
subBlockRebR.reserve(bOriginalTransferOp->Cols());
int curBlockId = 0;
Teuchos::RCP<const Import> rebalanceImporter = Teuchos::null;
std::vector<Teuchos::RCP<const FactoryManagerBase> >::const_iterator it;
for (it = FactManager_.begin(); it != FactManager_.end(); ++it) {
// begin SubFactoryManager environment
SetFactoryManager fineSFM (rcpFromRef(fineLevel), *it);
SetFactoryManager coarseSFM(rcpFromRef(coarseLevel), *it);
rebalanceImporter = coarseLevel.Get<Teuchos::RCP<const Import> >("Importer", (*it)->GetFactory("Importer").get());
// extract matrix block
Teuchos::RCP<CrsMatrix> Rmii = bOriginalTransferOp->getMatrix(curBlockId, curBlockId);
Teuchos::RCP<CrsMatrixWrap> Rwii = Teuchos::rcp(new CrsMatrixWrap(Rmii));
Teuchos::RCP<Matrix> Rii = Teuchos::rcp_dynamic_cast<Matrix>(Rwii);
Teuchos::RCP<Matrix> rebRii;
if(rebalanceImporter != Teuchos::null) {
std::stringstream ss; ss << "Rebalancing restriction block R(" << curBlockId << "," << curBlockId << ")";
SubFactoryMonitor m1(*this, ss.str(), coarseLevel);
{
SubFactoryMonitor subM(*this, "Rebalancing restriction -- fusedImport", coarseLevel);
// Note: The 3rd argument says to use originalR's domain map.
RCP<Map> dummy;
rebRii = MatrixFactory::Build(Rii,*rebalanceImporter,dummy,rebalanceImporter->getTargetMap());
}
RCP<ParameterList> params = rcp(new ParameterList());
params->set("printLoadBalancingInfo", true);
std::stringstream ss2; ss2 << "R(" << curBlockId << "," << curBlockId << ") rebalanced:";
GetOStream(Statistics0) << PerfUtils::PrintMatrixInfo(*rebRii, ss2.str(), params);
} else {
rebRii = Rii;
RCP<ParameterList> params = rcp(new ParameterList());
params->set("printLoadBalancingInfo", true);
std::stringstream ss2; ss2 << "R(" << curBlockId << "," << curBlockId << ") not rebalanced:";
GetOStream(Statistics0) << PerfUtils::PrintMatrixInfo(*rebRii, ss2.str(), params);
}
// fix striding information for rebalanced diagonal block rebRii
RCP<const Xpetra::MapExtractor<Scalar, LocalOrdinal, GlobalOrdinal, Node> > rgRMapExtractor = bOriginalTransferOp->getRangeMapExtractor(); // original map extractor
Teuchos::RCP<const StridedMap> orig_stridedRgMap = Teuchos::rcp_dynamic_cast<const StridedMap>(rgRMapExtractor->getMap(Teuchos::as<size_t>(curBlockId)));
Teuchos::RCP<const Map> stridedRgMap = Teuchos::null;
if(orig_stridedRgMap != Teuchos::null) {
std::vector<size_t> stridingData = orig_stridedRgMap->getStridingData();
Teuchos::ArrayView< const GlobalOrdinal > nodeRangeMapii = rebRii->getRangeMap()->getNodeElementList();
stridedRgMap = StridedMapFactory::Build(
originalTransferOp->getRangeMap()->lib(),
Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid(),
nodeRangeMapii,
rebRii->getRangeMap()->getIndexBase(),
stridingData,
originalTransferOp->getRangeMap()->getComm(),
orig_stridedRgMap->getStridedBlockId(),
orig_stridedRgMap->getOffset());
}
RCP<const Xpetra::MapExtractor<Scalar, LocalOrdinal, GlobalOrdinal, Node> > doRMapExtractor = bOriginalTransferOp->getDomainMapExtractor(); // original map extractor
Teuchos::RCP<const StridedMap> orig_stridedDoMap = Teuchos::rcp_dynamic_cast<const StridedMap>(doRMapExtractor->getMap(Teuchos::as<size_t>(curBlockId)));
Teuchos::RCP<const Map> stridedDoMap = Teuchos::null;
if(orig_stridedDoMap != Teuchos::null) {
std::vector<size_t> stridingData = orig_stridedDoMap->getStridingData();
Teuchos::ArrayView< const GlobalOrdinal > nodeDomainMapii = rebRii->getDomainMap()->getNodeElementList();
stridedDoMap = StridedMapFactory::Build(
originalTransferOp->getDomainMap()->lib(),
Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid(),
nodeDomainMapii,
rebRii->getDomainMap()->getIndexBase(),
stridingData,
originalTransferOp->getDomainMap()->getComm(),
orig_stridedDoMap->getStridedBlockId(),
orig_stridedDoMap->getOffset());
}
TEUCHOS_TEST_FOR_EXCEPTION(stridedRgMap == Teuchos::null,Exceptions::RuntimeError, "MueLu::RebalanceBlockRestrictionFactory::Build: failed to generate striding information. error.");
TEUCHOS_TEST_FOR_EXCEPTION(stridedDoMap == Teuchos::null,Exceptions::RuntimeError, "MueLu::RebalanceBlockRestrictionFactory::Build: failed to generate striding information. error.");
// replace stridedMaps view in diagonal sub block
if(rebRii->IsView("stridedMaps")) rebRii->RemoveView("stridedMaps");
rebRii->CreateView("stridedMaps", stridedRgMap, stridedDoMap);
// store rebalanced subblock
subBlockRebR.push_back(rebRii);
// append strided row map (= range map) to list of range maps.
Teuchos::RCP<const Map> rangeMapii = rebRii->getRowMap("stridedMaps"); //rebRii->getRangeMap();
subBlockRRangeMaps.push_back(rangeMapii);
Teuchos::ArrayView< const GlobalOrdinal > nodeRangeMapii = rebRii->getRangeMap()->getNodeElementList();
fullRangeMapVector.insert(fullRangeMapVector.end(), nodeRangeMapii.begin(), nodeRangeMapii.end());
sort(fullRangeMapVector.begin(), fullRangeMapVector.end());
// append strided col map (= domain map) to list of range maps.
Teuchos::RCP<const Map> domainMapii = rebRii->getColMap("stridedMaps"); //rebRii->getDomainMap();
subBlockRDomainMaps.push_back(domainMapii);
Teuchos::ArrayView< const GlobalOrdinal > nodeDomainMapii = rebRii->getDomainMap()->getNodeElementList();
fullDomainMapVector.insert(fullDomainMapVector.end(), nodeDomainMapii.begin(), nodeDomainMapii.end());
sort(fullDomainMapVector.begin(), fullDomainMapVector.end());
////////////////////////////////////////////////////////////
// rebalance null space
if(rebalanceImporter != Teuchos::null)
{ // rebalance null space
std::stringstream ss2; ss2 << "Rebalancing nullspace block(" << curBlockId << "," << curBlockId << ")";
SubFactoryMonitor subM(*this, ss2.str(), coarseLevel);
RCP<MultiVector> nullspace = coarseLevel.Get<RCP<MultiVector> >("Nullspace", (*it)->GetFactory("Nullspace").get());
RCP<MultiVector> permutedNullspace = MultiVectorFactory::Build(rebalanceImporter->getTargetMap(), nullspace->getNumVectors());
permutedNullspace->doImport(*nullspace, *rebalanceImporter, Xpetra::INSERT);
// TODO think about this
//if (pL.get<bool>("useSubcomm") == true) // TODO either useSubcomm is enabled everywhere or nowhere
//permutedNullspace->replaceMap(permutedNullspace->getMap()->removeEmptyProcesses());
coarseLevel.Set<RCP<MultiVector> >("Nullspace", permutedNullspace, (*it)->GetFactory("Nullspace").get());
} // end rebalance null space
else { // do nothing
RCP<MultiVector> nullspace = coarseLevel.Get<RCP<MultiVector> >("Nullspace", (*it)->GetFactory("Nullspace").get());
coarseLevel.Set<RCP<MultiVector> >("Nullspace", nullspace, (*it)->GetFactory("Nullspace").get());
}
////////////////////////////////////////////////////////////
curBlockId++;
} // end for loop
// extract map index base from maps of blocked P
GO rangeIndexBase = originalTransferOp->getRangeMap()->getIndexBase();
GO domainIndexBase= originalTransferOp->getDomainMap()->getIndexBase();
// check this
RCP<const Xpetra::MapExtractor<Scalar, LocalOrdinal, GlobalOrdinal, Node> > rangeRMapExtractor = bOriginalTransferOp->getRangeMapExtractor(); // original map extractor
Teuchos::ArrayView<GO> fullRangeMapGIDs(&fullRangeMapVector[0],fullRangeMapVector.size());
Teuchos::RCP<const StridedMap> stridedRgFullMap = Teuchos::rcp_dynamic_cast<const StridedMap>(rangeRMapExtractor->getFullMap());
Teuchos::RCP<const Map > fullRangeMap = Teuchos::null;
if(stridedRgFullMap != Teuchos::null) {
std::vector<size_t> stridedData = stridedRgFullMap->getStridingData();
fullRangeMap =
StridedMapFactory::Build(
originalTransferOp->getRangeMap()->lib(),
Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid(),
fullRangeMapGIDs,
rangeIndexBase,
stridedData,
originalTransferOp->getRangeMap()->getComm(),
stridedRgFullMap->getStridedBlockId(),
stridedRgFullMap->getOffset());
} else {
fullRangeMap =
MapFactory::Build(
originalTransferOp->getRangeMap()->lib(),
Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid(),
fullRangeMapGIDs,
rangeIndexBase,
originalTransferOp->getRangeMap()->getComm());
}
RCP<const Xpetra::MapExtractor<Scalar, LocalOrdinal, GlobalOrdinal, Node> > domainAMapExtractor = bOriginalTransferOp->getDomainMapExtractor();
Teuchos::ArrayView<GO> fullDomainMapGIDs(&fullDomainMapVector[0],fullDomainMapVector.size());
Teuchos::RCP<const StridedMap> stridedDoFullMap = Teuchos::rcp_dynamic_cast<const StridedMap>(domainAMapExtractor->getFullMap());
Teuchos::RCP<const Map > fullDomainMap = Teuchos::null;
if(stridedDoFullMap != Teuchos::null) {
TEUCHOS_TEST_FOR_EXCEPTION(stridedDoFullMap==Teuchos::null, Exceptions::BadCast, "MueLu::BlockedPFactory::Build: full map in domain map extractor has no striding information! error.");
std::vector<size_t> stridedData2 = stridedDoFullMap->getStridingData();
fullDomainMap =
StridedMapFactory::Build(
originalTransferOp->getDomainMap()->lib(),
Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid(),
fullDomainMapGIDs,
domainIndexBase,
stridedData2,
originalTransferOp->getDomainMap()->getComm(),
stridedDoFullMap->getStridedBlockId(),
stridedDoFullMap->getOffset());
} else {
fullDomainMap =
MapFactory::Build(
originalTransferOp->getDomainMap()->lib(),
Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid(),
fullDomainMapGIDs,
domainIndexBase,
originalTransferOp->getDomainMap()->getComm());
}
// build map extractors
Teuchos::RCP<const Xpetra::MapExtractor<Scalar, LocalOrdinal, GlobalOrdinal, Node> > rangeMapExtractor =
Xpetra::MapExtractorFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node>::Build(fullRangeMap, subBlockRRangeMaps);
Teuchos::RCP<const Xpetra::MapExtractor<Scalar, LocalOrdinal, GlobalOrdinal, Node> > domainMapExtractor =
Xpetra::MapExtractorFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node>::Build(fullDomainMap, subBlockRDomainMaps);
Teuchos::RCP<BlockedCrsMatrix> bRebR = Teuchos::rcp(new BlockedCrsMatrix(rangeMapExtractor,domainMapExtractor,10));
for(size_t i = 0; i<subBlockRRangeMaps.size(); i++) {
Teuchos::RCP<const CrsMatrixWrap> crsOpii = Teuchos::rcp_dynamic_cast<const CrsMatrixWrap>(subBlockRebR[i]);
Teuchos::RCP<CrsMatrix> crsMatii = crsOpii->getCrsMatrix();
bRebR->setMatrix(i,i,crsMatii);
}
bRebR->fillComplete();
Set(coarseLevel, "R", Teuchos::rcp_dynamic_cast<Matrix>(bRebR)); // do nothing // TODO remove this!
} // Build
