TEUCHOS_UNIT_TEST(CoarseMap, NonStandardCaseA )
{
out << "version: " << MueLu::Version() << std::endl;
Level myLevel;
myLevel.SetLevelID(0);
RCP<Matrix> A = TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::Build1DPoisson(15);
myLevel.Set("A", A);
// build dummy aggregate structure
Teuchos::RCP<Aggregates> aggs = Teuchos::rcp(new Aggregates(A->getRowMap()));
aggs->SetNumAggregates(10); // set (local!) number of aggregates
myLevel.Set("Aggregates", aggs);
// build dummy nullspace vector
Teuchos::RCP<MultiVector> nsp = MultiVectorFactory::Build(A->getRowMap(),1);
nsp->putScalar(1.0);
myLevel.Set("Nullspace", nsp);
RCP<CoarseMapFactory> myCMF = Teuchos::rcp(new CoarseMapFactory());
myLevel.Request("CoarseMap",myCMF.get());
myCMF->SetParameter("Domain GID offsets",Teuchos::ParameterEntry(std::string("{100,50}")));
myCMF->SetFactory("Aggregates",MueLu::NoFactory::getRCP());
myCMF->SetFactory("Nullspace",MueLu::NoFactory::getRCP());
myCMF->Build(myLevel);
Teuchos::RCP<const Map> myCoarseMap = myLevel.Get<Teuchos::RCP<const Map> >("CoarseMap",myCMF.get());
TEST_EQUALITY(myCoarseMap->getMinAllGlobalIndex() == 100, true);
TEST_EQUALITY(myCoarseMap->getMaxLocalIndex()==9,true);
myLevel.Release("CoarseMap",myCMF.get());
myLevel.SetLevelID(1);
myLevel.Request("CoarseMap",myCMF.get());
myCMF->SetParameter("Domain GID offsets",Teuchos::ParameterEntry(std::string("{100,50}")));
myCMF->SetFactory("Aggregates",MueLu::NoFactory::getRCP());
myCMF->SetFactory("Nullspace",MueLu::NoFactory::getRCP());
myCMF->Build(myLevel);
myCoarseMap = myLevel.Get<Teuchos::RCP<const Map> >("CoarseMap",myCMF.get());
TEST_EQUALITY(myCoarseMap->getMinAllGlobalIndex() == 50, true);
TEST_EQUALITY(myCoarseMap->getMaxLocalIndex()==9,true);
myLevel.Release("CoarseMap",myCMF.get());
myLevel.SetLevelID(2);
myLevel.Request("CoarseMap",myCMF.get());
myCMF->SetParameter("Domain GID offsets",Teuchos::ParameterEntry(std::string("{100,50}")));
myCMF->SetFactory("Aggregates",MueLu::NoFactory::getRCP());
myCMF->SetFactory("Nullspace",MueLu::NoFactory::getRCP());
myCMF->Build(myLevel);
myCoarseMap = myLevel.Get<Teuchos::RCP<const Map> >("CoarseMap",myCMF.get());
TEST_EQUALITY(myCoarseMap->getMinAllGlobalIndex() == 0, true);
TEST_EQUALITY(myCoarseMap->getMaxLocalIndex()==9,true);
}
void testBuildSmootherDefaultArg(RCP<SmootherPrototype> smooProtoA, RCP<SmootherPrototype> smooProtoB, Teuchos::FancyOStream & out, bool & success) {
RCP<SmootherFactory> smooFact = rcp( new SmootherFactory(smooProtoA, smooProtoB) );
Level level; //level.SetupPhase(true);
TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::createSingleLevelHierarchy(level);
level.Request("PreSmoother",smooFact.get());
level.Request("PostSmoother", smooFact.get());
smooFact->BuildSmoother(level);
testBuildCheck(smooFact, level, smooProtoA, smooProtoB, MueLu::BOTH, out, success);
}
TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(ThresholdAFilterFactory, Basic, Scalar, LocalOrdinal, GlobalOrdinal, Node)
{
# include <MueLu_UseShortNames.hpp>
MUELU_TESTING_SET_OSTREAM;
MUELU_TESTING_LIMIT_EPETRA_SCOPE(Scalar,GlobalOrdinal,Node);
out << "version: " << MueLu::Version() << std::endl;
Level aLevel;
TestHelpers::TestFactory<SC, LO, GO, NO>::createSingleLevelHierarchy(aLevel);
RCP<Matrix> A = TestHelpers::TestFactory<SC, LO, GO, NO>::Build1DPoisson(20); //can be an empty operator
RCP<ThresholdAFilterFactory> AfilterFactory0 = rcp(new ThresholdAFilterFactory("A",0.1)); // keep all
RCP<ThresholdAFilterFactory> AfilterFactory1 = rcp(new ThresholdAFilterFactory("A",1.1)); // keep only diagonal
RCP<ThresholdAFilterFactory> AfilterFactory2 = rcp(new ThresholdAFilterFactory("A",3)); // keep only diagonal
aLevel.Set("A",A);
aLevel.Request("A",AfilterFactory0.get());
AfilterFactory0->Build(aLevel);
TEST_EQUALITY(aLevel.IsAvailable("A",AfilterFactory0.get()), true);
RCP<Matrix> A0 = aLevel.Get< RCP<Matrix> >("A",AfilterFactory0.get());
aLevel.Release("A",AfilterFactory0.get());
TEST_EQUALITY(aLevel.IsAvailable("A",AfilterFactory0.get()), false);
TEST_EQUALITY(A0->getNodeNumEntries(), A->getNodeNumEntries());
TEST_EQUALITY(A0->getGlobalNumEntries(), A->getGlobalNumEntries());
aLevel.Request("A",AfilterFactory1.get());
AfilterFactory1->Build(aLevel);
TEST_EQUALITY(aLevel.IsAvailable("A",AfilterFactory1.get()), true);
RCP<Matrix> A1 = aLevel.Get< RCP<Matrix> >("A",AfilterFactory1.get());
aLevel.Release("A",AfilterFactory1.get());
TEST_EQUALITY(aLevel.IsAvailable("A",AfilterFactory1.get()), false);
TEST_EQUALITY(A1->getGlobalNumEntries(), A1->getGlobalNumRows());
aLevel.Request("A",AfilterFactory2.get());
AfilterFactory2->Build(aLevel);
TEST_EQUALITY(aLevel.IsAvailable("A",AfilterFactory2.get()), true);
RCP<Matrix> A2 = aLevel.Get< RCP<Matrix> >("A",AfilterFactory2.get());
aLevel.Release("A",AfilterFactory2.get());
TEST_EQUALITY(aLevel.IsAvailable("A",AfilterFactory2.get()), false);
TEST_EQUALITY(A2->getGlobalNumEntries(), A2->getGlobalNumRows());
}
Teuchos::RCP<MueLu::Aggregates_kokkos<LocalOrdinal, GlobalOrdinal, Node>>
gimmeUncoupledAggregates(const Teuchos::RCP<Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>>& A,
Teuchos::RCP<MueLu::AmalgamationInfo<LocalOrdinal, GlobalOrdinal, Node>>& amalgInfo,
bool bPhase1 = true, bool bPhase2a = true, bool bPhase2b = true, bool bPhase3 = true) {
# include "MueLu_UseShortNames.hpp"
Level level;
TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(level);
level.Set("A", A);
RCP<AmalgamationFactory> amalgFact = rcp(new AmalgamationFactory());
RCP<CoalesceDropFactory_kokkos> dropFact = rcp(new CoalesceDropFactory_kokkos());
dropFact->SetFactory("UnAmalgamationInfo", amalgFact);
using Teuchos::ParameterEntry;
// Setup aggregation factory (use default factory for graph)
RCP<UncoupledAggregationFactory_kokkos> aggFact = rcp(new UncoupledAggregationFactory_kokkos());
aggFact->SetFactory("Graph", dropFact);
aggFact->SetParameter("aggregation: max agg size", ParameterEntry(3));
aggFact->SetParameter("aggregation: min agg size", ParameterEntry(3));
aggFact->SetParameter("aggregation: max selected neighbors", ParameterEntry(0));
aggFact->SetParameter("aggregation: ordering", ParameterEntry(std::string("natural")));
aggFact->SetParameter("aggregation: enable phase 1", ParameterEntry(bPhase1));
aggFact->SetParameter("aggregation: enable phase 2a", ParameterEntry(bPhase2a));
aggFact->SetParameter("aggregation: enable phase 2b", ParameterEntry(bPhase2b));
aggFact->SetParameter("aggregation: enable phase 3", ParameterEntry(bPhase3));
level.Request("Aggregates", aggFact.get());
level.Request("UnAmalgamationInfo", amalgFact.get());
level.Request(*aggFact);
aggFact->Build(level);
auto aggregates = level.Get<RCP<Aggregates_kokkos> >("Aggregates", aggFact.get());
amalgInfo = level.Get<RCP<AmalgamationInfo> >("UnAmalgamationInfo", amalgFact.get());
level.Release("UnAmalgamationInfo", amalgFact.get());
level.Release("Aggregates", aggFact.get());
return aggregates;
}
TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CoarseMap_kokkos, StandardCase, Scalar, LocalOrdinal, GlobalOrdinal, Node)
{
# include "MueLu_UseShortNames.hpp"
RUN_EPETRA_ONLY_WITH_SERIAL_NODE(Node);
MueLu::VerboseObject::SetDefaultOStream(Teuchos::rcpFromRef(out));
out << "version: " << MueLu::Version() << std::endl;
Level fineLevel;
TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(fineLevel);
RCP<Matrix> A = TestHelpers_kokkos::TestFactory<SC, LO, GO, NO>::Build1DPoisson(15);
fineLevel.Set("A", A);
// build dummy aggregate structure
RCP<Aggregates_kokkos> aggs = Teuchos::rcp(new Aggregates_kokkos(A->getRowMap()));
aggs->SetNumAggregates(10); // set (local!) number of aggregates
fineLevel.Set("Aggregates", aggs);
// build dummy nullspace vector
RCP<MultiVector> nsp = MultiVectorFactory::Build(A->getRowMap(),1);
nsp->putScalar(1.0);
fineLevel.Set("Nullspace", nsp);
RCP<CoarseMapFactory_kokkos> coarseMapFactory = Teuchos::rcp(new CoarseMapFactory_kokkos());
coarseMapFactory->SetFactory("Aggregates", MueLu::NoFactory::getRCP());
coarseMapFactory->SetFactory("Nullspace", MueLu::NoFactory::getRCP());
fineLevel.Request("CoarseMap", coarseMapFactory.get());
coarseMapFactory->Build(fineLevel);
auto myCoarseMap = fineLevel.Get<Teuchos::RCP<const Map> >("CoarseMap", coarseMapFactory.get());
TEST_EQUALITY(myCoarseMap->getMinAllGlobalIndex() == 0, true);
TEST_EQUALITY(myCoarseMap->getMaxLocalIndex() == 9, true);
}
TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CoalesceDropFactory_kokkos, ClassicScalarWithoutFiltering, Scalar, LocalOrdinal, GlobalOrdinal, Node)
{
# include "MueLu_UseShortNames.hpp"
MUELU_TESTING_SET_OSTREAM;
MUELU_TESTING_LIMIT_SCOPE(Scalar,GlobalOrdinal,NO);
out << "version: " << MueLu::Version() << std::endl;
RCP<const Teuchos::Comm<int> > comm = Parameters::getDefaultComm();
Level fineLevel;
TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(fineLevel);
RCP<Matrix> A = TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::Build1DPoisson(36);
fineLevel.Set("A", A);
CoalesceDropFactory_kokkos dropFact;
fineLevel.Request("Graph", &dropFact);
fineLevel.Request("DofsPerNode", &dropFact);
fineLevel.Request("Filtering", &dropFact);
dropFact.Build(fineLevel);
auto graph = fineLevel.Get<RCP<LWGraph_kokkos> >("Graph", &dropFact);
auto myDofsPerNode = fineLevel.Get<LO> ("DofsPerNode", &dropFact);
auto filtering = fineLevel.Get<bool> ("Filtering", &dropFact);
TEST_EQUALITY(as<int>(myDofsPerNode) == 1, true);
TEST_EQUALITY(filtering, false);
bool bCorrectGraph = false;
if (comm->getSize() == 1) {
auto v0 = graph->getNeighborVertices(0);
auto v1 = graph->getNeighborVertices(1);
auto v2 = graph->getNeighborVertices(2);
if (v0.size() == 2 && ((v0(0) == 0 && v0(1) == 1) || (v0(0) == 1 && v0(1) == 0)) &&
v1.size() == 3 && v2.size() == 3)
bCorrectGraph = true;
} else {
if (comm->getRank() == 0 ) {
if (graph->getNeighborVertices(0).size() == 2)
bCorrectGraph = true;
} else {
if (graph->getNeighborVertices(0).size() == 3)
bCorrectGraph = true;
}
}
TEST_EQUALITY(bCorrectGraph, true);
auto myImportMap = graph->GetImportMap(); // < note that the ImportMap is built from the column map of the matrix A WITHOUT dropping!
auto myDomainMap = graph->GetDomainMap();
TEST_EQUALITY(myImportMap->getMaxAllGlobalIndex(), 35);
TEST_EQUALITY(myImportMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myImportMap->getMinLocalIndex(), 0);
TEST_EQUALITY(myImportMap->getGlobalNumElements(), as<size_t>(36 + (comm->getSize()-1)*2));
TEST_EQUALITY(myDomainMap->getMaxAllGlobalIndex(), 35);
TEST_EQUALITY(myDomainMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myDomainMap->getMinLocalIndex(), 0);
TEST_EQUALITY(myDomainMap->getGlobalNumElements(), 36);
}
TEUCHOS_UNIT_TEST(CoalesceDropFactory, AmalgamationStridedOffsetDropping2LW)
{
// unit test for block size 9 = (2,3,4). wrap block 1.
// drop small entries
// lightweight wrap = true
out << "version: " << MueLu::Version() << std::endl;
RCP<const Teuchos::Comm<int> > comm = Parameters::getDefaultComm();
Xpetra::UnderlyingLib lib = TestHelpers::Parameters::getLib();
// create strided map information
std::vector<size_t> stridingInfo;
stridingInfo.push_back(as<size_t>(2));
stridingInfo.push_back(as<size_t>(3));
stridingInfo.push_back(as<size_t>(4));
LocalOrdinal stridedBlockId = 1;
GlobalOrdinal offset = 19;
RCP<const StridedMap> dofMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(lib, 9*comm->getSize(), 0,
stridingInfo, comm,
stridedBlockId, offset);
/////////////////////////////////////////////////////
Teuchos::RCP<Matrix> mtx = TestHelpers::TestFactory<SC,LO,GO,NO>::BuildTridiag(dofMap, 2.0, 1.0, 0.0001);
Level fineLevel;
TestHelpers::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(fineLevel);
RCP<const Map> stridedRangeMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(
mtx->getRangeMap(),
stridingInfo,
stridedBlockId,
offset
);
RCP<const Map> stridedDomainMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(
mtx->getDomainMap(),
stridingInfo,
stridedBlockId,
offset
);
if(mtx->IsView("stridedMaps") == true) mtx->RemoveView("stridedMaps");
mtx->CreateView("stridedMaps", stridedRangeMap, stridedDomainMap);
fineLevel.Set("A", mtx);
CoalesceDropFactory dropFact = CoalesceDropFactory();
dropFact.SetParameter("lightweight wrap",Teuchos::ParameterEntry(true));
dropFact.SetParameter("aggregation: drop tol",Teuchos::ParameterEntry(0.3));
fineLevel.Request("Graph", &dropFact);
fineLevel.Request("DofsPerNode", &dropFact);
dropFact.Build(fineLevel);
fineLevel.print(out);
RCP<GraphBase> graph = fineLevel.Get<RCP<GraphBase> >("Graph", &dropFact);
LO myDofsPerNode = fineLevel.Get<LO>("DofsPerNode", &dropFact);
TEST_EQUALITY(as<int>(graph->GetDomainMap()->getGlobalNumElements()) == comm->getSize(), true);
TEST_EQUALITY(as<int>(myDofsPerNode) == 9, true);
bool bCorrectGraph = false;
if (comm->getSize() == 1 && graph->getNeighborVertices(0).size() == 1) {
bCorrectGraph = true;
} else {
if (comm->getRank() == 0) {
if (graph->getNeighborVertices(0).size() == 1) bCorrectGraph = true;
}
else {
if (graph->getNeighborVertices(0).size() == 2) bCorrectGraph = true;
}
}
TEST_EQUALITY(bCorrectGraph, true);
const RCP<const Map> myImportMap = graph->GetImportMap(); // < note that the ImportMap is built from the column map of the matrix A WITHOUT dropping!
const RCP<const Map> myDomainMap = graph->GetDomainMap();
TEST_EQUALITY(myImportMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myImportMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myImportMap->getMinLocalIndex(),0);
TEST_EQUALITY(myImportMap->getGlobalNumElements(),as<size_t>(comm->getSize()+2*(comm->getSize()-1)));
if (comm->getSize()>1) {
size_t numLocalRowMapElts = graph->GetNodeNumVertices();
size_t numLocalImportElts = myImportMap->getNodeNumElements();
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
TEST_EQUALITY(as<bool>(numLocalImportElts==numLocalRowMapElts+1), true);
} else {
TEST_EQUALITY(as<bool>(numLocalImportElts==numLocalRowMapElts+2), true);
}
}
TEST_EQUALITY(myDomainMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myDomainMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myDomainMap->getMinLocalIndex(),0);
TEST_EQUALITY(myDomainMap->getGlobalNumElements(),as<size_t>(comm->getSize()));
TEST_EQUALITY(as<bool>(myDomainMap->getNodeNumElements()==1), true);
} // AmalgamationStridedOffsetDropping2LW
TEUCHOS_UNIT_TEST(CoalesceDropFactory, AmalgamationStrided2LW)
{
# include "MueLu_UseShortNames.hpp"
MUELU_TESTING_SET_OSTREAM;
MUELU_TESTING_LIMIT_SCOPE(Scalar,GlobalOrdinal,NO);
out << "version: " << MueLu::Version() << std::endl;
// unit test for block size 3 = (2,1). wrap block 0
// lightweight wrap = true
RCP<const Teuchos::Comm<int> > comm = Parameters::getDefaultComm();
Xpetra::UnderlyingLib lib = TestHelpers::Parameters::getLib();
// create strided map information
std::vector<size_t> stridingInfo;
stridingInfo.push_back(as<size_t>(2));
stridingInfo.push_back(as<size_t>(1));
LocalOrdinal stridedBlockId = 0;
int blockSize=3;
RCP<const StridedMap> dofMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(lib, blockSize*comm->getSize(), 0,
stridingInfo, comm,
stridedBlockId /*blockId*/, 0 /*offset*/);
/////////////////////////////////////////////////////
Teuchos::RCP<Matrix> mtx = TestHelpers::TestFactory<SC,LO,GO,NO>::BuildTridiag(dofMap, 2.0, -1.0, -1.0);
Level fineLevel;
TestHelpers::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(fineLevel);
RCP<const Xpetra::StridedMap<LocalOrdinal, GlobalOrdinal, Node> > stridedRangeMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(
mtx->getRangeMap(),
stridingInfo,
stridedBlockId,
0 /*offset*/
);
RCP<const Map> stridedDomainMap = Xpetra::StridedMapFactory<LocalOrdinal, GlobalOrdinal, Node>::Build(
mtx->getDomainMap(),
stridingInfo,
stridedBlockId,
0 /*offset*/
);
if(mtx->IsView("stridedMaps") == true) mtx->RemoveView("stridedMaps");
mtx->CreateView("stridedMaps", stridedRangeMap, stridedDomainMap);
fineLevel.Set("A", mtx);
CoalesceDropFactory dropFact = CoalesceDropFactory();
dropFact.SetParameter("lightweight wrap",Teuchos::ParameterEntry(true));
fineLevel.Request("Graph", &dropFact);
fineLevel.Request("DofsPerNode", &dropFact);
dropFact.Build(fineLevel);
fineLevel.print(out);
RCP<GraphBase> graph = fineLevel.Get<RCP<GraphBase> >("Graph", &dropFact);
LO myDofsPerNode = fineLevel.Get<LO>("DofsPerNode", &dropFact);
TEST_EQUALITY(as<int>(graph->GetDomainMap()->getGlobalNumElements()) == comm->getSize(), true);
TEST_EQUALITY(as<int>(myDofsPerNode) == blockSize, true);
bool bCorrectGraph = false;
if (comm->getSize() == 1 && graph->getNeighborVertices(0).size() == 1) {
bCorrectGraph = true;
} else {
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
if (graph->getNeighborVertices(0).size() == 2) bCorrectGraph = true;
}
else {
if (graph->getNeighborVertices(0).size() == blockSize) bCorrectGraph = true;
}
}
TEST_EQUALITY(bCorrectGraph, true);
const RCP<const Map> myImportMap = graph->GetImportMap(); // < note that the ImportMap is built from the column map of the matrix A WITHOUT dropping!
const RCP<const Map> myDomainMap = graph->GetDomainMap();
TEST_EQUALITY(myImportMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myImportMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myImportMap->getMinLocalIndex(),0);
TEST_EQUALITY(myImportMap->getGlobalNumElements(),as<size_t>(comm->getSize()+2*(comm->getSize()-1)));
if (comm->getSize()>1) {
size_t numLocalRowMapElts = graph->GetNodeNumVertices();
size_t numLocalImportElts = myImportMap->getNodeNumElements();
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
TEST_EQUALITY(as<bool>(numLocalImportElts==numLocalRowMapElts+1), true);
} else {
TEST_EQUALITY(as<bool>(numLocalImportElts==numLocalRowMapElts+2), true);
}
}
if (comm->getSize()>1) {
size_t numLocalRowMapElts = graph->GetNodeNumVertices();
size_t maxLocalIndex = myImportMap->getMaxLocalIndex();
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
TEST_EQUALITY(as<bool>(maxLocalIndex==numLocalRowMapElts*blockSize-2), true);
} else {
TEST_EQUALITY(as<bool>(maxLocalIndex==numLocalRowMapElts*blockSize-1), true);
}
}
TEST_EQUALITY(myDomainMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myDomainMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myDomainMap->getMinLocalIndex(),0);
TEST_EQUALITY(myDomainMap->getMaxLocalIndex(),0);
TEST_EQUALITY(myDomainMap->getGlobalNumElements(),as<size_t>(comm->getSize()));
TEST_EQUALITY(as<bool>(myDomainMap->getNodeNumElements()==1), true);
} // AmalgamationStrided2LW
TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CoalesceDropFactory_kokkos, ClassicBlockWithFiltering, Scalar, LocalOrdinal, GlobalOrdinal, Node)
{
# include "MueLu_UseShortNames.hpp"
MUELU_TESTING_SET_OSTREAM;
MUELU_TESTING_LIMIT_SCOPE(Scalar,GlobalOrdinal,NO);
out << "version: " << MueLu::Version() << std::endl;
RCP<const Teuchos::Comm<int> > comm = Parameters::getDefaultComm();
Xpetra::UnderlyingLib lib = TestHelpers_kokkos::Parameters::getLib();
Level fineLevel;
TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(fineLevel);
auto dofMap = MapFactory::Build(lib, 3*comm->getSize(), 0, comm);
auto mtx = TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::BuildTridiag(dofMap, 2.0, -1.0, 0.00001);
mtx->SetFixedBlockSize(3, 0);
fineLevel.Set("A", mtx);
CoalesceDropFactory_kokkos dropFact = CoalesceDropFactory_kokkos();
dropFact.SetParameter("aggregation: drop tol", Teuchos::ParameterEntry(1.0));
fineLevel.Request("Graph", &dropFact);
fineLevel.Request("DofsPerNode", &dropFact);
fineLevel.Request("Filtering", &dropFact);
dropFact.Build(fineLevel);
auto graph = fineLevel.Get<RCP<LWGraph_kokkos> >("Graph", &dropFact);
auto myDofsPerNode = fineLevel.Get<LO> ("DofsPerNode", &dropFact);
auto filtering = fineLevel.Get<bool> ("Filtering", &dropFact);
TEST_EQUALITY(as<int>(myDofsPerNode) == 3, true);
TEST_EQUALITY(filtering, true);
TEST_EQUALITY(as<int>(graph->GetDomainMap()->getGlobalNumElements()) == comm->getSize(), true);
TEST_EQUALITY(graph->getNeighborVertices(0).size(), 1);
auto myImportMap = graph->GetImportMap(); // < note that the ImportMap is built from the column map of the matrix A WITHOUT dropping!
auto myDomainMap = graph->GetDomainMap();
TEST_EQUALITY(myImportMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myImportMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myImportMap->getMinLocalIndex(), 0);
TEST_EQUALITY(myImportMap->getGlobalNumElements(), as<size_t>(comm->getSize()+2*(comm->getSize()-1)));
if (comm->getSize() > 1) {
size_t numLocalRowMapElts = graph->GetNodeNumVertices();
size_t numLocalImportElts = myImportMap->getNodeNumElements();
if (comm->getRank() == 0 || comm->getRank() == comm->getSize()-1) {
TEST_EQUALITY(numLocalImportElts, numLocalRowMapElts+1);
} else {
TEST_EQUALITY(numLocalImportElts, numLocalRowMapElts+2);
}
}
TEST_EQUALITY(myDomainMap->getMaxAllGlobalIndex(), comm->getSize()-1);
TEST_EQUALITY(myDomainMap->getMinAllGlobalIndex(), 0);
TEST_EQUALITY(myDomainMap->getMaxLocalIndex(), 0);
TEST_EQUALITY(myDomainMap->getMinLocalIndex(), 0);
TEST_EQUALITY(myDomainMap->getGlobalNumElements(), as<size_t>(comm->getSize()));
TEST_EQUALITY(myDomainMap->getNodeNumElements(), 1);
}
TEUCHOS_UNIT_TEST(Zoltan, Build)
{
typedef Teuchos::ScalarTraits<Scalar> ST;
out << "version: " << MueLu::Version() << std::endl;
out << std::endl;
out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl;
out << "that has a random number of nonzeros per row. Good results have been precomputed" << std::endl;
out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl;
out << "once the Zoltan repartitioning has been applied." << std::endl;
out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl;
RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();
if (comm->getSize() > 5) {
out << std::endl;
out << "This test must be run on 1 to 5 processes." << std::endl;
TEST_EQUALITY(true, true);
return;
}
Level level;
RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
int nx=7;
int ny=nx;
GO numGlobalElements = nx*ny;
size_t maxEntriesPerRow=30;
// Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row.
RCP<const Map> map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm);
const size_t numMyElements = map->getNodeNumElements();
Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList();
RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more
// memory on the fly. While not super efficient, this
// ensures that no zeros are being stored. Thus, from
// Zoltan's perspective the matrix is imbalanced.
// Create a vector with random integer entries in [1,maxEntriesPerRow].
ST::seedrandom(666*comm->getRank());
RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false);
Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0);
for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) {
*i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1));
}
RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
fos->setOutputToRootOnly(-1);
Teuchos::Array<Scalar> vals(maxEntriesPerRow);
Teuchos::Array<GO> cols(maxEntriesPerRow);
for (size_t i = 0; i < numMyElements; ++i) {
Teuchos::ArrayView<SC> av(&vals[0],eprData[i]);
Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]);
//stick in ones for values
for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one();
//figure out valid column indices
GO start = std::max(myGlobalElements[i]-eprData[i]+1,0);
for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j;
A->insertGlobalValues(myGlobalElements[i], iv, av);
}
A->fillComplete();
level.Set("A",A);
//build coordinates
RCP<const Map> rowMap = A->getRowMap();
Teuchos::ParameterList list;
list.set("nx",nx);
list.set("ny",ny);
RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",rowMap,list);
level.Set("Coordinates",XYZ);
LO numPartitions = comm->getSize();
level.Set("number of partitions",numPartitions);
RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface());
//zoltan->SetNumberOfPartitions(numPartitions);
//zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints
level.Request("Partition",zoltan.get());
zoltan->Build(level);
RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get());
/* //TODO temporary code to have the trivial decomposition (no change)
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
*i = comm->getRank();
decompEntries=Teuchos::null;
*/ //TODO end of temporary code
//Create vector whose local length is the global number of partitions.
//This vector will record the local number of nonzeros associated with each partition.
Teuchos::Array<GO> parts(numPartitions);
for (int i=0; i<numPartitions; ++i) parts[i] = i;
Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions);
RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView,
map->getIndexBase(),comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap);
//For the local rows in each partition, tally up the number of nonzeros. This is what
//Zoltan should be load-balancing.
Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0);
Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0);
for (size_t i=0; i<decomposition->getLocalLength();++i) {
Teuchos::ArrayView<const LO> c;
Teuchos::ArrayView<const SC> v;
A->getLocalRowView(i,c,v);
lpvData[decompData[i]] += v.size();
}
lpvData = Teuchos::null;
decompData = Teuchos::null;
//localPartsVec->describe(*fos,Teuchos::VERB_EXTREME);
//Send the local nnz tallies to pid 0, which can report the global sums.
size_t mysize=1;
if (comm->getRank() == 0) mysize = numPartitions;
RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(),
mysize,
map->getIndexBase(),
comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap);
RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap);
globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD);
ArrayRCP<GO> expectedResults(numPartitions);
switch (comm->getSize()) {
case 1:
expectedResults[0] = 807;
break;
case 2:
expectedResults[0] = 364;
expectedResults[1] = 363;
break;
case 3:
expectedResults[0] = 277;
expectedResults[1] = 261;
expectedResults[2] = 269;
break;
case 4:
expectedResults[0] = 195;
expectedResults[1] = 186;
expectedResults[2] = 177;
expectedResults[3] = 168;
break;
case 5:
expectedResults[0] = 161;
expectedResults[1] = 145;
expectedResults[2] = 148;
expectedResults[3] = 159;
expectedResults[4] = 157;
break;
default:
break;
};
//FIXME cool ... this next line causes a hang if locally the globalyTallyVec has no data.
//FIXME I get around this by making mysize (above) 1 instead of 0. Is this a bug or feature
//FIXME in getData?
ArrayRCP<const LO> gtvData = globalTallyVec->getData(0);
#ifdef __linux__
out << "Checking results..." << std::endl;
for (int i=0; i<numPartitions; ++i) {
if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]);
}
#endif
#ifdef MUELU_VISUALIZE_REPARTITIONING
//
//Now write everything to a comma-separate list that ParaView can grok
//
Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0);
Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1);
Teuchos::ArrayRCP<const GO> D = decomposition->getData(0);
RCP<std::ofstream> outFile;
std::string fileName = "zoltanResults.csv";
//write header information
if (comm->getRank() == 0) {
outFile = rcp(new std::ofstream(fileName.c_str()));
*outFile << "x coord, y coord, z coord, scalar" << std::endl;
}
comm->barrier();
//append coordinates
for (int j=0; j<comm->getSize(); ++j) {
int mypid = comm->getRank();
if (mypid == j) {
outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app));
for (int i=0; i < D.size(); ++i) {
*outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", " << D[i] << std::endl;
}
}
} //for (int i=0; i<comm->getSize(); ++i)
out << std::endl;
out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl;
out << " 1) Load the data file " << fileName << "." << std::endl;
out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl;
out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl;
out << " 4) Split screen horizontally (Icon, top right)." << std::endl;
out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl;
out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl;
out << std::endl;
out << " To display row weights next to each point:" << std::endl;
out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl;
out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl;
out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl;
#endif
} //Build
TEUCHOS_UNIT_TEST(Zoltan, Build3PDEs)
{
typedef Teuchos::ScalarTraits<Scalar> ST;
out << "version: " << MueLu::Version() << std::endl;
out << std::endl;
out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl;
out << "that has a random number of nonzeros per row and 3 DOFs per mesh point. Good results have been precomputed" << std::endl;
out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl;
out << "once the Zoltan repartitioning has been applied." << std::endl;
out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl;
RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();
if (comm->getSize() > 5) {
out << std::endl;
out << "This test must be run on 1 to 5 processes." << std::endl;
TEST_EQUALITY(true, true);
return;
}
Level level;
RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
int nx=9;
int ny=nx;
int dofsPerNode = 3;
GO numGlobalElements = nx*ny*dofsPerNode;
size_t maxEntriesPerRow=30;
RCP<const Map> map;
int numMyNodes = numGlobalElements / dofsPerNode;
if (comm->getSize() > 1) {
// In parallel, make sure that the dof's associated with a node all
// reside on the same processor.
int numNodes = numGlobalElements / dofsPerNode;
TEUCHOS_TEST_FOR_EXCEPTION( (numGlobalElements - numNodes * dofsPerNode) != 0, MueLu::Exceptions::RuntimeError,
"Number of matrix rows is not divisible by #dofs" );
int nproc = comm->getSize();
if (comm->getRank() < nproc-1) numMyNodes = numNodes / nproc;
else numMyNodes = numNodes - (numNodes/nproc) * (nproc-1);
map = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, numMyNodes*dofsPerNode, comm);
} else {
map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm);
}
const size_t numMyElements = map->getNodeNumElements();
Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList();
RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more
// memory on the fly. While not super efficient, this
// ensures that no zeros are being stored. Thus, from
// Zoltan's perspective the matrix is imbalanced.
// Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row.
// Create a vector with random integer entries in [1,maxEntriesPerRow].
ST::seedrandom(666*comm->getRank());
RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false);
Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0);
for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) {
*i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1));
}
RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
fos->setOutputToRootOnly(-1);
Teuchos::Array<Scalar> vals(maxEntriesPerRow);
Teuchos::Array<GO> cols(maxEntriesPerRow);
for (size_t i = 0; i < numMyElements; ++i) {
Teuchos::ArrayView<SC> av(&vals[0],eprData[i]);
Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]);
//stick in ones for values
for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one();
//figure out valid column indices
GO start = std::max(myGlobalElements[i]-eprData[i]+1,0);
for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j;
A->insertGlobalValues(myGlobalElements[i], iv, av);
}
A->fillComplete();
// Now treat the matrix as if it has 3 DOFs per node.
A->SetFixedBlockSize(dofsPerNode);
level.Set("A",A);
//build coordinates
Teuchos::ParameterList list;
list.set("nx",nx);
list.set("ny",ny);
RCP<const Map> coalescedMap = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements/dofsPerNode, numMyNodes, comm);
RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",coalescedMap,list);
// XYZ are the "coalesce" coordinates as it has been generated for 1 DOF/node and we are using them for 3 DOFS/node
// level.Set("Coordinates",XYZ); "Coordinates" == uncoalesce. "X,Y,ZCoordinates" == coalesce
{
RCP<MultiVector> coordinates = XYZ;
// making a copy because I don't want to keep 'open' the Xpetra_MultiVector
if (coordinates->getNumVectors() >= 1) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(0);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("XCoordinates", coordCpy);
//std::cout << coordCpy << std::endl;
}
if (coordinates->getNumVectors() >= 2) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(1);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("YCoordinates", coordCpy);
}
/*if (coordinates->getNumVectors() >= 3) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(2);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("ZCoordinates", coordCpy);
}*/
}
//coalescedMap->describe(*fos,Teuchos::VERB_EXTREME);
//sleep(1); comm->barrier();
//XYZ->describe(*fos,Teuchos::VERB_EXTREME);
LO numPartitions = comm->getSize();
level.Set("number of partitions",numPartitions);
RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface());
//zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints
level.Request("Partition",zoltan.get());
zoltan->Build(level);
RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get());
/* //temporary code to have the trivial decomposition (no change)
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
*i = comm->getRank();
decompEntries=Teuchos::null;
*/
//Create vector whose local length is the global number of partitions.
//This vector will record the local number of nonzeros associated with each partition.
Teuchos::Array<GO> parts(numPartitions);
for (int i=0; i<numPartitions; ++i) parts[i] = i;
Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions);
RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView,
map->getIndexBase(),comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap);
RCP<Xpetra::Vector<LO,LO,GO,NO> > nnzPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(A->getRowMap());
Teuchos::ArrayRCP<GO> nnzData = nnzPerRow->getDataNonConst(0);
//For the local rows in each partition, tally up the number of nonzeros. This is what
//Zoltan should be load-balancing.
Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0);
Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0);
for (size_t i=0; i<decomposition->getLocalLength();++i) {
Teuchos::ArrayView<const LO> c;
Teuchos::ArrayView<const SC> v;
A->getLocalRowView(i,c,v);
lpvData[decompData[i]] += v.size();
nnzData[i] = v.size();
}
lpvData = Teuchos::null;
decompData = Teuchos::null;
nnzData = Teuchos::null;
/*
if (comm->getRank() == 0)
std::cout << "nnz per row" << std::endl;
nnzPerRow->describe(*fos,Teuchos::VERB_EXTREME);
if (comm->getRank() == 0)
std::cout << "Row-to-partition assignment (from Zoltan)" << std::endl;
decomposition->describe(*fos,Teuchos::VERB_EXTREME);
if (comm->getRank() == 0)
std::cout << "#nonzeros per partition" << std::endl;
localPartsVec->describe(*fos,Teuchos::VERB_EXTREME);
*/
//Send the local nnz tallies to pid 0, which can report the global sums.
size_t mysize=1;
if (comm->getRank() == 0) mysize = numPartitions;
RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(),
mysize,
map->getIndexBase(),
comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap);
RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap);
globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD);
ArrayRCP<GO> expectedResults(numPartitions);
switch (comm->getSize()) {
case 1:
expectedResults[0] = 3951;
break;
case 2:
expectedResults[0] = 1955;
expectedResults[1] = 1910;
break;
case 3:
expectedResults[0] = 1326;
expectedResults[1] = 1340;
expectedResults[2] = 1321;
break;
case 4:
expectedResults[0] = 950;
expectedResults[1] = 922;
expectedResults[2] = 908;
expectedResults[3] = 936;
break;
case 5:
expectedResults[0] = 774;
expectedResults[1] = 735;
expectedResults[2] = 726;
expectedResults[3] = 771;
expectedResults[4] = 759;
break;
default:
break;
};
ArrayRCP<const LO> gtvData = globalTallyVec->getData(0);
#ifdef __linux__
out << "Checking results..." << std::endl;
for (int i=0; i<numPartitions; ++i) {
if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]);
}
#endif
#ifdef MUELU_VISUALIZE_REPARTITIONING
//
//Now write everything to a comma-separate list that ParaView can grok
//
Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0);
Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1);
Teuchos::ArrayRCP<const GO> D = decomposition->getData(0);
RCP<std::ofstream> outFile;
std::string fileName = "zoltanResults.csv";
//write header information
if (comm->getRank() == 0) {
outFile = rcp(new std::ofstream(fileName.c_str()));
*outFile << "x coord, y coord, z coord, partition, row weight" << std::endl;
}
comm->barrier();
//append coordinates
nnzData = nnzPerRow->getDataNonConst(0);
for (int j=0; j<comm->getSize(); ++j) {
int mypid = comm->getRank();
if (mypid == j) {
outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app));
int blockSize = A->GetFixedBlockSize();
//Coordinates are for coalesced system, D is for uncoalesced
for (int i=0; i < D.size()/blockSize; ++i) {
int nnz=0;
for (int k=0; k<blockSize; ++k) nnz += nnzData[i*blockSize+k];
*outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", "
<< D[i*blockSize] << ", " << nnz << std::endl;
}
}
} //for (int i=0; i<comm->getSize(); ++i)
out << std::endl;
out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl;
out << " 1) Load the data file " << fileName << "." << std::endl;
out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl;
out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl;
out << " 4) Split screen horizontally (Icon, top right)." << std::endl;
out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl;
out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl;
out << std::endl;
out << " To display row weights next to each point:" << std::endl;
out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl;
out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl;
out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl;
#endif
} //Build3PDEs
int main(int argc, char *argv[]) {
using Teuchos::RCP;
Teuchos::oblackholestream blackhole;
Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
#ifndef HAVE_TEUCHOS_LONG_LONG_INT
*out << "Warning: scaling test was not compiled with long long int support" << std::endl;
#endif
/**********************************************************************************/
/* SET TEST PARAMETERS */
/**********************************************************************************/
// Note: use --help to list available options.
Teuchos::CommandLineProcessor clp(false);
// Default is Laplace1D with nx = 8748.
// It's a nice size for 1D and perfect aggregation. (6561=3^8)
//Nice size for 1D and perfect aggregation on small numbers of processors. (8748=4*3^7)
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra
// custom parameters
std::string aggOrdering = "natural";
int minPerAgg=2;
int maxNbrAlreadySelected=0;
clp.setOption("aggOrdering",&aggOrdering,"aggregation ordering strategy (natural,random,graph)");
clp.setOption("minPerAgg",&minPerAgg,"minimum #DOFs per aggregate");
clp.setOption("maxNbrSel",&maxNbrAlreadySelected,"maximum # of nbrs allowed to be in other aggregates");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
matrixParameters.check();
xpetraParameters.check();
// TODO: check custom parameters
if (comm->getRank() == 0) {
std::cout << matrixParameters << xpetraParameters << std::endl;
// TODO: print custom parameters
}
/**********************************************************************************/
/* CREATE INITIAL MATRIX */
/**********************************************************************************/
const RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap
RCP<Matrix> A = Pr->BuildMatrix();
// return EXIT_SUCCESS;
/**********************************************************************************/
/* */
/**********************************************************************************/
Level Finest;
Finest.SetLevelID(0); // must be level 0 for NullspaceFactory
Finest.Set("A", A);
Finest.SetFactoryManager( rcp( new FactoryManager() ));
CoupledAggregationFactory CoupledAggFact;
Finest.Request(CoupledAggFact);
*out << "========================= Aggregate option summary =========================" << std::endl;
*out << "min DOFs per aggregate : " << minPerAgg << std::endl;
*out << "min # of root nbrs already aggregated : " << maxNbrAlreadySelected << std::endl;
CoupledAggFact.SetMinNodesPerAggregate(minPerAgg); //TODO should increase if run anything other than 1D
CoupledAggFact.SetMaxNeighAlreadySelected(maxNbrAlreadySelected);
std::transform(aggOrdering.begin(), aggOrdering.end(), aggOrdering.begin(), ::tolower);
if (aggOrdering == "natural") {
*out << "aggregate ordering : NATURAL" << std::endl;
CoupledAggFact.SetOrdering(MueLu::AggOptions::NATURAL);
} else if (aggOrdering == "random") {
*out << "aggregate ordering : RANDOM" << std::endl;
CoupledAggFact.SetOrdering(MueLu::AggOptions::RANDOM);
} else if (aggOrdering == "graph") {
*out << "aggregate ordering : GRAPH" << std::endl;
CoupledAggFact.SetOrdering(MueLu::AggOptions::GRAPH);
} else {
std::string msg = "main: bad aggregation option """ + aggOrdering + """.";
throw(MueLu::Exceptions::RuntimeError(msg));
}
CoupledAggFact.SetPhase3AggCreation(0.5);
*out << "=============================================================================" << std::endl;
CoupledAggFact.Build(Finest);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[]) {
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false); // Note:
GO nx,ny,nz;
nx=500;
ny=500;
nz=100;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'scalingTest.xml'");
int amgAsPrecond=1; clp.setOption("precond",&amgAsPrecond,"apply multigrid as preconditioner");
int amgAsSolver=0; clp.setOption("fixPoint",&amgAsSolver,"apply multigrid as solver");
bool printTimings=true; clp.setOption("timings","notimings",&printTimings,"print timings to screen");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if (comm->getRank() == 0) {
std::cout << "========================================================" << std::endl
<< xpetraParameters << matrixParameters;
}
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
// read aggregation options from file
Teuchos::FileInputSource fileSrc(xmlFileName);
Teuchos::XMLObject fileXML = fileSrc.getObject();
Teuchos::XMLParameterListReader listReader;
Teuchos::ParameterList aggList = listReader.toParameterList(fileXML);
//std::cout << "===========aggList start===========" << std::endl;
//aggList.print(std::cout);
//std::cout << "===========aggList end===========" << std::endl;
// instantiate aggregate factory, set options from parameter list
RCP<MueLu::SingleLevelFactoryBase> aggFact;
if (aggList.name() == "UncoupledAggregationFactory") {
RCP<UncoupledAggregationFactory> ucFact = rcp( new UncoupledAggregationFactory() );
//ucFact->SetParameterList(aggList);
//FIXME hack until UCAgg uses PL interface
std::string ordering = aggList.get<std::string>("Ordering");
MueLu::AggOptions::Ordering eordering;
if (ordering=="Natural") eordering = MueLu::AggOptions::NATURAL;
if (ordering=="Graph") eordering = MueLu::AggOptions::GRAPH;
if (ordering=="Random") eordering = MueLu::AggOptions::RANDOM;
ucFact->SetOrdering(eordering);
ucFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected"));
ucFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate"));
aggFact = ucFact;
} else if (aggList.name() == "CoupledAggregationFactory") {
RCP<CoupledAggregationFactory> cFact = rcp( new CoupledAggregationFactory() );
//cFact->SetParameterList(aggList);
//FIXME hack until CoupledAgg uses PL interface
//cFact->SetOrdering(aggList.get<std::string>("Ordering"));
cFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected"));
cFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate"));
aggFact = cFact;
} else {
throw(MueLu::Exceptions::RuntimeError("List's name does not correspond to a known aggregation factory."));
}
//Teuchos::ParameterList tlist = aggFact->GetParameterList();
//std::cout << "===========verify List start===========" << std::endl;
//tlist.print(std::cout);
//std::cout << "===========verify List end===========" << std::endl;
// build matrix
RCP<TimeMonitor> tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<const Map> map;
RCP<MultiVector> coordinates;
// Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri.
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx",nx));
galeriList.set("ny", pl.get("ny",ny));
//galeriList.set("mx", comm->getSize());
//galeriList.set("my", 1);
if (matrixParameters.GetMatrixType() == "Laplace1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D",map,matrixParameters.GetParameterList());
//map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList); //TODO when available in Galeri
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
}
if (comm->getRank() == 0) {
GO mx = galeriList.get("mx", -1);
GO my = galeriList.get("my", -1);
std::cout << "Processor subdomains in x direction: " << mx << std::endl
<< "Processor subdomains in y direction: " << my << std::endl
<< "========================================================" << std::endl;
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A = Pr->BuildMatrix();
tm = Teuchos::null;
Level level;
RCP<MueLu::FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
level.SetLevelID(0);
level.Set("A", A);
level.Request("Aggregates", aggFact.get());
level.Request(*aggFact);
level.setVerbLevel(Teuchos::VERB_NONE);
aggFact->setVerbLevel(Teuchos::VERB_NONE);
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("aggregation time")));
aggFact->Build(level);
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize();
} //main
