TEUCHOS_UNIT_TEST(CoarseMap, NonStandardCaseA )
{
out << "version: " << MueLu::Version() << std::endl;
Level myLevel;
myLevel.SetLevelID(0);
RCP<Matrix> A = TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::Build1DPoisson(15);
myLevel.Set("A", A);
// build dummy aggregate structure
Teuchos::RCP<Aggregates> aggs = Teuchos::rcp(new Aggregates(A->getRowMap()));
aggs->SetNumAggregates(10); // set (local!) number of aggregates
myLevel.Set("Aggregates", aggs);
// build dummy nullspace vector
Teuchos::RCP<MultiVector> nsp = MultiVectorFactory::Build(A->getRowMap(),1);
nsp->putScalar(1.0);
myLevel.Set("Nullspace", nsp);
RCP<CoarseMapFactory> myCMF = Teuchos::rcp(new CoarseMapFactory());
myLevel.Request("CoarseMap",myCMF.get());
myCMF->SetParameter("Domain GID offsets",Teuchos::ParameterEntry(std::string("{100,50}")));
myCMF->SetFactory("Aggregates",MueLu::NoFactory::getRCP());
myCMF->SetFactory("Nullspace",MueLu::NoFactory::getRCP());
myCMF->Build(myLevel);
Teuchos::RCP<const Map> myCoarseMap = myLevel.Get<Teuchos::RCP<const Map> >("CoarseMap",myCMF.get());
TEST_EQUALITY(myCoarseMap->getMinAllGlobalIndex() == 100, true);
TEST_EQUALITY(myCoarseMap->getMaxLocalIndex()==9,true);
myLevel.Release("CoarseMap",myCMF.get());
myLevel.SetLevelID(1);
myLevel.Request("CoarseMap",myCMF.get());
myCMF->SetParameter("Domain GID offsets",Teuchos::ParameterEntry(std::string("{100,50}")));
myCMF->SetFactory("Aggregates",MueLu::NoFactory::getRCP());
myCMF->SetFactory("Nullspace",MueLu::NoFactory::getRCP());
myCMF->Build(myLevel);
myCoarseMap = myLevel.Get<Teuchos::RCP<const Map> >("CoarseMap",myCMF.get());
TEST_EQUALITY(myCoarseMap->getMinAllGlobalIndex() == 50, true);
TEST_EQUALITY(myCoarseMap->getMaxLocalIndex()==9,true);
myLevel.Release("CoarseMap",myCMF.get());
myLevel.SetLevelID(2);
myLevel.Request("CoarseMap",myCMF.get());
myCMF->SetParameter("Domain GID offsets",Teuchos::ParameterEntry(std::string("{100,50}")));
myCMF->SetFactory("Aggregates",MueLu::NoFactory::getRCP());
myCMF->SetFactory("Nullspace",MueLu::NoFactory::getRCP());
myCMF->Build(myLevel);
myCoarseMap = myLevel.Get<Teuchos::RCP<const Map> >("CoarseMap",myCMF.get());
TEST_EQUALITY(myCoarseMap->getMinAllGlobalIndex() == 0, true);
TEST_EQUALITY(myCoarseMap->getMaxLocalIndex()==9,true);
}
void testBuild(RCP<SmootherPrototype> smooProtoA, RCP<SmootherPrototype> smooProtoB, Teuchos::FancyOStream & out, bool & success) {
RCP<SmootherFactory> smooFact = rcp( new SmootherFactory(smooProtoA, smooProtoB) );
Level level; //level.SetupPhase(true);
TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::createSingleLevelHierarchy(level);
level.Request("PreSmoother",smooFact.get());
level.Request("PostSmoother", smooFact.get());
smooFact->Build(level);
testBuildCheck(smooFact, level, smooProtoA, smooProtoB, MueLu::BOTH, out, success);
}
TEUCHOS_UNIT_TEST(MultiVectorTransferFactory, Build)
{
out << "version: " << MueLu::Version() << std::endl;
out << "Tests the action of the transfer factory on a vector. In this test, the transfer is the tentative" << std::endl;
out << "prolongator, and the vector is all ones. So the norm of the resulting coarse grid vector should be" << std::endl;
out << "equal to the number of fine degrees of freedom." << std::endl;
Level fineLevel, coarseLevel;
TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::createTwoLevelHierarchy(fineLevel, coarseLevel);
GO nx = 199;
RCP<Matrix> A = TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::Build1DPoisson(nx);
fineLevel.Set("A",A);
RCP<MultiVector> fineOnes = MultiVectorFactory::Build(A->getRowMap(),1);
fineOnes->putScalar(1.0);
fineLevel.Set("onesVector",fineOnes);
RCP<TentativePFactory> TentativePFact = rcp(new TentativePFactory());
RCP<TransPFactory> RFact = rcp(new TransPFactory());
RCP<FactoryManager> M = rcp(new FactoryManager());
M->SetFactory("P", TentativePFact);
M->SetFactory("Ptent", TentativePFact);
M->SetFactory("R", RFact);
// fineLevel.SetFactoryManager(M);
coarseLevel.SetFactoryManager(M);
RCP<MueLu::MultiVectorTransferFactory<SC, LO, GO, NO, LMO> > mvtf = rcp(new MueLu::MultiVectorTransferFactory<SC, LO, GO, NO, LMO>("onesVector"));
mvtf->SetFactory("R",RFact);
coarseLevel.Request("onesVector",mvtf.get());
coarseLevel.Request("R",RFact.get());
coarseLevel.Request("P",TentativePFact.get());
mvtf->Build(fineLevel,coarseLevel);
RCP<MultiVector> coarseOnes = coarseLevel.Get<RCP<MultiVector> >("onesVector",mvtf.get());
Teuchos::Array<Teuchos::ScalarTraits<SC>::magnitudeType> vn(1);
coarseOnes->norm2(vn);
TEST_FLOATING_EQUALITY(vn[0]*vn[0],((SC)fineOnes->getGlobalLength()),1e-12);
} // Build test
Teuchos::RCP<MueLu::Aggregates_kokkos<LocalOrdinal, GlobalOrdinal, Node>>
gimmeUncoupledAggregates(const Teuchos::RCP<Xpetra::Matrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>>& A,
Teuchos::RCP<MueLu::AmalgamationInfo<LocalOrdinal, GlobalOrdinal, Node>>& amalgInfo,
bool bPhase1 = true, bool bPhase2a = true, bool bPhase2b = true, bool bPhase3 = true) {
# include "MueLu_UseShortNames.hpp"
Level level;
TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(level);
level.Set("A", A);
RCP<AmalgamationFactory> amalgFact = rcp(new AmalgamationFactory());
RCP<CoalesceDropFactory_kokkos> dropFact = rcp(new CoalesceDropFactory_kokkos());
dropFact->SetFactory("UnAmalgamationInfo", amalgFact);
using Teuchos::ParameterEntry;
// Setup aggregation factory (use default factory for graph)
RCP<UncoupledAggregationFactory_kokkos> aggFact = rcp(new UncoupledAggregationFactory_kokkos());
aggFact->SetFactory("Graph", dropFact);
aggFact->SetParameter("aggregation: max agg size", ParameterEntry(3));
aggFact->SetParameter("aggregation: min agg size", ParameterEntry(3));
aggFact->SetParameter("aggregation: max selected neighbors", ParameterEntry(0));
aggFact->SetParameter("aggregation: ordering", ParameterEntry(std::string("natural")));
aggFact->SetParameter("aggregation: enable phase 1", ParameterEntry(bPhase1));
aggFact->SetParameter("aggregation: enable phase 2a", ParameterEntry(bPhase2a));
aggFact->SetParameter("aggregation: enable phase 2b", ParameterEntry(bPhase2b));
aggFact->SetParameter("aggregation: enable phase 3", ParameterEntry(bPhase3));
level.Request("Aggregates", aggFact.get());
level.Request("UnAmalgamationInfo", amalgFact.get());
level.Request(*aggFact);
aggFact->Build(level);
auto aggregates = level.Get<RCP<Aggregates_kokkos> >("Aggregates", aggFact.get());
amalgInfo = level.Get<RCP<AmalgamationInfo> >("UnAmalgamationInfo", amalgFact.get());
level.Release("UnAmalgamationInfo", amalgFact.get());
level.Release("Aggregates", aggFact.get());
return aggregates;
}
TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(CoarseMap_kokkos, StandardCase, Scalar, LocalOrdinal, GlobalOrdinal, Node)
{
# include "MueLu_UseShortNames.hpp"
RUN_EPETRA_ONLY_WITH_SERIAL_NODE(Node);
MueLu::VerboseObject::SetDefaultOStream(Teuchos::rcpFromRef(out));
out << "version: " << MueLu::Version() << std::endl;
Level fineLevel;
TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::createSingleLevelHierarchy(fineLevel);
RCP<Matrix> A = TestHelpers_kokkos::TestFactory<SC, LO, GO, NO>::Build1DPoisson(15);
fineLevel.Set("A", A);
// build dummy aggregate structure
RCP<Aggregates_kokkos> aggs = Teuchos::rcp(new Aggregates_kokkos(A->getRowMap()));
aggs->SetNumAggregates(10); // set (local!) number of aggregates
fineLevel.Set("Aggregates", aggs);
// build dummy nullspace vector
RCP<MultiVector> nsp = MultiVectorFactory::Build(A->getRowMap(),1);
nsp->putScalar(1.0);
fineLevel.Set("Nullspace", nsp);
RCP<CoarseMapFactory_kokkos> coarseMapFactory = Teuchos::rcp(new CoarseMapFactory_kokkos());
coarseMapFactory->SetFactory("Aggregates", MueLu::NoFactory::getRCP());
coarseMapFactory->SetFactory("Nullspace", MueLu::NoFactory::getRCP());
fineLevel.Request("CoarseMap", coarseMapFactory.get());
coarseMapFactory->Build(fineLevel);
auto myCoarseMap = fineLevel.Get<Teuchos::RCP<const Map> >("CoarseMap", coarseMapFactory.get());
TEST_EQUALITY(myCoarseMap->getMinAllGlobalIndex() == 0, true);
TEST_EQUALITY(myCoarseMap->getMaxLocalIndex() == 9, true);
}
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::Time;
using Teuchos::TimeMonitor;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false);
GO nx, ny, nz;
nx=50;
ny=50;
nz=50;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
int optNraps = 5; clp.setOption("nraps", &optNraps, "number of RAPS to perform");
bool optTimings = true; clp.setOption("timings", "notimings", &optTimings, "print timings to screen");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if (comm->getRank() == 0) {
std::cout << "========================================================" << std::endl
<< xpetraParameters << matrixParameters;
}
//
// Construct the problem
//
{
TimeMonitor globalTimeMonitor(*TimeMonitor::getNewTimer("RAPScalingTest: S - Global Time"));
RCP<Matrix> A;
RCP<MultiVector> coordinates;
{
TimeMonitor tm(*TimeMonitor::getNewTimer("RAPScalingTest: 1 - Matrix creation"));
RCP<const Map> map;
// Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri.
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx", nx));
galeriList.set("ny", pl.get("ny", ny));
galeriList.set("nz", pl.get("nz", nz));
if (matrixParameters.GetMatrixType() == "Laplace1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
}
//FIXME
if (comm->getRank() == 0) {
GO mx = galeriList.get("mx", -1);
GO my = galeriList.get("my", -1);
std::cout << "Processor subdomains in x direction: " << mx << std::endl
<< "Processor subdomains in y direction: " << my << std::endl
<< "========================================================" << std::endl;
}
Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
A = Pr->BuildMatrix();
}
Level fineLevel, coarseLevel;
RAPFactory AcFact;
AcFact.DisableMultipleCallCheck();
RCP<SaPFactory> PFact;
// Setup P
{
TimeMonitor tm(*TimeMonitor::getNewTimer("RAPScalingTest: 2 - P Setup"));
MueLuTests::TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::createTwoLevelHierarchy(fineLevel, coarseLevel); // set a default FactoryManager
fineLevel.Set("A", A);
PFact = rcp(new SaPFactory());
coarseLevel.Request("P", PFact.get());
PFact->Build(fineLevel, coarseLevel);
AcFact.SetFactory("P", PFact);
}
// Setup R, repeatedly
RCP<Time> RKernelTimer = TimeMonitor::getNewTimer("RAPScalingTest: 3 - R Setup"); // re-use the same timer in the loop
RCP<TransPFactory> RFact = rcp(new TransPFactory());
RFact->SetFactory("P", PFact);
for (int i=0; i<optNraps; ++i) {
coarseLevel.Request("R", RFact.get());
{
TimeMonitor tm(*RKernelTimer);
RFact->Build(fineLevel, coarseLevel);
}
coarseLevel.Release("R", RFact.get());
}
} // end of globalTimeMonitor
if (optTimings) {
Teuchos::TableFormat &format = TimeMonitor::format();
format.setPrecision(25);
TimeMonitor::summarize();
}
} //main
TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(SaPFactory_kokkos, Build, Scalar, LocalOrdinal, GlobalOrdinal, Node)
{
# include "MueLu_UseShortNames.hpp"
MUELU_TESTING_SET_OSTREAM;
MUELU_TESTING_LIMIT_SCOPE(Scalar,GlobalOrdinal,Node);
out << "version: " << MueLu::Version() << std::endl;
RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// construct two levels
Level fineLevel, coarseLevel;
TestHelpers_kokkos::TestFactory<SC, LO, GO, NO>::createTwoLevelHierarchy(fineLevel, coarseLevel);
// construct matrices
const SC lambdaMax = 5;
RCP<Matrix> A = TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::Build2DPoisson(27*comm->getSize());
A->SetMaxEigenvalueEstimate(lambdaMax);
RCP<Matrix> Ptent = TestHelpers_kokkos::TestFactory<SC,LO,GO,NO>::Build2DPoisson(27*comm->getSize());
// set level matrices
fineLevel .Set("A", A);
coarseLevel.Set("P", Ptent);
// construct the factory to be tested
const double dampingFactor = 0.5;
RCP<SaPFactory_kokkos> sapFactory = rcp(new SaPFactory_kokkos);
ParameterList Pparams;
Pparams.set("sa: damping factor", dampingFactor);
sapFactory->SetParameterList(Pparams);
sapFactory->SetFactory("A", MueLu::NoFactory::getRCP());
sapFactory->SetFactory("P", MueLu::NoFactory::getRCP());
// build the data
coarseLevel.Request("P", sapFactory.get());
sapFactory->Build(fineLevel, coarseLevel);
// fetch the data
RCP<Matrix> Pfact = coarseLevel.Get<RCP<Matrix>>("P", sapFactory.get());
// construct the data to compare
SC omega = dampingFactor / lambdaMax;
RCP<Vector> invDiag = Utilities_kokkos::GetMatrixDiagonalInverse(*A);
RCP<ParameterList> APparams = rcp(new ParameterList);
RCP<Matrix> Ptest = Xpetra::IteratorOps<SC,LO,GO,NO>::Jacobi(omega, *invDiag, *A, *Ptent, Teuchos::null, out, "label", APparams);
// compare matrices by multiplying them by a random vector
RCP<MultiVector> X = MultiVectorFactory::Build(A->getDomainMap(), 1);
X->setSeed(846930886);
X->randomize();
RCP<MultiVector> Bfact = MultiVectorFactory::Build(A->getRangeMap(), 1);
RCP<MultiVector> Btest = MultiVectorFactory::Build(A->getRangeMap(), 1);
typedef Teuchos::ScalarTraits<SC> STS;
SC zero = STS::zero(), one = STS::one();
Pfact->apply(*X, *Bfact, Teuchos::NO_TRANS, one, zero);
Ptest->apply(*X, *Btest, Teuchos::NO_TRANS, one, zero);
Btest->update(-one, *Bfact, one);
Array<typename STS::magnitudeType> norms(1);
Btest->norm2(norms);
out << "|| B_factory - B_test || = " << norms[0] << std::endl;
TEST_EQUALITY(norms[0] < 1e-12, true);
}
TEUCHOS_UNIT_TEST(Zoltan, Build)
{
typedef Teuchos::ScalarTraits<Scalar> ST;
out << "version: " << MueLu::Version() << std::endl;
out << std::endl;
out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl;
out << "that has a random number of nonzeros per row. Good results have been precomputed" << std::endl;
out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl;
out << "once the Zoltan repartitioning has been applied." << std::endl;
out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl;
RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();
if (comm->getSize() > 5) {
out << std::endl;
out << "This test must be run on 1 to 5 processes." << std::endl;
TEST_EQUALITY(true, true);
return;
}
Level level;
RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
int nx=7;
int ny=nx;
GO numGlobalElements = nx*ny;
size_t maxEntriesPerRow=30;
// Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row.
RCP<const Map> map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm);
const size_t numMyElements = map->getNodeNumElements();
Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList();
RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more
// memory on the fly. While not super efficient, this
// ensures that no zeros are being stored. Thus, from
// Zoltan's perspective the matrix is imbalanced.
// Create a vector with random integer entries in [1,maxEntriesPerRow].
ST::seedrandom(666*comm->getRank());
RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false);
Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0);
for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) {
*i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1));
}
RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
fos->setOutputToRootOnly(-1);
Teuchos::Array<Scalar> vals(maxEntriesPerRow);
Teuchos::Array<GO> cols(maxEntriesPerRow);
for (size_t i = 0; i < numMyElements; ++i) {
Teuchos::ArrayView<SC> av(&vals[0],eprData[i]);
Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]);
//stick in ones for values
for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one();
//figure out valid column indices
GO start = std::max(myGlobalElements[i]-eprData[i]+1,0);
for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j;
A->insertGlobalValues(myGlobalElements[i], iv, av);
}
A->fillComplete();
level.Set("A",A);
//build coordinates
RCP<const Map> rowMap = A->getRowMap();
Teuchos::ParameterList list;
list.set("nx",nx);
list.set("ny",ny);
RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",rowMap,list);
level.Set("Coordinates",XYZ);
LO numPartitions = comm->getSize();
level.Set("number of partitions",numPartitions);
RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface());
//zoltan->SetNumberOfPartitions(numPartitions);
//zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints
level.Request("Partition",zoltan.get());
zoltan->Build(level);
RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get());
/* //TODO temporary code to have the trivial decomposition (no change)
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
*i = comm->getRank();
decompEntries=Teuchos::null;
*/ //TODO end of temporary code
//Create vector whose local length is the global number of partitions.
//This vector will record the local number of nonzeros associated with each partition.
Teuchos::Array<GO> parts(numPartitions);
for (int i=0; i<numPartitions; ++i) parts[i] = i;
Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions);
RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView,
map->getIndexBase(),comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap);
//For the local rows in each partition, tally up the number of nonzeros. This is what
//Zoltan should be load-balancing.
Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0);
Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0);
for (size_t i=0; i<decomposition->getLocalLength();++i) {
Teuchos::ArrayView<const LO> c;
Teuchos::ArrayView<const SC> v;
A->getLocalRowView(i,c,v);
lpvData[decompData[i]] += v.size();
}
lpvData = Teuchos::null;
decompData = Teuchos::null;
//localPartsVec->describe(*fos,Teuchos::VERB_EXTREME);
//Send the local nnz tallies to pid 0, which can report the global sums.
size_t mysize=1;
if (comm->getRank() == 0) mysize = numPartitions;
RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(),
mysize,
map->getIndexBase(),
comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap);
RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap);
globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD);
ArrayRCP<GO> expectedResults(numPartitions);
switch (comm->getSize()) {
case 1:
expectedResults[0] = 807;
break;
case 2:
expectedResults[0] = 364;
expectedResults[1] = 363;
break;
case 3:
expectedResults[0] = 277;
expectedResults[1] = 261;
expectedResults[2] = 269;
break;
case 4:
expectedResults[0] = 195;
expectedResults[1] = 186;
expectedResults[2] = 177;
expectedResults[3] = 168;
break;
case 5:
expectedResults[0] = 161;
expectedResults[1] = 145;
expectedResults[2] = 148;
expectedResults[3] = 159;
expectedResults[4] = 157;
break;
default:
break;
};
//FIXME cool ... this next line causes a hang if locally the globalyTallyVec has no data.
//FIXME I get around this by making mysize (above) 1 instead of 0. Is this a bug or feature
//FIXME in getData?
ArrayRCP<const LO> gtvData = globalTallyVec->getData(0);
#ifdef __linux__
out << "Checking results..." << std::endl;
for (int i=0; i<numPartitions; ++i) {
if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]);
}
#endif
#ifdef MUELU_VISUALIZE_REPARTITIONING
//
//Now write everything to a comma-separate list that ParaView can grok
//
Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0);
Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1);
Teuchos::ArrayRCP<const GO> D = decomposition->getData(0);
RCP<std::ofstream> outFile;
std::string fileName = "zoltanResults.csv";
//write header information
if (comm->getRank() == 0) {
outFile = rcp(new std::ofstream(fileName.c_str()));
*outFile << "x coord, y coord, z coord, scalar" << std::endl;
}
comm->barrier();
//append coordinates
for (int j=0; j<comm->getSize(); ++j) {
int mypid = comm->getRank();
if (mypid == j) {
outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app));
for (int i=0; i < D.size(); ++i) {
*outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", " << D[i] << std::endl;
}
}
} //for (int i=0; i<comm->getSize(); ++i)
out << std::endl;
out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl;
out << " 1) Load the data file " << fileName << "." << std::endl;
out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl;
out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl;
out << " 4) Split screen horizontally (Icon, top right)." << std::endl;
out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl;
out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl;
out << std::endl;
out << " To display row weights next to each point:" << std::endl;
out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl;
out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl;
out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl;
#endif
} //Build
TEUCHOS_UNIT_TEST(Zoltan, Build3PDEs)
{
typedef Teuchos::ScalarTraits<Scalar> ST;
out << "version: " << MueLu::Version() << std::endl;
out << std::endl;
out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl;
out << "that has a random number of nonzeros per row and 3 DOFs per mesh point. Good results have been precomputed" << std::endl;
out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl;
out << "once the Zoltan repartitioning has been applied." << std::endl;
out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl;
RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();
if (comm->getSize() > 5) {
out << std::endl;
out << "This test must be run on 1 to 5 processes." << std::endl;
TEST_EQUALITY(true, true);
return;
}
Level level;
RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
int nx=9;
int ny=nx;
int dofsPerNode = 3;
GO numGlobalElements = nx*ny*dofsPerNode;
size_t maxEntriesPerRow=30;
RCP<const Map> map;
int numMyNodes = numGlobalElements / dofsPerNode;
if (comm->getSize() > 1) {
// In parallel, make sure that the dof's associated with a node all
// reside on the same processor.
int numNodes = numGlobalElements / dofsPerNode;
TEUCHOS_TEST_FOR_EXCEPTION( (numGlobalElements - numNodes * dofsPerNode) != 0, MueLu::Exceptions::RuntimeError,
"Number of matrix rows is not divisible by #dofs" );
int nproc = comm->getSize();
if (comm->getRank() < nproc-1) numMyNodes = numNodes / nproc;
else numMyNodes = numNodes - (numNodes/nproc) * (nproc-1);
map = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, numMyNodes*dofsPerNode, comm);
} else {
map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm);
}
const size_t numMyElements = map->getNodeNumElements();
Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList();
RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more
// memory on the fly. While not super efficient, this
// ensures that no zeros are being stored. Thus, from
// Zoltan's perspective the matrix is imbalanced.
// Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row.
// Create a vector with random integer entries in [1,maxEntriesPerRow].
ST::seedrandom(666*comm->getRank());
RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false);
Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0);
for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) {
*i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1));
}
RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
fos->setOutputToRootOnly(-1);
Teuchos::Array<Scalar> vals(maxEntriesPerRow);
Teuchos::Array<GO> cols(maxEntriesPerRow);
for (size_t i = 0; i < numMyElements; ++i) {
Teuchos::ArrayView<SC> av(&vals[0],eprData[i]);
Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]);
//stick in ones for values
for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one();
//figure out valid column indices
GO start = std::max(myGlobalElements[i]-eprData[i]+1,0);
for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j;
A->insertGlobalValues(myGlobalElements[i], iv, av);
}
A->fillComplete();
// Now treat the matrix as if it has 3 DOFs per node.
A->SetFixedBlockSize(dofsPerNode);
level.Set("A",A);
//build coordinates
Teuchos::ParameterList list;
list.set("nx",nx);
list.set("ny",ny);
RCP<const Map> coalescedMap = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements/dofsPerNode, numMyNodes, comm);
RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",coalescedMap,list);
// XYZ are the "coalesce" coordinates as it has been generated for 1 DOF/node and we are using them for 3 DOFS/node
// level.Set("Coordinates",XYZ); "Coordinates" == uncoalesce. "X,Y,ZCoordinates" == coalesce
{
RCP<MultiVector> coordinates = XYZ;
// making a copy because I don't want to keep 'open' the Xpetra_MultiVector
if (coordinates->getNumVectors() >= 1) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(0);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("XCoordinates", coordCpy);
//std::cout << coordCpy << std::endl;
}
if (coordinates->getNumVectors() >= 2) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(1);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("YCoordinates", coordCpy);
}
/*if (coordinates->getNumVectors() >= 3) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(2);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("ZCoordinates", coordCpy);
}*/
}
//coalescedMap->describe(*fos,Teuchos::VERB_EXTREME);
//sleep(1); comm->barrier();
//XYZ->describe(*fos,Teuchos::VERB_EXTREME);
LO numPartitions = comm->getSize();
level.Set("number of partitions",numPartitions);
RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface());
//zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints
level.Request("Partition",zoltan.get());
zoltan->Build(level);
RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get());
/* //temporary code to have the trivial decomposition (no change)
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
*i = comm->getRank();
decompEntries=Teuchos::null;
*/
//Create vector whose local length is the global number of partitions.
//This vector will record the local number of nonzeros associated with each partition.
Teuchos::Array<GO> parts(numPartitions);
for (int i=0; i<numPartitions; ++i) parts[i] = i;
Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions);
RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView,
map->getIndexBase(),comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap);
RCP<Xpetra::Vector<LO,LO,GO,NO> > nnzPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(A->getRowMap());
Teuchos::ArrayRCP<GO> nnzData = nnzPerRow->getDataNonConst(0);
//For the local rows in each partition, tally up the number of nonzeros. This is what
//Zoltan should be load-balancing.
Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0);
Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0);
for (size_t i=0; i<decomposition->getLocalLength();++i) {
Teuchos::ArrayView<const LO> c;
Teuchos::ArrayView<const SC> v;
A->getLocalRowView(i,c,v);
lpvData[decompData[i]] += v.size();
nnzData[i] = v.size();
}
lpvData = Teuchos::null;
decompData = Teuchos::null;
nnzData = Teuchos::null;
/*
if (comm->getRank() == 0)
std::cout << "nnz per row" << std::endl;
nnzPerRow->describe(*fos,Teuchos::VERB_EXTREME);
if (comm->getRank() == 0)
std::cout << "Row-to-partition assignment (from Zoltan)" << std::endl;
decomposition->describe(*fos,Teuchos::VERB_EXTREME);
if (comm->getRank() == 0)
std::cout << "#nonzeros per partition" << std::endl;
localPartsVec->describe(*fos,Teuchos::VERB_EXTREME);
*/
//Send the local nnz tallies to pid 0, which can report the global sums.
size_t mysize=1;
if (comm->getRank() == 0) mysize = numPartitions;
RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(),
mysize,
map->getIndexBase(),
comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap);
RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap);
globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD);
ArrayRCP<GO> expectedResults(numPartitions);
switch (comm->getSize()) {
case 1:
expectedResults[0] = 3951;
break;
case 2:
expectedResults[0] = 1955;
expectedResults[1] = 1910;
break;
case 3:
expectedResults[0] = 1326;
expectedResults[1] = 1340;
expectedResults[2] = 1321;
break;
case 4:
expectedResults[0] = 950;
expectedResults[1] = 922;
expectedResults[2] = 908;
expectedResults[3] = 936;
break;
case 5:
expectedResults[0] = 774;
expectedResults[1] = 735;
expectedResults[2] = 726;
expectedResults[3] = 771;
expectedResults[4] = 759;
break;
default:
break;
};
ArrayRCP<const LO> gtvData = globalTallyVec->getData(0);
#ifdef __linux__
out << "Checking results..." << std::endl;
for (int i=0; i<numPartitions; ++i) {
if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]);
}
#endif
#ifdef MUELU_VISUALIZE_REPARTITIONING
//
//Now write everything to a comma-separate list that ParaView can grok
//
Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0);
Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1);
Teuchos::ArrayRCP<const GO> D = decomposition->getData(0);
RCP<std::ofstream> outFile;
std::string fileName = "zoltanResults.csv";
//write header information
if (comm->getRank() == 0) {
outFile = rcp(new std::ofstream(fileName.c_str()));
*outFile << "x coord, y coord, z coord, partition, row weight" << std::endl;
}
comm->barrier();
//append coordinates
nnzData = nnzPerRow->getDataNonConst(0);
for (int j=0; j<comm->getSize(); ++j) {
int mypid = comm->getRank();
if (mypid == j) {
outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app));
int blockSize = A->GetFixedBlockSize();
//Coordinates are for coalesced system, D is for uncoalesced
for (int i=0; i < D.size()/blockSize; ++i) {
int nnz=0;
for (int k=0; k<blockSize; ++k) nnz += nnzData[i*blockSize+k];
*outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", "
<< D[i*blockSize] << ", " << nnz << std::endl;
}
}
} //for (int i=0; i<comm->getSize(); ++i)
out << std::endl;
out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl;
out << " 1) Load the data file " << fileName << "." << std::endl;
out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl;
out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl;
out << " 4) Split screen horizontally (Icon, top right)." << std::endl;
out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl;
out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl;
out << std::endl;
out << " To display row weights next to each point:" << std::endl;
out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl;
out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl;
out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl;
#endif
} //Build3PDEs
int main(int argc, char *argv[]) {
#include <MueLu_UseShortNames.hpp>
using Teuchos::RCP;
using Teuchos::rcp;
using Teuchos::Time;
using Teuchos::TimeMonitor;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
// Test name
const std::string testName("ProlongatorConstruction Test");
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false);
GO nx, ny, nz;
nx=50;
ny=50;
nz=50;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
int optNits = 5; clp.setOption("nits", &optNits, "number of kernel operations to perform");
bool optTimings = true; clp.setOption("timings", "notimings", &optTimings, "print timings to screen");
std::string xmlFileName; clp.setOption("xml", &xmlFileName, "xml option file");
Scalar optDampingFactor = 0.; clp.setOption("omega", &optDampingFactor, "smoothed prolongator damping factor");
switch (clp.parse(argc, argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if (comm->getRank() == 0) {
std::cout << "========================================================" << std::endl
<< xpetraParameters << matrixParameters;
}
//
// Construct the problem
//
{
std::string timerName = testName + ": S - Global Time";
TimeMonitor globalTimeMonitor(*TimeMonitor::getNewTimer(timerName));
RCP<Matrix> A;
RCP<MultiVector> coordinates;
{
timerName = testName + ": 1 - Matrix creation";
TimeMonitor tm(*TimeMonitor::getNewTimer(timerName));
RCP<const Map> map;
// Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri.
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx", nx));
galeriList.set("ny", pl.get("ny", ny));
galeriList.set("nz", pl.get("nz", nz));
if (matrixParameters.GetMatrixType() == "Laplace1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("1D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("2D", map, matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC, LO, GO, Map, MultiVector>("3D", map, matrixParameters.GetParameterList());
}
//FIXME
if (comm->getRank() == 0) {
GO mx = galeriList.get("mx", -1);
GO my = galeriList.get("my", -1);
std::cout << "Processor subdomains in x direction: " << mx << std::endl
<< "Processor subdomains in y direction: " << my << std::endl
<< "========================================================" << std::endl;
}
Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
A = Pr->BuildMatrix();
} //matrix creation
Level fineLevel, coarseLevel;
//RCP<SaPFactory> PFact;
RCP<UncoupledAggregationFactory> aggFact;
RCP<TentativePFactory> PFact;
RCP<CoalesceDropFactory> cdFact;
{
timerName = testName + ": 2 - Setup";
TimeMonitor tm(*TimeMonitor::getNewTimer(timerName));
Teuchos::ParameterList paramList;
if (!xmlFileName.empty())
Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr<Teuchos::ParameterList>(¶mList), *comm);
MueLuTests::TestHelpers::TestFactory<SC, LO, GO, NO, LMO>::createTwoLevelHierarchy(fineLevel, coarseLevel);
fineLevel.Set("A", A);
cdFact = rcp( new CoalesceDropFactory());
aggFact = rcp( new UncoupledAggregationFactory());
//PFact = rcp( new SaPFactory());
PFact = rcp( new TentativePFactory());
// set factory options according to the XML input file
Teuchos::ParameterList cdList = paramList.sublist("CoalesceDrop");
cdFact->SetParameterList(cdList);
Teuchos::ParameterList aggregationList = paramList.sublist("Aggregates");
aggFact->SetParameterList(aggregationList);
Teuchos::ParameterList prolongatorList = paramList.sublist("Prolongator");
PFact->SetParameterList(prolongatorList);
// overwrite default FactoryManager
RCP<FactoryManager> M = rcp(new FactoryManager());
M->SetFactory("Graph",cdFact);
M->SetFactory("Aggregates",aggFact);
fineLevel.SetFactoryManager(M);
coarseLevel.SetFactoryManager(M);
// IMPORTANT: The request for P must occur *after* setting the coarse level's FactoryManager.
//coarseLevel.Request("Aggregates", aggFact.get());
//aggFact->Build(fineLevel, coarseLevel);
//coarseLevel.Release("Aggregates", PFact.get());
} //setup
timerName = testName + ": 3 - kernel";
RCP<Time> kernelTimer = TimeMonitor::getNewTimer(timerName); // re-use the same timer in the loop
for (int i=0; i<optNits; ++i) {
coarseLevel.Request("Graph", cdFact.get());
{
TimeMonitor tm(*kernelTimer);
cdFact->Build(coarseLevel);
}
comm->barrier();
coarseLevel.Release("Graph", cdFact.get());
} //kernel apply
} // end of globalTimeMonitor
if (optTimings) {
//Teuchos::TableFormat &format = TimeMonitor::format();
//format.setPrecision(25);
TimeMonitor::summarize();
}
} //main
void TestPseudoPoisson(Teuchos::FancyOStream &out, int num_nodes, int degree, std::vector<Scalar> &pn_gold_in, std::vector<Scalar> &pn_gold_out,const std::string & hi_basis)
{
# include "MueLu_UseShortNames.hpp"
MUELU_TESTING_SET_OSTREAM;
MUELU_TESTING_LIMIT_SCOPE(Scalar,GlobalOrdinal,Node);
typedef Scalar SC;
typedef GlobalOrdinal GO;
typedef LocalOrdinal LO;
typedef Node NO;
typedef TestHelpers::TestFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node> test_factory;
typedef typename Teuchos::ScalarTraits<Scalar>::magnitudeType MT;
#ifdef HAVE_MUELU_INTREPID2_REFACTOR
typedef Kokkos::DynRankView<LocalOrdinal,typename Node::device_type> FCi;
#else
typedef Intrepid2::FieldContainer<LO> FCi;
#endif
out << "version: " << MueLu::Version() << std::endl;
Xpetra::UnderlyingLib lib = MueLuTests::TestHelpers::Parameters::getLib();
RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();
GO gst_invalid = Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid();
GO lo_invalid = Teuchos::OrdinalTraits<LO>::invalid();
int MyPID = comm->getRank();
// Setup Levels
Level fineLevel, coarseLevel;
test_factory::createTwoLevelHierarchy(fineLevel, coarseLevel);
fineLevel.SetFactoryManager(Teuchos::null); // factory manager is not used on this test
coarseLevel.SetFactoryManager(Teuchos::null);
// Build a pseudo-poisson test matrix
FCi elem_to_node;
RCP<Matrix> A = test_factory::Build1DPseudoPoissonHigherOrder(num_nodes,degree,elem_to_node,lib);
fineLevel.Set("A",A);
fineLevel.Set("ipc: element to node map",rcp(&elem_to_node,false));
// only one NS vector
LocalOrdinal NSdim = 1;
RCP<MultiVector> nullSpace = MultiVectorFactory::Build(A->getRowMap(),NSdim);
nullSpace->setSeed(846930886);
nullSpace->randomize();
fineLevel.Set("Nullspace",nullSpace);
// ParameterList
ParameterList Params;
Params.set("ipc: hi basis",hi_basis);
Params.set("ipc: lo basis","hgrad_line_c1");
// Build P
RCP<MueLu::IntrepidPCoarsenFactory<SC,LO,GO,NO> > IPCFact = rcp(new MueLu::IntrepidPCoarsenFactory<SC,LO,GO,NO>());
IPCFact->SetParameterList(Params);
coarseLevel.Request("P",IPCFact.get()); // request Ptent
coarseLevel.Request("Nullspace",IPCFact.get());
coarseLevel.Request("CoarseMap",IPCFact.get());
coarseLevel.Request(*IPCFact);
IPCFact->Build(fineLevel,coarseLevel);
// Get P
RCP<Matrix> P;
coarseLevel.Get("P",P,IPCFact.get());
RCP<CrsMatrix> Pcrs = rcp_dynamic_cast<CrsMatrixWrap>(P)->getCrsMatrix();
if(!MyPID) printf("P size = %d x %d\n",(int)P->getRangeMap()->getGlobalNumElements(),(int)P->getDomainMap()->getGlobalNumElements());
// Build serial comparison maps
GO pn_num_global_dofs = A->getRowMap()->getGlobalNumElements();
GO pn_num_serial_elements = !MyPID ? pn_num_global_dofs : 0;
RCP<Map> pn_SerialMap = MapFactory::Build(lib,pn_num_global_dofs,pn_num_serial_elements,0,comm);
GO p1_num_global_dofs = P->getDomainMap()->getGlobalNumElements();
GO p1_num_serial_elements = !MyPID ? p1_num_global_dofs : 0;
RCP<Map> p1_SerialMap = MapFactory::Build(lib, p1_num_global_dofs,p1_num_serial_elements,0,comm);
RCP<Export> p1_importer = ExportFactory::Build(p1_SerialMap,P->getDomainMap());
RCP<Export> pn_importer = ExportFactory::Build(A->getRowMap(),pn_SerialMap);
// Allocate some vectors
RCP<Vector> s_InVec = VectorFactory::Build(p1_SerialMap);
RCP<Vector> p_InVec = VectorFactory::Build(P->getDomainMap());
RCP<Vector> s_OutVec = VectorFactory::Build(pn_SerialMap);
RCP<Vector> s_codeOutput = VectorFactory::Build(pn_SerialMap);
RCP<Vector> p_codeOutput = VectorFactory::Build(A->getRowMap());
// Fill serial GOLD vecs on Proc 0
if(!MyPID) {
for(size_t i=0; i<(size_t)pn_gold_in.size(); i++)
s_InVec->replaceLocalValue(i,pn_gold_in[i]);
for(size_t i=0; i<(size_t)pn_gold_out.size(); i++)
s_OutVec->replaceLocalValue(i,pn_gold_out[i]);
}
// Migrate input data
p_InVec->doExport(*s_InVec,*p1_importer,Xpetra::ADD);
// Apply P
P->apply(*p_InVec,*p_codeOutput);
// Migrate Output data
s_codeOutput->doExport(*p_codeOutput,*pn_importer,Xpetra::ADD);
// Compare vs. GOLD
s_codeOutput->update(-1.0,*s_OutVec,1.0);
Teuchos::Array<MT> norm2(1);
s_codeOutput->norm2(norm2());
if(!MyPID) printf("Diff norm = %10.4e\n",norm2[0]);
}
int main(int argc, char *argv[]) {
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false); // Note:
GO nx,ny,nz;
nx=500;
ny=500;
nz=100;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'scalingTest.xml'");
int amgAsPrecond=1; clp.setOption("precond",&amgAsPrecond,"apply multigrid as preconditioner");
int amgAsSolver=0; clp.setOption("fixPoint",&amgAsSolver,"apply multigrid as solver");
bool printTimings=true; clp.setOption("timings","notimings",&printTimings,"print timings to screen");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if (comm->getRank() == 0) {
std::cout << "========================================================" << std::endl
<< xpetraParameters << matrixParameters;
}
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
// read aggregation options from file
Teuchos::FileInputSource fileSrc(xmlFileName);
Teuchos::XMLObject fileXML = fileSrc.getObject();
Teuchos::XMLParameterListReader listReader;
Teuchos::ParameterList aggList = listReader.toParameterList(fileXML);
//std::cout << "===========aggList start===========" << std::endl;
//aggList.print(std::cout);
//std::cout << "===========aggList end===========" << std::endl;
// instantiate aggregate factory, set options from parameter list
RCP<MueLu::SingleLevelFactoryBase> aggFact;
if (aggList.name() == "UncoupledAggregationFactory") {
RCP<UncoupledAggregationFactory> ucFact = rcp( new UncoupledAggregationFactory() );
//ucFact->SetParameterList(aggList);
//FIXME hack until UCAgg uses PL interface
std::string ordering = aggList.get<std::string>("Ordering");
MueLu::AggOptions::Ordering eordering;
if (ordering=="Natural") eordering = MueLu::AggOptions::NATURAL;
if (ordering=="Graph") eordering = MueLu::AggOptions::GRAPH;
if (ordering=="Random") eordering = MueLu::AggOptions::RANDOM;
ucFact->SetOrdering(eordering);
ucFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected"));
ucFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate"));
aggFact = ucFact;
} else if (aggList.name() == "CoupledAggregationFactory") {
RCP<CoupledAggregationFactory> cFact = rcp( new CoupledAggregationFactory() );
//cFact->SetParameterList(aggList);
//FIXME hack until CoupledAgg uses PL interface
//cFact->SetOrdering(aggList.get<std::string>("Ordering"));
cFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected"));
cFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate"));
aggFact = cFact;
} else {
throw(MueLu::Exceptions::RuntimeError("List's name does not correspond to a known aggregation factory."));
}
//Teuchos::ParameterList tlist = aggFact->GetParameterList();
//std::cout << "===========verify List start===========" << std::endl;
//tlist.print(std::cout);
//std::cout << "===========verify List end===========" << std::endl;
// build matrix
RCP<TimeMonitor> tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<const Map> map;
RCP<MultiVector> coordinates;
// Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri.
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx",nx));
galeriList.set("ny", pl.get("ny",ny));
//galeriList.set("mx", comm->getSize());
//galeriList.set("my", 1);
if (matrixParameters.GetMatrixType() == "Laplace1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D",map,matrixParameters.GetParameterList());
//map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList); //TODO when available in Galeri
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
}
if (comm->getRank() == 0) {
GO mx = galeriList.get("mx", -1);
GO my = galeriList.get("my", -1);
std::cout << "Processor subdomains in x direction: " << mx << std::endl
<< "Processor subdomains in y direction: " << my << std::endl
<< "========================================================" << std::endl;
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A = Pr->BuildMatrix();
tm = Teuchos::null;
Level level;
RCP<MueLu::FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
level.SetLevelID(0);
level.Set("A", A);
level.Request("Aggregates", aggFact.get());
level.Request(*aggFact);
level.setVerbLevel(Teuchos::VERB_NONE);
aggFact->setVerbLevel(Teuchos::VERB_NONE);
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("aggregation time")));
aggFact->Build(level);
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize();
} //main