TEUCHOS_UNIT_TEST(Zoltan, Build) { typedef Teuchos::ScalarTraits<Scalar> ST; out << "version: " << MueLu::Version() << std::endl; out << std::endl; out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl; out << "that has a random number of nonzeros per row. Good results have been precomputed" << std::endl; out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl; out << "once the Zoltan repartitioning has been applied." << std::endl; out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl; RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm(); if (comm->getSize() > 5) { out << std::endl; out << "This test must be run on 1 to 5 processes." << std::endl; TEST_EQUALITY(true, true); return; } Level level; RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager()); level.SetFactoryManager(factoryHandler); int nx=7; int ny=nx; GO numGlobalElements = nx*ny; size_t maxEntriesPerRow=30; // Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row. RCP<const Map> map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm); const size_t numMyElements = map->getNodeNumElements(); Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList(); RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more // memory on the fly. While not super efficient, this // ensures that no zeros are being stored. Thus, from // Zoltan's perspective the matrix is imbalanced. // Create a vector with random integer entries in [1,maxEntriesPerRow]. ST::seedrandom(666*comm->getRank()); RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false); Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0); for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) { *i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1)); } RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); fos->setOutputToRootOnly(-1); Teuchos::Array<Scalar> vals(maxEntriesPerRow); Teuchos::Array<GO> cols(maxEntriesPerRow); for (size_t i = 0; i < numMyElements; ++i) { Teuchos::ArrayView<SC> av(&vals[0],eprData[i]); Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]); //stick in ones for values for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one(); //figure out valid column indices GO start = std::max(myGlobalElements[i]-eprData[i]+1,0); for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j; A->insertGlobalValues(myGlobalElements[i], iv, av); } A->fillComplete(); level.Set("A",A); //build coordinates RCP<const Map> rowMap = A->getRowMap(); Teuchos::ParameterList list; list.set("nx",nx); list.set("ny",ny); RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",rowMap,list); level.Set("Coordinates",XYZ); LO numPartitions = comm->getSize(); level.Set("number of partitions",numPartitions); RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface()); //zoltan->SetNumberOfPartitions(numPartitions); //zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints level.Request("Partition",zoltan.get()); zoltan->Build(level); RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get()); /* //TODO temporary code to have the trivial decomposition (no change) ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0); for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i) *i = comm->getRank(); decompEntries=Teuchos::null; */ //TODO end of temporary code //Create vector whose local length is the global number of partitions. //This vector will record the local number of nonzeros associated with each partition. Teuchos::Array<GO> parts(numPartitions); for (int i=0; i<numPartitions; ++i) parts[i] = i; Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions); RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(), Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView, map->getIndexBase(),comm); RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap); //For the local rows in each partition, tally up the number of nonzeros. This is what //Zoltan should be load-balancing. Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0); Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0); for (size_t i=0; i<decomposition->getLocalLength();++i) { Teuchos::ArrayView<const LO> c; Teuchos::ArrayView<const SC> v; A->getLocalRowView(i,c,v); lpvData[decompData[i]] += v.size(); } lpvData = Teuchos::null; decompData = Teuchos::null; //localPartsVec->describe(*fos,Teuchos::VERB_EXTREME); //Send the local nnz tallies to pid 0, which can report the global sums. size_t mysize=1; if (comm->getRank() == 0) mysize = numPartitions; RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(), Teuchos::OrdinalTraits<global_size_t>::invalid(), mysize, map->getIndexBase(), comm); RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap); RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap); globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD); ArrayRCP<GO> expectedResults(numPartitions); switch (comm->getSize()) { case 1: expectedResults[0] = 807; break; case 2: expectedResults[0] = 364; expectedResults[1] = 363; break; case 3: expectedResults[0] = 277; expectedResults[1] = 261; expectedResults[2] = 269; break; case 4: expectedResults[0] = 195; expectedResults[1] = 186; expectedResults[2] = 177; expectedResults[3] = 168; break; case 5: expectedResults[0] = 161; expectedResults[1] = 145; expectedResults[2] = 148; expectedResults[3] = 159; expectedResults[4] = 157; break; default: break; }; //FIXME cool ... this next line causes a hang if locally the globalyTallyVec has no data. //FIXME I get around this by making mysize (above) 1 instead of 0. Is this a bug or feature //FIXME in getData? ArrayRCP<const LO> gtvData = globalTallyVec->getData(0); #ifdef __linux__ out << "Checking results..." << std::endl; for (int i=0; i<numPartitions; ++i) { if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]); } #endif #ifdef MUELU_VISUALIZE_REPARTITIONING // //Now write everything to a comma-separate list that ParaView can grok // Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0); Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1); Teuchos::ArrayRCP<const GO> D = decomposition->getData(0); RCP<std::ofstream> outFile; std::string fileName = "zoltanResults.csv"; //write header information if (comm->getRank() == 0) { outFile = rcp(new std::ofstream(fileName.c_str())); *outFile << "x coord, y coord, z coord, scalar" << std::endl; } comm->barrier(); //append coordinates for (int j=0; j<comm->getSize(); ++j) { int mypid = comm->getRank(); if (mypid == j) { outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app)); for (int i=0; i < D.size(); ++i) { *outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", " << D[i] << std::endl; } } } //for (int i=0; i<comm->getSize(); ++i) out << std::endl; out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl; out << " 1) Load the data file " << fileName << "." << std::endl; out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl; out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl; out << " 4) Split screen horizontally (Icon, top right)." << std::endl; out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl; out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl; out << std::endl; out << " To display row weights next to each point:" << std::endl; out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl; out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl; out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl; #endif } //Build
int main(int argc, char *argv[]) { using Teuchos::RCP; Teuchos::oblackholestream blackhole; Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole); RCP<const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm(); RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); out->setOutputToRootOnly(0); #ifndef HAVE_TEUCHOS_LONG_LONG_INT *out << "Warning: scaling test was not compiled with long long int support" << std::endl; #endif /**********************************************************************************/ /* SET TEST PARAMETERS */ /**********************************************************************************/ // Note: use --help to list available options. Teuchos::CommandLineProcessor clp(false); // Default is Laplace1D with nx = 8748. // It's a nice size for 1D and perfect aggregation. (6561=3^8) //Nice size for 1D and perfect aggregation on small numbers of processors. (8748=4*3^7) Galeri::Xpetra::Parameters<GO> matrixParameters(clp, 8748); // manage parameters of the test case Xpetra::Parameters xpetraParameters(clp); // manage parameters of xpetra // custom parameters std::string aggOrdering = "natural"; int minPerAgg=2; int maxNbrAlreadySelected=0; clp.setOption("aggOrdering",&aggOrdering,"aggregation ordering strategy (natural,random,graph)"); clp.setOption("minPerAgg",&minPerAgg,"minimum #DOFs per aggregate"); clp.setOption("maxNbrSel",&maxNbrAlreadySelected,"maximum # of nbrs allowed to be in other aggregates"); switch (clp.parse(argc,argv)) { case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break; case Teuchos::CommandLineProcessor::PARSE_ERROR: case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break; case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break; } matrixParameters.check(); xpetraParameters.check(); // TODO: check custom parameters if (comm->getRank() == 0) { std::cout << matrixParameters << xpetraParameters << std::endl; // TODO: print custom parameters } /**********************************************************************************/ /* CREATE INITIAL MATRIX */ /**********************************************************************************/ const RCP<const Map> map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm); Teuchos::RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr = Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); //TODO: Matrix vs. CrsMatrixWrap RCP<Matrix> A = Pr->BuildMatrix(); // return EXIT_SUCCESS; /**********************************************************************************/ /* */ /**********************************************************************************/ Level Finest; Finest.SetLevelID(0); // must be level 0 for NullspaceFactory Finest.Set("A", A); Finest.SetFactoryManager( rcp( new FactoryManager() )); CoupledAggregationFactory CoupledAggFact; Finest.Request(CoupledAggFact); *out << "========================= Aggregate option summary =========================" << std::endl; *out << "min DOFs per aggregate : " << minPerAgg << std::endl; *out << "min # of root nbrs already aggregated : " << maxNbrAlreadySelected << std::endl; CoupledAggFact.SetMinNodesPerAggregate(minPerAgg); //TODO should increase if run anything other than 1D CoupledAggFact.SetMaxNeighAlreadySelected(maxNbrAlreadySelected); std::transform(aggOrdering.begin(), aggOrdering.end(), aggOrdering.begin(), ::tolower); if (aggOrdering == "natural") { *out << "aggregate ordering : NATURAL" << std::endl; CoupledAggFact.SetOrdering(MueLu::AggOptions::NATURAL); } else if (aggOrdering == "random") { *out << "aggregate ordering : RANDOM" << std::endl; CoupledAggFact.SetOrdering(MueLu::AggOptions::RANDOM); } else if (aggOrdering == "graph") { *out << "aggregate ordering : GRAPH" << std::endl; CoupledAggFact.SetOrdering(MueLu::AggOptions::GRAPH); } else { std::string msg = "main: bad aggregation option """ + aggOrdering + """."; throw(MueLu::Exceptions::RuntimeError(msg)); } CoupledAggFact.SetPhase3AggCreation(0.5); *out << "=============================================================================" << std::endl; CoupledAggFact.Build(Finest); return EXIT_SUCCESS; }
TEUCHOS_UNIT_TEST(Zoltan, Build3PDEs) { typedef Teuchos::ScalarTraits<Scalar> ST; out << "version: " << MueLu::Version() << std::endl; out << std::endl; out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl; out << "that has a random number of nonzeros per row and 3 DOFs per mesh point. Good results have been precomputed" << std::endl; out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl; out << "once the Zoltan repartitioning has been applied." << std::endl; out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl; RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm(); if (comm->getSize() > 5) { out << std::endl; out << "This test must be run on 1 to 5 processes." << std::endl; TEST_EQUALITY(true, true); return; } Level level; RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager()); level.SetFactoryManager(factoryHandler); int nx=9; int ny=nx; int dofsPerNode = 3; GO numGlobalElements = nx*ny*dofsPerNode; size_t maxEntriesPerRow=30; RCP<const Map> map; int numMyNodes = numGlobalElements / dofsPerNode; if (comm->getSize() > 1) { // In parallel, make sure that the dof's associated with a node all // reside on the same processor. int numNodes = numGlobalElements / dofsPerNode; TEUCHOS_TEST_FOR_EXCEPTION( (numGlobalElements - numNodes * dofsPerNode) != 0, MueLu::Exceptions::RuntimeError, "Number of matrix rows is not divisible by #dofs" ); int nproc = comm->getSize(); if (comm->getRank() < nproc-1) numMyNodes = numNodes / nproc; else numMyNodes = numNodes - (numNodes/nproc) * (nproc-1); map = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, numMyNodes*dofsPerNode, comm); } else { map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm); } const size_t numMyElements = map->getNodeNumElements(); Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList(); RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more // memory on the fly. While not super efficient, this // ensures that no zeros are being stored. Thus, from // Zoltan's perspective the matrix is imbalanced. // Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row. // Create a vector with random integer entries in [1,maxEntriesPerRow]. ST::seedrandom(666*comm->getRank()); RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false); Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0); for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) { *i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1)); } RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); fos->setOutputToRootOnly(-1); Teuchos::Array<Scalar> vals(maxEntriesPerRow); Teuchos::Array<GO> cols(maxEntriesPerRow); for (size_t i = 0; i < numMyElements; ++i) { Teuchos::ArrayView<SC> av(&vals[0],eprData[i]); Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]); //stick in ones for values for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one(); //figure out valid column indices GO start = std::max(myGlobalElements[i]-eprData[i]+1,0); for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j; A->insertGlobalValues(myGlobalElements[i], iv, av); } A->fillComplete(); // Now treat the matrix as if it has 3 DOFs per node. A->SetFixedBlockSize(dofsPerNode); level.Set("A",A); //build coordinates Teuchos::ParameterList list; list.set("nx",nx); list.set("ny",ny); RCP<const Map> coalescedMap = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements/dofsPerNode, numMyNodes, comm); RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",coalescedMap,list); // XYZ are the "coalesce" coordinates as it has been generated for 1 DOF/node and we are using them for 3 DOFS/node // level.Set("Coordinates",XYZ); "Coordinates" == uncoalesce. "X,Y,ZCoordinates" == coalesce { RCP<MultiVector> coordinates = XYZ; // making a copy because I don't want to keep 'open' the Xpetra_MultiVector if (coordinates->getNumVectors() >= 1) { Teuchos::ArrayRCP<const SC> coord = coordinates->getData(0); Teuchos::ArrayRCP<SC> coordCpy(coord.size()); for(int i=0; i<coord.size(); i++) { coordCpy[i] = coord[i]; } level.Set("XCoordinates", coordCpy); //std::cout << coordCpy << std::endl; } if (coordinates->getNumVectors() >= 2) { Teuchos::ArrayRCP<const SC> coord = coordinates->getData(1); Teuchos::ArrayRCP<SC> coordCpy(coord.size()); for(int i=0; i<coord.size(); i++) { coordCpy[i] = coord[i]; } level.Set("YCoordinates", coordCpy); } /*if (coordinates->getNumVectors() >= 3) { Teuchos::ArrayRCP<const SC> coord = coordinates->getData(2); Teuchos::ArrayRCP<SC> coordCpy(coord.size()); for(int i=0; i<coord.size(); i++) { coordCpy[i] = coord[i]; } level.Set("ZCoordinates", coordCpy); }*/ } //coalescedMap->describe(*fos,Teuchos::VERB_EXTREME); //sleep(1); comm->barrier(); //XYZ->describe(*fos,Teuchos::VERB_EXTREME); LO numPartitions = comm->getSize(); level.Set("number of partitions",numPartitions); RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface()); //zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints level.Request("Partition",zoltan.get()); zoltan->Build(level); RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get()); /* //temporary code to have the trivial decomposition (no change) ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0); for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i) *i = comm->getRank(); decompEntries=Teuchos::null; */ //Create vector whose local length is the global number of partitions. //This vector will record the local number of nonzeros associated with each partition. Teuchos::Array<GO> parts(numPartitions); for (int i=0; i<numPartitions; ++i) parts[i] = i; Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions); RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(), Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView, map->getIndexBase(),comm); RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap); RCP<Xpetra::Vector<LO,LO,GO,NO> > nnzPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(A->getRowMap()); Teuchos::ArrayRCP<GO> nnzData = nnzPerRow->getDataNonConst(0); //For the local rows in each partition, tally up the number of nonzeros. This is what //Zoltan should be load-balancing. Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0); Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0); for (size_t i=0; i<decomposition->getLocalLength();++i) { Teuchos::ArrayView<const LO> c; Teuchos::ArrayView<const SC> v; A->getLocalRowView(i,c,v); lpvData[decompData[i]] += v.size(); nnzData[i] = v.size(); } lpvData = Teuchos::null; decompData = Teuchos::null; nnzData = Teuchos::null; /* if (comm->getRank() == 0) std::cout << "nnz per row" << std::endl; nnzPerRow->describe(*fos,Teuchos::VERB_EXTREME); if (comm->getRank() == 0) std::cout << "Row-to-partition assignment (from Zoltan)" << std::endl; decomposition->describe(*fos,Teuchos::VERB_EXTREME); if (comm->getRank() == 0) std::cout << "#nonzeros per partition" << std::endl; localPartsVec->describe(*fos,Teuchos::VERB_EXTREME); */ //Send the local nnz tallies to pid 0, which can report the global sums. size_t mysize=1; if (comm->getRank() == 0) mysize = numPartitions; RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(), Teuchos::OrdinalTraits<global_size_t>::invalid(), mysize, map->getIndexBase(), comm); RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap); RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap); globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD); ArrayRCP<GO> expectedResults(numPartitions); switch (comm->getSize()) { case 1: expectedResults[0] = 3951; break; case 2: expectedResults[0] = 1955; expectedResults[1] = 1910; break; case 3: expectedResults[0] = 1326; expectedResults[1] = 1340; expectedResults[2] = 1321; break; case 4: expectedResults[0] = 950; expectedResults[1] = 922; expectedResults[2] = 908; expectedResults[3] = 936; break; case 5: expectedResults[0] = 774; expectedResults[1] = 735; expectedResults[2] = 726; expectedResults[3] = 771; expectedResults[4] = 759; break; default: break; }; ArrayRCP<const LO> gtvData = globalTallyVec->getData(0); #ifdef __linux__ out << "Checking results..." << std::endl; for (int i=0; i<numPartitions; ++i) { if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]); } #endif #ifdef MUELU_VISUALIZE_REPARTITIONING // //Now write everything to a comma-separate list that ParaView can grok // Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0); Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1); Teuchos::ArrayRCP<const GO> D = decomposition->getData(0); RCP<std::ofstream> outFile; std::string fileName = "zoltanResults.csv"; //write header information if (comm->getRank() == 0) { outFile = rcp(new std::ofstream(fileName.c_str())); *outFile << "x coord, y coord, z coord, partition, row weight" << std::endl; } comm->barrier(); //append coordinates nnzData = nnzPerRow->getDataNonConst(0); for (int j=0; j<comm->getSize(); ++j) { int mypid = comm->getRank(); if (mypid == j) { outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app)); int blockSize = A->GetFixedBlockSize(); //Coordinates are for coalesced system, D is for uncoalesced for (int i=0; i < D.size()/blockSize; ++i) { int nnz=0; for (int k=0; k<blockSize; ++k) nnz += nnzData[i*blockSize+k]; *outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", " << D[i*blockSize] << ", " << nnz << std::endl; } } } //for (int i=0; i<comm->getSize(); ++i) out << std::endl; out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl; out << " 1) Load the data file " << fileName << "." << std::endl; out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl; out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl; out << " 4) Split screen horizontally (Icon, top right)." << std::endl; out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl; out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl; out << std::endl; out << " To display row weights next to each point:" << std::endl; out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl; out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl; out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl; #endif } //Build3PDEs
int main(int argc, char *argv[]) { using Teuchos::RCP; // reference count pointers using Teuchos::rcp; using Teuchos::TimeMonitor; // // MPI initialization using Teuchos // Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL); RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm(); // // Parameters // Teuchos::CommandLineProcessor clp(false); // Note: GO nx,ny,nz; nx=500; ny=500; nz=100; Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'scalingTest.xml'"); int amgAsPrecond=1; clp.setOption("precond",&amgAsPrecond,"apply multigrid as preconditioner"); int amgAsSolver=0; clp.setOption("fixPoint",&amgAsSolver,"apply multigrid as solver"); bool printTimings=true; clp.setOption("timings","notimings",&printTimings,"print timings to screen"); switch (clp.parse(argc,argv)) { case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break; case Teuchos::CommandLineProcessor::PARSE_ERROR: case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break; case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break; } if (comm->getRank() == 0) { std::cout << "========================================================" << std::endl << xpetraParameters << matrixParameters; } RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time"))); // read aggregation options from file Teuchos::FileInputSource fileSrc(xmlFileName); Teuchos::XMLObject fileXML = fileSrc.getObject(); Teuchos::XMLParameterListReader listReader; Teuchos::ParameterList aggList = listReader.toParameterList(fileXML); //std::cout << "===========aggList start===========" << std::endl; //aggList.print(std::cout); //std::cout << "===========aggList end===========" << std::endl; // instantiate aggregate factory, set options from parameter list RCP<MueLu::SingleLevelFactoryBase> aggFact; if (aggList.name() == "UncoupledAggregationFactory") { RCP<UncoupledAggregationFactory> ucFact = rcp( new UncoupledAggregationFactory() ); //ucFact->SetParameterList(aggList); //FIXME hack until UCAgg uses PL interface std::string ordering = aggList.get<std::string>("Ordering"); MueLu::AggOptions::Ordering eordering; if (ordering=="Natural") eordering = MueLu::AggOptions::NATURAL; if (ordering=="Graph") eordering = MueLu::AggOptions::GRAPH; if (ordering=="Random") eordering = MueLu::AggOptions::RANDOM; ucFact->SetOrdering(eordering); ucFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected")); ucFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate")); aggFact = ucFact; } else if (aggList.name() == "CoupledAggregationFactory") { RCP<CoupledAggregationFactory> cFact = rcp( new CoupledAggregationFactory() ); //cFact->SetParameterList(aggList); //FIXME hack until CoupledAgg uses PL interface //cFact->SetOrdering(aggList.get<std::string>("Ordering")); cFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected")); cFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate")); aggFact = cFact; } else { throw(MueLu::Exceptions::RuntimeError("List's name does not correspond to a known aggregation factory.")); } //Teuchos::ParameterList tlist = aggFact->GetParameterList(); //std::cout << "===========verify List start===========" << std::endl; //tlist.print(std::cout); //std::cout << "===========verify List end===========" << std::endl; // build matrix RCP<TimeMonitor> tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build"))); RCP<const Map> map; RCP<MultiVector> coordinates; // Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri. // Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g., // d1 d2 d3 // d4 d5 d6 // d7 d8 d9 // d10 d11 d12 // A perfect distribution is only possible when the #processors is a perfect square. // This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in // size. For example, np=14 will give a 7-by-2 distribution. // If you don't want Galeri to do this, specify mx or my on the galeriList. Teuchos::ParameterList pl = matrixParameters.GetParameterList(); Teuchos::ParameterList galeriList; galeriList.set("nx", pl.get("nx",nx)); galeriList.set("ny", pl.get("ny",ny)); //galeriList.set("mx", comm->getSize()); //galeriList.set("my", 1); if (matrixParameters.GetMatrixType() == "Laplace1D") { map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm); coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D",map,matrixParameters.GetParameterList()); } else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D") { map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList); coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",map,matrixParameters.GetParameterList()); } else if (matrixParameters.GetMatrixType() == "Laplace3D") { coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D",map,matrixParameters.GetParameterList()); //map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList); //TODO when available in Galeri map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm); } if (comm->getRank() == 0) { GO mx = galeriList.get("mx", -1); GO my = galeriList.get("my", -1); std::cout << "Processor subdomains in x direction: " << mx << std::endl << "Processor subdomains in y direction: " << my << std::endl << "========================================================" << std::endl; } RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr = Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList()); RCP<Matrix> A = Pr->BuildMatrix(); tm = Teuchos::null; Level level; RCP<MueLu::FactoryManagerBase> factoryHandler = rcp(new FactoryManager()); level.SetFactoryManager(factoryHandler); level.SetLevelID(0); level.Set("A", A); level.Request("Aggregates", aggFact.get()); level.Request(*aggFact); level.setVerbLevel(Teuchos::VERB_NONE); aggFact->setVerbLevel(Teuchos::VERB_NONE); tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("aggregation time"))); aggFact->Build(level); tm = Teuchos::null; globalTimeMonitor = Teuchos::null; if (printTimings) TimeMonitor::summarize(); } //main