Teuchos::RCP<const Tpetra::Map<LO,GO,Node> >
createMeshMap (const LO& blockSize, const Tpetra::Map<LO,GO,Node>& pointMap)
{
typedef Teuchos::OrdinalTraits<Tpetra::global_size_t> TOT;
typedef Tpetra::Map<LO,GO,Node> map_type;
//calculate mesh GIDs
Teuchos::ArrayView<const GO> pointGids = pointMap.getNodeElementList();
Teuchos::Array<GO> meshGids;
GO indexBase = pointMap.getIndexBase();
// Use hash table to track whether we've encountered this GID previously. This will happen
// when striding through the point DOFs in a block. It should not happen otherwise.
// I don't use sort/make unique because I don't want to change the ordering.
meshGids.reserve(pointGids.size());
Tpetra::Details::HashTable<GO,int> hashTable(pointGids.size());
for (int i=0; i<pointGids.size(); ++i) {
GO meshGid = (pointGids[i]-indexBase) / blockSize + indexBase;
if (hashTable.get(meshGid) == -1) {
hashTable.add(meshGid,1); //(key,value)
meshGids.push_back(meshGid);
}
}
Teuchos::RCP<const map_type> meshMap = Teuchos::rcp( new map_type(TOT::invalid(), meshGids(), 0, pointMap.getComm()) );
return meshMap;
}
Teuchos::Array<EpetraGlobalIndex> getMyLIDs(
const Epetra_BlockMap &map,
const Teuchos::ArrayView<const EpetraGlobalIndex> &selectedGIDs)
{
Teuchos::Array<EpetraGlobalIndex> sortedMyGIDs(map.MyGlobalElements(), map.MyGlobalElements() + map.NumMyElements());
std::sort(sortedMyGIDs.begin(), sortedMyGIDs.end());
Teuchos::Array<EpetraGlobalIndex> sortedSelectedGIDs(selectedGIDs);
std::sort(sortedSelectedGIDs.begin(), sortedSelectedGIDs.end());
Teuchos::Array<EpetraGlobalIndex> mySelectedGIDs;
std::set_intersection(sortedMyGIDs.begin(), sortedMyGIDs.end(),
sortedSelectedGIDs.begin(), sortedSelectedGIDs.end(),
std::back_inserter(mySelectedGIDs));
Teuchos::Array<EpetraGlobalIndex> result;
result.reserve(mySelectedGIDs.size());
std::transform(
mySelectedGIDs.begin(), mySelectedGIDs.end(),
std::back_inserter(result),
std::bind1st(std::mem_fun_ref(static_cast<int(Epetra_BlockMap::*)(EpetraGlobalIndex) const>(&Epetra_BlockMap::LID)), map));
return result;
}
Teuchos::Array<bool> createResponseTable(
int count,
const std::string selectionType,
int index,
const Teuchos::ArrayView<const int> &list)
{
Teuchos::Array<bool> result;
if (count > 0) {
if (selectionType == "All") {
result.resize(count, true);
} else if (selectionType == "Last") {
result = createResponseTableFromIndex(count - 1, count);
} else if (selectionType == "AllButLast") {
result.reserve(count);
result.resize(count - 1, true);
result.push_back(false);
} else if (selectionType == "Index") {
result = createResponseTableFromIndex(index, count);
} else if (selectionType == "List") {
result.resize(count, false);
for (Teuchos::ArrayView<const int>::const_iterator it = list.begin(), it_end = list.end(); it != it_end; ++it) {
result.at(*it) = true;
}
} else {
TEUCHOS_TEST_FOR_EXCEPT(false);
}
}
return result;
}
void epetraFromThyra(
const Teuchos::RCP<const Epetra_Comm> &comm,
const Teuchos::Array<Teuchos::RCP<const Thyra::VectorBase<double> > > &thyraResponses,
const Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Thyra::MultiVectorBase<double> > > > &thyraSensitivities,
Teuchos::Array<Teuchos::RCP<const Epetra_Vector> > &responses,
Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Epetra_MultiVector> > > &sensitivities)
{
responses.clear();
responses.reserve(thyraResponses.size());
typedef Teuchos::Array<Teuchos::RCP<const Thyra::VectorBase<double> > > ThyraResponseArray;
for (ThyraResponseArray::const_iterator it_begin = thyraResponses.begin(),
it_end = thyraResponses.end(),
it = it_begin;
it != it_end;
++it) {
responses.push_back(epetraVectorFromThyra(comm, *it));
}
sensitivities.clear();
sensitivities.reserve(thyraSensitivities.size());
typedef Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Thyra::MultiVectorBase<double> > > > ThyraSensitivityArray;
for (ThyraSensitivityArray::const_iterator it_begin = thyraSensitivities.begin(),
it_end = thyraSensitivities.end(),
it = it_begin;
it != it_end;
++it) {
ThyraSensitivityArray::const_reference sens_thyra = *it;
Teuchos::Array<Teuchos::RCP<const Epetra_MultiVector> > sens;
sens.reserve(sens_thyra.size());
for (ThyraSensitivityArray::value_type::const_iterator jt = sens_thyra.begin(),
jt_end = sens_thyra.end();
jt != jt_end;
++jt) {
sens.push_back(epetraMultiVectorFromThyra(comm, *jt));
}
sensitivities.push_back(sens);
}
}
void tpetraFromThyra(
const Teuchos::Array<Teuchos::RCP<const Thyra::VectorBase<ST> > > &thyraResponses,
const Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Thyra::MultiVectorBase<ST> > > > &thyraSensitivities,
Teuchos::Array<Teuchos::RCP<const Tpetra_Vector> > &responses,
Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Tpetra_MultiVector> > > &sensitivities)
{
responses.clear();
responses.reserve(thyraResponses.size());
typedef Teuchos::Array<Teuchos::RCP<const Thyra::VectorBase<ST> > > ThyraResponseArray;
for (ThyraResponseArray::const_iterator it_begin = thyraResponses.begin(),
it_end = thyraResponses.end(),
it = it_begin;
it != it_end;
++it) {
responses.push_back(Teuchos::nonnull(*it) ? ConverterT::getConstTpetraVector(*it) : Teuchos::null);
}
sensitivities.clear();
sensitivities.reserve(thyraSensitivities.size());
typedef Teuchos::Array<Teuchos::Array<Teuchos::RCP<const Thyra::MultiVectorBase<ST> > > > ThyraSensitivityArray;
for (ThyraSensitivityArray::const_iterator it_begin = thyraSensitivities.begin(),
it_end = thyraSensitivities.end(),
it = it_begin;
it != it_end;
++it) {
ThyraSensitivityArray::const_reference sens_thyra = *it;
Teuchos::Array<Teuchos::RCP<const Tpetra_MultiVector> > sens;
sens.reserve(sens_thyra.size());
for (ThyraSensitivityArray::value_type::const_iterator jt = sens_thyra.begin(),
jt_end = sens_thyra.end();
jt != jt_end;
++jt) {
sens.push_back(Teuchos::nonnull(*jt) ? ConverterT::getConstTpetraMultiVector(*jt) : Teuchos::null);
}
sensitivities.push_back(sens);
}
}
void blockCrsMatrixWriter(BlockCrsMatrix<Scalar,LO,GO,Node> const &A, std::ostream &os, Teuchos::ParameterList const ¶ms) {
using Teuchos::RCP;
using Teuchos::rcp;
typedef Teuchos::OrdinalTraits<GO> TOT;
typedef BlockCrsMatrix<Scalar, LO, GO, Node> block_crs_matrix_type;
typedef Tpetra::Import<LO, GO, Node> import_type;
typedef Tpetra::Map<LO, GO, Node> map_type;
typedef Tpetra::MultiVector<GO, LO, GO, Node> mv_type;
typedef Tpetra::CrsGraph<LO, GO, Node> crs_graph_type;
RCP<const map_type> const rowMap = A.getRowMap(); //"mesh" map
RCP<const Teuchos::Comm<int> > comm = rowMap->getComm();
const int myRank = comm->getRank();
const size_t numProcs = comm->getSize();
// If true, force use of the import strip-mining infrastructure. This is useful for debugging on one process.
bool alwaysUseParallelAlgorithm = false;
if (params.isParameter("always use parallel algorithm"))
alwaysUseParallelAlgorithm = params.get<bool>("always use parallel algorithm");
bool printMatrixMarketHeader = true;
if (params.isParameter("print MatrixMarket header"))
printMatrixMarketHeader = params.get<bool>("print MatrixMarket header");
if (printMatrixMarketHeader && myRank==0) {
std::time_t now = std::time(NULL);
const std::string dataTypeStr =
Teuchos::ScalarTraits<Scalar>::isComplex ? "complex" : "real";
// Explanation of first line of file:
// - "%%MatrixMarket" is the tag for Matrix Market format.
// - "matrix" is what we're printing.
// - "coordinate" means sparse (triplet format), rather than dense.
// - "real" / "complex" is the type (in an output format sense,
// not in a C++ sense) of each value of the matrix. (The
// other two possibilities are "integer" (not really necessary
// here) and "pattern" (no values, just graph).)
os << "%%MatrixMarket matrix coordinate " << dataTypeStr << " general" << std::endl;
os << "% time stamp: " << ctime(&now);
os << "% written from " << numProcs << " processes" << std::endl;
os << "% point representation of Tpetra::Experimental::BlockCrsMatrix" << std::endl;
size_t numRows = A.getGlobalNumRows();
size_t numCols = A.getGlobalNumCols();
os << "% " << numRows << " block rows, " << numCols << " block columns" << std::endl;
const LO blockSize = A.getBlockSize();
os << "% block size " << blockSize << std::endl;
os << numRows*blockSize << " " << numCols*blockSize << " " << A.getGlobalNumEntries()*blockSize*blockSize << std::endl;
}
if (numProcs==1 && !alwaysUseParallelAlgorithm) {
writeMatrixStrip(A,os,params);
} else {
size_t numRows = rowMap->getNodeNumElements();
//Create source map
RCP<const map_type> allMeshGidsMap = rcp(new map_type(TOT::invalid(), numRows, A.getIndexBase(), comm));
//Create and populate vector of mesh GIDs corresponding to this pid's rows.
//This vector will be imported one pid's worth of information at a time to pid 0.
mv_type allMeshGids(allMeshGidsMap,1);
Teuchos::ArrayRCP<GO> allMeshGidsData = allMeshGids.getDataNonConst(0);
for (size_t i=0; i<numRows; i++)
allMeshGidsData[i] = rowMap->getGlobalElement(i);
allMeshGidsData = Teuchos::null;
// Now construct a RowMatrix on PE 0 by strip-mining the rows of the input matrix A.
size_t stripSize = allMeshGids.getGlobalLength() / numProcs;
size_t remainder = allMeshGids.getGlobalLength() % numProcs;
size_t curStart = 0;
size_t curStripSize = 0;
Teuchos::Array<GO> importMeshGidList;
for (size_t i=0; i<numProcs; i++) {
if (myRank==0) { // Only PE 0 does this part
curStripSize = stripSize;
if (i<remainder) curStripSize++; // handle leftovers
importMeshGidList.resize(curStripSize); // Set size of vector to max needed
for (size_t j=0; j<curStripSize; j++) importMeshGidList[j] = j + curStart + A.getIndexBase();
curStart += curStripSize;
}
// The following import map should be non-trivial only on PE 0.
TEUCHOS_TEST_FOR_EXCEPTION(myRank>0 && curStripSize!=0,
std::runtime_error, "Tpetra::Experimental::blockCrsMatrixWriter: (pid "
<< myRank << ") map size should be zero, but is " << curStripSize);
RCP<map_type> importMeshGidMap = rcp(new map_type(TOT::invalid(), importMeshGidList(), A.getIndexBase(), comm));
import_type gidImporter(allMeshGidsMap, importMeshGidMap);
mv_type importMeshGids(importMeshGidMap, 1);
importMeshGids.doImport(allMeshGids, gidImporter, INSERT);
// importMeshGids now has a list of GIDs for the current strip of matrix rows.
// Use these values to build another importer that will get rows of the matrix.
// The following import map will be non-trivial only on PE 0.
Teuchos::ArrayRCP<const GO> importMeshGidsData = importMeshGids.getData(0);
Teuchos::Array<GO> importMeshGidsGO;
importMeshGidsGO.reserve(importMeshGidsData.size());
for (typename Teuchos::ArrayRCP<const GO>::size_type j=0; j<importMeshGidsData.size(); ++j)
importMeshGidsGO.push_back(importMeshGidsData[j]);
RCP<const map_type> importMap = rcp(new map_type(TOT::invalid(), importMeshGidsGO(), rowMap->getIndexBase(), comm) );
import_type importer(rowMap,importMap );
size_t numEntriesPerRow = A.getCrsGraph().getGlobalMaxNumRowEntries();
RCP<crs_graph_type> graph = createCrsGraph(importMap,numEntriesPerRow);
RCP<const map_type> domainMap = A.getCrsGraph().getDomainMap();
graph->doImport(A.getCrsGraph(), importer, INSERT);
graph->fillComplete(domainMap, importMap);
block_crs_matrix_type importA(*graph, A.getBlockSize());
importA.doImport(A, importer, INSERT);
// Finally we are ready to write this strip of the matrix
writeMatrixStrip(importA, os, params);
}
}
}
int main(int argc, char *argv[])
{
// Communicators
Teuchos::GlobalMPISession mpiSession(&argc, &argv);
const Albany_MPI_Comm nativeComm = Albany_MPI_COMM_WORLD;
const RCP<const Teuchos::Comm<int> > teuchosComm = Albany::createTeuchosCommFromMpiComm(nativeComm);
// Standard output
const RCP<Teuchos::FancyOStream> out = Teuchos::VerboseObjectBase::getDefaultOStream();
// Parse command-line argument for input file
const std::string firstArg = (argc > 1) ? argv[1] : "";
if (firstArg.empty() || firstArg == "--help") {
*out << "AlbanyRBGen input-file-path\n";
return 0;
}
const std::string inputFileName = argv[1];
// Parse XML input file
const RCP<Teuchos::ParameterList> topLevelParams = Teuchos::createParameterList("Albany Parameters");
Teuchos::updateParametersFromXmlFileAndBroadcast(inputFileName, topLevelParams.ptr(), *teuchosComm);
const bool sublistMustExist = true;
// Setup discretization factory
const RCP<Teuchos::ParameterList> discParams = Teuchos::sublist(topLevelParams, "Discretization", sublistMustExist);
TEUCHOS_TEST_FOR_EXCEPT(discParams->get<std::string>("Method") != "Ioss");
const std::string outputParamLabel = "Exodus Output File Name";
const std::string sampledOutputParamLabel = "Reference Exodus Output File Name";
const RCP<const Teuchos::ParameterEntry> reducedOutputParamEntry = getEntryCopy(*discParams, outputParamLabel);
const RCP<const Teuchos::ParameterEntry> sampledOutputParamEntry = getEntryCopy(*discParams, sampledOutputParamLabel);
discParams->remove(outputParamLabel, /*throwIfNotExists =*/ false);
discParams->remove(sampledOutputParamLabel, /*throwIfNotExists =*/ false);
const RCP<const Teuchos::ParameterList> discParamsCopy = Teuchos::rcp(new Teuchos::ParameterList(*discParams));
const RCP<Teuchos::ParameterList> problemParams = Teuchos::sublist(topLevelParams, "Problem", sublistMustExist);
const RCP<const Teuchos::ParameterList> problemParamsCopy = Teuchos::rcp(new Teuchos::ParameterList(*problemParams));
// Create original (full) discretization
const RCP<Albany::AbstractDiscretization> disc = Albany::discretizationNew(topLevelParams, teuchosComm);
// Determine mesh sample
const RCP<Teuchos::ParameterList> samplingParams = Teuchos::sublist(topLevelParams, "Mesh Sampling", sublistMustExist);
const int firstVectorRank = samplingParams->get("First Vector Rank", 0);
const Teuchos::Ptr<const int> basisSizeMax = Teuchos::ptr(samplingParams->getPtr<int>("Basis Size Max"));
const int sampleSize = samplingParams->get("Sample Size", 0);
*out << "Sampling " << sampleSize << " nodes";
if (Teuchos::nonnull(basisSizeMax)) {
*out << " based on no more than " << *basisSizeMax << " basis vectors";
}
if (firstVectorRank != 0) {
*out << " starting from vector rank " << firstVectorRank;
}
*out << "\n";
const RCP<Albany::STKDiscretization> stkDisc =
Teuchos::rcp_dynamic_cast<Albany::STKDiscretization>(disc, /*throw_on_fail =*/ true);
const RCP<MOR::AtomicBasisSource> rawBasisSource = Teuchos::rcp(new Albany::StkNodalBasisSource(stkDisc));
const RCP<MOR::AtomicBasisSource> basisSource = Teuchos::rcp(
Teuchos::nonnull(basisSizeMax) ?
new MOR::WindowedAtomicBasisSource(rawBasisSource, firstVectorRank, *basisSizeMax) :
new MOR::WindowedAtomicBasisSource(rawBasisSource, firstVectorRank)
);
MOR::CollocationMetricCriterionFactory criterionFactory(samplingParams);
const Teuchos::RCP<const MOR::CollocationMetricCriterion> criterion =
criterionFactory.instanceNew(basisSource->entryCountMax());
const Teuchos::RCP<MOR::GreedyAtomicBasisSample> sampler(new MOR::GreedyAtomicBasisSample(*basisSource, criterion));
sampler->sampleSizeInc(sampleSize);
Teuchos::Array<stk::mesh::EntityId> sampleNodeIds;
const Teuchos::ArrayView<const int> sampleAtoms = sampler->sample();
sampleNodeIds.reserve(sampleAtoms.size());
for (Teuchos::ArrayView<const int>::const_iterator it = sampleAtoms.begin(), it_end = sampleAtoms.end(); it != it_end; ++it) {
sampleNodeIds.push_back(*it + 1);
}
*out << "Sample = " << sampleNodeIds << "\n";
// Choose first sample node as sensor
const Teuchos::ArrayView<const stk::mesh::EntityId> sensorNodeIds = sampleNodeIds.view(0, 1);
const Teuchos::Array<std::string> additionalNodeSets =
Teuchos::tuple(std::string("sample_nodes"), std::string("sensors"));
// Create sampled discretization
if (Teuchos::nonnull(sampledOutputParamEntry)) {
const RCP<Teuchos::ParameterList> discParamsLocalCopy = Teuchos::rcp(new Teuchos::ParameterList(*discParamsCopy));
discParamsLocalCopy->setEntry("Exodus Output File Name", *sampledOutputParamEntry);
discParamsLocalCopy->set("Additional Node Sets", additionalNodeSets);
topLevelParams->set("Discretization", *discParamsLocalCopy);
topLevelParams->set("Problem", *problemParamsCopy);
const bool performReduction = false;
const RCP<Albany::AbstractDiscretization> sampledDisc =
sampledDiscretizationNew(topLevelParams, teuchosComm, sampleNodeIds, sensorNodeIds, performReduction);
if (Teuchos::nonnull(basisSizeMax)) {
transferSolutionHistory(*stkDisc, *sampledDisc, *basisSizeMax + firstVectorRank);
} else {
transferSolutionHistory(*stkDisc, *sampledDisc);
}
}
// Create reduced discretization
if (Teuchos::nonnull(reducedOutputParamEntry)) {
const RCP<Teuchos::ParameterList> discParamsLocalCopy = Teuchos::rcp(new Teuchos::ParameterList(*discParamsCopy));
discParamsLocalCopy->setEntry("Exodus Output File Name", *reducedOutputParamEntry);
discParamsLocalCopy->set("Additional Node Sets", additionalNodeSets);
topLevelParams->set("Discretization", *discParamsLocalCopy);
topLevelParams->set("Problem", *problemParamsCopy);
const bool performReduction = true;
const RCP<Albany::AbstractDiscretization> reducedDisc =
sampledDiscretizationNew(topLevelParams, teuchosComm, sampleNodeIds, sensorNodeIds, performReduction);
if (Teuchos::nonnull(basisSizeMax)) {
transferSolutionHistory(*stkDisc, *reducedDisc, *basisSizeMax + firstVectorRank);
} else {
transferSolutionHistory(*stkDisc, *reducedDisc);
}
}
}