void UserAggregationFactory<LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level ¤tLevel) const {
FactoryMonitor m(*this, "Build", currentLevel);
const ParameterList & pL = GetParameterList();
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
const int myRank = comm->getRank();
std::string fileName = pL.get<std::string>("filePrefix") + toString(currentLevel.GetLevelID()) + "_" + toString(myRank) + "." + pL.get<std::string>("fileExt");
std::ifstream ifs(fileName.c_str());
if (!ifs.good())
throw Exceptions::RuntimeError("Cannot read data from \"" + fileName + "\"");
LO numVertices, numAggregates;
ifs >> numVertices >> numAggregates;
// FIXME: what is the map?
Xpetra::UnderlyingLib lib = Xpetra::UseEpetra;
const int indexBase = 0;
RCP<Map> map = MapFactory::Build(lib, numVertices, indexBase, comm);
RCP<Aggregates> aggregates = rcp(new Aggregates(map));
aggregates->setObjectLabel("User");
aggregates->SetNumAggregates(numAggregates);
Teuchos::ArrayRCP<LO> vertex2AggId = aggregates->GetVertex2AggId()->getDataNonConst(0);
Teuchos::ArrayRCP<LO> procWinner = aggregates->GetProcWinner() ->getDataNonConst(0);
for (LO i = 0; i < numAggregates; i++) {
int aggSize = 0;
ifs >> aggSize;
std::vector<LO> list(aggSize);
for (int k = 0; k < aggSize; k++) {
// FIXME: File contains GIDs, we need LIDs
// for now, works on a single processor
ifs >> list[k];
}
// Mark first node as root node for the aggregate
aggregates->SetIsRoot(list[0]);
// Fill vertex2AggId and procWinner structure with information
for (int k = 0; k < aggSize; k++) {
vertex2AggId[list[k]] = i;
procWinner [list[k]] = myRank;
}
}
// FIXME: do the proper check whether aggregates cross interprocessor boundary
aggregates->AggregatesCrossProcessors(false);
Set(currentLevel, "Aggregates", aggregates);
GetOStream(Statistics0, 0) << aggregates->description() << std::endl;
}
void BrickAggregationFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node>::Build(Level& currentLevel) const {
FactoryMonitor m(*this, "Build", currentLevel);
typedef Xpetra::MultiVector<double,LO,GO,NO> MultiVector_d;
const ParameterList& pL = GetParameterList();
RCP<MultiVector_d> coords = Get<RCP<MultiVector_d> >(currentLevel, "Coordinates");
RCP<Matrix> A = Get< RCP<Matrix> > (currentLevel, "A");
RCP<const Map> rowMap = A->getRowMap();
RCP<const Map> colMap = A->getColMap();
RCP<const Teuchos::Comm<int> > comm = rowMap->getComm();
int numProcs = comm->getSize();
int myRank = comm->getRank();
int numPoints = colMap->getNodeNumElements();
bx_ = pL.get<int>("aggregation: brick x size");
by_ = pL.get<int>("aggregation: brick y size");
bz_ = pL.get<int>("aggregation: brick z size");
if (numProcs > 1) {
// TODO: deal with block size > 1 (see comments above)
TEUCHOS_TEST_FOR_EXCEPTION(bx_ > 3 || by_ > 3 || bz_ > 3, Exceptions::RuntimeError, "Currently cannot deal with brick size > 3");
}
RCP<MultiVector_d> overlappedCoords = coords;
RCP<const Import> importer = ImportFactory::Build(coords->getMap(), colMap);
if (!importer.is_null()) {
overlappedCoords = Xpetra::MultiVectorFactory<double,LO,GO,NO>::Build(colMap, coords->getNumVectors());
overlappedCoords->doImport(*coords, *importer, Xpetra::INSERT);
}
// Setup misc structures
// Logically, we construct enough data to query topological information of a rectangular grid
Setup(comm, overlappedCoords, colMap);
GetOStream(Runtime0) << "Using brick size: " << bx_
<< (nDim_ > 1 ? "x " + toString(by_) : "")
<< (nDim_ > 2 ? "x " + toString(bz_) : "") << std::endl;
// Construct aggregates
RCP<Aggregates> aggregates = rcp(new Aggregates(colMap));
aggregates->setObjectLabel("Brick");
ArrayRCP<LO> vertex2AggId = aggregates->GetVertex2AggId()->getDataNonConst(0);
ArrayRCP<LO> procWinner = aggregates->GetProcWinner() ->getDataNonConst(0);
// In the first pass, we set a mapping from a vertex to aggregate global id. We deal with a structured
// rectangular mesh, therefore we know the structure of aggregates. For each vertex we can tell exactly
// which aggregate it belongs to.
// If we determine that the aggregate does not belong to us (i.e. the root vertex does not belong to this
// processor, or is outside and we lost "" arbitration), we record the global aggregate id in order to
// fetch the local info from the processor owning the aggregate. This is required for aggregates, as it
// uses the local aggregate ids of the owning processor.
std::set<GO> myAggGIDs, remoteAggGIDs;
for (LO LID = 0; LID < numPoints; LID++) {
GO aggGID = getAggGID(LID);
if ((revMap_.find(getRoot(LID)) != revMap_.end()) && rowMap->isNodeGlobalElement(colMap->getGlobalElement(revMap_[getRoot(LID)]))) {
// Root of the brick aggregate containing GID (<- LID) belongs to us
vertex2AggId[LID] = aggGID;
myAggGIDs.insert(aggGID);
if (isRoot(LID))
aggregates->SetIsRoot(LID);
} else {
remoteAggGIDs.insert(aggGID);
}
}
size_t numAggregates = myAggGIDs .size();
size_t numRemote = remoteAggGIDs.size();
aggregates->SetNumAggregates(numAggregates);
std::map<GO,LO> AggG2L; // Map: Agg GID -> Agg LID (possibly on a different processor)
std::map<GO,int> AggG2R; // Map: Agg GID -> processor rank owning aggregate
Array<GO> myAggGIDsArray(numAggregates), remoteAggGIDsArray(numRemote);
// Fill in the maps for aggregates that we own
size_t ind = 0;
for (typename std::set<GO>::const_iterator it = myAggGIDs.begin(); it != myAggGIDs.end(); it++) {
AggG2L[*it] = ind;
AggG2R[*it] = myRank;
myAggGIDsArray[ind++] = *it;
}
// The map is a convenient way to fetch remote local indices from global indices.
RCP<Map> aggMap = MapFactory::Build(rowMap->lib(), Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid(),
myAggGIDsArray, 0, comm);
ind = 0;
for (typename std::set<GO>::const_iterator it = remoteAggGIDs.begin(); it != remoteAggGIDs.end(); it++)
remoteAggGIDsArray[ind++] = *it;
// Fetch the required aggregate local ids and ranks
Array<int> remoteProcIDs(numRemote);
Array<LO> remoteLIDs (numRemote);
aggMap->getRemoteIndexList(remoteAggGIDsArray, remoteProcIDs, remoteLIDs);
// Fill in the maps for aggregates that we don't own but which have some of our vertices
for (size_t i = 0; i < numRemote; i++) {
AggG2L[remoteAggGIDsArray[i]] = remoteLIDs [i];
AggG2R[remoteAggGIDsArray[i]] = remoteProcIDs[i];
}
// Remap aggregate GIDs to LIDs and set up owning processors
for (LO LID = 0; LID < numPoints; LID++) {
if (revMap_.find(getRoot(LID)) != revMap_.end() && rowMap->isNodeGlobalElement(colMap->getGlobalElement(revMap_[getRoot(LID)]))) {
GO aggGID = vertex2AggId[LID];
vertex2AggId[LID] = AggG2L[aggGID];
procWinner [LID] = AggG2R[aggGID];
}
}
GO numGlobalRemote;
MueLu_sumAll(comm, as<GO>(numRemote), numGlobalRemote);
aggregates->AggregatesCrossProcessors(numGlobalRemote);
Set(currentLevel, "Aggregates", aggregates);
GetOStream(Statistics0) << aggregates->description() << std::endl;
}
void CoordinatesTransferFactory<Scalar, LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::Build(Level & fineLevel, Level &coarseLevel) const {
FactoryMonitor m(*this, "Build", coarseLevel);
GetOStream(Runtime0, 0) << "Transferring coordinates" << std::endl;
const ParameterList & pL = GetParameterList();
int writeStart = pL.get< int >("write start");
int writeEnd = pL.get< int >("write end");
RCP<Aggregates> aggregates = Get< RCP<Aggregates> > (fineLevel, "Aggregates");
RCP<MultiVector> fineCoords = Get< RCP<MultiVector> >(fineLevel, "Coordinates");
RCP<const Map> coarseMap = Get< RCP<const Map> > (fineLevel, "CoarseMap");
// coarseMap is being used to set up the domain map of tentative P, and therefore, the row map of Ac
// Therefore, if we amalgamate coarseMap, logical nodes in the coordinates vector would correspond to
// logical blocks in the matrix
ArrayView<const GO> elementAList = coarseMap->getNodeElementList();
LO blkSize = 1;
if (rcp_dynamic_cast<const StridedMap>(coarseMap) != Teuchos::null)
blkSize = rcp_dynamic_cast<const StridedMap>(coarseMap)->getFixedBlockSize();
GO indexBase = coarseMap->getIndexBase();
size_t numElements = elementAList.size() / blkSize;
Array<GO> elementList(numElements);
// Amalgamate the map
for (LO i = 0; i < Teuchos::as<LO>(numElements); i++)
elementList[i] = (elementAList[i*blkSize]-indexBase)/blkSize + indexBase;
RCP<const Map> coarseCoordMap = MapFactory ::Build(coarseMap->lib(), Teuchos::OrdinalTraits<Xpetra::global_size_t>::invalid(), elementList, indexBase, coarseMap->getComm());
RCP<MultiVector> coarseCoords = MultiVectorFactory::Build(coarseCoordMap, fineCoords->getNumVectors());
// Maps
RCP<const Map> uniqueMap = fineCoords->getMap();
RCP<const Map> nonUniqueMap = aggregates->GetMap();
// Create overlapped fine coordinates to reduce global communication
RCP<const Import> importer = ImportFactory ::Build(uniqueMap, nonUniqueMap);
RCP<MultiVector> ghostedCoords = MultiVectorFactory::Build(nonUniqueMap, fineCoords->getNumVectors());
ghostedCoords->doImport(*fineCoords, *importer, Xpetra::INSERT);
// Get some info about aggregates
int myPID = uniqueMap->getComm()->getRank();
LO numAggs = aggregates->GetNumAggregates();
ArrayRCP<LO> aggSizes = aggregates->ComputeAggregateSizes();
const ArrayRCP<const LO> vertex2AggID = aggregates->GetVertex2AggId()->getData(0);
const ArrayRCP<const LO> procWinner = aggregates->GetProcWinner()->getData(0);
// Fill in coarse coordinates
for (size_t j = 0; j < fineCoords->getNumVectors(); j++) {
ArrayRCP<const Scalar> fineCoordsData = ghostedCoords->getData(j);
ArrayRCP<Scalar> coarseCoordsData = coarseCoords->getDataNonConst(j);
for (LO lnode = 0; lnode < vertex2AggID.size(); lnode++)
if (procWinner[lnode] == myPID)
coarseCoordsData[vertex2AggID[lnode]] += fineCoordsData[lnode];
for (LO agg = 0; agg < numAggs; agg++)
coarseCoordsData[agg] /= aggSizes[agg];
}
Set<RCP<MultiVector> >(coarseLevel, "Coordinates", coarseCoords);
if (writeStart == 0 && fineLevel.GetLevelID() == 0 && writeStart <= writeEnd) {
std::ostringstream buf;
buf << fineLevel.GetLevelID();
std::string fileName = "coordinates_before_rebalance_level_" + buf.str() + ".m";
Utils::Write(fileName,*fineCoords);
}
if (writeStart <= coarseLevel.GetLevelID() && coarseLevel.GetLevelID() <= writeEnd) {
std::ostringstream buf;
buf << coarseLevel.GetLevelID();
std::string fileName = "coordinates_before_rebalance_level_" + buf.str() + ".m";
Utils::Write(fileName,*coarseCoords);
}
} // Build
