void seissol::checkpoint::sionlib::Wavefield::write(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
int file = open(dataFile(odd()), writeMode());
checkErr(file);
// Write the header
SCOREP_USER_REGION_DEFINE(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(sion_coll_fwrite(header, headerSize, 1, file), 1);
SCOREP_USER_REGION_END(r_write_header);
// Save data
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(sion_coll_fwrite(dofs(), sizeof(real), numDofs(), file), numDofs());
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint(file);
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::mpio::WavefieldAsync::writePrepare(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_writePrepare");
SCOREP_USER_REGION("CheckPoint_writePrepare", SCOREP_USER_REGION_TYPE_FUNCTION);
// Write the header
writeHeader(time, timestepWaveField);
// Create copy of the dofs
memcpy(m_dofsCopy, dofs(), numDofs()*sizeof(real));
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_begin_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_begin_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all_begin(file(), m_dofsCopy, numDofs(), MPI_DOUBLE));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
m_started = true;
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::mpio::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
// Write the header
writeHeader(time, timestepWaveField);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
checkMPIErr(MPI_File_write_all(file(), dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void seissol::checkpoint::mpio::Wavefield::load(real* dofs)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::setLoaded();
MPI_File file = open();
if (file == MPI_FILE_NULL)
logError() << "Could not open checkpoint file";
// Read and broadcast header
checkMPIErr(setHeaderView(file));
if (rank() == 0)
checkMPIErr(MPI_File_read(file, header().data(), 1, headerType(), MPI_STATUS_IGNORE));
MPI_Bcast(header().data(), 1, headerType(), 0, comm());
// Read dofs
checkMPIErr(setDataView(file));
checkMPIErr(MPI_File_read_all(file, dofs, numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE));
// Close the file
checkMPIErr(MPI_File_close(&file));
}
void seissol::checkpoint::mpio::Wavefield::load(double &time, int ×tepWaveField)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::load();
MPI_File file = open();
if (file == MPI_FILE_NULL)
logError() << "Could not open checkpoint file";
// Read and broadcast header
checkMPIErr(setHeaderView(file));
Header header;
if (rank() == 0)
checkMPIErr(MPI_File_read(file, &header, 1, headerType(), MPI_STATUS_IGNORE));
MPI_Bcast(&header, 1, headerType(), 0, comm());
time = header.time;
timestepWaveField = header.timestepWavefield;
// Read dofs
checkMPIErr(setDataView(file));
checkMPIErr(MPI_File_read_all(file, dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE));
// Close the file
checkMPIErr(MPI_File_close(&file));
}
void seissol::checkpoint::posix::Wavefield::load(double &time, int ×tepWaveField, real* dofs)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::setLoaded();
int file = open();
checkErr(file);
// Read header
WavefieldHeader header;
readHeader(file, header);
time = header.time;
timestepWaveField = header.timestepWaveField;
// Skip other processes before this in the group
checkErr(lseek64(file, groupOffset() * sizeof(real), SEEK_CUR));
// Convert to char* to do pointer arithmetic
char* buffer = reinterpret_cast<char*>(dofs);
unsigned long left = numDofs()*sizeof(real);
// Read dofs
while (left > 0) {
unsigned long readSize = read(file, buffer, left);
if (readSize <= 0)
checkErr(readSize, left);
buffer += readSize;
left -= readSize;
}
// Close the file
checkErr(::close(file));
}
void seissol::checkpoint::posix::Wavefield::load(double &time, int ×tepWaveField)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::load();
int file = open();
checkErr(file);
// Skip identifier
checkErr(lseek64(file, sizeof(unsigned long), SEEK_SET));
// Read header
checkErr(read(file, &time, sizeof(time)), sizeof(time));
checkErr(read(file, ×tepWaveField, sizeof(timestepWaveField)),
sizeof(timestepWaveField));
// Convert to char* to do pointer arithmetic
char* buffer = reinterpret_cast<char*>(dofs());
unsigned long left = numDofs()*sizeof(real);
// Read dofs
while (left > 0) {
unsigned long readSize = read(file, buffer, left);
if (readSize <= 0)
checkErr(readSize, left);
buffer += readSize;
left -= readSize;
}
// Close the file
checkErr(::close(file));
}
void seissol::checkpoint::sionlib::Wavefield::load(real* dofs)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::setLoaded();
int file = open(linkFile(), readMode());
checkErr(file);
// Read header
checkErr(sion_coll_fread(header().data(), header().size(), 1, file), 1);
// Read dofs
checkErr(sion_coll_fread(dofs, sizeof(double), numDofs(), file), numDofs());
// Close the file
sionClose(file);
}
void seissol::checkpoint::posix::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
// Start at the beginning
checkErr(lseek64(file(), 0, SEEK_SET));
// Write the header
EPIK_USER_REG(r_write_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_write_header);
EPIK_USER_START(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
WavefieldHeader header;
header.time = time;
header.timestepWaveField = timestepWaveField;
writeHeader(file(), header);
EPIK_USER_END(r_write_header);
SCOREP_USER_REGION_END(r_write_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Convert to char* to do pointer arithmetic
const char* buffer = reinterpret_cast<const char*>(dofs());
unsigned long left = numDofs()*sizeof(real);
if (alignment()) {
left = (left + alignment() - 1) / alignment();
left *= alignment();
}
while (left > 0) {
unsigned long written = ::write(file(), buffer, left);
if (written <= 0)
checkErr(written, left);
buffer += written;
left -= written;
}
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::posix::Wavefield::write(double time, int timestepWaveField)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
// Skip identifier
checkErr(lseek64(file(), sizeof(unsigned long), SEEK_SET));
// Write the header
EPIK_USER_REG(r_write_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_write_header);
EPIK_USER_START(r_write_header);
SCOREP_USER_REGION_BEGIN(r_write_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
checkErr(::write(file(), &time, sizeof(time)), sizeof(time));
checkErr(::write(file(), ×tepWaveField, sizeof(timestepWaveField)),
sizeof(timestepWaveField));
EPIK_USER_END(r_write_header);
SCOREP_USER_REGION_END(r_write_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Convert to char* to do pointer arithmetic
const char* buffer = reinterpret_cast<const char*>(dofs());
unsigned long left = numDofs()*sizeof(real);
while (left > 0) {
unsigned long written = ::write(file(), buffer, left);
if (written <= 0)
checkErr(written, left);
buffer += written;
left -= written;
}
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void seissol::checkpoint::mpio::Wavefield::write(const void* header, size_t headerSize)
{
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Checkpoint backend: Writing.";
// Write the header
writeHeader(header, headerSize);
// Save data
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
checkMPIErr(setDataView(file()));
unsigned int totalIter = totalIterations();
unsigned int iter = iterations();
unsigned int count = dofsPerIteration();
if (m_useLargeBuffer) {
totalIter = (totalIter + sizeof(real) - 1) / sizeof(real);
iter = (iter + sizeof(real) - 1) / sizeof(real);
count *= sizeof(real);
}
unsigned long offset = 0;
for (unsigned int i = 0; i < totalIter; i++) {
if (i == iter-1)
// Last iteration
count = numDofs() - (iter-1) * count;
checkMPIErr(MPI_File_write_all(file(), const_cast<real*>(&dofs()[offset]), count, MPI_DOUBLE, MPI_STATUS_IGNORE));
if (i < iter-1)
offset += count;
// otherwise we just continue writing the last chunk over and over
else if (i != totalIter-1)
checkMPIErr(MPI_File_seek(file(), -count * sizeof(real), MPI_SEEK_CUR));
}
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Checkpoint backend: Writing. Done.";
}
void seissol::checkpoint::h5::Wavefield::load(double &time, int ×tepWavefield, real* dofs)
{
logInfo(rank()) << "Loading wave field checkpoint";
seissol::checkpoint::CheckPoint::setLoaded();
hid_t h5file = open(linkFile());
checkH5Err(h5file);
// Attributes
hid_t h5attr = H5Aopen(h5file, "time", H5P_DEFAULT);
checkH5Err(h5attr);
checkH5Err(H5Aread(h5attr, H5T_NATIVE_DOUBLE, &time));
checkH5Err(H5Aclose(h5attr));
h5attr = H5Aopen(h5file, "timestep_wavefield", H5P_DEFAULT);
checkH5Err(h5attr);
checkH5Err(H5Aread(h5attr, H5T_NATIVE_INT, ×tepWavefield));
checkH5Err(H5Aclose(h5attr));
// Get dataset
hid_t h5data = H5Dopen(h5file, "values", H5P_DEFAULT);
checkH5Err(h5data);
hid_t h5fSpace = H5Dget_space(h5data);
checkH5Err(h5fSpace);
// Read the data
unsigned int offset = 0;
hsize_t fStart = fileOffset();
hsize_t count = dofsPerIteration();
hid_t h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
for (unsigned int i = 0; i < totalIterations()-1; i++) {
checkH5Err(H5Sselect_hyperslab(h5fSpace, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dread(h5data, H5T_NATIVE_DOUBLE, h5memSpace, h5fSpace,
h5XferList(), &dofs[offset]));
// We are finished in less iterations, read data twice
// so everybody needs the same number of iterations
if (i < iterations()-1) {
fStart += count;
offset += count;
}
}
checkH5Err(H5Sclose(h5memSpace));
// Read reminding data in the last iteration
count = numDofs() - (iterations() - 1) * count;
h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(h5fSpace, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dread(h5data, H5T_NATIVE_DOUBLE, h5memSpace, h5fSpace,
h5XferList(), &dofs[offset]));
checkH5Err(H5Sclose(h5memSpace));
checkH5Err(H5Sclose(h5fSpace));
checkH5Err(H5Dclose(h5data));
checkH5Err(H5Fclose(h5file));
}
void seissol::checkpoint::h5::Wavefield::write(double time, int waveFieldTimeStep)
{
EPIK_TRACER("CheckPoint_write");
SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION);
logInfo(rank()) << "Writing check point.";
EPIK_USER_REG(r_header, "checkpoint_write_header");
SCOREP_USER_REGION_DEFINE(r_header);
EPIK_USER_START(r_header);
SCOREP_USER_REGION_BEGIN(r_header, "checkpoint_write_header", SCOREP_USER_REGION_TYPE_COMMON);
// Time
checkH5Err(H5Awrite(m_h5time[odd()], H5T_NATIVE_DOUBLE, &time));
// Wavefield writer
checkH5Err(H5Awrite(m_h5timestepWavefield[odd()], H5T_NATIVE_INT, &waveFieldTimeStep));
EPIK_USER_END(r_header);
SCOREP_USER_REGION_END(r_header);
// Save data
EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield");
SCOREP_USER_REGION_DEFINE(r_write_wavefield);
EPIK_USER_START(r_write_wavefield);
SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON);
// Write the wave field
unsigned int offset = 0;
hsize_t fStart = fileOffset();
hsize_t count = dofsPerIteration();
hid_t h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
for (unsigned int i = 0; i < totalIterations()-1; i++) {
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), &const_cast<real*>(dofs())[offset]));
// We are finished in less iterations, read data twice
// so everybody needs the same number of iterations
if (i < iterations()-1) {
fStart += count;
offset += count;
}
}
checkH5Err(H5Sclose(h5memSpace));
// Save reminding data in the last iteration
count = numDofs() - (iterations() - 1) * count;
h5memSpace = H5Screate_simple(1, &count, 0L);
checkH5Err(h5memSpace);
checkH5Err(H5Sselect_all(h5memSpace));
checkH5Err(H5Sselect_hyperslab(m_h5fSpaceData, H5S_SELECT_SET, &fStart, 0L, &count, 0L));
checkH5Err(H5Dwrite(m_h5data[odd()], H5T_NATIVE_DOUBLE, h5memSpace, m_h5fSpaceData,
h5XferList(), &dofs()[offset]));
checkH5Err(H5Sclose(h5memSpace));
EPIK_USER_END(r_write_wavefield);
SCOREP_USER_REGION_END(r_write_wavefield);
// Finalize the checkpoint
finalizeCheckpoint();
logInfo(rank()) << "Writing check point. Done.";
}
void AmalgamationInfo<LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::UnamalgamateAggregates(const Aggregates& aggregates,
Teuchos::ArrayRCP<LocalOrdinal>& aggStart, Teuchos::ArrayRCP<GlobalOrdinal>& aggToRowMap) const {
int myPid = aggregates.GetMap()->getComm()->getRank();
Teuchos::ArrayView<const GO> nodeGlobalElts = aggregates.GetMap()->getNodeElementList();
Teuchos::ArrayRCP<LO> procWinner = aggregates.GetProcWinner()->getDataNonConst(0);
Teuchos::ArrayRCP<LO> vertex2AggId = aggregates.GetVertex2AggId()->getDataNonConst(0);
LO size = procWinner.size();
GO numAggregates = aggregates.GetNumAggregates();
std::vector<LO> sizes(numAggregates);
if (stridedblocksize_ == 1) {
for (LO lnode = 0; lnode < size; ++lnode) {
LO myAgg = vertex2AggId[lnode];
if (procWinner[lnode] == myPid)
sizes[myAgg] += 1;
}
} else {
for (LO lnode = 0; lnode < size; ++lnode) {
LO myAgg = vertex2AggId[lnode];
if (procWinner[lnode] == myPid) {
GO gnodeid = nodeGlobalElts[lnode];
for (LocalOrdinal k = 0; k < stridedblocksize_; k++) {
GlobalOrdinal gDofIndex = ComputeGlobalDOF(gnodeid,k);
if (columnMap_->isNodeGlobalElement(gDofIndex))
sizes[myAgg] += 1;
}
}
}
}
aggStart = ArrayRCP<LO>(numAggregates+1,0);
aggStart[0]=0;
for (GO i=0; i<numAggregates; ++i) {
aggStart[i+1] = aggStart[i] + sizes[i];
}
aggToRowMap = ArrayRCP<GO>(aggStart[numAggregates],0);
// count, how many dofs have been recorded for each aggregate so far
Array<LO> numDofs(numAggregates, 0); // empty array with number of Dofs for each aggregate
if (stridedblocksize_ == 1) {
for (LO lnode = 0; lnode < size; ++lnode) {
LO myAgg = vertex2AggId[lnode];
if (procWinner[lnode] == myPid) {
aggToRowMap[ aggStart[myAgg] + numDofs[myAgg] ] = ComputeGlobalDOF(nodeGlobalElts[lnode]);
++(numDofs[myAgg]);
}
}
} else {
for (LO lnode = 0; lnode < size; ++lnode) {
LO myAgg = vertex2AggId[lnode];
if (procWinner[lnode] == myPid) {
GO gnodeid = nodeGlobalElts[lnode];
for (LocalOrdinal k = 0; k < stridedblocksize_; k++) {
GlobalOrdinal gDofIndex = ComputeGlobalDOF(gnodeid,k);
if (columnMap_->isNodeGlobalElement(gDofIndex)) {
aggToRowMap[ aggStart[myAgg] + numDofs[myAgg] ] = gDofIndex;
++(numDofs[myAgg]);
}
}
}
}
}
// todo plausibility check: entry numDofs[k] == aggToRowMap[k].size()
} //UnamalgamateAggregates
void AmalgamationInfo<LocalOrdinal, GlobalOrdinal, Node, LocalMatOps>::UnamalgamateAggregatesLO(const Aggregates& aggregates,
Teuchos::ArrayRCP<LO>& aggStart, Teuchos::ArrayRCP<LO>& aggToRowMap) const {
int myPid = aggregates.GetMap()->getComm()->getRank();
Teuchos::ArrayView<const GO> nodeGlobalElts = aggregates.GetMap()->getNodeElementList();
Teuchos::ArrayRCP<LO> procWinner = aggregates.GetProcWinner() ->getDataNonConst(0);
Teuchos::ArrayRCP<LO> vertex2AggId = aggregates.GetVertex2AggId()->getDataNonConst(0);
const GO numAggregates = aggregates.GetNumAggregates();
// FIXME: Do we need to compute size here? Or can we use existing?
LO size = procWinner.size();
std::vector<LO> sizes(numAggregates);
if (stridedblocksize_ == 1) {
for (LO lnode = 0; lnode < size; lnode++)
if (procWinner[lnode] == myPid)
sizes[vertex2AggId[lnode]]++;
} else {
for (LO lnode = 0; lnode < size; lnode++)
if (procWinner[lnode] == myPid) {
GO nodeGID = nodeGlobalElts[lnode];
for (LO k = 0; k < stridedblocksize_; k++) {
GO GID = ComputeGlobalDOF(nodeGID, k);
if (columnMap_->isNodeGlobalElement(GID))
sizes[vertex2AggId[lnode]]++;
}
}
}
aggStart = ArrayRCP<LO>(numAggregates+1); // FIXME: useless initialization with zeros
aggStart[0] = 0;
for (GO i = 0; i < numAggregates; i++)
aggStart[i+1] = aggStart[i] + sizes[i];
aggToRowMap = ArrayRCP<LO>(aggStart[numAggregates], 0);
// count, how many dofs have been recorded for each aggregate so far
Array<LO> numDofs(numAggregates, 0); // empty array with number of DOFs for each aggregate
if (stridedblocksize_ == 1) {
for (LO lnode = 0; lnode < size; ++lnode)
if (procWinner[lnode] == myPid) {
LO myAgg = vertex2AggId[lnode];
aggToRowMap[aggStart[myAgg] + numDofs[myAgg]] = lnode;
numDofs[myAgg]++;
}
} else {
for (LO lnode = 0; lnode < size; ++lnode)
if (procWinner[lnode] == myPid) {
LO myAgg = vertex2AggId[lnode];
GO nodeGID = nodeGlobalElts[lnode];
for (LO k = 0; k < stridedblocksize_; k++) {
GO GID = ComputeGlobalDOF(nodeGID, k);
if (columnMap_->isNodeGlobalElement(GID)) {
aggToRowMap[aggStart[myAgg] + numDofs[myAgg]] = lnode*stridedblocksize_ + k;
numDofs[myAgg]++;
}
}
}
}
// todo plausibility check: entry numDofs[k] == aggToRowMap[k].size()
} //UnamalgamateAggregates
