void
SystemBase::augmentSendList(std::vector<dof_id_type> & send_list)
{
std::set<dof_id_type> & ghosted_elems = _subproblem.ghostedElems();
DofMap & dof_map = dofMap();
std::vector<dof_id_type> dof_indices;
System & sys = system();
unsigned int sys_num = sys.number();
unsigned int n_vars = sys.n_vars();
for (std::set<dof_id_type>::iterator elem_id = ghosted_elems.begin();
elem_id != ghosted_elems.end();
++elem_id)
{
Elem * elem = _mesh.elem(*elem_id);
if (elem->active())
{
dof_map.dof_indices(elem, dof_indices);
for (unsigned int i=0; i<dof_indices.size(); i++)
{
dof_id_type dof = dof_indices[i];
// Only need to ghost it if it's actually not on this processor
if (dof < dof_map.first_dof() || dof >= dof_map.end_dof())
send_list.push_back(dof);
}
// Now add the DoFs from all of the nodes. This is necessary because of block
// restricted variables. A variable might not live _on_ this element but it
// might live on nodes connected to this element.
for (unsigned int n=0; n<elem->n_nodes(); n++)
{
Node * node = elem->get_node(n);
// Have to get each variable's dofs
for (unsigned int v=0; v<n_vars; v++)
{
const Variable & var = sys.variable(v);
unsigned int var_num = var.number();
unsigned int n_comp = var.n_components();
// See if this variable has any dofs at this node
if (node->n_dofs(sys_num, var_num) > 0)
{
// Loop over components of the variable
for (unsigned int c=0; c<n_comp; c++)
send_list.push_back(node->dof_number(sys_num, var_num, c));
}
}
}
}
}
}
void RieszRep::distributeDofs(){
int myRank = Teuchos::GlobalMPISession::getRank();
int numRanks = Teuchos::GlobalMPISession::getNProc();
#ifdef HAVE_MPI
Epetra_MpiComm Comm(MPI_COMM_WORLD);
//cout << "rank: " << rank << " of " << numProcs << endl;
#else
Epetra_SerialComm Comm;
#endif
// the code below could stand to be reworked; I'm pretty sure this is not the best way to distribute the data, and it would also be best to get rid of the iteration over the global set of active elements. But a similar point could be made about this method as a whole: do we really need to distribute all the dofs to every rank? It may be best to eliminate this method altogether.
vector<GlobalIndexType> cellIDsByPartitionOrdering;
for (int rank=0; rank<numRanks; rank++) {
set<GlobalIndexType> cellIDsForRank = _mesh->globalDofAssignment()->cellsInPartition(rank);
cellIDsByPartitionOrdering.insert(cellIDsByPartitionOrdering.end(), cellIDsForRank.begin(), cellIDsForRank.end());
}
// determine inverse map:
map<GlobalIndexType,int> ordinalForCellID;
for (int ordinal=0; ordinal<cellIDsByPartitionOrdering.size(); ordinal++) {
GlobalIndexType cellID = cellIDsByPartitionOrdering[ordinal];
ordinalForCellID[cellID] = ordinal;
// cout << "ordinalForCellID[" << cellID << "] = " << ordinal << endl;
}
for (int cellOrdinal=0; cellOrdinal<cellIDsByPartitionOrdering.size(); cellOrdinal++) {
GlobalIndexType cellID = cellIDsByPartitionOrdering[cellOrdinal];
ElementTypePtr elemTypePtr = _mesh->getElementType(cellID);
DofOrderingPtr testOrderingPtr = elemTypePtr->testOrderPtr;
int numDofs = testOrderingPtr->totalDofs();
int cellIDPartition = _mesh->partitionForCellID(cellID);
bool isInPartition = (cellIDPartition == myRank);
int numMyDofs;
FieldContainer<double> dofs(numDofs);
if (isInPartition){ // if in partition
numMyDofs = numDofs;
dofs = _rieszRepDofs[cellID];
} else{
numMyDofs = 0;
}
Epetra_Map dofMap(numDofs,numMyDofs,0,Comm);
Epetra_Vector distributedRieszDofs(dofMap);
if (isInPartition) {
for (int i = 0;i<numMyDofs;i++) { // shouldn't activate on off-proc partitions
distributedRieszDofs.ReplaceGlobalValues(1,&dofs(i),&i);
}
}
Epetra_Map importMap(numDofs,numDofs,0,Comm); // every proc should own their own copy of the dofs
Epetra_Import testDofImporter(importMap, dofMap);
Epetra_Vector globalRieszDofs(importMap);
globalRieszDofs.Import(distributedRieszDofs, testDofImporter, Insert);
if (!isInPartition){
for (int i = 0;i<numDofs;i++){
dofs(i) = globalRieszDofs[i];
}
}
_rieszRepDofsGlobal[cellID] = dofs;
// { // debugging
// ostringstream cellIDlabel;
// cellIDlabel << "cell " << cellID << " _rieszRepDofsGlobal, after global import";
// TestSuite::serializeOutput(cellIDlabel.str(), _rieszRepDofsGlobal[cellID]);
// }
}
// distribute norms as well
GlobalIndexType numElems = _mesh->numActiveElements();
set<GlobalIndexType> rankLocalCellIDs = _mesh->cellIDsInPartition();
IndexType numMyElems = rankLocalCellIDs.size();
GlobalIndexType myElems[numMyElems];
// build cell index
GlobalIndexType myCellOrdinal = 0;
double rankLocalRieszNorms[numMyElems];
for (set<GlobalIndexType>::iterator cellIDIt = rankLocalCellIDs.begin(); cellIDIt != rankLocalCellIDs.end(); cellIDIt++) {
GlobalIndexType cellID = *cellIDIt;
myElems[myCellOrdinal] = ordinalForCellID[cellID];
rankLocalRieszNorms[myCellOrdinal] = _rieszRepNormSquared[cellID];
myCellOrdinal++;
}
Epetra_Map normMap((GlobalIndexTypeToCast)numElems,(int)numMyElems,(GlobalIndexTypeToCast *)myElems,(GlobalIndexTypeToCast)0,Comm);
Epetra_Vector distributedRieszNorms(normMap);
int err = distributedRieszNorms.ReplaceGlobalValues(numMyElems,rankLocalRieszNorms,(GlobalIndexTypeToCast *)myElems);
if (err != 0) {
cout << "RieszRep::distributeDofs(): on rank" << myRank << ", ReplaceGlobalValues returned error code " << err << endl;
}
Epetra_Map normImportMap((GlobalIndexTypeToCast)numElems,(GlobalIndexTypeToCast)numElems,0,Comm);
Epetra_Import normImporter(normImportMap,normMap);
Epetra_Vector globalNorms(normImportMap);
globalNorms.Import(distributedRieszNorms, normImporter, Add); // add should be OK (everything should be zeros)
for (int cellOrdinal=0; cellOrdinal<cellIDsByPartitionOrdering.size(); cellOrdinal++) {
GlobalIndexType cellID = cellIDsByPartitionOrdering[cellOrdinal];
_rieszRepNormSquaredGlobal[cellID] = globalNorms[cellOrdinal];
// if (myRank==0) cout << "_rieszRepNormSquaredGlobal[" << cellID << "] = " << globalNorms[cellOrdinal] << endl;
}
}
