//----------------------------------------------------------------------------
Epetra_FECrsMatrix::Epetra_FECrsMatrix(Epetra_DataAccess CV,
const Epetra_Map& rowMap,
int NumEntriesPerRow,
bool ignoreNonLocalEntries)
: Epetra_CrsMatrix(CV, rowMap, NumEntriesPerRow),
myFirstRow_(0),
myNumRows_(0),
ignoreNonLocalEntries_(ignoreNonLocalEntries),
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
nonlocalRows_int_(),
nonlocalCols_int_(),
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
nonlocalRows_LL_(),
nonlocalCols_LL_(),
#endif
nonlocalCoefs_(),
workData_(128),
useNonlocalMatrix_ (false),
nonlocalMatrix_ (NULL),
sourceMap_(NULL),
colMap_(NULL),
exporter_(NULL),
tempMat_(NULL)
{
myFirstRow_ = rowMap.MinMyGID64();
myNumRows_ = rowMap.NumMyElements();
}
// =============================================================================
Teuchos::RCP<Epetra_Map>
VIO::EpetraMesh::Reader::
createComplexValuesMap_ ( const Epetra_Map & nodesMap
) const
{
// get view for the global indices of the global elements
int numMyElements = nodesMap.NumMyElements();
Teuchos::ArrayRCP<int> myGlobalElements( numMyElements );
nodesMap.MyGlobalElements( myGlobalElements.getRawPtr() );
// Construct the map in such a way that all complex entries on processor K
// are split up into real and imaginary part, which will both reside on
// processor K again.
int numMyComplexElements = 2*numMyElements;
Teuchos::ArrayRCP<int> myComplexGlobalElements ( numMyComplexElements );
for ( int k = 0; k < numMyElements; k++ )
{
myComplexGlobalElements[2*k ] = 2 * myGlobalElements[k];
myComplexGlobalElements[2*k+1] = 2 * myGlobalElements[k] + 1;
}
return Teuchos::rcp ( new Epetra_Map ( -1,
myComplexGlobalElements.size(),
myComplexGlobalElements.getRawPtr(),
nodesMap.IndexBase(),
nodesMap.Comm()
)
);
}
void buildGIDs(std::vector<std::vector<int> > & gids,const Epetra_Map & map)
{
int numLocal = map.NumMyElements();
int numHalf = numLocal/2;
numHalf += ((numHalf % 2 == 0) ? 0 : 1);
gids.clear();
gids.resize(3);
std::vector<int> & blk0 = gids[0];
std::vector<int> & blk1 = gids[1];
std::vector<int> & blk2 = gids[2];
// loop over global IDs: treat first block as strided
int gid = -1;
for(int i=0;i<numHalf;i+=2) {
gid = map.GID(i);
blk0.push_back(gid);
gid = map.GID(i+1);
blk1.push_back(gid);
}
// loop over global IDs: treat remainder as contiguous
for(int i=numHalf;i<numLocal;i++) {
gid = map.GID(i);
blk2.push_back(gid);
}
}
Teuchos::RCP<Epetra_Map>
Albany::SolutionResponseFunction::
buildCulledMap(const Epetra_Map& x_map,
const Teuchos::Array<int>& keepDOF) const
{
int numKeepDOF = std::accumulate(keepDOF.begin(), keepDOF.end(), 0);
int Neqns = keepDOF.size();
int N = x_map.NumMyElements(); // x_map is map for solution vector
TEUCHOS_ASSERT( !(N % Neqns) ); // Assume that all the equations for
// a given node are on the assigned
// processor. I.e. need to ensure
// that N is exactly Neqns-divisible
int nnodes = N / Neqns; // number of fem nodes
int N_new = nnodes * numKeepDOF; // length of local x_new
int *gids = x_map.MyGlobalElements(); // Fill local x_map into gids array
Teuchos::Array<int> gids_new(N_new);
int idx = 0;
for ( int inode = 0; inode < N/Neqns ; ++inode) // For every node
for ( int ieqn = 0; ieqn < Neqns; ++ieqn ) // Check every dof on the node
if ( keepDOF[ieqn] == 1 ) // then want to keep this dof
gids_new[idx++] = gids[(inode*Neqns)+ieqn];
// end cull
Teuchos::RCP<Epetra_Map> x_new_map =
Teuchos::rcp( new Epetra_Map( -1, N_new, &gids_new[0], 0, x_map.Comm() ) );
return x_new_map;
}
RCP<Epetra_CrsMatrix> getMyEpetraMatrix(int numRows, int numCols, double shift=0.0)
{
const RCP<const Epetra_Comm> comm = getEpetraComm();
const Epetra_Map rowMap(numRows, 0, *comm);
const Epetra_Map domainMap(numCols, numCols, 0, *comm);
const RCP<Epetra_CrsMatrix> epetraCrsM =
rcp(new Epetra_CrsMatrix(Copy, rowMap,domainMap,0));
Array<double> rowEntries(numCols);
Array<int> columnIndices(numCols);
for (int j = 0; j < numCols; ++j)
columnIndices[j] = j;
const int numLocalRows = rowMap.NumMyElements();
for (int i = 0; i < numLocalRows; ++i) {
for (int j = 0; j < numCols; ++j) {
rowEntries[j] = as<double>(i+1) + as<double>(j+1) / 10 + shift;
}
epetraCrsM->InsertMyValues( i, numCols, &rowEntries[0], &columnIndices[0] );
}
epetraCrsM->FillComplete(domainMap,rowMap);
return epetraCrsM;
}
//==============================================================================
int Komplex_LinearProblem::MakeKomplexMap(const Epetra_Map & Map, Teuchos::RCP<Epetra_Map> & KMap) {
if (Map.LinearMap()) {
KMap = Teuchos::rcp(new Epetra_Map(-1, Map.NumMyElements()*2, Map.IndexBase(), Map.Comm()));
}
else {
Epetra_IntSerialDenseVector KMapGIDs(Map.NumMyElements()*2);
int * myGlobalElements = Map.MyGlobalElements();
for (int i=0; i<Map.NumMyElements(); i++) {
KMapGIDs[2*i] = 2*myGlobalElements[i];
KMapGIDs[2*i+1] = 2*myGlobalElements[i]+1;
}
KMap = Teuchos::rcp(new Epetra_Map(-1, KMapGIDs.Length(), KMapGIDs.Values(), Map.IndexBase(), Map.Comm()));
}
return(0);
}
// B here is the "reduced" matrix. Square matrices w/ Row=Domain=Range only.
double test_with_matvec_reduced_maps(const Epetra_CrsMatrix &A, const Epetra_CrsMatrix &B, const Epetra_Map & Bfullmap){
const Epetra_Map & Amap = A.DomainMap();
Epetra_Vector Xa(Amap), Ya(Amap), Diff(Amap);
const Epetra_Map *Bmap = Bfullmap.NumMyElements() > 0 ? &B.DomainMap() : 0;
Epetra_Vector *Xb = Bmap ? new Epetra_Vector(*Bmap) : 0;
Epetra_Vector *Yb = Bmap ? new Epetra_Vector(*Bmap) : 0;
Epetra_Vector Xb_alias(View,Bfullmap, Bmap ? Xb->Values(): 0);
Epetra_Vector Yb_alias(View,Bfullmap, Bmap ? Yb->Values(): 0);
Epetra_Import Ximport(Bfullmap,Amap);
// Set the input vector
Xa.SetSeed(24601);
Xa.Random();
Xb_alias.Import(Xa,Ximport,Insert);
// Do the multiplies
A.Apply(Xa,Ya);
if(Bmap) B.Apply(*Xb,*Yb);
// Check solution
Epetra_Import Yimport(Amap,Bfullmap);
Diff.Import(Yb_alias,Yimport,Insert);
Diff.Update(-1.0,Ya,1.0);
double norm;
Diff.Norm2(&norm);
delete Xb; delete Yb;
return norm;
}
int RowMatrixToHandle(FILE * handle, const Epetra_RowMatrix & A) {
Epetra_Map map = A.RowMatrixRowMap();
const Epetra_Comm & comm = map.Comm();
int numProc = comm.NumProc();
if (numProc==1 || !A.Map().DistributedGlobal())
writeRowMatrix(handle, A);
else {
int numRows = map.NumMyElements();
Epetra_Map allGidsMap((int_type) -1, numRows, (int_type) 0,comm);
typename Epetra_GIDTypeVector<int_type>::impl allGids(allGidsMap);
for (int i=0; i<numRows; i++) allGids[i] = (int_type) map.GID64(i);
// Now construct a RowMatrix on PE 0 by strip-mining the rows of the input matrix A.
int numChunks = numProc;
int stripSize = allGids.GlobalLength64()/numChunks;
int remainder = allGids.GlobalLength64()%numChunks;
int curStart = 0;
int curStripSize = 0;
typename Epetra_GIDTypeSerialDenseVector<int_type>::impl importGidList;
if (comm.MyPID()==0)
importGidList.Size(stripSize+1); // Set size of vector to max needed
for (int i=0; i<numChunks; i++) {
if (comm.MyPID()==0) { // Only PE 0 does this part
curStripSize = stripSize;
if (i<remainder) curStripSize++; // handle leftovers
for (int j=0; j<curStripSize; j++) importGidList[j] = j + curStart;
curStart += curStripSize;
}
// The following import map will be non-trivial only on PE 0.
if (comm.MyPID()>0) assert(curStripSize==0);
Epetra_Map importGidMap(-1, curStripSize, importGidList.Values(), 0, comm);
Epetra_Import gidImporter(importGidMap, allGidsMap);
typename Epetra_GIDTypeVector<int_type>::impl importGids(importGidMap);
if (importGids.Import(allGids, gidImporter, Insert)!=0) {EPETRA_CHK_ERR(-1); }
// importGids now has a list of GIDs for the current strip of matrix rows.
// Use these values to build another importer that will get rows of the matrix.
// The following import map will be non-trivial only on PE 0.
Epetra_Map importMap(-1, importGids.MyLength(), importGids.Values(), map.IndexBase64(), comm);
Epetra_Import importer(importMap, map);
Epetra_CrsMatrix importA(Copy, importMap, 0);
if (importA.Import(A, importer, Insert)!=0) {EPETRA_CHK_ERR(-1); }
if (importA.FillComplete(A.OperatorDomainMap(), importMap)!=0) {EPETRA_CHK_ERR(-1);}
// Finally we are ready to write this strip of the matrix to ostream
if (writeRowMatrix(handle, importA)!=0) {EPETRA_CHK_ERR(-1);}
}
}
return(0);
}
TridiagonalCrsMatrix(
const Epetra_Map & Map,
double a,
double diag,
double c
) :
Epetra_CrsMatrix(Copy,Map,3)
{
// global number of rows
int NumGlobalElements = Map.NumGlobalElements();
// local number of rows
int NumMyElements = Map.NumMyElements();
// get update list
int * MyGlobalElements = new int [NumMyElements];
Map.MyGlobalElements( MyGlobalElements );
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
double *Values = new double[2];
Values[0] = a;
Values[1] = c;
int *Indices = new int[2];
int NumEntries;
for( int i=0 ; i<NumMyElements; ++i ) {
if (MyGlobalElements[i]==0) {
Indices[0] = 1;
NumEntries = 1;
} else if (MyGlobalElements[i] == NumGlobalElements-1) {
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
} else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices);
// Put in the diagonal entry
InsertGlobalValues(MyGlobalElements[i], 1, &diag, MyGlobalElements+i);
}
// Finish up
FillComplete();
delete [] MyGlobalElements;
delete [] Values;
delete [] Indices;
}
//EpetraMap_To_TpetraMap: takes in Epetra_Map object, converts it to its equivalent Tpetra::Map object,
//and returns an RCP pointer to this Tpetra::Map
Teuchos::RCP<const Tpetra_Map> Petra::EpetraMap_To_TpetraMap(const Epetra_Map& epetraMap_,
const Teuchos::RCP<const Teuchos::Comm<int> >& commT_)
{
const std::size_t numElements = Teuchos::as<std::size_t>(epetraMap_.NumMyElements());
const auto indexBase = Teuchos::as<GO>(epetraMap_.IndexBase());
if (epetraMap_.DistributedGlobal() || epetraMap_.Comm().NumProc() == Teuchos::OrdinalTraits<int>::one()) {
Teuchos::Array<Tpetra_GO> indices(numElements);
int *epetra_indices = epetraMap_.MyGlobalElements();
for(LO i=0; i < numElements; i++)
indices[i] = epetra_indices[i];
const Tpetra::global_size_t computeGlobalElements = Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid();
return Teuchos::rcp(new Tpetra_Map(computeGlobalElements, indices, indexBase, commT_));
} else {
return Teuchos::rcp(new Tpetra_Map(numElements, indexBase, commT_, Tpetra::LocallyReplicated));
}
}
// FIXME long long
Epetra_Map
Epetra_Util::Create_OneToOne_Map(const Epetra_Map& usermap,
bool high_rank_proc_owns_shared)
{
//if usermap is already 1-to-1 then we'll just return a copy of it.
if (usermap.IsOneToOne()) {
Epetra_Map newmap(usermap);
return(newmap);
}
int myPID = usermap.Comm().MyPID();
Epetra_Directory* directory = usermap.Comm().CreateDirectory(usermap);
int numMyElems = usermap.NumMyElements();
const int* myElems = usermap.MyGlobalElements();
int* owner_procs = new int[numMyElems];
directory->GetDirectoryEntries(usermap, numMyElems, myElems, owner_procs,
0, 0, high_rank_proc_owns_shared);
//we'll fill a list of map-elements which belong on this processor
int* myOwnedElems = new int[numMyElems];
int numMyOwnedElems = 0;
for(int i=0; i<numMyElems; ++i) {
int GID = myElems[i];
int owner = owner_procs[i];
if (myPID == owner) {
myOwnedElems[numMyOwnedElems++] = GID;
}
}
Epetra_Map one_to_one_map(-1, numMyOwnedElems, myOwnedElems,
usermap.IndexBase(), usermap.Comm());
delete [] myOwnedElems;
delete [] owner_procs;
delete directory;
return(one_to_one_map);
}
Teuchos::RCP<Epetra_CrsMatrix> Epetra_Operator_to_Epetra_Matrix::constructInverseMatrix(const Epetra_Operator &op, const Epetra_Map &map)
{
int numEntriesPerRow = 0;
Teuchos::RCP<Epetra_FECrsMatrix> matrix = Teuchos::rcp(new Epetra_FECrsMatrix(::Copy, map, numEntriesPerRow));
int numRows = map.NumGlobalElements();
Epetra_Vector X(map);
Epetra_Vector Y(map);
double tol = 1e-15; // values below this will be considered 0
for (int rowIndex=0; rowIndex<numRows; rowIndex++)
{
int lid = map.LID(rowIndex);
if (lid != -1)
{
X[lid] = 1.0;
}
op.ApplyInverse(X, Y);
if (lid != -1)
{
X[lid] = 0.0;
}
std::vector<double> values;
std::vector<int> indices;
for (int i=0; i<map.NumMyElements(); i++)
{
if (abs(Y[i]) > tol)
{
values.push_back(Y[i]);
indices.push_back(map.GID(i));
}
}
matrix->InsertGlobalValues(rowIndex, values.size(), &values[0], &indices[0]);
}
matrix->GlobalAssemble();
return matrix;
}
int CopyRowMatrix(mxArray* matlabA, const Epetra_RowMatrix& A) {
int valueCount = 0;
//int* valueCount = &temp;
Epetra_Map map = A.RowMatrixRowMap();
const Epetra_Comm & comm = map.Comm();
int numProc = comm.NumProc();
if (numProc==1)
DoCopyRowMatrix(matlabA, valueCount, A);
else {
int numRows = map.NumMyElements();
//cout << "creating allGidsMap\n";
Epetra_Map allGidsMap(-1, numRows, 0,comm);
//cout << "done creating allGidsMap\n";
Epetra_IntVector allGids(allGidsMap);
for (int i=0; i<numRows; i++) allGids[i] = map.GID(i);
// Now construct a RowMatrix on PE 0 by strip-mining the rows of the input matrix A.
int numChunks = numProc;
int stripSize = allGids.GlobalLength()/numChunks;
int remainder = allGids.GlobalLength()%numChunks;
int curStart = 0;
int curStripSize = 0;
Epetra_IntSerialDenseVector importGidList;
int numImportGids = 0;
if (comm.MyPID()==0)
importGidList.Size(stripSize+1); // Set size of vector to max needed
for (int i=0; i<numChunks; i++) {
if (comm.MyPID()==0) { // Only PE 0 does this part
curStripSize = stripSize;
if (i<remainder) curStripSize++; // handle leftovers
for (int j=0; j<curStripSize; j++) importGidList[j] = j + curStart;
curStart += curStripSize;
}
// The following import map will be non-trivial only on PE 0.
//cout << "creating importGidMap\n";
Epetra_Map importGidMap(-1, curStripSize, importGidList.Values(), 0, comm);
//cout << "done creating importGidMap\n";
Epetra_Import gidImporter(importGidMap, allGidsMap);
Epetra_IntVector importGids(importGidMap);
if (importGids.Import(allGids, gidImporter, Insert)) return(-1);
// importGids now has a list of GIDs for the current strip of matrix rows.
// Use these values to build another importer that will get rows of the matrix.
// The following import map will be non-trivial only on PE 0.
//cout << "creating importMap\n";
//cout << "A.RowMatrixRowMap().MinAllGID: " << A.RowMatrixRowMap().MinAllGID() << "\n";
Epetra_Map importMap(-1, importGids.MyLength(), importGids.Values(), A.RowMatrixRowMap().MinAllGID(), comm);
//cout << "done creating importMap\n";
Epetra_Import importer(importMap, map);
Epetra_CrsMatrix importA(Copy, importMap, 0);
if (importA.Import(A, importer, Insert)) return(-1);
if (importA.FillComplete()) return(-1);
// Finally we are ready to write this strip of the matrix to ostream
if (DoCopyRowMatrix(matlabA, valueCount, importA)) return(-1);
}
}
if (A.RowMatrixRowMap().Comm().MyPID() == 0) {
// set max cap
int* matlabAcolumnIndicesPtr = mxGetJc(matlabA);
matlabAcolumnIndicesPtr[A.NumGlobalRows()] = valueCount;
}
return(0);
}
/* Apply an identity matrix to the Schur complement operator. Drop the entries
entries using a relative threshold. Assemble the result in a Crs Matrix
which will be our approximate Schur complement.
*/
Teuchos::RCP<Epetra_CrsMatrix> computeApproxSchur(shylu_config *config,
shylu_symbolic *sym,
Epetra_CrsMatrix *G, Epetra_CrsMatrix *R,
Epetra_LinearProblem *LP, Amesos_BaseSolver *solver,
Ifpack_Preconditioner *ifSolver, Epetra_CrsMatrix *C,
Epetra_Map *localDRowMap)
{
double relative_thres = config->relative_threshold;
int nvectors = 16;
ShyLU_Probing_Operator probeop(config, sym, G, R, LP, solver, ifSolver, C,
localDRowMap, nvectors);
// Get row map
Epetra_Map rMap = G->RowMap();
int *rows = rMap.MyGlobalElements();
int totalElems = rMap.NumGlobalElements();
int localElems = rMap.NumMyElements();
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
ftime.start();
#endif
int prefixSum;
G->Comm().ScanSum(&localElems, &prefixSum, 1);
//cout << " prefixSum" << prefixSum << endl;
// Start the index in prefixSum-localElems
int *mySGID = new int[totalElems]; // vector of size Schur complement !
int *allSGID = new int[totalElems]; // vector of size Schur complement !
int i, j;
for (i = 0, j = 0; i < totalElems ; i++)
{
if (i >= prefixSum - localElems && i < prefixSum)
{
mySGID[i] = rows[j];
j++;
}
else
{
mySGID[i] = 0;
}
allSGID[i] = 0;
}
C->Comm().SumAll(mySGID, allSGID, totalElems);
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time to Compute RowIDS" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
// Now everyone knows the GIDs in the Schur complement
//cout << rMap << endl;
j = 0;
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, rMap, localElems));
int nentries;
double *values = new double[localElems]; // Need to adjust this for more
int *indices = new int[localElems]; // than one vector
double *vecvalues;
int dropped = 0;
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("Apply time");
#endif
int findex = totalElems / nvectors ;
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
Epetra_MultiVector probevec(rMap, nvectors);
Epetra_MultiVector Scol(rMap, nvectors);
probevec.PutScalar(0.0);
int cindex;
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
if (cindex >= prefixSum - localElems && cindex < prefixSum)
{
probevec.ReplaceGlobalValue(allSGID[cindex], k, 1.0);
}
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol.MaxValue(maxvalue);
for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops
{
cindex = k+i;
vecvalues = Scol[k];
//cout << "MAX" << maxvalue << endl;
for (j = 0 ; j < localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (allSGID[cindex] == rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[cindex];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[cindex];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
}
probeop.ResetTempVectors(1);
for ( ; i < totalElems ; i++)
{
Epetra_MultiVector probevec(rMap, 1); // TODO: Try doing more than one
Epetra_MultiVector Scol(rMap, 1); // vector at a time
probevec.PutScalar(0.0);
if (i >= prefixSum - localElems && i < prefixSum)
{
probevec.ReplaceGlobalValue(allSGID[i], 0, 1.0);
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
vecvalues = Scol[0];
Scol.MaxValue(maxvalue);
//cout << "MAX" << maxvalue << endl;
for (j = 0 ; j < localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (allSGID[i] == rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[i];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = allSGID[i];
nentries++;
Sbar->InsertGlobalValues(rows[j], nentries, values, indices);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
#ifdef TIMING_OUTPUT
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
#endif
Sbar->FillComplete();
cout << "#dropped entries" << dropped << endl;
delete[] allSGID;
delete[] mySGID;
delete[] values;
delete[] indices;
delete[] maxvalue;
return Sbar;
}
/* Computes the approximate Schur complement for the wide separator
using guided probing*/
Teuchos::RCP<Epetra_CrsMatrix> computeSchur_GuidedProbing
(
shylu_config *config,
shylu_symbolic *ssym, // symbolic structure
shylu_data *data, // numeric structure
Epetra_Map *localDRowMap
)
{
int i;
double relative_thres = config->relative_threshold;
Epetra_CrsMatrix *G = ssym->G.getRawPtr();
Epetra_CrsMatrix *R = ssym->R.getRawPtr();
Epetra_LinearProblem *LP = ssym->LP.getRawPtr();
Amesos_BaseSolver *solver = ssym->Solver.getRawPtr();
Ifpack_Preconditioner *ifSolver = ssym->ifSolver.getRawPtr();
Epetra_CrsMatrix *C = ssym->C.getRawPtr();
// Need to create local G (block diagonal portion) , R, C
// Get row map of G
Epetra_Map CrMap = C->RowMap();
int *c_rows = CrMap.MyGlobalElements();
int *c_cols = (C->ColMap()).MyGlobalElements();
//int c_totalElems = CrMap.NumGlobalElements();
int c_localElems = CrMap.NumMyElements();
int c_localcolElems = (C->ColMap()).NumMyElements();
Epetra_Map GrMap = G->RowMap();
int *g_rows = GrMap.MyGlobalElements();
//int g_totalElems = GrMap.NumGlobalElements();
int g_localElems = GrMap.NumMyElements();
Epetra_Map RrMap = R->RowMap();
int *r_rows = RrMap.MyGlobalElements();
int *r_cols = (R->ColMap()).MyGlobalElements();
//int r_totalElems = RrMap.NumGlobalElements();
int r_localElems = RrMap.NumMyElements();
int r_localcolElems = (R->ColMap()).NumMyElements();
Epetra_SerialComm LComm;
Epetra_Map C_localRMap (-1, c_localElems, c_rows, 0, LComm);
Epetra_Map C_localCMap (-1, c_localcolElems, c_cols, 0, LComm);
Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm);
Epetra_Map R_localRMap (-1, r_localElems, r_rows, 0, LComm);
Epetra_Map R_localCMap (-1, r_localcolElems, r_cols, 0, LComm);
//cout << "#local rows" << g_localElems << "#non zero local cols" << c_localcolElems << endl;
#ifdef DEBUG
cout << "DEBUG MODE" << endl;
int nrows = C->RowMap().NumMyElements();
assert(nrows == localDRowMap->NumGlobalElements());
int gids[nrows], gids1[nrows];
C_localRMap.MyGlobalElements(gids);
localDRowMap->MyGlobalElements(gids1);
cout << "Comparing R's domain map with D's row map" << endl;
for (int i = 0; i < nrows; i++)
{
assert(gids[i] == gids1[i]);
}
#endif
int nentries1, gid;
// maxentries is the maximum of all three possible matrices as the arrays
// are reused between the three
int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries());
maxentries = max(maxentries, G->MaxNumEntries());
double *values1 = new double[maxentries];
double *values2 = new double[maxentries];
double *values3 = new double[maxentries];
int *indices1 = new int[maxentries];
int *indices2 = new int[maxentries];
int *indices3 = new int[maxentries];
//cout << "Creating local matrices" << endl;
int err;
Epetra_CrsMatrix localC(Copy, C_localRMap, C->MaxNumEntries(), false);
for (i = 0; i < c_localElems ; i++)
{
gid = c_rows[i];
err = C->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1,
indices1);
assert (err == 0);
//if (nentries1 > 0) // TODO: Later
//{
err = localC.InsertGlobalValues(gid, nentries1, values1, indices1);
assert (err == 0);
//}
}
localC.FillComplete(G_localRMap, C_localRMap);
//cout << "Created local C matrix" << endl;
Epetra_CrsMatrix localR(Copy, R_localRMap, R->MaxNumEntries(), false);
for (i = 0; i < r_localElems ; i++)
{
gid = r_rows[i];
R->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
localR.InsertGlobalValues(gid, nentries1, values1, indices1);
}
localR.FillComplete(*localDRowMap, R_localRMap);
//cout << "Created local R matrix" << endl;
// Sbar - Approximate Schur complement
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, GrMap, g_localElems));
// Include only the block diagonal elements of G in localG
Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false);
int cnt, scnt;
for (i = 0; i < g_localElems ; i++)
{
gid = g_rows[i];
G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
cnt = 0;
scnt = 0;
for (int j = 0 ; j < nentries1 ; j++)
{
if (G->LRID(indices1[j]) != -1)
{
values2[cnt] = values1[j];
indices2[cnt++] = indices1[j];
}
else
{
// Add it to Sbar immediately
values3[scnt] = values1[j];
indices3[scnt++] = indices1[j];
}
}
localG.InsertGlobalValues(gid, cnt, values2, indices2);
Sbar->InsertGlobalValues(gid, scnt, values3, indices3);
}
localG.FillComplete();
cout << "Created local G matrix" << endl;
int nvectors = 16;
ShyLU_Probing_Operator probeop(config, ssym, &localG, &localR, LP, solver,
ifSolver, &localC, localDRowMap, nvectors);
#ifdef DUMP_MATRICES
//ostringstream fnamestr;
//fnamestr << "localC" << C->Comm().MyPID() << ".mat";
//string Cfname = fnamestr.str();
//EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC);
//Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm());
//EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg =
//new EpetraExt::CrsMatrix_Reindex( defMapg );
//Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG );
//ReIdx_MatTransg->fwd();
//EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G);
#endif
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("setup time");
#endif
Teuchos::RCP<Epetra_CrsGraph> lSGraph = Teuchos::RCP<Epetra_CrsGraph> (
new Epetra_CrsGraph(Copy, G_localRMap, maxentries));
if (data->num_compute % config->reset_iter == 0)
{
int nentries;
// size > maxentries as there could be fill
// TODO: Currently the size of the two arrays can be one, Even if we switch
// the loop below the size of the array required is nvectors. Fix it
double *values = new double[g_localElems];
int *indices = new int[g_localElems];
double *vecvalues;
int dropped = 0;
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
int findex = g_localElems / nvectors ;
int cindex;
// int mypid = C->Comm().MyPID(); // unused
Epetra_MultiVector probevec(G_localRMap, nvectors);
Epetra_MultiVector Scol(G_localRMap, nvectors);
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
probevec.PutScalar(0.0); // TODO: Move it out
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
// Not much of use for Shasta 2x2 .. Later.
probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
//if (mypid == 0)
//cout << "Changing row to 1.0 " << g_rows[cindex] << endl;
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol.MaxValue(maxvalue);
for (int k = 0; k < nvectors; k++) //TODO:Need to switch these loops
{
cindex = k+i;
vecvalues = Scol[k];
//cout << "MAX" << maxvalue << endl;
for (int j = 0 ; j < g_localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (g_rows[cindex] == g_rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[cindex];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries,
indices);
assert(err >= 0);
}
else if (abs(vecvalues[j]/maxvalue[k]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[cindex];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries,
indices);
assert(err >= 0);
}
else
{
if (vecvalues[j] != 0.0)
{
dropped++;
//cout << "vecvalues[j]" << vecvalues[j] <<
// " max" << maxvalue[k] << endl;
}
}
}
}
}
probeop.ResetTempVectors(1);
for ( ; i < g_localElems ; i++)
{
// TODO: Can move the next two decalarations outside the loop
Epetra_MultiVector probevec(G_localRMap, 1);
Epetra_MultiVector Scol(G_localRMap, 1);
probevec.PutScalar(0.0);
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec.ReplaceGlobalValue(g_rows[i], 0, 1.0);
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
vecvalues = Scol[0];
Scol.MaxValue(maxvalue);
//cout << "MAX" << maxvalue << endl;
for (int j = 0 ; j < g_localElems ; j++)
{
nentries = 0; // inserting one entry in each row for now
if (g_rows[i] == g_rows[j]) // diagonal entry
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[i];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices);
assert(err >= 0);
}
else if (abs(vecvalues[j]/maxvalue[0]) > relative_thres)
{
values[nentries] = vecvalues[j];
indices[nentries] = g_rows[i];
nentries++;
err = Sbar->InsertGlobalValues(g_rows[j], nentries, values, indices);
assert(err >= 0);
err = lSGraph->InsertGlobalIndices(g_rows[j], nentries, indices);
assert(err >= 0);
}
else
{
if (vecvalues[j] != 0.0) dropped++;
}
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
#ifdef TIMING_OUTPUT
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
#endif
probeop.PrintTimingInfo();
Sbar->FillComplete();
lSGraph->FillComplete();
data->localSbargraph = lSGraph;
#ifdef DUMP_MATRICES
Epetra_Map defMap2(-1, g_localElems, 0, C->Comm());
EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 =
new EpetraExt::CrsMatrix_Reindex( defMap2 );
Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar );
ReIdx_MatTrans2->fwd();
EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S);
#endif
cout << "#dropped entries" << dropped << endl;
delete[] values;
delete[] indices;
delete[] maxvalue;
}
else
{
if (((data->num_compute-1) % config->reset_iter) == 0)
{
// We recomputed the Schur complement with dropping for the last
// compute. Reset the prober with the new orthogonal vectors for
// the Sbar from the previous iteration.
Teuchos::ParameterList pList;
Teuchos::RCP<Isorropia::Epetra::Prober> gprober =
Teuchos::RCP<Isorropia::Epetra::Prober> (new
Isorropia::Epetra::Prober(
data->localSbargraph.getRawPtr(), pList, false));
gprober->color();
data->guided_prober = gprober;
}
// Use the prober to probe the probeop for the sparsity pattern
// add that to Sbar and call Fill complete
int nvectors = data->guided_prober->getNumOrthogonalVectors();
cout << "Number of Orthogonal Vectors for guided probing" << nvectors
<< endl;
probeop.ResetTempVectors(nvectors);
Teuchos::RCP<Epetra_CrsMatrix> blockdiag_Sbar =
data->guided_prober->probe(probeop);
int maxentries = blockdiag_Sbar->GlobalMaxNumEntries();
int *indices = new int[maxentries];
double *values = new double[maxentries];
int numentries;
for (int i = 0; i < blockdiag_Sbar->NumGlobalRows() ; i++)
{
int gid = blockdiag_Sbar->GRID(i);
blockdiag_Sbar->ExtractGlobalRowCopy(gid, maxentries, numentries,
values, indices);
Sbar->InsertGlobalValues(gid, numentries, values, indices);
}
Sbar->FillComplete();
delete[] indices;
delete[] values;
}
delete[] values1;
delete[] indices1;
delete[] values2;
delete[] indices2;
delete[] values3;
delete[] indices3;
return Sbar;
}
/* Computes the approximate Schur complement for the wide separator */
Teuchos::RCP<Epetra_CrsMatrix> computeApproxWideSchur(shylu_config *config,
shylu_symbolic *ssym, // symbolic structure
Epetra_CrsMatrix *G, Epetra_CrsMatrix *R,
Epetra_LinearProblem *LP, Amesos_BaseSolver *solver,
Ifpack_Preconditioner *ifSolver, Epetra_CrsMatrix *C,
Epetra_Map *localDRowMap)
{
int i;
double relative_thres = config->relative_threshold;
// Need to create local G (block diagonal portion) , R, C
// Get row map of G
//Epetra_Map CrMap = C->RowMap();
//int *c_rows = CrMap.MyGlobalElements();
//int *c_cols = (C->ColMap()).MyGlobalElements();
//int c_totalElems = CrMap.NumGlobalElements();
//int c_localElems = CrMap.NumMyElements();
//int c_localcolElems = (C->ColMap()).NumMyElements();
Epetra_Map GrMap = G->RowMap();
int *g_rows = GrMap.MyGlobalElements();
//int g_totalElems = GrMap.NumGlobalElements();
int g_localElems = GrMap.NumMyElements();
//Epetra_Map RrMap = R->RowMap();
//int *r_rows = RrMap.MyGlobalElements();
//int *r_cols = (R->ColMap()).MyGlobalElements();
//int r_totalElems = RrMap.NumGlobalElements();
//int r_localElems = RrMap.NumMyElements();
//int r_localcolElems = (R->ColMap()).NumMyElements();
Epetra_SerialComm LComm;
Epetra_Map G_localRMap (-1, g_localElems, g_rows, 0, LComm);
int nentries1, gid;
// maxentries is the maximum of all three possible matrices as the arrays
// are reused between the three
int maxentries = max(C->MaxNumEntries(), R->MaxNumEntries());
maxentries = max(maxentries, G->MaxNumEntries());
double *values1 = new double[maxentries];
double *values2 = new double[maxentries];
double *values3 = new double[maxentries];
int *indices1 = new int[maxentries];
int *indices2 = new int[maxentries];
int *indices3 = new int[maxentries];
// Sbar - Approximate Schur complement
Teuchos::RCP<Epetra_CrsMatrix> Sbar = Teuchos::rcp(new Epetra_CrsMatrix(
Copy, GrMap, g_localElems));
// Include only the block diagonal elements of G in localG
Epetra_CrsMatrix localG(Copy, G_localRMap, G->MaxNumEntries(), false);
int cnt, scnt;
for (i = 0; i < g_localElems ; i++)
{
gid = g_rows[i];
G->ExtractGlobalRowCopy(gid, maxentries, nentries1, values1, indices1);
cnt = 0;
scnt = 0;
for (int j = 0 ; j < nentries1 ; j++)
{
if (G->LRID(indices1[j]) != -1)
{
values2[cnt] = values1[j];
indices2[cnt++] = indices1[j];
}
else
{
// Add it to Sbar immediately
values3[scnt] = values1[j];
indices3[scnt++] = indices1[j];
}
}
localG.InsertGlobalValues(gid, cnt, values2, indices2);
Sbar->InsertGlobalValues(gid, scnt, values3, indices3);
}
localG.FillComplete();
//cout << "Created local G matrix" << endl;
int nvectors = 16;
/*ShyLU_Probing_Operator probeop(&localG, &localR, LP, solver, &localC,
localDRowMap, nvectors);*/
ShyLU_Local_Schur_Operator probeop(config, ssym, &localG, R, LP, solver,
ifSolver, C, localDRowMap, nvectors);
#ifdef DUMP_MATRICES
//ostringstream fnamestr;
//fnamestr << "localC" << C->Comm().MyPID() << ".mat";
//string Cfname = fnamestr.str();
//EpetraExt::RowMatrixToMatlabFile(Cfname.c_str(), localC);
//Epetra_Map defMapg(-1, g_localElems, 0, localG.Comm());
//EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTransg =
//new EpetraExt::CrsMatrix_Reindex( defMapg );
//Epetra_CrsMatrix t2G = (*ReIdx_MatTransg)( localG );
//ReIdx_MatTransg->fwd();
//EpetraExt::RowMatrixToMatlabFile("localG.mat", t2G);
#endif
//cout << " totalElems in Schur Complement" << totalElems << endl;
//cout << myPID << " localElems" << localElems << endl;
// **************** Two collectives here *********************
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
ftime.start();
#endif
#ifdef TIMING_OUTPUT
Teuchos::Time app_time("setup time");
#endif
int nentries;
// size > maxentries as there could be fill
// TODO: Currently the size of the two arrays can be one, Even if we switch
// the loop below the size of the array required is nvectors. Fix it
double *values = new double[nvectors];
int *indices = new int[nvectors];
double *vecvalues;
#ifdef SHYLU_DEBUG
// mfh 25 May 2015: Don't declare this variable if it's not used.
// It's only used if SHYLU_DEBUG is defined.
int dropped = 0;
#endif // SHYLU_DEBUG
double *maxvalue = new double[nvectors];
#ifdef TIMING_OUTPUT
ftime.start();
#endif
int findex = g_localElems / nvectors ;
int cindex;
// int mypid = C->Comm().MyPID(); // unused
Epetra_MultiVector probevec (G_localRMap, nvectors);
Epetra_MultiVector Scol (G_localRMap, nvectors);
probevec.PutScalar(0.0);
for (i = 0 ; i < findex*nvectors ; i+=nvectors)
{
// Set the probevec to find block columns of S.
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
//if (mypid == 0)
//cout << "Changing row to 1.0 " << g_rows[cindex] << endl;
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec, Scol);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
// Reset the probevec to all zeros.
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
probevec.ReplaceGlobalValue(g_rows[cindex], k, 0.0);
}
Scol.MaxValue(maxvalue);
nentries = 0;
for (int j = 0 ; j < g_localElems ; j++)
{
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
vecvalues = Scol[k];
if ((g_rows[cindex] == g_rows[j]) ||
(abs(vecvalues[j]/maxvalue[k]) > relative_thres))
// diagonal entry or large entry.
{
values[nentries] = vecvalues[j];
indices[nentries++] = g_rows[cindex];
}
#ifdef SHYLU_DEBUG
else if (vecvalues[j] != 0.0)
{
dropped++;
}
#endif // SHYLU_DEBUG
}
Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
nentries = 0;
}
}
if (i < g_localElems)
{
nvectors = g_localElems - i;
probeop.ResetTempVectors(nvectors);
Epetra_MultiVector probevec1 (G_localRMap, nvectors);
Epetra_MultiVector Scol1 (G_localRMap, nvectors);
probevec1.PutScalar(0.0);
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
// TODO: Can do better than this, just need to go to the column map
// of C, there might be null columns in C
probevec1.ReplaceGlobalValue(g_rows[cindex], k, 1.0);
}
#ifdef TIMING_OUTPUT
app_time.start();
#endif
probeop.Apply(probevec1, Scol1);
#ifdef TIMING_OUTPUT
app_time.stop();
#endif
Scol1.MaxValue(maxvalue);
nentries = 0;
for (int j = 0 ; j < g_localElems ; j++)
{
//cout << "MAX" << maxvalue << endl;
for (int k = 0; k < nvectors; k++)
{
cindex = k+i;
vecvalues = Scol1[k];
//nentries = 0; // inserting one entry in each row for now
if ((g_rows[cindex] == g_rows[j]) ||
(abs(vecvalues[j]/maxvalue[k]) > relative_thres))
// diagonal entry or large entry.
{
values[nentries] = vecvalues[j];
indices[nentries++] = g_rows[cindex];
}
#ifdef SHYLU_DEBUG
else if (vecvalues[j] != 0.0)
{
dropped++;
}
#endif // SHYLU_DEBUG
}
Sbar->InsertGlobalValues(g_rows[j], nentries, values,
indices);
nentries = 0;
}
}
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time in finding and dropping entries" << ftime.totalElapsedTime()
<< endl;
ftime.reset();
cout << "Time in Apply of probing" << app_time.totalElapsedTime() << endl;
probeop.PrintTimingInfo();
#endif
Sbar->FillComplete();
#ifdef DUMP_MATRICES
Epetra_Map defMap2(-1, g_localElems, 0, C->Comm());
EpetraExt::ViewTransform<Epetra_CrsMatrix> * ReIdx_MatTrans2 =
new EpetraExt::CrsMatrix_Reindex( defMap2 );
Epetra_CrsMatrix t2S = (*ReIdx_MatTrans2)( *Sbar );
ReIdx_MatTrans2->fwd();
EpetraExt::RowMatrixToMatlabFile("Schur.mat", t2S);
#endif
#ifdef SHYLU_DEBUG
cout << "#dropped entries" << dropped << endl;
#endif
delete[] values;
delete[] indices;
delete[] values1;
delete[] indices1;
delete[] values2;
delete[] indices2;
delete[] values3;
delete[] indices3;
delete[] maxvalue;
return Sbar;
}
int CreateTridi(Epetra_CrsMatrix& A)
{
Epetra_Map Map = A.RowMap();
int NumMyElements = Map.NumMyElements();
int NumGlobalElements = Map.NumGlobalElements();
int * MyGlobalElements = new int[NumMyElements];
Map.MyGlobalElements(MyGlobalElements);
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
double *Values = new double[3];
int *Indices = new int[3];
int NumEntries;
for (int i=0; i<NumMyElements; i++)
{
if (MyGlobalElements[i]==0)
{
Indices[0] = 0;
Indices[1] = 1;
Values[0] = 2.0;
Values[1] = -1.0;
NumEntries = 2;
}
else if (MyGlobalElements[i] == NumGlobalElements-1)
{
Indices[0] = NumGlobalElements-1;
Indices[1] = NumGlobalElements-2;
Values[0] = 2.0;
Values[1] = -1.0;
NumEntries = 2;
}
else
{
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i];
Indices[2] = MyGlobalElements[i]+1;
Values[0] = -1.0;
Values[1] = 2.0;
Values[2] = -1.0;
NumEntries = 3;
}
assert(A.InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices)==0);
// Put in the diagonal entry
// assert(A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i])==0);
}
// Finish up
assert(A.FillComplete()==0);
delete[] MyGlobalElements;
delete[] Values;
delete[] Indices;
return 0;
}
int TLowCommunicationMakeColMapAndReindex(int N, const int * rowptr, int * colind_LID, const int_type *colind_GID, const Epetra_Map& domainMap, const int * owningPIDs, bool SortGhostsAssociatedWithEachProcessor, std::vector<int>& RemotePIDs, MapType1 & NewColMap)
{
int i,j;
// Sanity checks
bool UseLL;
if(domainMap.GlobalIndicesLongLong()) UseLL=true;
else if(domainMap.GlobalIndicesInt()) UseLL=false;
else throw std::runtime_error("LowCommunicationMakeColMapAndReindex: cannot detect int type.");
// Scan all column indices and sort into two groups:
// Local: those whose GID matches a GID of the domain map on this processor and
// Remote: All others.
int numDomainElements = domainMap.NumMyElements();
bool * LocalGIDs = 0;
if (numDomainElements>0) LocalGIDs = new bool[numDomainElements];
for (i=0; i<numDomainElements; i++) LocalGIDs[i] = false; // Assume domain GIDs are not local
bool DoSizes = !domainMap.ConstantElementSize(); // If not constant element size, then error
if(DoSizes) throw std::runtime_error("LowCommunicationMakeColMapAndReindex: cannot handle non-constant sized domainMap.");
// In principle it is good to have RemoteGIDs and RemotGIDList be as long as the number of remote GIDs
// on this processor, but this would require two passes through the column IDs, so we make it the max of 100
// and the number of block rows.
const int numMyBlockRows = N;
int hashsize = numMyBlockRows; if (hashsize < 100) hashsize = 100;
Epetra_HashTable<int_type> RemoteGIDs(hashsize);
std::vector<int_type> RemoteGIDList; RemoteGIDList.reserve(hashsize);
std::vector<int> PIDList; PIDList.reserve(hashsize);
// Here we start using the *int* colind array. If int_type==int this clobbers the GIDs, if
// int_type==long long, then this is the first use of the colind array.
// For *local* GID's set colind with with their LID in the domainMap. For *remote* GIDs,
// we set colind with (numDomainElements+NumRemoteColGIDs) before the increment of
// the remote count. These numberings will be separate because no local LID is greater
// than numDomainElements.
int NumLocalColGIDs = 0;
int NumRemoteColGIDs = 0;
for(i = 0; i < numMyBlockRows; i++) {
for(j = rowptr[i]; j < rowptr[i+1]; j++) {
int_type GID = colind_GID[j];
// Check if GID matches a row GID
int LID = domainMap.LID(GID);
if(LID != -1) {
bool alreadyFound = LocalGIDs[LID];
if (!alreadyFound) {
LocalGIDs[LID] = true; // There is a column in the graph associated with this domain map GID
NumLocalColGIDs++;
}
colind_LID[j] = LID;
}
else {
int_type hash_value=RemoteGIDs.Get(GID);
if(hash_value == -1) { // This means its a new remote GID
int PID = owningPIDs[j];
if(PID==-1) throw std::runtime_error("LowCommunicationMakeColMapAndReindex: Cannot figure out if PID is owned.");
colind_LID[j] = numDomainElements + NumRemoteColGIDs;
RemoteGIDs.Add(GID, NumRemoteColGIDs);
RemoteGIDList.push_back(GID);
PIDList.push_back(PID);
NumRemoteColGIDs++;
}
else
colind_LID[j] = numDomainElements + hash_value;
}
}
}
// Possible short-circuit: If all domain map GIDs are present as column indices, then set ColMap=domainMap and quit
if (domainMap.Comm().NumProc()==1) {
if (NumRemoteColGIDs!=0) {
throw std::runtime_error("Some column IDs are not in domainMap. If matrix is rectangular, you must pass in a domainMap");
// Sanity test: When one processor,there can be no remoteGIDs
}
if (NumLocalColGIDs==numDomainElements) {
if (LocalGIDs!=0) delete [] LocalGIDs;
// In this case, we just use the domainMap's indices, which is, not coincidently, what we clobbered colind with up above anyway.
// No further reindexing is needed.
NewColMap = domainMap;
return 0;
}
}
// Now build integer array containing column GIDs
// Build back end, containing remote GIDs, first
int numMyBlockCols = NumLocalColGIDs + NumRemoteColGIDs;
std::vector<int_type> ColIndices;
int_type * RemoteColIndices=0;
if(numMyBlockCols > 0) {
ColIndices.resize(numMyBlockCols);
if(NumLocalColGIDs!=numMyBlockCols) RemoteColIndices = &ColIndices[NumLocalColGIDs]; // Points to back end of ColIndices
else RemoteColIndices=0;
}
for(i = 0; i < NumRemoteColGIDs; i++)
RemoteColIndices[i] = RemoteGIDList[i];
// Build permute array for *remote* reindexing.
std::vector<int> RemotePermuteIDs(NumRemoteColGIDs);
for(i=0; i<NumRemoteColGIDs; i++) RemotePermuteIDs[i]=i;
// Sort External column indices so that all columns coming from a given remote processor are contiguous
int NumListsInt=0;
int NumListsLL =0;
int * IntSortLists[2];
long long * LLSortLists[2];
int * RemotePermuteIDs_ptr = RemotePermuteIDs.size() ? &RemotePermuteIDs[0] : 0;
if(!UseLL) {
// int version
IntSortLists[0] = (int*) RemoteColIndices;
IntSortLists[1] = RemotePermuteIDs_ptr;
NumListsInt=2;
}
else {
//LL version
LLSortLists[0] = (long long*) RemoteColIndices;
IntSortLists[0] = RemotePermuteIDs_ptr;
NumListsInt = NumListsLL = 1;
}
int * PIDList_ptr = PIDList.size() ? &PIDList[0] : 0;
Epetra_Util::Sort(true, NumRemoteColGIDs, PIDList_ptr, 0, 0, NumListsInt, IntSortLists,NumListsLL,LLSortLists);
// Stash the RemotePIDs
PIDList.resize(NumRemoteColGIDs);
RemotePIDs = PIDList;
if (SortGhostsAssociatedWithEachProcessor) {
// Sort external column indices so that columns from a given remote processor are not only contiguous
// but also in ascending order. NOTE: I don't know if the number of externals associated
// with a given remote processor is known at this point ... so I count them here.
// NTS: Only sort the RemoteColIndices this time...
int StartCurrent, StartNext;
StartCurrent = 0; StartNext = 1;
while ( StartNext < NumRemoteColGIDs ) {
if (PIDList[StartNext]==PIDList[StartNext-1]) StartNext++;
else {
IntSortLists[0] = &RemotePermuteIDs[StartCurrent];
Epetra_Util::Sort(true,StartNext-StartCurrent, &(RemoteColIndices[StartCurrent]),0,0,1,IntSortLists,0,0);
StartCurrent = StartNext; StartNext++;
}
}
IntSortLists[0] = &RemotePermuteIDs[StartCurrent];
Epetra_Util::Sort(true, StartNext-StartCurrent, &(RemoteColIndices[StartCurrent]), 0, 0, 1,IntSortLists,0,0);
}
// Reverse the permutation to get the information we actually care about
std::vector<int> ReverseRemotePermuteIDs(NumRemoteColGIDs);
for(i=0; i<NumRemoteColGIDs; i++) ReverseRemotePermuteIDs[RemotePermuteIDs[i]]=i;
// Build permute array for *local* reindexing.
bool use_local_permute=false;
std::vector<int> LocalPermuteIDs(numDomainElements);
// Now fill front end. Two cases:
// (1) If the number of Local column GIDs is the same as the number of Local domain GIDs, we
// can simply read the domain GIDs into the front part of ColIndices, otherwise
// (2) We step through the GIDs of the domainMap, checking to see if each domain GID is a column GID.
// we want to do this to maintain a consistent ordering of GIDs between the columns and the domain.
if(NumLocalColGIDs == domainMap.NumMyElements()) {
if(NumLocalColGIDs > 0) {
domainMap.MyGlobalElements(&ColIndices[0]); // Load Global Indices into first numMyBlockCols elements column GID list
}
}
else {
int_type* MyGlobalElements = 0;
domainMap.MyGlobalElementsPtr(MyGlobalElements);
int* ElementSizeList = 0;
if(DoSizes)
ElementSizeList = domainMap.ElementSizeList();
int NumLocalAgain = 0;
use_local_permute = true;
for(i = 0; i < numDomainElements; i++) {
if(LocalGIDs[i]) {
LocalPermuteIDs[i] = NumLocalAgain;
ColIndices[NumLocalAgain++] = MyGlobalElements[i];
}
}
assert(NumLocalAgain==NumLocalColGIDs); // Sanity test
}
// Done with this array
if (LocalGIDs!=0) delete [] LocalGIDs;
// Make Column map with same element sizes as Domain map
int_type * ColIndices_ptr = ColIndices.size() ? &ColIndices[0] : 0;
MapType2 temp((int_type)(-1), numMyBlockCols, ColIndices_ptr, (int_type)domainMap.IndexBase64(), domainMap.Comm());
NewColMap = temp;
// Low-cost reindex of the matrix
for(i=0; i<numMyBlockRows; i++){
for(j=rowptr[i]; j<rowptr[i+1]; j++){
int ID=colind_LID[j];
if(ID < numDomainElements){
if(use_local_permute) colind_LID[j] = LocalPermuteIDs[colind_LID[j]];
// In the case where use_local_permute==false, we just copy the DomainMap's ordering, which it so happens
// is what we put in colind to begin with.
}
else
colind_LID[j] = NumLocalColGIDs + ReverseRemotePermuteIDs[colind_LID[j]-numDomainElements];
}
}
return 0;
}
void build_simple_matrix(
Epetra_Comm &comm, // Communicator to use
Epetra_CrsMatrix *&A, // OUTPUT: Matrix returned
itype nGlobalRows, // Number of global matrix rows and columns
bool testEpetra64, // if true, add 2*INT_MAX to each global ID
// to exercise Epetra64
bool verbose // if true, print out matrix information
)
{
Epetra_Map *rowMap = NULL; // Row map for the created matrix
Epetra_Map *colMap = NULL; // Col map for the created matrix
Epetra_Map *vectorMap = NULL; // Range/Domain map for the created matrix
long long offsetEpetra64;
build_maps(nGlobalRows, testEpetra64, comm,
&vectorMap, &rowMap, &colMap, offsetEpetra64, verbose);
// Create an integer vector nnzPerRow that is used to build the Epetra Matrix.
// nnzPerRow[i] is the number of entries for the ith global equation
int nMyRows = rowMap->NumMyElements();
std::vector<int> nnzPerRow(nMyRows+1, 0);
// Also create lists of the nonzeros to be assigned to processors.
// To save programming time and complexity, these vectors are allocated
// bigger than they may actually be needed.
std::vector<itype> iv(3*nMyRows+1);
std::vector<itype> jv(3*nMyRows+1);
std::vector<double> vv(3*nMyRows+1);
itype nMyNonzeros = 0;
for (itype i = 0, myrowcnt = 0; i < nGlobalRows; i++) {
if (rowMap->MyGID(i+offsetEpetra64)) {
// This processor owns part of this row; see whether it owns the nonzeros
if (i > 0 && (!colMap || colMap->MyGID(i-1+offsetEpetra64))) {
iv[nMyNonzeros] = i + offsetEpetra64;
jv[nMyNonzeros] = i-1 + offsetEpetra64;
vv[nMyNonzeros] = -1;
nMyNonzeros++;
nnzPerRow[myrowcnt]++;
}
if (!colMap || colMap->MyGID(i+offsetEpetra64)) {
iv[nMyNonzeros] = i + offsetEpetra64;
jv[nMyNonzeros] = i + offsetEpetra64;
vv[nMyNonzeros] = ((i == 0 || i == nGlobalRows-1) ? 1. : 2.);
nMyNonzeros++;
nnzPerRow[myrowcnt]++;
}
if (i < nGlobalRows - 1 && (!colMap || colMap->MyGID(i+1+offsetEpetra64))) {
iv[nMyNonzeros] = i + offsetEpetra64;
jv[nMyNonzeros] = i+1 + offsetEpetra64;
vv[nMyNonzeros] = -1;
nMyNonzeros++;
nnzPerRow[myrowcnt]++;
}
myrowcnt++;
}
}
// Create an Epetra_Matrix
A = new Epetra_CrsMatrix(Copy, *rowMap, &nnzPerRow[0], true);
int info;
for (int sum = 0, i=0; i < nMyRows; i++) {
if (nnzPerRow[i]) {
info = A->InsertGlobalValues(iv[sum],nnzPerRow[i],&vv[sum],&jv[sum]);
assert(info==0);
sum += nnzPerRow[i];
}
}
// Finish up
if (vectorMap)
info = A->FillComplete(*vectorMap, *vectorMap);
else
info = A->FillComplete();
assert(info==0);
}
void
LOCA::Epetra::AugmentedOp::buildExtendedMap(const Epetra_BlockMap& uMap,
Epetra_Map*& eMapPtr,
bool buildImporter,
bool haveParam)
{
Epetra_BlockMap& nonconstUnderlyingMap = const_cast<Epetra_BlockMap&>(uMap);
// Convert underlying map to point map if necessary
Epetra_Map* uPointMapPtr =
dynamic_cast<Epetra_Map*>(&nonconstUnderlyingMap);
bool allocatedPointMap = false;
if (uPointMapPtr == NULL) {
allocatedPointMap = true;
blockMap2PointMap(uMap, uPointMapPtr);
}
int max_gid = uPointMapPtr->MaxAllGID();
int num_global_elements = uPointMapPtr->NumGlobalElements();
int num_my_elements = uPointMapPtr->NumMyElements();
int *global_elements = uPointMapPtr->MyGlobalElements();
const Epetra_Comm& comm = uPointMapPtr->Comm();
int index_base = uPointMapPtr->IndexBase();
int ext_num_global_elements;
int ext_num_my_elements;
int *ext_global_elements;
// Compute number of extended global elements
if (buildImporter)
ext_num_global_elements =
num_global_elements + numConstraints*comm.NumProc();
else
ext_num_global_elements = num_global_elements + numConstraints;
// Compute number of extended local elements
if (buildImporter || haveParam)
ext_num_my_elements = num_my_elements + numConstraints;
else
ext_num_my_elements = num_my_elements;
// Allocate extended global elements array
ext_global_elements = new int[ext_num_my_elements];
// Set extended global elements
for (int i=0; i<num_my_elements; i++) {
ext_global_elements[i] = global_elements[i];
}
if (buildImporter || haveParam)
for (int i=0; i<numConstraints; i++)
ext_global_elements[num_my_elements+i] = max_gid + 1 + i;
// Create extended point map
eMapPtr = new Epetra_Map(ext_num_global_elements, ext_num_my_elements,
ext_global_elements, index_base, comm);
// Free global elements array
delete [] ext_global_elements;
if (allocatedPointMap)
delete uPointMapPtr;
}
int checkmap(Epetra_Map & Map, int NumGlobalElements, int NumMyElements,
int *MyGlobalElements, int IndexBase, Epetra_Comm& Comm,
bool DistributedGlobal)
{
int i, ierr=0, forierr = 0;
EPETRA_TEST_ERR(!Map.ConstantElementSize(),ierr);
EPETRA_TEST_ERR(DistributedGlobal!=Map.DistributedGlobal(),ierr);
EPETRA_TEST_ERR(Map.ElementSize()!=1,ierr);
int *MyElementSizeList = new int[NumMyElements];
EPETRA_TEST_ERR(Map.ElementSizeList(MyElementSizeList)!=0,ierr);
forierr = 0;
for (i=0; i<NumMyElements; i++) forierr += MyElementSizeList[i]!=1;
EPETRA_TEST_ERR(forierr,ierr);
delete [] MyElementSizeList;
const Epetra_Comm & Comm1 = Map.Comm();
EPETRA_TEST_ERR(Comm1.NumProc()!=Comm.NumProc(),ierr);
EPETRA_TEST_ERR(Comm1.MyPID()!=Comm.MyPID(),ierr);
EPETRA_TEST_ERR(Map.IndexBase()!=IndexBase,ierr);
EPETRA_TEST_ERR(!Map.LinearMap() && MyGlobalElements==0,ierr);
EPETRA_TEST_ERR(Map.LinearMap() && MyGlobalElements!=0,ierr);
EPETRA_TEST_ERR(Map.MaxAllGID()!=NumGlobalElements-1+IndexBase,ierr);
EPETRA_TEST_ERR(Map.MaxElementSize()!=1,ierr);
int MaxLID = Map.MaxLID();
EPETRA_TEST_ERR(MaxLID!=NumMyElements-1,ierr);
int MaxMyGID = (Comm.MyPID()+1)*NumMyElements-1+IndexBase;
if (Comm.MyPID()>2) MaxMyGID+=3;
if (!DistributedGlobal) MaxMyGID = NumMyElements-1+IndexBase;
EPETRA_TEST_ERR(Map.MaxMyGID()!=MaxMyGID,ierr);
EPETRA_TEST_ERR(Map.MinAllGID()!=IndexBase,ierr);
EPETRA_TEST_ERR(Map.MinElementSize()!=1,ierr);
EPETRA_TEST_ERR(Map.MinLID()!=0,ierr);
int MinMyGID = Comm.MyPID()*NumMyElements+IndexBase;
if (Comm.MyPID()>2) MinMyGID+=3;
if (!DistributedGlobal) MinMyGID = 0;
EPETRA_TEST_ERR(Map.MinMyGID()!=MinMyGID,ierr);
int * MyGlobalElements1 = new int[NumMyElements];
EPETRA_TEST_ERR(Map.MyGlobalElements(MyGlobalElements1)!=0,ierr);
forierr = 0;
if (MyGlobalElements==0)
{
for (i=0; i<NumMyElements; i++)
forierr += MyGlobalElements1[i]!=MinMyGID+i;
EPETRA_TEST_ERR(forierr,ierr);
}
else {
for (i=0; i<NumMyElements; i++)
forierr += MyGlobalElements[i]!=MyGlobalElements1[i];
EPETRA_TEST_ERR(forierr,ierr);
}
EPETRA_TEST_ERR(Map.NumGlobalElements()!=NumGlobalElements,ierr);
EPETRA_TEST_ERR(Map.NumGlobalPoints()!=NumGlobalElements,ierr);
EPETRA_TEST_ERR(Map.NumMyElements()!=NumMyElements,ierr);
EPETRA_TEST_ERR(Map.NumMyPoints()!=NumMyElements,ierr);
int MaxMyGID2 = Map.GID(Map.LID(MaxMyGID));
EPETRA_TEST_ERR(MaxMyGID2 != MaxMyGID,ierr);
int MaxLID2 = Map.LID(Map.GID(MaxLID));
EPETRA_TEST_ERR(MaxLID2 != MaxLID,ierr);
EPETRA_TEST_ERR(Map.GID(MaxLID+1) != IndexBase-1,ierr);// MaxLID+1 doesn't exist
EPETRA_TEST_ERR(Map.LID(MaxMyGID+1) != -1,ierr);// MaxMyGID+1 doesn't exist or is on a different processor
EPETRA_TEST_ERR(!Map.MyGID(MaxMyGID),ierr);
EPETRA_TEST_ERR(Map.MyGID(MaxMyGID+1),ierr);
EPETRA_TEST_ERR(!Map.MyLID(MaxLID),ierr);
EPETRA_TEST_ERR(Map.MyLID(MaxLID+1),ierr);
EPETRA_TEST_ERR(!Map.MyGID(Map.GID(MaxLID)),ierr);
EPETRA_TEST_ERR(Map.MyGID(Map.GID(MaxLID+1)),ierr);
EPETRA_TEST_ERR(!Map.MyLID(Map.LID(MaxMyGID)),ierr);
EPETRA_TEST_ERR(Map.MyLID(Map.LID(MaxMyGID+1)),ierr);
// Check RemoteIDList function
// Get some GIDs off of each processor to test
int TotalNumEle, NumElePerProc, NumProc = Comm.NumProc();
int MinNumEleOnProc;
int NumMyEle=Map.NumMyElements();
Comm.MinAll(&NumMyEle,&MinNumEleOnProc,1);
if (MinNumEleOnProc > 5) NumElePerProc = 6;
else NumElePerProc = MinNumEleOnProc;
if (NumElePerProc > 0) {
TotalNumEle = NumElePerProc*NumProc;
int * MyGIDlist = new int[NumElePerProc];
int * GIDlist = new int[TotalNumEle];
int * PIDlist = new int[TotalNumEle];
int * LIDlist = new int[TotalNumEle];
for (i=0; i<NumElePerProc; i++)
MyGIDlist[i] = MyGlobalElements1[i];
Comm.GatherAll(MyGIDlist,GIDlist,NumElePerProc);// Get a few values from each proc
Map.RemoteIDList(TotalNumEle, GIDlist, PIDlist, LIDlist);
int MyPID= Comm.MyPID();
forierr = 0;
for (i=0; i<TotalNumEle; i++) {
if (Map.MyGID(GIDlist[i])) {
forierr += PIDlist[i] != MyPID;
forierr += !Map.MyLID(Map.LID(GIDlist[i])) || Map.LID(GIDlist[i]) != LIDlist[i] || Map.GID(LIDlist[i]) != GIDlist[i];
}
else {
forierr += PIDlist[i] == MyPID; // If MyGID comes back false, the PID listed should be that of another proc
}
}
EPETRA_TEST_ERR(forierr,ierr);
delete [] MyGIDlist;
delete [] GIDlist;
delete [] PIDlist;
delete [] LIDlist;
}
delete [] MyGlobalElements1;
// Check RemoteIDList function (assumes all maps are linear, even if not stored that way)
if (Map.LinearMap()) {
int * GIDList = new int[3];
int * PIDList = new int[3];
int * LIDList = new int[3];
int MyPID = Map.Comm().MyPID();
int NumIDs = 0;
//GIDList[NumIDs++] = Map.MaxAllGID()+1; // Should return -1 for both PID and LID
if (Map.MinMyGID()-1>=Map.MinAllGID()) GIDList[NumIDs++] = Map.MinMyGID()-1;
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) GIDList[NumIDs++] = Map.MaxMyGID()+1;
Map.RemoteIDList(NumIDs, GIDList, PIDList, LIDList);
NumIDs = 0;
//EPETRA_TEST_ERR(!(PIDList[NumIDs]==-1),ierr);
//EPETRA_TEST_ERR(!(LIDList[NumIDs++]==-1),ierr);
if (Map.MinMyGID()-1>=Map.MinAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs++]==MyPID-1),ierr);
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs]==MyPID+1),ierr);
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(LIDList[NumIDs++]==0),ierr);
delete [] GIDList;
delete [] PIDList;
delete [] LIDList;
}
return (ierr);
}
/* Find the DBBD form */
int shylu_symbolic_factor
(
Epetra_CrsMatrix *A, // i/p: A matrix
shylu_symbolic *ssym, // symbolic structure
shylu_data *data, // numeric structure, TODO: Required ?
shylu_config *config // i/p: library configuration
)
{
#ifdef TIMING_OUTPUT
Teuchos::Time symtime("symbolic time");
symtime.start();
#endif
int myPID = A->Comm().MyPID();
int n = A->NumGlobalRows();
int Dnr;
int Snr;
int *DRowElems;
int *SRowElems;
int sym = config->sym;
checkMaps(A);
// Get column map
Epetra_Map AColMap = A->ColMap();
int ncols = AColMap.NumMyElements();
int *cols = AColMap.MyGlobalElements();
// Get row map
Epetra_Map ARowMap = A->RowMap();
int nrows = ARowMap.NumMyElements();
int *rows = ARowMap.MyGlobalElements();
// Find all columns in this proc
int *gvals = new int[n]; // vector of size n, not ncols !
// gvals[local cols] = 1, gvals[shared cols] > 1.
int SNumGlobalCols;
findLocalColumns(A, gvals, SNumGlobalCols);
// See if you can shrink the separator by assigning more rows/columns to
// the block diagonals
// TODO: This is because of a bug in coloring remove the if once that is
// fixed
//if (config->schurApproxMethod == 2)
if (config->sep_type == 2)
findNarrowSeparator(A, gvals);
// 3. Assemble diagonal block and the border in convenient form [
/* In each processor, we have (in a permuted form)
* | D_i C_i |
* | R_i S_i |
* D_i - diagonal block, C_i - Column Separator, R_i - Row separator
* S_i - A22 block corresponding to Schur complement part of A
* Assemble all four blocks in local matrices. */
ostringstream ssmsg1;
ssmsg1 << "PID =" << myPID << " ";
string msg = ssmsg1.str();
ssmsg1.clear(); ssmsg1.str("");
// Find #cols in each block
int Dnc = 0; // #cols in diagonal block
int Snc = 0; // #cols in the col. separator
/* Looping on cols will work only for wide separator
* as for narrow sep there will be some sep cols with gvals[col] ==1
* */
/*for (int i = 0; i < ncols ; i++)
{
if (gvals[cols[i]] == 1)
Dnc++;
else
Snc++;
}
// Find #rows in each block
Dnr = Dnc; // #rows in square diagonal block
Snr = nrows - Dnr; // #rows in the row separator*/
// Find #rows in each block
Dnr = 0;
Snr = 0;
for (int i = 0; i < nrows ; i++)
{
if (gvals[rows[i]] == 1)
Dnr++;
else
Snr++;
}
Dnc = Dnr;
// TODO: Snc is no longer useful, should remove it
for (int i = 0; i < ncols ; i++)
{
if (gvals[cols[i]] != 1)
Snc++;
}
assert(Snc >= 0);
// TODO : The above assignment may not be correct in the unsymetric case
////config->dm.print(2, msg + " Mycols=");
cout << msg << " Mycols="<< ncols << "Myrows ="<< nrows << endl;
cout << msg << " #rows and #cols in diagonal blk ="<< Dnr << endl;
cout << msg << " #columns in S ="<< Snc << endl;
cout << msg << " #rows in S ="<< Snr << endl;
ostringstream pidstr;
pidstr << myPID ;
// Create a row map for the D and S blocks [
DRowElems = new int[Dnr];
SRowElems = new int[Snr];
int gid;
// Assemble row ids in two arrays (for D and R blocks)
if (sym)
{
findBlockElems(A, nrows, rows, gvals, Dnr, DRowElems, Snr, SRowElems,
"D"+pidstr.str()+"Rows", "S"+pidstr.str()+"Rows", false) ;
}
else
{
// SRowElems are not known until factorization, TODO
assert(0 == 1);
}
data->Dnr = Dnr;
data->Snr = Snr;
data->Dnc = Dnc;
data->DRowElems = DRowElems;
data->SRowElems = SRowElems;
// Create a column map for the D and S blocks [
int *DColElems = new int[Dnc]; // Elems in column map of D
int *SColElems = new int[Snc]; // Elems in column map of C TODO: Unused
// Assemble column ids in two arrays (for D and C blocks)
findBlockElems(A, ncols, cols, gvals, Dnc, DColElems, Snc, SColElems,
"D"+pidstr.str()+"Cols", "S"+pidstr.str()+"Cols", true) ;
data->DColElems = DColElems;
data->gvals = gvals;
for (int i = 0; i < Snr; i++)
{
// Epetra guarentees columns corresponding to local rows will be first
// in the column map.
assert(SRowElems[i] == SColElems[i]);
}
// ]
/*--Create the Epetra Matrices with the maps (does not insert values) --- */
create_matrices(A, ssym, data, config);
/*--Extract the Epetra Matrices and call fillComplete --- */
extract_matrices(A, ssym, data, config, true);
delete[] SColElems;
Amesos Factory;
const char* SolverType = config->diagonalBlockSolver.c_str();
bool IsAvailable = Factory.Query(SolverType);
assert(IsAvailable == true);
Teuchos::RCP<Epetra_LinearProblem> LP = Teuchos::RCP<Epetra_LinearProblem>
(new Epetra_LinearProblem());
LP->SetOperator((ssym->D).getRawPtr());
//LP->SetOperator((ssym->DT).getRawPtr()); // for transpose
// Create temp vectors
ssym->Dlhs = Teuchos::RCP<Epetra_MultiVector>
(new Epetra_MultiVector(ssym->D->RowMap(), 16));
ssym->Drhs = Teuchos::RCP<Epetra_MultiVector>
(new Epetra_MultiVector(ssym->D->RowMap(), 16));
ssym->Gvec = Teuchos::RCP<Epetra_MultiVector>
(new Epetra_MultiVector(ssym->G->RowMap(), 16));
LP->SetRHS(ssym->Drhs.getRawPtr());
LP->SetLHS(ssym->Dlhs.getRawPtr());
ssym->ReIdx_LP = Teuchos::RCP<
EpetraExt::ViewTransform<Epetra_LinearProblem> >
(new EpetraExt::LinearProblem_Reindex2(0));
ssym->LP = Teuchos::RCP<Epetra_LinearProblem>(&((*(ssym->ReIdx_LP))(*LP)),
false);
Teuchos::RCP<Amesos_BaseSolver> Solver = Teuchos::RCP<Amesos_BaseSolver>
(Factory.Create(SolverType, *(ssym->LP)));
//config->dm.print(5, "Created the diagonal solver");
#ifdef TIMING_OUTPUT
Teuchos::Time ftime("setup time");
ftime.start();
#endif
//Solver->SetUseTranspose(true); // for transpose
Teuchos::ParameterList aList;
aList.set("TrustMe", true);
Solver->SetParameters(aList);
Solver->SymbolicFactorization();
//config->dm.print(3, "Symbolic Factorization done");
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Symbolic Factorization Time" << ftime.totalElapsedTime() << endl;
ftime.reset();
#endif
ssym->OrigLP = LP;
//ssym->LP = LP;
ssym->Solver = Solver;
if (config->schurApproxMethod == 1)
{
Teuchos::ParameterList pList;
Teuchos::RCP<Isorropia::Epetra::Prober> prober =
Teuchos::RCP<Isorropia::Epetra::Prober> (new
Isorropia::Epetra::Prober((ssym->Sg).getRawPtr(),
pList, false));
//config->dm.print(3, "Doing Coloring");
#ifdef TIMING_OUTPUT
ftime.start();
#endif
prober->color();
#ifdef TIMING_OUTPUT
ftime.stop();
cout << "Time to color" << ftime.totalElapsedTime() << endl;
ftime.reset();
ftime.start();
#endif
ssym->prober = prober;
}
#ifdef TIMING_OUTPUT
symtime.stop();
cout << "Symbolic Time" << symtime.totalElapsedTime() << endl;
symtime.reset();
#endif
}
int main(int argc, char *argv[]) {
int ierr=0, returnierr=0;
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
bool verbose = false;
// Check if we should print results to standard out
if (argc>1) if (argv[1][0]=='-' && argv[1][1]=='v') verbose = true;
if (!verbose) {
Comm.SetTracebackMode(0); // This should shut down any error traceback reporting
}
int MyPID = Comm.MyPID();
int NumProc = Comm.NumProc();
if (verbose && MyPID==0)
cout << Epetra_Version() << endl << endl;
if (verbose) cout << Comm << endl;
bool verbose1 = verbose;
if (verbose) verbose = (MyPID==0);
int NumMyElements = 10000;
int NumMyElements1 = NumMyElements; // Used for local map
long long NumGlobalElements = NumMyElements*NumProc+EPETRA_MIN(NumProc,3);
if (MyPID < 3) NumMyElements++;
int IndexBase = 0;
bool DistributedGlobal = (NumGlobalElements>NumMyElements);
Epetra_Map* Map;
// Test exceptions
if (verbose)
cout << "*******************************************************************************************" << endl
<< " Testing Exceptions (Expect error messages if EPETRA_NO_ERROR_REPORTS is not defined" << endl
<< "*******************************************************************************************" << endl
<< endl << endl;
try {
if (verbose) cout << "Checking Epetra_Map(-2, IndexBase, Comm)" << endl;
Map = new Epetra_Map((long long)-2, IndexBase, Comm);
}
catch (int Error) {
if (Error!=-1) {
if (Error!=0) {
EPETRA_TEST_ERR(Error,returnierr);
if (verbose) cout << "Error code should be -1" << endl;
}
else {
cout << "Error code = " << Error << "Should be -1" << endl;
returnierr+=1;
}
}
else if (verbose) cout << "Checked OK\n\n" << endl;
}
try {
if (verbose) cout << "Checking Epetra_Map(2, 3, IndexBase, Comm)" << endl;
Map = new Epetra_Map((long long)2, 3, IndexBase, Comm);
}
catch (int Error) {
if (Error!=-4) {
if (Error!=0) {
EPETRA_TEST_ERR(Error,returnierr);
if (verbose) cout << "Error code should be -4" << endl;
}
else {
cout << "Error code = " << Error << "Should be -4" << endl;
returnierr+=1;
}
}
else if (verbose) cout << "Checked OK\n\n" << endl;
}
if (verbose) cerr << flush;
if (verbose) cout << flush;
Comm.Barrier();
if (verbose)
cout << endl << endl
<< "*******************************************************************************************" << endl
<< " Testing valid constructor now......................................................" << endl
<< "*******************************************************************************************" << endl
<< endl << endl;
// Test Epetra-defined uniform linear distribution constructor
Map = new Epetra_Map(NumGlobalElements, IndexBase, Comm);
if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, IndexBase, Comm)" << endl;
ierr = checkmap(*Map, NumGlobalElements, NumMyElements, 0,
IndexBase, Comm, DistributedGlobal);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
delete Map;
// Test User-defined linear distribution constructor
Map = new Epetra_Map(NumGlobalElements, NumMyElements, IndexBase, Comm);
if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, NumMyElements, IndexBase, Comm)" << endl;
ierr = checkmap(*Map, NumGlobalElements, NumMyElements, 0,
IndexBase, Comm, DistributedGlobal);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
delete Map;
// Test User-defined arbitrary distribution constructor
// Generate Global Element List. Do in reverse for fun!
long long * MyGlobalElements = new long long[NumMyElements];
int MaxMyGID = (Comm.MyPID()+1)*NumMyElements-1+IndexBase;
if (Comm.MyPID()>2) MaxMyGID+=3;
for (int i = 0; i<NumMyElements; i++) MyGlobalElements[i] = MaxMyGID-i;
Map = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements,
IndexBase, Comm);
if (verbose) cout << "Checking Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm)" << endl;
ierr = checkmap(*Map, NumGlobalElements, NumMyElements, MyGlobalElements,
IndexBase, Comm, DistributedGlobal);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
// Test Copy constructor
Epetra_Map* Map1 = new Epetra_Map(*Map);
// Test SameAs() method
bool same = Map1->SameAs(*Map);
EPETRA_TEST_ERR(!(same==true),ierr);// should return true since Map1 is a copy of Map
Epetra_BlockMap* Map2 = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase, Comm);
same = Map2->SameAs(*Map);
EPETRA_TEST_ERR(!(same==true),ierr); // Map and Map2 were created with the same sets of parameters
delete Map2;
// now test SameAs() on a map that is different
Map2 = new Epetra_Map(NumGlobalElements, NumMyElements, MyGlobalElements, IndexBase-1, Comm);
same = Map2->SameAs(*Map);
EPETRA_TEST_ERR(!(same==false),ierr); // IndexBases are different
delete Map2;
// Back to testing copy constructor
if (verbose) cout << "Checking Epetra_Map(*Map)" << endl;
ierr = checkmap(*Map1, NumGlobalElements, NumMyElements, MyGlobalElements,
IndexBase, Comm, DistributedGlobal);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
Epetra_Map* SmallMap = 0;
if (verbose1) {
// Build a small map for test cout. Use 10 elements from current map
long long* MyEls = Map->MyGlobalElements64();
int IndBase = Map->IndexBase();
int MyLen = EPETRA_MIN(10+Comm.MyPID(),Map->NumMyElements());
SmallMap = new Epetra_Map((long long)-1, MyLen, MyEls, IndBase, Comm);
}
delete [] MyGlobalElements;
delete Map;
delete Map1;
// Test reference-counting in Epetra_Map
if (verbose) cout << "Checking Epetra_Map reference counting" << endl;
ierr = checkMapDataClass(Comm, verbose);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
// Test LocalMap constructor
Epetra_LocalMap* LocalMap = new Epetra_LocalMap((long long)NumMyElements1, IndexBase, Comm);
if (verbose) cout << "Checking Epetra_LocalMap(NumMyElements1, IndexBase, Comm)" << endl;
ierr = checkmap(*LocalMap, NumMyElements1, NumMyElements1, 0, IndexBase, Comm, false);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
// Test Copy constructor
Epetra_LocalMap* LocalMap1 = new Epetra_LocalMap(*LocalMap);
if (verbose) cout << "Checking Epetra_LocalMap(*LocalMap)" << endl;
ierr = checkmap(*LocalMap1, NumMyElements1, NumMyElements1, 0, IndexBase, Comm, false);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
delete LocalMap1;
delete LocalMap;
// Test reference-counting in Epetra_LocalMap
if (verbose) cout << "Checking Epetra_LocalMap reference counting" << endl;
ierr = checkLocalMapDataClass(Comm, verbose);
EPETRA_TEST_ERR(ierr,returnierr);
if (verbose && ierr==0) cout << "Checked OK\n\n" <<endl;
// Test output
if (verbose1) {
if (verbose) cout << "Test ostream << operator" << endl << flush;
cout << *SmallMap;
delete SmallMap;
}
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
return returnierr;
}
// ============================================================================
void EpetraExt::XMLWriter::
Write(const std::string& Label, const Epetra_Map& Map)
{
TEUCHOS_TEST_FOR_EXCEPTION(IsOpen_ == false, std::logic_error,
"No file has been opened");
int NumGlobalElements = Map.NumGlobalElements();
int* MyGlobalElements = Map.MyGlobalElements();
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "<Map Label=\"" << Label
<< "\" NumElements=\"" << NumGlobalElements << '"'
<< " IndexBase=\"" << Map.IndexBase() << '"'
<< " NumProc=\"" << Comm_.NumProc() << '"';
of.close();
}
for (int iproc = 0; iproc < Comm_.NumProc(); ++iproc)
{
if (iproc == Comm_.MyPID())
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << " ElementsOnProc" << iproc << "=\"" << Map.NumMyElements() << '"';
of.close();
}
Comm_.Barrier();
}
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << '>' << std::endl;
of.close();
}
for (int iproc = 0; iproc < Comm_.NumProc(); iproc++)
{
if (iproc == Comm_.MyPID())
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "<Proc ID=\"" << Comm_.MyPID() << "\">" << std::endl;
for (int i = 0; i < Map.NumMyElements(); ++i)
{
of << MyGlobalElements[i] << std::endl;
}
of << "</Proc>" << std::endl;
of.close();
}
Comm_.Barrier();
}
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "</Map>" << std::endl;
of.close();
}
}
int
main (int argc, char *argv[])
{
// These "using" statements make the code a bit more concise.
using std::cout;
using std::endl;
int ierr = 0, i;
// If Trilinos was built with MPI, initialize MPI, otherwise
// initialize the serial "communicator" that stands in for MPI.
#ifdef EPETRA_MPI
MPI_Init (&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
const int MyPID = Comm.MyPID();
const int NumProc = Comm.NumProc();
// We only allow (MPI) Process 0 to write to stdout.
const bool verbose = (MyPID == 0);
const int NumGlobalElements = 100;
if (verbose)
cout << Epetra_Version() << endl << endl;
// Asking the Epetra_Comm to print itself is a good test for whether
// you are running in an MPI environment. However, it will print
// something on all MPI processes, so you should remove it for a
// large-scale parallel run.
cout << Comm << endl;
if (NumGlobalElements < NumProc)
{
if (verbose)
cout << "numGlobalBlocks = " << NumGlobalElements
<< " cannot be < number of processors = " << NumProc << endl;
std::exit (EXIT_FAILURE);
}
// Construct a Map that puts approximately the same number of rows
// of the matrix A on each processor.
Epetra_Map Map (NumGlobalElements, 0, Comm);
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map.NumMyElements();
std::vector<int> MyGlobalElements(NumMyElements);
Map.MyGlobalElements(&MyGlobalElements[0]);
// NumNz[i] is the number of nonzero elements in row i of the sparse
// matrix on this MPI process. Epetra_CrsMatrix uses this to figure
// out how much space to allocate.
std::vector<int> NumNz (NumMyElements);
// We are building a tridiagonal matrix where each row contains the
// nonzero elements (-1 2 -1). Thus, we need 2 off-diagonal terms,
// except for the first and last row of the matrix.
for (int i = 0; i < NumMyElements; ++i)
if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1)
NumNz[i] = 2; // First or last row
else
NumNz[i] = 3; // Not the (first or last row)
// Create the Epetra_CrsMatrix.
Epetra_CrsMatrix A (Copy, Map, &NumNz[0]);
//
// Add rows to the sparse matrix one at a time.
//
std::vector<double> Values(2);
Values[0] = -1.0; Values[1] = -1.0;
std::vector<int> Indices(2);
const double two = 2.0;
int NumEntries;
for (int i = 0; i < NumMyElements; ++i)
{
if (MyGlobalElements[i] == 0)
{ // The first row of the matrix.
Indices[0] = 1;
NumEntries = 1;
}
else if (MyGlobalElements[i] == NumGlobalElements - 1)
{ // The last row of the matrix.
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
}
else
{ // Any row of the matrix other than the first or last.
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
ierr = A.InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert (ierr==0);
// Insert the diagonal entry.
ierr = A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i]);
assert(ierr==0);
}
// Finish up. We can call FillComplete() with no arguments, because
// the matrix is square.
ierr = A.FillComplete ();
assert (ierr==0);
// Parameters for the power method.
const int niters = NumGlobalElements*10;
const double tolerance = 1.0e-2;
//
// Run the power method. Keep track of the flop count and the total
// elapsed time.
//
Epetra_Flops counter;
A.SetFlopCounter(counter);
Epetra_Time timer(Comm);
double lambda = 0.0;
ierr += powerMethod (lambda, A, niters, tolerance, verbose);
double elapsedTime = timer.ElapsedTime ();
double totalFlops =counter.Flops ();
// Mflop/s: Million floating-point arithmetic operations per second.
double Mflop_per_s = totalFlops / elapsedTime / 1000000.0;
if (verbose)
cout << endl << endl << "Total Mflop/s for first solve = "
<< Mflop_per_s << endl<< endl;
// Increase the first (0,0) diagonal entry of the matrix.
if (verbose)
cout << endl << "Increasing magnitude of first diagonal term, solving again"
<< endl << endl << endl;
if (A.MyGlobalRow (0)) {
int numvals = A.NumGlobalEntries (0);
std::vector<double> Rowvals (numvals);
std::vector<int> Rowinds (numvals);
A.ExtractGlobalRowCopy (0, numvals, numvals, &Rowvals[0], &Rowinds[0]); // Get A(0,0)
for (int i = 0; i < numvals; ++i)
if (Rowinds[i] == 0)
Rowvals[i] *= 10.0;
A.ReplaceGlobalValues (0, numvals, &Rowvals[0], &Rowinds[0]);
}
//
// Run the power method again. Keep track of the flop count and the
// total elapsed time.
//
lambda = 0.0;
timer.ResetStartTime();
counter.ResetFlops();
ierr += powerMethod (lambda, A, niters, tolerance, verbose);
elapsedTime = timer.ElapsedTime();
totalFlops = counter.Flops();
Mflop_per_s = totalFlops / elapsedTime / 1000000.0;
if (verbose)
cout << endl << endl << "Total Mflop/s for second solve = "
<< Mflop_per_s << endl << endl;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return ierr;
}
//
// Diagonal: 0=no change, 1=eliminate entry
// from the map for the largest row element in process 0
// 2=add diagonal entries to the matrix, with a zero value
// (assume row map contains all diagonal entries).
//
// ReindexRowMap:
// 0=no change, 1= add 2 (still contiguous), 2=non-contiguous
//
// ReindexColMap
// 0=same as RowMap, 1=add 4 - Different From RowMap, but contiguous)
//
// RangeMap:
// 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map
//
// DomainMap:
// 0=no change, 1=serial map, 2=bizarre distribution, 3=replicated map
//
RCP<Epetra_CrsMatrix> NewMatNewMap(Epetra_CrsMatrix& In,
int Diagonal,
int ReindexRowMap,
int ReindexColMap,
int RangeMapType,
int DomainMapType
)
{
//
// If we are making no change, return the original matrix (which has a linear map)
//
#if 0
std::cout << __FILE__ << "::" << __LINE__ << " "
<< Diagonal << " "
<< ReindexRowMap << " "
<< ReindexColMap << " "
<< RangeMapType << " "
<< DomainMapType << " " << std::endl ;
#endif
if ( Diagonal + ReindexRowMap + ReindexColMap + RangeMapType + DomainMapType == 0 ) {
RCP<Epetra_CrsMatrix> ReturnOrig = rcp( &In, false );
return ReturnOrig ;
}
//
// Diagonal==2 is used for a different purpose -
// Making sure that the diagonal of the matrix is non-empty.
// Note: The diagonal must exist in In.RowMap().
//
if ( Diagonal == 2 ) {
assert( ReindexRowMap==0 && ReindexColMap == 0 ) ;
}
int (*RowPermute)(int in) = 0;
int (*ColPermute)(int in) = 0;
assert( Diagonal >= 0 && Diagonal <= 2 );
assert( ReindexRowMap>=0 && ReindexRowMap<=2 );
assert( ReindexColMap>=0 && ReindexColMap<=1 );
assert( RangeMapType>=0 && RangeMapType<=3 );
assert( DomainMapType>=0 && DomainMapType<=3 );
Epetra_Map DomainMap = In.DomainMap();
Epetra_Map RangeMap = In.RangeMap();
Epetra_Map ColMap = In.ColMap();
Epetra_Map RowMap = In.RowMap();
int NumMyRowElements = RowMap.NumMyElements();
int NumMyColElements = ColMap.NumMyElements();
int NumMyRangeElements = RangeMap.NumMyElements();
int NumMyDomainElements = DomainMap.NumMyElements();
int NumGlobalRowElements = RowMap.NumGlobalElements();
int NumGlobalColElements = ColMap.NumGlobalElements();
int NumGlobalRangeElements = RangeMap.NumGlobalElements();
int NumGlobalDomainElements = DomainMap.NumGlobalElements();
assert( NumGlobalRangeElements == NumGlobalDomainElements ) ;
std::vector<int> MyGlobalRowElements( NumMyRowElements ) ;
std::vector<int> NumEntriesPerRow( NumMyRowElements ) ;
std::vector<int> MyPermutedGlobalRowElements( NumMyRowElements ) ;
std::vector<int> MyGlobalColElements( NumMyColElements ) ;
std::vector<int> MyPermutedGlobalColElements( NumMyColElements ) ; // Used to create the column map
std::vector<int> MyPermutedGlobalColElementTable( NumMyColElements ) ; // To convert local indices to global
std::vector<int> MyGlobalRangeElements( NumMyRangeElements ) ;
std::vector<int> MyPermutedGlobalRangeElements( NumMyRangeElements ) ;
std::vector<int> MyGlobalDomainElements( NumMyDomainElements ) ;
std::vector<int> MyPermutedGlobalDomainElements( NumMyDomainElements ) ;
RowMap.MyGlobalElements(&MyGlobalRowElements[0]);
ColMap.MyGlobalElements(&MyGlobalColElements[0]);
RangeMap.MyGlobalElements(&MyGlobalRangeElements[0]);
DomainMap.MyGlobalElements(&MyGlobalDomainElements[0]);
switch( ReindexRowMap ) {
case 0:
RowPermute = &NoPermute ;
break;
case 1:
RowPermute = &SmallRowPermute ;
break;
case 2:
RowPermute = BigRowPermute ;
break;
}
switch( ReindexColMap ) {
case 0:
ColPermute = RowPermute ;
break;
case 1:
ColPermute = &SmallColPermute ;
break;
}
//
// Create Serial Range and Domain Maps based on the permuted indexing
//
int nlocal = 0;
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements;
std::vector<int> AllIDs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDs[i] = (*RowPermute)( i ) ;
Epetra_Map SerialRangeMap( -1, nlocal, &AllIDs[0], 0, In.Comm());
std::vector<int> AllIDBs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDBs[i] = (*ColPermute)( i ) ;
Epetra_Map SerialDomainMap( -1, nlocal, &AllIDBs[0], 0, In.Comm());
//
// Create Bizarre Range and Domain Maps based on the permuted indexing
// These are nearly serial, having all but one element on process 0
// The goal here is to make sure that we can use Domain and Range maps
// that are neither serial, nor distributed in the normal manner.
//
std::vector<int> AllIDCs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDCs[i] = (*ColPermute)( i ) ;
if ( In.Comm().NumProc() > 1 ) {
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1;
if (In.Comm().MyPID()==1) {
nlocal = 1;
AllIDCs[0] = (*ColPermute)( NumGlobalRangeElements - 1 );
}
}
int iam = In.Comm().MyPID();
Epetra_Map BizarreDomainMap( -1, nlocal, &AllIDCs[0], 0, In.Comm());
std::vector<int> AllIDDs( NumGlobalRangeElements ) ;
for ( int i = 0; i < NumGlobalRangeElements ; i++ ) AllIDDs[i] = (*RowPermute)( i ) ;
if ( In.Comm().NumProc() > 1 ) {
if (In.Comm().MyPID()==0) nlocal = NumGlobalRangeElements-1;
if (In.Comm().MyPID()==1) {
nlocal = 1;
AllIDDs[0] = (*RowPermute)( NumGlobalRangeElements -1 ) ;
}
}
Epetra_Map BizarreRangeMap( -1, nlocal, &AllIDDs[0], 0, In.Comm());
//
// Compute the column map
//
// If Diagonal==1, remove the column corresponding to the last row owned
// by process 0. Removing this column from a tridiagonal matrix, leaves
// a disconnected, but non-singular matrix.
//
int NumMyColElementsOut = 0 ;
int NumGlobalColElementsOut ;
if ( Diagonal == 1 )
NumGlobalColElementsOut = NumGlobalColElements-1;
else
NumGlobalColElementsOut = NumGlobalColElements;
if ( Diagonal == 1 && iam==0 ) {
for ( int i=0; i < NumMyColElements ; i++ ) {
if ( MyGlobalColElements[i] != MyGlobalRowElements[NumMyRowElements-1] ) {
MyPermutedGlobalColElements[NumMyColElementsOut++] =
(*ColPermute)( MyGlobalColElements[i] ) ;
}
}
assert( NumMyColElementsOut == NumMyColElements-1 );
} else {
for ( int i=0; i < NumMyColElements ; i++ )
MyPermutedGlobalColElements[i] =
(*ColPermute)( MyGlobalColElements[i] ) ;
NumMyColElementsOut = NumMyColElements ;
if ( Diagonal == 2 ) {
// For each row, make sure that the column map has this row in it,
// if it doesn't, add it to the column map.
// Note: MyPermutedGlobalColElements == MyGlobalColElements when
// Diagonal==2 because ( Diagonal == 2 ) implies:
// ReindexRowMap==0 && ReindexColMap == 0 - see assert above
for ( int i=0; i < NumMyRowElements ; i++ ) {
bool MissingDiagonal = true;
for ( int j=0; j < NumMyColElements; j++ ) {
if ( MyGlobalRowElements[i] == MyGlobalColElements[j] ) {
MissingDiagonal = false;
}
}
if ( MissingDiagonal ) {
MyPermutedGlobalColElements.resize(NumMyColElements+1);
MyPermutedGlobalColElements[NumMyColElementsOut] = MyGlobalRowElements[i];
NumMyColElementsOut++;
}
}
In.Comm().SumAll(&NumMyColElementsOut,&NumGlobalColElementsOut,1);
}
}
//
// These tables are used both as the permutation tables and to create the maps.
//
for ( int i=0; i < NumMyColElements ; i++ )
MyPermutedGlobalColElementTable[i] =
(*ColPermute)( MyGlobalColElements[i] ) ;
for ( int i=0; i < NumMyRowElements ; i++ )
MyPermutedGlobalRowElements[i] =
(*RowPermute)( MyGlobalRowElements[i] ) ;
for ( int i=0; i < NumMyRangeElements ; i++ )
MyPermutedGlobalRangeElements[i] =
(*RowPermute)( MyGlobalRangeElements[i] ) ;
for ( int i=0; i < NumMyDomainElements ; i++ )
MyPermutedGlobalDomainElements[i] =
(*ColPermute)( MyGlobalDomainElements[i] ) ;
RCP<Epetra_Map> PermutedRowMap =
rcp( new Epetra_Map( NumGlobalRowElements, NumMyRowElements,
&MyPermutedGlobalRowElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedColMap =
rcp( new Epetra_Map( NumGlobalColElementsOut, NumMyColElementsOut,
&MyPermutedGlobalColElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedRangeMap =
rcp( new Epetra_Map( NumGlobalRangeElements, NumMyRangeElements,
&MyPermutedGlobalRangeElements[0], 0, In.Comm() ) );
RCP<Epetra_Map> PermutedDomainMap =
rcp( new Epetra_Map( NumGlobalDomainElements, NumMyDomainElements,
&MyPermutedGlobalDomainElements[0], 0, In.Comm() ) );
//
// These vectors are filled and then passed to InsertGlobalValues
//
std::vector<int> ThisRowIndices( In.MaxNumEntries() );
std::vector<double> ThisRowValues( In.MaxNumEntries() );
std::vector<int> PermutedGlobalColIndices( In.MaxNumEntries() );
//std::cout << __FILE__ << "::" <<__LINE__ << std::endl ;
RCP<Epetra_CrsMatrix> Out =
rcp( new Epetra_CrsMatrix( Copy, *PermutedRowMap, *PermutedColMap, 0 ) );
for (int i=0; i<NumMyRowElements; i++)
{
int NumIndicesThisRow = 0;
assert( In.ExtractMyRowCopy( i,
In.MaxNumEntries(),
NumIndicesThisRow,
&ThisRowValues[0],
&ThisRowIndices[0] ) == 0 ) ;
for (int j = 0 ; j < NumIndicesThisRow ; j++ )
{
PermutedGlobalColIndices[j] = MyPermutedGlobalColElementTable[ ThisRowIndices[j] ] ;
}
bool MissingDiagonal = false;
if ( Diagonal==2 ) {
//
assert( MyGlobalRowElements[i] == MyPermutedGlobalRowElements[i] );
MissingDiagonal = true;
for( int j =0 ; j < NumIndicesThisRow ; j++ ) {
if ( PermutedGlobalColIndices[j] == MyPermutedGlobalRowElements[i] ) {
MissingDiagonal = false ;
}
}
#if 0
std::cout << __FILE__ << "::" << __LINE__
<< " i = " << i
<< " MyPermutedGlobalRowElements[i] = " << MyPermutedGlobalRowElements[i]
<< " MissingDiagonal = " << MissingDiagonal << std::endl ;
#endif
}
if ( MissingDiagonal ) {
ThisRowValues.resize(NumIndicesThisRow+1) ;
ThisRowValues[NumIndicesThisRow] = 0.0;
PermutedGlobalColIndices.resize(NumIndicesThisRow+1);
PermutedGlobalColIndices[NumIndicesThisRow] = MyPermutedGlobalRowElements[i] ;
#if 0
std::cout << __FILE__ << "::" << __LINE__
<< " i = " << i
<< "NumIndicesThisRow = " << NumIndicesThisRow
<< "ThisRowValues[NumIndicesThisRow = " << ThisRowValues[NumIndicesThisRow]
<< " PermutedGlobalColIndices[NumIndcesThisRow] = " << PermutedGlobalColIndices[NumIndicesThisRow]
<< std::endl ;
#endif
NumIndicesThisRow++ ;
}
assert( Out->InsertGlobalValues( MyPermutedGlobalRowElements[i],
NumIndicesThisRow,
&ThisRowValues[0],
&PermutedGlobalColIndices[0] ) >= 0 );
}
//
Epetra_LocalMap ReplicatedMap( NumGlobalRangeElements, 0, In.Comm() );
RCP<Epetra_Map> OutRangeMap ;
RCP<Epetra_Map> OutDomainMap ;
switch( RangeMapType ) {
case 0:
OutRangeMap = PermutedRangeMap ;
break;
case 1:
OutRangeMap = rcp(&SerialRangeMap, false);
break;
case 2:
OutRangeMap = rcp(&BizarreRangeMap, false);
break;
case 3:
OutRangeMap = rcp(&ReplicatedMap, false);
break;
}
// switch( DomainMapType ) {
switch( DomainMapType ) {
case 0:
OutDomainMap = PermutedDomainMap ;
break;
case 1:
OutDomainMap = rcp(&SerialDomainMap, false);
break;
case 2:
OutDomainMap = rcp(&BizarreDomainMap, false);
break;
case 3:
OutDomainMap = rcp(&ReplicatedMap, false);
break;
}
#if 0
assert(Out->FillComplete( *PermutedDomainMap, *PermutedRangeMap )==0);
#else
assert(Out->FillComplete( *OutDomainMap, *OutRangeMap )==0);
#endif
#if 0
std::cout << __FILE__ << "::" << __LINE__ << std::endl ;
Out->Print( std::cout ) ;
#endif
return Out;
}
// build maps to make other conversions
void buildSubMaps(const Epetra_Map & globalMap,const std::vector<int> & vars,const Epetra_Comm & comm,
std::vector<std::pair<int,Teuchos::RCP<Epetra_Map> > > & subMaps)
{
buildSubMaps(globalMap.NumGlobalElements(),globalMap.NumMyElements(),globalMap.MinMyGID(),
vars,comm,subMaps);
}
// FIXME long long
Epetra_Map
Epetra_Util::Create_Root_Map(const Epetra_Map& usermap,
int root)
{
int numProc = usermap.Comm().NumProc();
if (numProc==1) {
Epetra_Map newmap(usermap);
return(newmap);
}
const Epetra_Comm & comm = usermap.Comm();
bool isRoot = usermap.Comm().MyPID()==root;
//if usermap is already completely owned by root then we'll just return a copy of it.
int quickreturn = 0;
int globalquickreturn = 0;
if (isRoot) {
if (usermap.NumMyElements()==usermap.NumGlobalElements64()) quickreturn = 1;
}
else {
if (usermap.NumMyElements()==0) quickreturn = 1;
}
usermap.Comm().MinAll(&quickreturn, &globalquickreturn, 1);
if (globalquickreturn==1) {
Epetra_Map newmap(usermap);
return(newmap);
}
// Linear map: Simple case, just put all GIDs linearly on root processor
if (usermap.LinearMap() && root!=-1) {
int numMyElements = 0;
if (isRoot) numMyElements = usermap.MaxAllGID64()+1; // FIXME long long
Epetra_Map newmap(-1, numMyElements, usermap.IndexBase(), comm);
return(newmap);
}
if (!usermap.UniqueGIDs())
throw usermap.ReportError("usermap must have unique GIDs",-1);
// General map
// Build IntVector of the GIDs, then ship them to root processor
int numMyElements = usermap.NumMyElements();
Epetra_Map allGidsMap(-1, numMyElements, 0, comm);
Epetra_IntVector allGids(allGidsMap);
for (int i=0; i<numMyElements; i++) allGids[i] = usermap.GID64(i);
int numGlobalElements = usermap.NumGlobalElements64();
if (root!=-1) {
int n1 = 0; if (isRoot) n1 = numGlobalElements;
Epetra_Map allGidsOnRootMap(-1, n1, 0, comm);
Epetra_Import importer(allGidsOnRootMap, allGidsMap);
Epetra_IntVector allGidsOnRoot(allGidsOnRootMap);
allGidsOnRoot.Import(allGids, importer, Insert);
Epetra_Map rootMap(-1, allGidsOnRoot.MyLength(), allGidsOnRoot.Values(), usermap.IndexBase(), comm);
return(rootMap);
}
else {
int n1 = numGlobalElements;
Epetra_LocalMap allGidsOnRootMap(n1, 0, comm);
Epetra_Import importer(allGidsOnRootMap, allGidsMap);
Epetra_IntVector allGidsOnRoot(allGidsOnRootMap);
allGidsOnRoot.Import(allGids, importer, Insert);
Epetra_Map rootMap(-1, allGidsOnRoot.MyLength(), allGidsOnRoot.Values(), usermap.IndexBase(), comm);
return(rootMap);
}
}
int
main (int argc, char *argv[])
{
using namespace Anasazi;
using Teuchos::RCP;
using Teuchos::rcp;
using std::endl;
#ifdef HAVE_MPI
// Initialize MPI
MPI_Init (&argc, &argv);
#endif // HAVE_MPI
// Create an Epetra communicator
#ifdef HAVE_MPI
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif // HAVE_MPI
// Create an Anasazi output manager
BasicOutputManager<double> printer;
printer.stream(Errors) << Anasazi_Version() << std::endl << std::endl;
// Get the sorting std::string from the command line
std::string which ("LM");
Teuchos::CommandLineProcessor cmdp (false, true);
cmdp.setOption("sort", &which, "Targetted eigenvalues (SM or LM).");
if (cmdp.parse (argc, argv) != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return -1;
}
// Dimension of the matrix
//
// Discretization points in any one direction.
const int nx = 10;
// Size of matrix nx*nx
const int NumGlobalElements = nx*nx;
// Construct a Map that puts approximately the same number of
// equations on each process.
Epetra_Map Map (NumGlobalElements, 0, Comm);
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map.NumMyElements ();
std::vector<int> MyGlobalElements (NumMyElements);
Map.MyGlobalElements (&MyGlobalElements[0]);
// Create an integer vector NumNz that is used to build the Petra
// matrix. NumNz[i] is the number of OFF-DIAGONAL terms for the
// i-th global equation on this process.
std::vector<int> NumNz (NumMyElements);
/* We are building a matrix of block structure:
| T -I |
|-I T -I |
| -I T |
| ... -I|
| -I T|
where each block is dimension nx by nx and the matrix is on the order of
nx*nx. The block T is a tridiagonal matrix.
*/
for (int i=0; i<NumMyElements; ++i) {
if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1 ||
MyGlobalElements[i] == nx-1 || MyGlobalElements[i] == nx*(nx-1) ) {
NumNz[i] = 3;
}
else if (MyGlobalElements[i] < nx || MyGlobalElements[i] > nx*(nx-1) ||
MyGlobalElements[i]%nx == 0 || (MyGlobalElements[i]+1)%nx == 0) {
NumNz[i] = 4;
}
else {
NumNz[i] = 5;
}
}
// Create an Epetra_Matrix
RCP<Epetra_CrsMatrix> A = rcp (new Epetra_CrsMatrix (Epetra_DataAccess::Copy, Map, &NumNz[0]));
// Compute coefficients for discrete convection-diffution operator
const double one = 1.0;
std::vector<double> Values(4);
std::vector<int> Indices(4);
double rho = 0.0;
double h = one /(nx+1);
double h2 = h*h;
double c = 5.0e-01*rho/ h;
Values[0] = -one/h2 - c; Values[1] = -one/h2 + c; Values[2] = -one/h2; Values[3]= -one/h2;
double diag = 4.0 / h2;
int NumEntries;
for (int i=0; i<NumMyElements; ++i) {
if (MyGlobalElements[i]==0) {
Indices[0] = 1;
Indices[1] = nx;
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == nx*(nx-1)) {
Indices[0] = nx*(nx-1)+1;
Indices[1] = nx*(nx-2);
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == nx-1) {
Indices[0] = nx-2;
NumEntries = 1;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = 2*nx-1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] == NumGlobalElements-1) {
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = nx*(nx-1)-1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] < nx) {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]+nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i] > nx*(nx-1)) {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]-nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if (MyGlobalElements[i]%nx == 0) {
Indices[0] = MyGlobalElements[i]+1;
Indices[1] = MyGlobalElements[i]-nx;
Indices[2] = MyGlobalElements[i]+nx;
NumEntries = 3;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[1], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else if ((MyGlobalElements[i]+1)%nx == 0) {
Indices[0] = MyGlobalElements[i]-nx;
Indices[1] = MyGlobalElements[i]+nx;
NumEntries = 2;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[2], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
Indices[0] = MyGlobalElements[i]-1;
NumEntries = 1;
info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
else {
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
Indices[2] = MyGlobalElements[i]-nx;
Indices[3] = MyGlobalElements[i]+nx;
NumEntries = 4;
int info = A->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
// Put in the diagonal entry
int info = A->InsertGlobalValues(MyGlobalElements[i], 1, &diag, &MyGlobalElements[i]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "InsertGlobalValues returned info = "
<< info << " != 0." );
}
// Finish up
int info = A->FillComplete ();
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "A->FillComplete() returned info = "
<< info << " != 0." );
A->SetTracebackMode (1); // Shutdown Epetra Warning tracebacks
// Create a identity matrix for the temporary mass matrix
RCP<Epetra_CrsMatrix> M = rcp (new Epetra_CrsMatrix (Epetra_DataAccess::Copy, Map, 1));
for (int i=0; i<NumMyElements; i++) {
Values[0] = one;
Indices[0] = i;
NumEntries = 1;
info = M->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "M->InsertGlobalValues() returned info = "
<< info << " != 0." );
}
// Finish up
info = M->FillComplete ();
TEUCHOS_TEST_FOR_EXCEPTION
(info != 0, std::runtime_error, "M->FillComplete() returned info = "
<< info << " != 0." );
M->SetTracebackMode (1); // Shutdown Epetra Warning tracebacks
//************************************
// Call the LOBPCG solver manager
//***********************************
//
// Variables used for the LOBPCG Method
const int nev = 10;
const int blockSize = 5;
const int maxIters = 500;
const double tol = 1.0e-8;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef MultiVecTraits<double, Epetra_MultiVector> MVT;
// Create an Epetra_MultiVector for an initial vector to start the
// solver. Note: This needs to have the same number of columns as
// the blocksize.
RCP<Epetra_MultiVector> ivec = rcp (new Epetra_MultiVector (Map, blockSize));
ivec->Random (); // fill the initial vector with random values
// Create the eigenproblem.
RCP<BasicEigenproblem<double, MV, OP> > MyProblem =
rcp (new BasicEigenproblem<double, MV, OP> (A, ivec));
// Inform the eigenproblem that the operator A is symmetric
MyProblem->setHermitian (true);
// Set the number of eigenvalues requested
MyProblem->setNEV (nev);
// Tell the eigenproblem that you are finishing passing it information.
const bool success = MyProblem->setProblem ();
if (! success) {
printer.print (Errors, "Anasazi::BasicEigenproblem::setProblem() reported an error.\n");
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return -1;
}
// Create parameter list to pass into the solver manager
Teuchos::ParameterList MyPL;
MyPL.set ("Which", which);
MyPL.set ("Block Size", blockSize);
MyPL.set ("Maximum Iterations", maxIters);
MyPL.set ("Convergence Tolerance", tol);
MyPL.set ("Full Ortho", true);
MyPL.set ("Use Locking", true);
// Create the solver manager
LOBPCGSolMgr<double, MV, OP> MySolverMan (MyProblem, MyPL);
// Solve the problem
ReturnType returnCode = MySolverMan.solve ();
// Get the eigenvalues and eigenvectors from the eigenproblem
Eigensolution<double,MV> sol = MyProblem->getSolution ();
std::vector<Value<double> > evals = sol.Evals;
RCP<MV> evecs = sol.Evecs;
// Compute residuals.
std::vector<double> normR (sol.numVecs);
if (sol.numVecs > 0) {
Teuchos::SerialDenseMatrix<int,double> T (sol.numVecs, sol.numVecs);
Epetra_MultiVector tempAevec (Map, sol.numVecs );
T.putScalar (0.0);
for (int i = 0; i < sol.numVecs; ++i) {
T(i,i) = evals[i].realpart;
}
A->Apply (*evecs, tempAevec);
MVT::MvTimesMatAddMv (-1.0, *evecs, T, 1.0, tempAevec);
MVT::MvNorm (tempAevec, normR);
}
// Print the results
std::ostringstream os;
os.setf (std::ios_base::right, std::ios_base::adjustfield);
os << "Solver manager returned "
<< (returnCode == Converged ? "converged." : "unconverged.") << endl;
os << endl;
os << "------------------------------------------------------" << endl;
os << std::setw(16) << "Eigenvalue"
<< std::setw(18) << "Direct Residual"
<< endl;
os << "------------------------------------------------------" << endl;
for (int i = 0; i < sol.numVecs; ++i) {
os << std::setw(16) << evals[i].realpart
<< std::setw(18) << normR[i] / evals[i].realpart
<< endl;
}
os << "------------------------------------------------------" << endl;
printer.print (Errors, os.str ());
#ifdef HAVE_MPI
MPI_Finalize ();
#endif // HAVE_MPI
return 0;
}
static int run_test(Teuchos::RCP<Epetra_CrsMatrix> matrix,
bool verbose, // display the graph before & after
bool contract, // set global number of partitions to 1/2 num procs
int partitioningType, // hypergraph or graph partitioning, or simple
int vertexWeightType, // use vertex weights?
int edgeWeightType, // use edge/hyperedge weights?
int objectType) // use isorropia's CrsMatrix or CrsGraph
{
int rc=0, fail = 0;
#ifdef HAVE_EPETRAEXT
int localProc = 0;
double balance1, balance2, cutn1, cutn2, cutl1, cutl2;
double balance3, cutn3, cutl3;
double cutWgt1, cutWgt2, cutWgt3;
int numCuts1, numCuts2, numCuts3, valid;
int numPartitions = 0;
int keepDenseEdges = 0;
int numProcs = 1;
#ifdef HAVE_MPI
const Epetra_MpiComm &Comm = dynamic_cast<const Epetra_MpiComm &>(matrix->Comm());
localProc = Comm.MyPID();
numProcs = Comm.NumProc();
#else
const Epetra_SerialComm &Comm = dynamic_cast<const Epetra_SerialComm &>(matrix->Comm());
#endif
int numRows = matrix->NumGlobalRows();
if (numRows < (numProcs * 100)){
// By default Zoltan throws out dense edges, defined as those
// whose number of non-zeros exceeds 25% of the number of vertices.
//
// If dense edges are thrown out of a small matrix, there may be nothing left.
keepDenseEdges = 1;
}
double myShareBefore = 1.0 / numProcs;
double myShare = myShareBefore;
if (contract){
numPartitions = numProcs / 2;
if (numPartitions > numRows)
numPartitions = numRows;
if (numPartitions > 0){
if (localProc < numPartitions){
myShare = 1.0 / numPartitions;
}
else{
myShare = 0.0;
}
}
else{
contract = 0;
}
}
// If we want Zoltan's or Isorropia's default weights, then we don't
// need to supply a CostDescriber object to createBalancedCopy,
// so we get to test the API functions that don't take a CostDescriber.
bool noCosts = ((vertexWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS) &&
(edgeWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS));
// Test the interface that has no parameters, if possible
bool noParams =
((partitioningType == HYPERGRAPH_PARTITIONING) && // default, so requires no params
(numPartitions == 0) && // >0 would require a parameter
(keepDenseEdges == 0)); // >0 would require a parameter
// Maps for original object
const Epetra_Map &sourceRowMap = matrix->RowMap();
const Epetra_Map &sourceRangeMap = matrix->RangeMap();
// const Epetra_Map &sourceColMap = matrix->ColMap();
const Epetra_Map &sourceDomainMap = matrix->DomainMap();
int numCols = matrix->NumGlobalCols();
int nMyRows = sourceRowMap.NumMyElements();
int base = sourceRowMap.IndexBase();
// Compute vertex and edge weights
Isorropia::Epetra::CostDescriber costs;
Teuchos::RCP<Epetra_Vector> vptr;
Teuchos::RCP<Epetra_CrsMatrix> eptr;
Teuchos::RCP<Epetra_Vector> hyperEdgeWeights;
if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){
if (partitioningType == GRAPH_PARTITIONING){
// Create graph edge weights.
eptr = Teuchos::rcp(new Epetra_CrsMatrix(*matrix));
if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){
eptr->PutScalar(1.0); // set all nonzeros to 1.0
}
else{
int maxRowSize = eptr->MaxNumEntries();
double *newVal = NULL;
if (maxRowSize > 0){
newVal = new double [maxRowSize];
for (int j=0; j<maxRowSize; j++){
newVal[j] = localProc + 1 + j;
}
}
int numEntries;
int *idx;
double *val;
for (int i=0; i<nMyRows; i++){
rc = eptr->ExtractMyRowView(i, numEntries, val, idx);
for (int j=0; j<numEntries; j++){
val[j] = newVal[j];
}
}
if (newVal) delete [] newVal;
}
eptr->FillComplete(sourceDomainMap, sourceRangeMap);
costs.setGraphEdgeWeights(eptr);
}
else{
// Create hyperedge weights. (Note that the list of hyperedges that a
// process provides weights for has no relation to the columns
// that it has non-zeroes for, or the rows that is has. Hypergraphs
// in general are not square. Also more than one process can provide
// a weight for the same edge. Zoltan combines the weights according
// to the value of the PHG_EDGE_WEIGHT_OPERATION parameter. The default
// for this parameter is to use the maximum edge weight provided by any
// process for a given hyperedge.)
Epetra_Map hyperEdgeMap(numCols, base, Comm);
hyperEdgeWeights = Teuchos::rcp(new Epetra_Vector(hyperEdgeMap));
int *edgeGIDs = NULL;
double *weights = NULL;
int numHEweights = hyperEdgeMap.NumMyElements();
if (numHEweights){
edgeGIDs = new int [numHEweights];
weights = new double [numHEweights];
if (edgeWeightType == SUPPLY_EQUAL_WEIGHTS){
for (int i=0; i<numHEweights; i++){
edgeGIDs[i] = hyperEdgeMap.GID(i);
weights[i] = 1.0;
}
}
else{
int hiVolumeStart = matrix->NumGlobalCols() / 3;
int hiVolumeEnd = hiVolumeStart * 2;
for (int i=0; i<numHEweights; i++){
edgeGIDs[i] = hyperEdgeMap.GID(i);
if ((edgeGIDs[i] < hiVolumeStart) || (edgeGIDs[i] >= hiVolumeEnd)){
weights[i] = 1.0;
}
else{
weights[i] = 3.0;
}
}
}
hyperEdgeWeights->ReplaceGlobalValues(numHEweights, weights, edgeGIDs);
}
if (weights){
delete [] weights;
delete [] edgeGIDs;
}
costs.setHypergraphEdgeWeights(hyperEdgeWeights);
}
}
bool need_importer = false;
if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){
need_importer = true; // to redistribute row weights
double *val = NULL;
if (nMyRows){
val = new double [nMyRows];
if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){
for (int i=0; i<nMyRows; i++){
val[i] = 1.0;
}
}
else if (vertexWeightType == SUPPLY_UNEQUAL_WEIGHTS){
for (int i=0; i<nMyRows; i++){
val[i] = 1.0 + ((localProc+1) / 2);
}
}
}
vptr = Teuchos::rcp(new Epetra_Vector(Copy, sourceRowMap, val));
if (val) delete [] val;
costs.setVertexWeights(vptr);
}
// Calculate partition quality metrics before calling Zoltan
if (partitioningType == GRAPH_PARTITIONING){
rc = ispatest::compute_graph_metrics(matrix->Graph(), costs,
myShare, balance1, numCuts1, cutWgt1, cutn1, cutl1);
if (contract){
// balance wrt target of balancing weight over *all* procs
rc = ispatest::compute_graph_metrics(matrix->Graph(), costs,
myShareBefore, balance3, numCuts3, cutWgt3, cutn3, cutl3);
}
}
else{
rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs,
myShare, balance1, cutn1, cutl1);
if (contract){
// balance wrt target of balancing weight over *all* procs
rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs,
myShareBefore, balance3, cutn3, cutl3);
}
}
if (rc){
ERROREXIT((localProc==0), "Error in computing partitioning metrics")
}
Teuchos::ParameterList params;
#ifdef HAVE_ISORROPIA_ZOLTAN
if (!noParams){
// We're using Zoltan for partitioning and supplying
// parameters, overriding defaults.
Teuchos::ParameterList &sublist = params.sublist("Zoltan");
if (partitioningType == GRAPH_PARTITIONING){
params.set("PARTITIONING METHOD", "GRAPH");
sublist.set("GRAPH_PACKAGE", "PHG");
}
else{
params.set("PARTITIONING METHOD", "HYPERGRAPH");
sublist.set("LB_APPROACH", "PARTITION");
sublist.set("PHG_CUT_OBJECTIVE", "CONNECTIVITY"); // "cutl"
}
if (keepDenseEdges){
// only throw out rows that have no zeroes, default is to
// throw out if .25 or more of the columns are non-zero
sublist.set("PHG_EDGE_SIZE_THRESHOLD", "1.0");
}
if (numPartitions > 0){
// test #Partitions < #Processes
std::ostringstream os;
os << numPartitions;
std::string s = os.str();
// sublist.set("NUM_GLOBAL_PARTS", s);
params.set("NUM PARTS", s);
}
//sublist.set("DEBUG_LEVEL", "1"); // Zoltan will print out parameters
//sublist.set("DEBUG_LEVEL", "5"); // proc 0 will trace Zoltan calls
//sublist.set("DEBUG_MEMORY", "2"); // Zoltan will trace alloc & free
}
#else
ERROREXIT((localProc==0),
"Zoltan partitioning required but Zoltan not available.")
#endif
// Function scope values
Teuchos::RCP<Epetra_Vector> newvwgts;
Teuchos::RCP<Epetra_CrsMatrix> newewgts;
// Function scope values required for LinearProblem
Epetra_LinearProblem *problem = NULL;
Epetra_Map *LHSmap = NULL;
Epetra_MultiVector *RHS = NULL;
Epetra_MultiVector *LHS = NULL;
// Reference counted pointer to balanced object
Epetra_CrsMatrix *matrixPtr=NULL;
Epetra_CrsGraph *graphPtr=NULL;
Epetra_RowMatrix *rowMatrixPtr=NULL;
Epetra_LinearProblem *problemPtr=NULL;
// Row map for balanced object
const Epetra_BlockMap *targetBlockRowMap=NULL; // for input CrsGraph
const Epetra_Map *targetRowMap=NULL; // for all other inputs
// Column map for balanced object
const Epetra_BlockMap *targetBlockColMap=NULL; // for input CrsGraph
const Epetra_Map *targetColMap=NULL; // for all other inputs
if (objectType == EPETRA_CRSMATRIX){
if (noParams && noCosts){
matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix);
}
else if (noCosts){
matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params);
}
targetRowMap = &(matrixPtr->RowMap());
targetColMap = &(matrixPtr->ColMap());
}
else if (objectType == EPETRA_CRSGRAPH){
const Epetra_CrsGraph graph = matrix->Graph();
if (noParams && noCosts){
graphPtr = Isorropia::Epetra::createBalancedCopy(graph);
}
else if (noCosts){
graphPtr = Isorropia::Epetra::createBalancedCopy(graph, params);
}
targetBlockRowMap = &(graphPtr->RowMap());
targetBlockColMap = &(graphPtr->ColMap());
}
else if (objectType == EPETRA_ROWMATRIX){
if (noParams && noCosts){
rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix);
}
else if (noCosts){
rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params);
}
targetRowMap = &(rowMatrixPtr->RowMatrixRowMap());
targetColMap = &(rowMatrixPtr->RowMatrixColMap());
}
else if (objectType == EPETRA_LINEARPROBLEM){
// Create a linear problem with this matrix.
LHSmap = new Epetra_Map(numCols, base, Comm);
int myRHSsize = sourceRowMap.NumMyElements();
int myLHSsize = LHSmap->NumMyElements();
int valSize = ((myRHSsize > myLHSsize) ? myRHSsize : myLHSsize);
double *vals = NULL;
if (valSize){
vals = new double [valSize];
}
if (valSize){
for (int i=0; i < valSize; i++){
// put my rank in my portion of LHS and my portion of RHS
vals[i] = localProc;
}
}
RHS = new Epetra_MultiVector(Copy, sourceRowMap, vals, 1, 1);
LHS = new Epetra_MultiVector(Copy, *LHSmap, vals, 1, 1);
if (valSize){
delete [] vals;
}
problem = new Epetra_LinearProblem(matrix.get(), LHS, RHS);
Epetra_LinearProblem lp = *problem;
if (lp.CheckInput()){
ERROREXIT((localProc==0), "Error creating a LinearProblem");
}
if (noParams && noCosts){
problemPtr = Isorropia::Epetra::createBalancedCopy(lp);
}
else if (noCosts){
problemPtr = Isorropia::Epetra::createBalancedCopy(lp, params);
}
targetRowMap = &(problemPtr->GetMatrix()->RowMatrixRowMap());
targetColMap = &(problemPtr->GetMatrix()->RowMatrixColMap());
}
// Redistribute the edge weights
// Comment this out since we don't redistribute columns
if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){
if (partitioningType == GRAPH_PARTITIONING){
Epetra_Import *importer = NULL;
if (objectType == EPETRA_CRSGRAPH){
newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *graphPtr));
targetRowMap = &(newewgts->RowMap());
targetColMap = &(newewgts->ColMap());
}
else{
newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *targetRowMap, *targetColMap, 0));
}
importer = new Epetra_Import(*targetRowMap, sourceRowMap);
newewgts->Import(*eptr, *importer, Insert);
newewgts->FillComplete(*targetColMap, *targetRowMap);
costs.setGraphEdgeWeights(newewgts);
}
}
// Redistribute the vertex weights
if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){
Epetra_Import *importer = NULL;
if (objectType == EPETRA_CRSGRAPH){
newvwgts = Teuchos::rcp(new Epetra_Vector(*targetBlockRowMap));
importer = new Epetra_Import(*targetBlockRowMap, sourceRowMap);
}
else{
newvwgts = Teuchos::rcp(new Epetra_Vector(*targetRowMap));
importer = new Epetra_Import(*targetRowMap, sourceRowMap);
}
newvwgts->Import(*vptr, *importer, Insert);
costs.setVertexWeights(newvwgts);
}
if (localProc == 0){
test_type(numPartitions, partitioningType, vertexWeightType, edgeWeightType, objectType);
}
if (verbose){
// Picture of problem before balancing
if (objectType == EPETRA_LINEARPROBLEM){
ispatest::show_matrix("Before load balancing", *problem, Comm);
}
else{
ispatest::show_matrix("Before load balancing", matrix->Graph(), Comm);
}
// Picture of problem after balancing
if (objectType == EPETRA_LINEARPROBLEM){
ispatest::show_matrix("After load balancing (x in Ax=b is not redistributed)", *problemPtr, Comm);
}
else if (objectType == EPETRA_ROWMATRIX){
ispatest::show_matrix("After load balancing", *rowMatrixPtr, Comm);
}
else if (objectType == EPETRA_CRSMATRIX){
ispatest::show_matrix("After load balancing", matrixPtr->Graph(), Comm);
}
else if (objectType == EPETRA_CRSGRAPH){
ispatest::show_matrix("After load balancing", *graphPtr, Comm);
}
}
// After partitioning, recompute the metrics
if (partitioningType == GRAPH_PARTITIONING){
if (objectType == EPETRA_LINEARPROBLEM){
rc = ispatest::compute_graph_metrics(*(problemPtr->GetMatrix()), costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else if (objectType == EPETRA_ROWMATRIX){
rc = ispatest::compute_graph_metrics(*rowMatrixPtr, costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else if (objectType == EPETRA_CRSMATRIX){
rc = ispatest::compute_graph_metrics(matrixPtr->Graph(), costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else {
rc = ispatest::compute_graph_metrics(*graphPtr, costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
}
else{
if (objectType == EPETRA_LINEARPROBLEM){
rc = ispatest::compute_hypergraph_metrics(*(problemPtr->GetMatrix()), costs,
myShare, balance2, cutn2, cutl2);
}
else if (objectType == EPETRA_ROWMATRIX){
rc = ispatest::compute_hypergraph_metrics(*rowMatrixPtr, costs,
myShare, balance2, cutn2, cutl2);
}
else if (objectType == EPETRA_CRSMATRIX){
rc = ispatest::compute_hypergraph_metrics(matrixPtr->Graph(), costs,
myShare, balance2, cutn2, cutl2);
}
else{
rc = ispatest::compute_hypergraph_metrics(*graphPtr, costs,
myShare, balance2, cutn2, cutl2);
}
}
if (rc){
ERROREXIT((localProc==0), "Error in computing partitioning metrics")
}
std::string why;
if (partitioningType == GRAPH_PARTITIONING){
fail = (cutWgt2 > cutWgt1);
why = "New weighted edge cuts are worse";
if (localProc == 0){
std::cout << "Before partitioning: Balance " << balance1 ;
std::cout << " cutn " << cutn1 ;
std::cout << " cutl " << cutl1 ;
if (contract){
std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl;
std::cout << "Before partitioning: Balance " << balance3 ;
std::cout << " cutn " << cutn3 ;
std::cout << " cutl " << cutl3 ;
std::cout << " (wrt balancing over " << numProcs << " partitions)" ;
}
std::cout << std::endl;
std::cout << " Total edge cuts: " << numCuts1;
std::cout << " Total weighted edge cuts: " << cutWgt1 << std::endl;
std::cout << "After partitioning: Balance " << balance2 ;
std::cout << " cutn " << cutn2 ;
std::cout << " cutl " << cutl2 << std::endl;
std::cout << " Total edge cuts: " << numCuts2;
std::cout << " Total weighted edge cuts: " << cutWgt2 << std::endl;
}
}
else{
fail = (cutl2 > cutl1);
why = "New cutl is worse";
if (localProc == 0){
std::cout << "Before partitioning: Balance " << balance1 ;
std::cout << " cutn " << cutn1 ;
std::cout << " cutl " << cutl1 ;
if (contract){
std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl;
std::cout << "Before partitioning: Balance " << balance3 ;
std::cout << " cutn " << cutn3 ;
std::cout << " cutl " << cutl3 ;
std::cout << " (wrt balancing over " << numProcs << " partitions)" ;
}
std::cout << std::endl;
std::cout << "After partitioning: Balance " << balance2 ;
std::cout << " cutn " << cutn2 ;
std::cout << " cutl " << cutl2 << std::endl;
}
}
if (fail){
if (localProc == 0) std::cout << "ERROR: "+why << std::endl;
}
// Check that input matrix is valid. This test constructs an "x"
// with the matrix->DomainMap() and a "y" with matrix->RangeMap()
// and then calculates y = Ax.
if (objectType == EPETRA_LINEARPROBLEM){
valid = ispatest::test_matrix_vector_multiply(*problemPtr);
}
else if (objectType == EPETRA_ROWMATRIX){
valid = ispatest::test_row_matrix_vector_multiply(*rowMatrixPtr);
}
else if (objectType == EPETRA_CRSMATRIX){
valid = ispatest::test_matrix_vector_multiply(*matrixPtr);
}
else{
valid = ispatest::test_matrix_vector_multiply(*graphPtr);
}
if (!valid){
if (localProc == 0) std::cout << "Rebalanced matrix is not a valid Epetra matrix" << std::endl;
fail = 1;
}
else{
if (localProc == 0) std::cout << "Rebalanced matrix is a valid Epetra matrix" << std::endl;
}
if (localProc == 0)
std::cout << std::endl;
#else
std::cout << "test_simple main: currently can only test "
<< "with Epetra and EpetraExt enabled." << std::endl;
rc = -1;
#endif
return fail;
}
