void buildGIDs(std::vector<std::vector<int> > & gids,const Epetra_Map & map)
{
int numLocal = map.NumMyElements();
int numHalf = numLocal/2;
numHalf += ((numHalf % 2 == 0) ? 0 : 1);
gids.clear();
gids.resize(3);
std::vector<int> & blk0 = gids[0];
std::vector<int> & blk1 = gids[1];
std::vector<int> & blk2 = gids[2];
// loop over global IDs: treat first block as strided
int gid = -1;
for(int i=0;i<numHalf;i+=2) {
gid = map.GID(i);
blk0.push_back(gid);
gid = map.GID(i+1);
blk1.push_back(gid);
}
// loop over global IDs: treat remainder as contiguous
for(int i=numHalf;i<numLocal;i++) {
gid = map.GID(i);
blk2.push_back(gid);
}
}
static void UpdateComponent( spaceT const& Xh, Epetra_MultiVector& sol, Epetra_MultiVector& comp )
{
Epetra_Map componentMap ( epetraMap( Xh->template functionSpace<index>()->map() ) );
Epetra_Map globalMap ( epetraMap( Xh->map() ) );
int shift = Xh->nDofStart( index );
int Length = comp.MyLength();
for ( int i=0; i < Length; i++ )
{
int compGlobalID = componentMap.GID( i );
if ( compGlobalID >= 0 )
{
int compLocalID = componentMap.LID( compGlobalID );
int localID = globalMap.LID( compGlobalID+shift );
// int globalID = globalMap.GID(localID);
DVLOG(2) << "Copy entry component[" << compLocalID << "] to sol[" << localID << "]="
<< sol[0][localID]
<< "]\n";
sol[0][localID] = comp[0][compLocalID] ;
DVLOG(2) << comp[0][compLocalID] << "\n";
}
}
}
static Epetra_MultiVector getComponent( spaceT const& Xh, Epetra_MultiVector const& sol )
{
Epetra_Map componentMap ( epetraMap( Xh->template functionSpace<index>()->map() ) );
Epetra_Map globalMap ( epetraMap( Xh->map() ) );
//DVLOG(2) << "Component map: " << componentMap << "\n";
Epetra_MultiVector component( componentMap, 1 );
int Length = component.MyLength();
int shift = Xh->nDofStart( index );
for ( int i=0; i < Length; i++ )
{
int compGlobalID = componentMap.GID( i );
if ( compGlobalID >= 0 )
{
int compLocalID = componentMap.LID( compGlobalID );
int localID = globalMap.LID( compGlobalID+shift );
// int globalID = globalMap.GID(localID);
DVLOG(2) << "[MyBackend] Copy entry sol[" << localID << "]=" << sol[0][localID]
<< " to component[" << compLocalID << "]\n";
component[0][compLocalID] = sol[0][localID];
DVLOG(2) << component[0][compLocalID] << "\n";
}
}
return component;
}
Teuchos::RCP<Epetra_CrsMatrix> Epetra_Operator_to_Epetra_Matrix::constructInverseMatrix(const Epetra_Operator &op, const Epetra_Map &map)
{
int numEntriesPerRow = 0;
Teuchos::RCP<Epetra_FECrsMatrix> matrix = Teuchos::rcp(new Epetra_FECrsMatrix(::Copy, map, numEntriesPerRow));
int numRows = map.NumGlobalElements();
Epetra_Vector X(map);
Epetra_Vector Y(map);
double tol = 1e-15; // values below this will be considered 0
for (int rowIndex=0; rowIndex<numRows; rowIndex++)
{
int lid = map.LID(rowIndex);
if (lid != -1)
{
X[lid] = 1.0;
}
op.ApplyInverse(X, Y);
if (lid != -1)
{
X[lid] = 0.0;
}
std::vector<double> values;
std::vector<int> indices;
for (int i=0; i<map.NumMyElements(); i++)
{
if (abs(Y[i]) > tol)
{
values.push_back(Y[i]);
indices.push_back(map.GID(i));
}
}
matrix->InsertGlobalValues(rowIndex, values.size(), &values[0], &indices[0]);
}
matrix->GlobalAssemble();
return matrix;
}
int CopyRowMatrix(mxArray* matlabA, const Epetra_RowMatrix& A) {
int valueCount = 0;
//int* valueCount = &temp;
Epetra_Map map = A.RowMatrixRowMap();
const Epetra_Comm & comm = map.Comm();
int numProc = comm.NumProc();
if (numProc==1)
DoCopyRowMatrix(matlabA, valueCount, A);
else {
int numRows = map.NumMyElements();
//cout << "creating allGidsMap\n";
Epetra_Map allGidsMap(-1, numRows, 0,comm);
//cout << "done creating allGidsMap\n";
Epetra_IntVector allGids(allGidsMap);
for (int i=0; i<numRows; i++) allGids[i] = map.GID(i);
// Now construct a RowMatrix on PE 0 by strip-mining the rows of the input matrix A.
int numChunks = numProc;
int stripSize = allGids.GlobalLength()/numChunks;
int remainder = allGids.GlobalLength()%numChunks;
int curStart = 0;
int curStripSize = 0;
Epetra_IntSerialDenseVector importGidList;
int numImportGids = 0;
if (comm.MyPID()==0)
importGidList.Size(stripSize+1); // Set size of vector to max needed
for (int i=0; i<numChunks; i++) {
if (comm.MyPID()==0) { // Only PE 0 does this part
curStripSize = stripSize;
if (i<remainder) curStripSize++; // handle leftovers
for (int j=0; j<curStripSize; j++) importGidList[j] = j + curStart;
curStart += curStripSize;
}
// The following import map will be non-trivial only on PE 0.
//cout << "creating importGidMap\n";
Epetra_Map importGidMap(-1, curStripSize, importGidList.Values(), 0, comm);
//cout << "done creating importGidMap\n";
Epetra_Import gidImporter(importGidMap, allGidsMap);
Epetra_IntVector importGids(importGidMap);
if (importGids.Import(allGids, gidImporter, Insert)) return(-1);
// importGids now has a list of GIDs for the current strip of matrix rows.
// Use these values to build another importer that will get rows of the matrix.
// The following import map will be non-trivial only on PE 0.
//cout << "creating importMap\n";
//cout << "A.RowMatrixRowMap().MinAllGID: " << A.RowMatrixRowMap().MinAllGID() << "\n";
Epetra_Map importMap(-1, importGids.MyLength(), importGids.Values(), A.RowMatrixRowMap().MinAllGID(), comm);
//cout << "done creating importMap\n";
Epetra_Import importer(importMap, map);
Epetra_CrsMatrix importA(Copy, importMap, 0);
if (importA.Import(A, importer, Insert)) return(-1);
if (importA.FillComplete()) return(-1);
// Finally we are ready to write this strip of the matrix to ostream
if (DoCopyRowMatrix(matlabA, valueCount, importA)) return(-1);
}
}
if (A.RowMatrixRowMap().Comm().MyPID() == 0) {
// set max cap
int* matlabAcolumnIndicesPtr = mxGetJc(matlabA);
matlabAcolumnIndicesPtr[A.NumGlobalRows()] = valueCount;
}
return(0);
}
int DoCopyRowMatrix(mxArray* matlabA, int& valueCount, const Epetra_RowMatrix& A) {
//cout << "doing DoCopyRowMatrix\n";
int ierr = 0;
int numRows = A.NumGlobalRows();
//cout << "numRows: " << numRows << "\n";
Epetra_Map rowMap = A.RowMatrixRowMap();
Epetra_Map colMap = A.RowMatrixColMap();
int minAllGID = rowMap.MinAllGID();
const Epetra_Comm & comm = rowMap.Comm();
//cout << "did global setup\n";
if (comm.MyPID()!=0) {
if (A.NumMyRows()!=0) ierr = -1;
if (A.NumMyCols()!=0) ierr = -1;
}
else {
// declare and get initial values of all matlabA pointers
double* matlabAvaluesPtr = mxGetPr(matlabA);
int* matlabAcolumnIndicesPtr = mxGetJc(matlabA);
int* matlabArowIndicesPtr = mxGetIr(matlabA);
// set all matlabA pointers to the proper offset
matlabAvaluesPtr += valueCount;
matlabArowIndicesPtr += valueCount;
if (numRows!=A.NumMyRows()) ierr = -1;
Epetra_SerialDenseVector values(A.MaxNumEntries());
Epetra_IntSerialDenseVector indices(A.MaxNumEntries());
//cout << "did proc0 setup\n";
for (int i=0; i<numRows; i++) {
//cout << "extracting a row\n";
int I = rowMap.GID(i);
int numEntries = 0;
if (A.ExtractMyRowCopy(i, values.Length(), numEntries,
values.Values(), indices.Values())) return(-1);
matlabAcolumnIndicesPtr[I - minAllGID] = valueCount; // set the starting index of column I
double* serialValuesPtr = values.Values();
for (int j=0; j<numEntries; j++) {
int J = colMap.GID(indices[j]);
*matlabAvaluesPtr = *serialValuesPtr++;
*matlabArowIndicesPtr = J;
// increment matlabA pointers
matlabAvaluesPtr++;
matlabArowIndicesPtr++;
valueCount++;
}
}
//cout << "proc0 row extraction for this chunck is done\n";
}
/*
if (comm.MyPID() == 0) {
cout << "printing matlabA pointers\n";
double* matlabAvaluesPtr = mxGetPr(matlabA);
int* matlabAcolumnIndicesPtr = mxGetJc(matlabA);
int* matlabArowIndicesPtr = mxGetIr(matlabA);
for(int i=0; i < numRows; i++) {
for(int j=0; j < A.MaxNumEntries(); j++) {
cout << "*matlabAvaluesPtr: " << *matlabAvaluesPtr++ << " *matlabAcolumnIndicesPtr: " << *matlabAcolumnIndicesPtr++ << " *matlabArowIndicesPtr" << *matlabArowIndicesPtr++ << "\n";
}
}
cout << "done printing matlabA pointers\n";
}
*/
int ierrGlobal;
comm.MinAll(&ierr, &ierrGlobal, 1); // If any processor has -1, all return -1
return(ierrGlobal);
}
int checkmap(Epetra_Map & Map, int NumGlobalElements, int NumMyElements,
int *MyGlobalElements, int IndexBase, Epetra_Comm& Comm,
bool DistributedGlobal)
{
int i, ierr=0, forierr = 0;
EPETRA_TEST_ERR(!Map.ConstantElementSize(),ierr);
EPETRA_TEST_ERR(DistributedGlobal!=Map.DistributedGlobal(),ierr);
EPETRA_TEST_ERR(Map.ElementSize()!=1,ierr);
int *MyElementSizeList = new int[NumMyElements];
EPETRA_TEST_ERR(Map.ElementSizeList(MyElementSizeList)!=0,ierr);
forierr = 0;
for (i=0; i<NumMyElements; i++) forierr += MyElementSizeList[i]!=1;
EPETRA_TEST_ERR(forierr,ierr);
delete [] MyElementSizeList;
const Epetra_Comm & Comm1 = Map.Comm();
EPETRA_TEST_ERR(Comm1.NumProc()!=Comm.NumProc(),ierr);
EPETRA_TEST_ERR(Comm1.MyPID()!=Comm.MyPID(),ierr);
EPETRA_TEST_ERR(Map.IndexBase()!=IndexBase,ierr);
EPETRA_TEST_ERR(!Map.LinearMap() && MyGlobalElements==0,ierr);
EPETRA_TEST_ERR(Map.LinearMap() && MyGlobalElements!=0,ierr);
EPETRA_TEST_ERR(Map.MaxAllGID()!=NumGlobalElements-1+IndexBase,ierr);
EPETRA_TEST_ERR(Map.MaxElementSize()!=1,ierr);
int MaxLID = Map.MaxLID();
EPETRA_TEST_ERR(MaxLID!=NumMyElements-1,ierr);
int MaxMyGID = (Comm.MyPID()+1)*NumMyElements-1+IndexBase;
if (Comm.MyPID()>2) MaxMyGID+=3;
if (!DistributedGlobal) MaxMyGID = NumMyElements-1+IndexBase;
EPETRA_TEST_ERR(Map.MaxMyGID()!=MaxMyGID,ierr);
EPETRA_TEST_ERR(Map.MinAllGID()!=IndexBase,ierr);
EPETRA_TEST_ERR(Map.MinElementSize()!=1,ierr);
EPETRA_TEST_ERR(Map.MinLID()!=0,ierr);
int MinMyGID = Comm.MyPID()*NumMyElements+IndexBase;
if (Comm.MyPID()>2) MinMyGID+=3;
if (!DistributedGlobal) MinMyGID = 0;
EPETRA_TEST_ERR(Map.MinMyGID()!=MinMyGID,ierr);
int * MyGlobalElements1 = new int[NumMyElements];
EPETRA_TEST_ERR(Map.MyGlobalElements(MyGlobalElements1)!=0,ierr);
forierr = 0;
if (MyGlobalElements==0)
{
for (i=0; i<NumMyElements; i++)
forierr += MyGlobalElements1[i]!=MinMyGID+i;
EPETRA_TEST_ERR(forierr,ierr);
}
else {
for (i=0; i<NumMyElements; i++)
forierr += MyGlobalElements[i]!=MyGlobalElements1[i];
EPETRA_TEST_ERR(forierr,ierr);
}
EPETRA_TEST_ERR(Map.NumGlobalElements()!=NumGlobalElements,ierr);
EPETRA_TEST_ERR(Map.NumGlobalPoints()!=NumGlobalElements,ierr);
EPETRA_TEST_ERR(Map.NumMyElements()!=NumMyElements,ierr);
EPETRA_TEST_ERR(Map.NumMyPoints()!=NumMyElements,ierr);
int MaxMyGID2 = Map.GID(Map.LID(MaxMyGID));
EPETRA_TEST_ERR(MaxMyGID2 != MaxMyGID,ierr);
int MaxLID2 = Map.LID(Map.GID(MaxLID));
EPETRA_TEST_ERR(MaxLID2 != MaxLID,ierr);
EPETRA_TEST_ERR(Map.GID(MaxLID+1) != IndexBase-1,ierr);// MaxLID+1 doesn't exist
EPETRA_TEST_ERR(Map.LID(MaxMyGID+1) != -1,ierr);// MaxMyGID+1 doesn't exist or is on a different processor
EPETRA_TEST_ERR(!Map.MyGID(MaxMyGID),ierr);
EPETRA_TEST_ERR(Map.MyGID(MaxMyGID+1),ierr);
EPETRA_TEST_ERR(!Map.MyLID(MaxLID),ierr);
EPETRA_TEST_ERR(Map.MyLID(MaxLID+1),ierr);
EPETRA_TEST_ERR(!Map.MyGID(Map.GID(MaxLID)),ierr);
EPETRA_TEST_ERR(Map.MyGID(Map.GID(MaxLID+1)),ierr);
EPETRA_TEST_ERR(!Map.MyLID(Map.LID(MaxMyGID)),ierr);
EPETRA_TEST_ERR(Map.MyLID(Map.LID(MaxMyGID+1)),ierr);
// Check RemoteIDList function
// Get some GIDs off of each processor to test
int TotalNumEle, NumElePerProc, NumProc = Comm.NumProc();
int MinNumEleOnProc;
int NumMyEle=Map.NumMyElements();
Comm.MinAll(&NumMyEle,&MinNumEleOnProc,1);
if (MinNumEleOnProc > 5) NumElePerProc = 6;
else NumElePerProc = MinNumEleOnProc;
if (NumElePerProc > 0) {
TotalNumEle = NumElePerProc*NumProc;
int * MyGIDlist = new int[NumElePerProc];
int * GIDlist = new int[TotalNumEle];
int * PIDlist = new int[TotalNumEle];
int * LIDlist = new int[TotalNumEle];
for (i=0; i<NumElePerProc; i++)
MyGIDlist[i] = MyGlobalElements1[i];
Comm.GatherAll(MyGIDlist,GIDlist,NumElePerProc);// Get a few values from each proc
Map.RemoteIDList(TotalNumEle, GIDlist, PIDlist, LIDlist);
int MyPID= Comm.MyPID();
forierr = 0;
for (i=0; i<TotalNumEle; i++) {
if (Map.MyGID(GIDlist[i])) {
forierr += PIDlist[i] != MyPID;
forierr += !Map.MyLID(Map.LID(GIDlist[i])) || Map.LID(GIDlist[i]) != LIDlist[i] || Map.GID(LIDlist[i]) != GIDlist[i];
}
else {
forierr += PIDlist[i] == MyPID; // If MyGID comes back false, the PID listed should be that of another proc
}
}
EPETRA_TEST_ERR(forierr,ierr);
delete [] MyGIDlist;
delete [] GIDlist;
delete [] PIDlist;
delete [] LIDlist;
}
delete [] MyGlobalElements1;
// Check RemoteIDList function (assumes all maps are linear, even if not stored that way)
if (Map.LinearMap()) {
int * GIDList = new int[3];
int * PIDList = new int[3];
int * LIDList = new int[3];
int MyPID = Map.Comm().MyPID();
int NumIDs = 0;
//GIDList[NumIDs++] = Map.MaxAllGID()+1; // Should return -1 for both PID and LID
if (Map.MinMyGID()-1>=Map.MinAllGID()) GIDList[NumIDs++] = Map.MinMyGID()-1;
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) GIDList[NumIDs++] = Map.MaxMyGID()+1;
Map.RemoteIDList(NumIDs, GIDList, PIDList, LIDList);
NumIDs = 0;
//EPETRA_TEST_ERR(!(PIDList[NumIDs]==-1),ierr);
//EPETRA_TEST_ERR(!(LIDList[NumIDs++]==-1),ierr);
if (Map.MinMyGID()-1>=Map.MinAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs++]==MyPID-1),ierr);
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs]==MyPID+1),ierr);
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(LIDList[NumIDs++]==0),ierr);
delete [] GIDList;
delete [] PIDList;
delete [] LIDList;
}
return (ierr);
}
int main(int argc, char *argv[])
{
int ierr = 0;
double elapsed_time;
double total_flops;
double MFLOPs;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm comm( MPI_COMM_WORLD );
#else
Epetra_SerialComm comm;
#endif
bool verbose = false;
bool summary = false;
// Check if we should print verbose results to standard out
if (argc>6) if (argv[6][0]=='-' && argv[6][1]=='v') verbose = true;
// Check if we should print verbose results to standard out
if (argc>6) if (argv[6][0]=='-' && argv[6][1]=='s') summary = true;
if(argc < 6) {
cerr << "Usage: " << argv[0]
<< " NumNodesX NumNodesY NumProcX NumProcY NumPoints [-v|-s]" << endl
<< "where:" << endl
<< "NumNodesX - Number of mesh nodes in X direction per processor" << endl
<< "NumNodesY - Number of mesh nodes in Y direction per processor" << endl
<< "NumProcX - Number of processors to use in X direction" << endl
<< "NumProcY - Number of processors to use in Y direction" << endl
<< "NumPoints - Number of points to use in stencil (5, 9 or 25 only)" << endl
<< "-v|-s - (Optional) Run in verbose mode if -v present or summary mode if -s present" << endl
<< " NOTES: NumProcX*NumProcY must equal the number of processors used to run the problem." << endl << endl
<< " Serial example:" << endl
<< argv[0] << " 16 12 1 1 25 -v" << endl
<< " Run this program in verbose mode on 1 processor using a 16 X 12 grid with a 25 point stencil."<< endl <<endl
<< " MPI example:" << endl
<< "mpirun -np 32 " << argv[0] << " 10 12 4 8 9 -v" << endl
<< " Run this program in verbose mode on 32 processors putting a 10 X 12 subgrid on each processor using 4 processors "<< endl
<< " in the X direction and 8 in the Y direction. Total grid size is 40 points in X and 96 in Y with a 9 point stencil."<< endl
<< endl;
return(1);
}
//char tmp;
//if (comm.MyPID()==0) cout << "Press any key to continue..."<< endl;
//if (comm.MyPID()==0) cin >> tmp;
//comm.Barrier();
comm.SetTracebackMode(0); // This should shut down any error traceback reporting
if (verbose && comm.MyPID()==0)
cout << Epetra_Version() << endl << endl;
if (summary && comm.MyPID()==0) {
if (comm.NumProc()==1)
cout << Epetra_Version() << endl << endl;
else
cout << endl << endl; // Print two blank line to keep output columns lined up
}
if (verbose) cout << comm <<endl;
// Redefine verbose to only print on PE 0
if (verbose && comm.MyPID()!=0) verbose = false;
if (summary && comm.MyPID()!=0) summary = false;
int numNodesX = atoi(argv[1]);
int numNodesY = atoi(argv[2]);
int numProcsX = atoi(argv[3]);
int numProcsY = atoi(argv[4]);
int numPoints = atoi(argv[5]);
if (verbose || (summary && comm.NumProc()==1)) {
cout << " Number of local nodes in X direction = " << numNodesX << endl
<< " Number of local nodes in Y direction = " << numNodesY << endl
<< " Number of global nodes in X direction = " << numNodesX*numProcsX << endl
<< " Number of global nodes in Y direction = " << numNodesY*numProcsY << endl
<< " Number of local nonzero entries = " << numNodesX*numNodesY*numPoints << endl
<< " Number of global nonzero entries = " << numNodesX*numNodesY*numPoints*numProcsX*numProcsY << endl
<< " Number of Processors in X direction = " << numProcsX << endl
<< " Number of Processors in Y direction = " << numProcsY << endl
<< " Number of Points in stencil = " << numPoints << endl << endl;
}
// Print blank line to keep output columns lined up
if (summary && comm.NumProc()>1)
cout << endl << endl << endl << endl << endl << endl << endl << endl<< endl << endl;
if (numProcsX*numProcsY!=comm.NumProc()) {
cerr << "Number of processors = " << comm.NumProc() << endl
<< " is not the product of " << numProcsX << " and " << numProcsY << endl << endl;
return(1);
}
if (numPoints!=5 && numPoints!=9 && numPoints!=25) {
cerr << "Number of points specified = " << numPoints << endl
<< " is not 5, 9, 25" << endl << endl;
return(1);
}
if (numNodesX*numNodesY<=0) {
cerr << "Product of number of nodes is <= zero" << endl << endl;
return(1);
}
Epetra_IntSerialDenseVector Xoff, XLoff, XUoff;
Epetra_IntSerialDenseVector Yoff, YLoff, YUoff;
if (numPoints==5) {
// Generate a 5-point 2D Finite Difference matrix
Xoff.Size(5);
Yoff.Size(5);
Xoff[0] = -1; Xoff[1] = 1; Xoff[2] = 0; Xoff[3] = 0; Xoff[4] = 0;
Yoff[0] = 0; Yoff[1] = 0; Yoff[2] = 0; Yoff[3] = -1; Yoff[4] = 1;
// Generate a 2-point 2D Lower triangular Finite Difference matrix
XLoff.Size(2);
YLoff.Size(2);
XLoff[0] = -1; XLoff[1] = 0;
YLoff[0] = 0; YLoff[1] = -1;
// Generate a 3-point 2D upper triangular Finite Difference matrix
XUoff.Size(3);
YUoff.Size(3);
XUoff[0] = 0; XUoff[1] = 1; XUoff[2] = 0;
YUoff[0] = 0; YUoff[1] = 0; YUoff[2] = 1;
}
else if (numPoints==9) {
// Generate a 9-point 2D Finite Difference matrix
Xoff.Size(9);
Yoff.Size(9);
Xoff[0] = -1; Xoff[1] = 0; Xoff[2] = 1;
Yoff[0] = -1; Yoff[1] = -1; Yoff[2] = -1;
Xoff[3] = -1; Xoff[4] = 0; Xoff[5] = 1;
Yoff[3] = 0; Yoff[4] = 0; Yoff[5] = 0;
Xoff[6] = -1; Xoff[7] = 0; Xoff[8] = 1;
Yoff[6] = 1; Yoff[7] = 1; Yoff[8] = 1;
// Generate a 5-point lower triangular 2D Finite Difference matrix
XLoff.Size(5);
YLoff.Size(5);
XLoff[0] = -1; XLoff[1] = 0; Xoff[2] = 1;
YLoff[0] = -1; YLoff[1] = -1; Yoff[2] = -1;
XLoff[3] = -1; XLoff[4] = 0;
YLoff[3] = 0; YLoff[4] = 0;
// Generate a 4-point upper triangular 2D Finite Difference matrix
XUoff.Size(4);
YUoff.Size(4);
XUoff[0] = 1;
YUoff[0] = 0;
XUoff[1] = -1; XUoff[2] = 0; XUoff[3] = 1;
YUoff[1] = 1; YUoff[2] = 1; YUoff[3] = 1;
}
else {
// Generate a 25-point 2D Finite Difference matrix
Xoff.Size(25);
Yoff.Size(25);
int xi = 0, yi = 0;
int xo = -2, yo = -2;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
xo = -2, yo++;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
xo = -2, yo++;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
xo = -2, yo++;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
xo = -2, yo++;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
// Generate a 13-point lower triangular 2D Finite Difference matrix
XLoff.Size(13);
YLoff.Size(13);
xi = 0, yi = 0;
xo = -2, yo = -2;
XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++;
YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ;
xo = -2, yo++;
XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++;
YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ;
xo = -2, yo++;
XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++;
YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ;
// Generate a 13-point upper triangular 2D Finite Difference matrix
XUoff.Size(13);
YUoff.Size(13);
xi = 0, yi = 0;
xo = 0, yo = 0;
XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++;
YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ;
xo = -2, yo++;
XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++;
YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ;
xo = -2, yo++;
XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++;
YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ;
}
Epetra_Map * map;
Epetra_Map * mapL;
Epetra_Map * mapU;
Epetra_CrsMatrix * A;
Epetra_CrsMatrix * L;
Epetra_CrsMatrix * U;
Epetra_MultiVector * b;
Epetra_MultiVector * bt;
Epetra_MultiVector * xexact;
Epetra_MultiVector * bL;
Epetra_MultiVector * btL;
Epetra_MultiVector * xexactL;
Epetra_MultiVector * bU;
Epetra_MultiVector * btU;
Epetra_MultiVector * xexactU;
Epetra_SerialDenseVector resvec(0);
//Timings
Epetra_Flops flopcounter;
Epetra_Time timer(comm);
#ifdef EPETRA_VERY_SHORT_PERFTEST
int jstop = 1;
#elif EPETRA_SHORT_PERFTEST
int jstop = 1;
#else
int jstop = 2;
#endif
for (int j=0; j<jstop; j++) {
for (int k=1; k<17; k++) {
#ifdef EPETRA_VERY_SHORT_PERFTEST
if (k<3 || (k%4==0 && k<9)) {
#elif EPETRA_SHORT_PERFTEST
if (k<6 || k%4==0) {
#else
if (k<7 || k%2==0) {
#endif
int nrhs=k;
if (verbose) cout << "\n*************** Results for " << nrhs << " RHS with ";
bool StaticProfile = (j!=0);
if (verbose)
if (StaticProfile) cout << " static profile\n";
else cout << " dynamic profile\n";
GenerateCrsProblem(numNodesX, numNodesY, numProcsX, numProcsY, numPoints,
Xoff.Values(), Yoff.Values(), nrhs, comm, verbose, summary,
map, A, b, bt, xexact, StaticProfile, false);
#ifdef EPETRA_HAVE_JADMATRIX
timer.ResetStartTime();
Epetra_JadMatrix JA(*A);
elapsed_time = timer.ElapsedTime();
if (verbose) cout << "Time to create Jagged diagonal matrix = " << elapsed_time << endl;
//cout << "A = " << *A << endl;
//cout << "JA = " << JA << endl;
runJadMatrixTests(&JA, b, bt, xexact, StaticProfile, verbose, summary);
#endif
runMatrixTests(A, b, bt, xexact, StaticProfile, verbose, summary);
delete A;
delete b;
delete bt;
delete xexact;
GenerateCrsProblem(numNodesX, numNodesY, numProcsX, numProcsY, XLoff.Length(),
XLoff.Values(), YLoff.Values(), nrhs, comm, verbose, summary,
mapL, L, bL, btL, xexactL, StaticProfile, true);
GenerateCrsProblem(numNodesX, numNodesY, numProcsX, numProcsY, XUoff.Length(),
XUoff.Values(), YUoff.Values(), nrhs, comm, verbose, summary,
mapU, U, bU, btU, xexactU, StaticProfile, true);
runLUMatrixTests(L, bL, btL, xexactL, U, bU, btU, xexactU, StaticProfile, verbose, summary);
delete L;
delete bL;
delete btL;
delete xexactL;
delete mapL;
delete U;
delete bU;
delete btU;
delete xexactU;
delete mapU;
Epetra_MultiVector q(*map, nrhs);
Epetra_MultiVector z(q);
Epetra_MultiVector r(q);
delete map;
q.SetFlopCounter(flopcounter);
z.SetFlopCounter(q);
r.SetFlopCounter(q);
resvec.Resize(nrhs);
flopcounter.ResetFlops();
timer.ResetStartTime();
//10 norms
for( int i = 0; i < 10; ++i )
q.Norm2( resvec.Values() );
elapsed_time = timer.ElapsedTime();
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "\nTotal MFLOPs for 10 Norm2's= " << MFLOPs << endl;
if (summary) {
if (comm.NumProc()==1) cout << "Norm2" << '\t';
cout << MFLOPs << endl;
}
flopcounter.ResetFlops();
timer.ResetStartTime();
//10 dot's
for( int i = 0; i < 10; ++i )
q.Dot(z, resvec.Values());
elapsed_time = timer.ElapsedTime();
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Total MFLOPs for 10 Dot's = " << MFLOPs << endl;
if (summary) {
if (comm.NumProc()==1) cout << "DotProd" << '\t';
cout << MFLOPs << endl;
}
flopcounter.ResetFlops();
timer.ResetStartTime();
//10 dot's
for( int i = 0; i < 10; ++i )
q.Update(1.0, z, 1.0, r, 0.0);
elapsed_time = timer.ElapsedTime();
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Total MFLOPs for 10 Updates= " << MFLOPs << endl;
if (summary) {
if (comm.NumProc()==1) cout << "Update" << '\t';
cout << MFLOPs << endl;
}
}
}
}
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return ierr ;
}
// Constructs a 2D PDE finite difference matrix using the list of x and y offsets.
//
// nx (In) - number of grid points in x direction
// ny (In) - number of grid points in y direction
// The total number of equations will be nx*ny ordered such that the x direction changes
// most rapidly:
// First equation is at point (0,0)
// Second at (1,0)
// ...
// nx equation at (nx-1,0)
// nx+1st equation at (0,1)
// numPoints (In) - number of points in finite difference stencil
// xoff (In) - stencil offsets in x direction (of length numPoints)
// yoff (In) - stencil offsets in y direction (of length numPoints)
// A standard 5-point finite difference stencil would be described as:
// numPoints = 5
// xoff = [-1, 1, 0, 0, 0]
// yoff = [ 0, 0, 0, -1, 1]
// nrhs - Number of rhs to generate. (First interface produces vectors, so nrhs is not needed
// comm (In) - an Epetra_Comm object describing the parallel machine (numProcs and my proc ID)
// map (Out) - Epetra_Map describing distribution of matrix and vectors/multivectors
// A (Out) - Epetra_CrsMatrix constructed for nx by ny grid using prescribed stencil
// Off-diagonal values are random between 0 and 1. If diagonal is part of stencil,
// diagonal will be slightly diag dominant.
// b (Out) - Generated RHS. Values satisfy b = A*xexact
// bt (Out) - Generated RHS. Values satisfy b = A'*xexact
// xexact (Out) - Generated exact solution to Ax = b and b' = A'xexact
// Note: Caller of this function is responsible for deleting all output objects.
void GenerateCrsProblem(int numNodesX, int numNodesY, int numProcsX, int numProcsY, int numPoints,
int * xoff, int * yoff,
const Epetra_Comm &comm, bool verbose, bool summary,
Epetra_Map *& map,
Epetra_CrsMatrix *& A,
Epetra_Vector *& b,
Epetra_Vector *& bt,
Epetra_Vector *&xexact, bool StaticProfile, bool MakeLocalOnly) {
Epetra_MultiVector * b1, * bt1, * xexact1;
GenerateCrsProblem(numNodesX, numNodesY, numProcsX, numProcsY, numPoints,
xoff, yoff, 1, comm, verbose, summary,
map, A, b1, bt1, xexact1, StaticProfile, MakeLocalOnly);
b = dynamic_cast<Epetra_Vector *>(b1);
bt = dynamic_cast<Epetra_Vector *>(bt1);
xexact = dynamic_cast<Epetra_Vector *>(xexact1);
return;
}
void GenerateCrsProblem(int numNodesX, int numNodesY, int numProcsX, int numProcsY, int numPoints,
int * xoff, int * yoff, int nrhs,
const Epetra_Comm &comm, bool verbose, bool summary,
Epetra_Map *& map,
Epetra_CrsMatrix *& A,
Epetra_MultiVector *& b,
Epetra_MultiVector *& bt,
Epetra_MultiVector *&xexact, bool StaticProfile, bool MakeLocalOnly) {
Epetra_Time timer(comm);
// Determine my global IDs
int * myGlobalElements;
GenerateMyGlobalElements(numNodesX, numNodesY, numProcsX, numProcsY, comm.MyPID(), myGlobalElements);
int numMyEquations = numNodesX*numNodesY;
map = new Epetra_Map(-1, numMyEquations, myGlobalElements, 0, comm); // Create map with 2D block partitioning.
delete [] myGlobalElements;
int numGlobalEquations = map->NumGlobalElements();
int profile = 0; if (StaticProfile) profile = numPoints;
#ifdef EPETRA_HAVE_STATICPROFILE
if (MakeLocalOnly)
A = new Epetra_CrsMatrix(Copy, *map, *map, profile, StaticProfile); // Construct matrix with rowmap=colmap
else
A = new Epetra_CrsMatrix(Copy, *map, profile, StaticProfile); // Construct matrix
#else
if (MakeLocalOnly)
A = new Epetra_CrsMatrix(Copy, *map, *map, profile); // Construct matrix with rowmap=colmap
else
A = new Epetra_CrsMatrix(Copy, *map, profile); // Construct matrix
#endif
int * indices = new int[numPoints];
double * values = new double[numPoints];
double dnumPoints = (double) numPoints;
int nx = numNodesX*numProcsX;
for (int i=0; i<numMyEquations; i++) {
int rowID = map->GID(i);
int numIndices = 0;
for (int j=0; j<numPoints; j++) {
int colID = rowID + xoff[j] + nx*yoff[j]; // Compute column ID based on stencil offsets
if (colID>-1 && colID<numGlobalEquations) {
indices[numIndices] = colID;
double value = - ((double) rand())/ ((double) RAND_MAX);
if (colID==rowID)
values[numIndices++] = dnumPoints - value; // Make diagonal dominant
else
values[numIndices++] = value;
}
}
//cout << "Building row " << rowID << endl;
A->InsertGlobalValues(rowID, numIndices, values, indices);
}
delete [] indices;
delete [] values;
double insertTime = timer.ElapsedTime();
timer.ResetStartTime();
A->FillComplete(false);
double fillCompleteTime = timer.ElapsedTime();
if (verbose)
cout << "Time to insert matrix values = " << insertTime << endl
<< "Time to complete fill = " << fillCompleteTime << endl;
if (summary) {
if (comm.NumProc()==1) cout << "InsertTime" << '\t';
cout << insertTime << endl;
if (comm.NumProc()==1) cout << "FillCompleteTime" << '\t';
cout << fillCompleteTime << endl;
}
if (nrhs<=1) {
b = new Epetra_Vector(*map);
bt = new Epetra_Vector(*map);
xexact = new Epetra_Vector(*map);
}
else {
b = new Epetra_MultiVector(*map, nrhs);
bt = new Epetra_MultiVector(*map, nrhs);
xexact = new Epetra_MultiVector(*map, nrhs);
}
xexact->Random(); // Fill xexact with random values
A->Multiply(false, *xexact, *b);
A->Multiply(true, *xexact, *bt);
return;
}
