//==============================================================================
Epetra_CrsMatrix * Poisson2dOperator::GeneratePrecMatrix() const {
// Generate a tridiagonal matrix as an Epetra_CrsMatrix
// This method illustrates how to generate a matrix that is an approximation to the true
// operator. Given this matrix, we can use any of the Aztec or IFPACK preconditioners.
// Create a Epetra_Matrix
Epetra_CrsMatrix * A = new Epetra_CrsMatrix(Copy, *map_, 3);
int NumMyElements = map_->NumMyElements();
long long NumGlobalElements = map_->NumGlobalElements64();
// Add rows one-at-a-time
double negOne = -1.0;
double posTwo = 4.0;
for (int i=0; i<NumMyElements; i++) {
long long GlobalRow = A->GRID64(i);
long long RowLess1 = GlobalRow - 1;
long long RowPlus1 = GlobalRow + 1;
if (RowLess1!=-1) A->InsertGlobalValues(GlobalRow, 1, &negOne, &RowLess1);
if (RowPlus1!=NumGlobalElements) A->InsertGlobalValues(GlobalRow, 1, &negOne, &RowPlus1);
A->InsertGlobalValues(GlobalRow, 1, &posTwo, &GlobalRow);
}
// Finish up
A->FillComplete();
return(A);
}
/* ml_epetra_data_pack::setup - This function does the setup phase for ML_Epetra, pulling
key parameters from the Teuchos list, and calling the aggregation routines
Parameters:
N - Number of unknowns [I]
rowind - Row indices of matrix (CSC format) [I]
colptr - Column indices of matrix (CSC format) [I]
vals - Nonzero values of matrix (CSC format) [I]
Returns: IS_TRUE if setup was succesful, IS_FALSE otherwise
*/
int ml_epetra_data_pack::setup(int N,int* rowind,int* colptr, double* vals){
int i,j;
int *rnz;
/* Nonzero counts for Epetra */
rnz=new int[N];
for(i=0;i<N;i++) rnz[i]=rowind[i+1]-rowind[i];
/* Epetra Setup */
Comm= new Epetra_SerialComm;
Map=new Epetra_Map(N,0,*Comm);
A=new Epetra_CrsMatrix(Copy,*Map,rnz);
/* Do the matrix assembly */
for(i=0;i<N;i++)
for(j=colptr[i];j<colptr[i+1];j++)
A->InsertGlobalValues(rowind[j],1,&vals[j],&i);
//NTS: Redo with block assembly, remembering to transpose
A->FillComplete();
/* Allocate Memory */
Prec=new MultiLevelPreconditioner(*A, *List,false);
/* Build the Heirarchy */
Prec->ComputePreconditioner();
/* Pull Operator Complexity */
operator_complexity = Prec->GetML_Aggregate()->operator_complexity / Prec->GetML_Aggregate()->fine_complexity;
/* Cleanup */
delete rnz;
return IS_TRUE;
}/*end setup*/
Epetra_CrsMatrix* create_and_fill_crs_matrix(const Epetra_Map& emap)
{
int localproc = emap.Comm().MyPID();
int local_n = emap.NumMyElements();
int global_n = emap.NumGlobalElements();
int myFirstGlobalRow = localproc*local_n;
int globalCols[3];
double values[3];
Epetra_CrsMatrix* A = new Epetra_CrsMatrix(Copy, emap, 3);
for(int i=0; i<local_n; ++i) {
int globalRow = myFirstGlobalRow +i;
int numcols = 0;
if (globalRow > 0) {
globalCols[numcols] = globalRow-1;
values[numcols++] = -1.0;
}
globalCols[numcols] = globalRow;
values[numcols++] = 4.0;
if (globalRow < global_n-1) {
globalCols[numcols] = globalRow+1;
values[numcols++] = -1.0;
}
A->InsertGlobalValues(globalRow, numcols, values, globalCols);
}
A->FillComplete();
return A;
}
/*----------------------------------------------------------------------*
| m.gee 11/05|
*----------------------------------------------------------------------*/
Epetra_CrsMatrix* ML_NOX::StripZeros(Epetra_CrsMatrix& in, double eps)
{
Epetra_CrsMatrix* out = new Epetra_CrsMatrix(Copy,in.RowMap(),200);
for (int lrow=0; lrow<in.NumMyRows(); ++lrow)
{
int grow = in.GRID(lrow);
if (grow<0) { cout << "ERROR: grow<0 \n"; exit(0); }
int numentries;
int* lindices;
double* values;
int err = in.ExtractMyRowView(lrow,numentries,values,lindices);
if (err) { cout << "ExtractMyRowView returned " << err << endl; exit(0);}
for (int j=0; j<numentries; ++j)
{
int lcol = lindices[j];
int gcol = in.GCID(lcol);
if (gcol<0) { cout << "ERROR: gcol<0 \n"; exit(0); }
if (abs(values[j])<eps && gcol != grow)
continue;
int err = out->InsertGlobalValues(grow,1,&values[j],&gcol);
if (err) { cout << "InsertGlobalValues returned " << err << endl; exit(0);}
}
}
out->FillComplete();
out->OptimizeStorage();
return out;
}
Epetra_CrsMatrix* TridiagMatrix(const Epetra_Map* Map, const double a, const double b, const double c)
{
Epetra_CrsMatrix* Matrix = new Epetra_CrsMatrix(Copy, *Map, 3);
int NumGlobalElements = Map->NumGlobalElements();
int NumMyElements = Map->NumMyElements();
int* MyGlobalElements = Map->MyGlobalElements();
std::vector<double> Values(2);
std::vector<int> Indices(2);
int NumEntries;
for (int i = 0; i < NumMyElements; ++i)
{
if (MyGlobalElements[i] == 0)
{
// off-diagonal for first row
Indices[0] = 1;
NumEntries = 1;
Values[0] = c;
}
else if (MyGlobalElements[i] == NumGlobalElements - 1)
{
// off-diagonal for last row
Indices[0] = NumGlobalElements - 2;
NumEntries = 1;
Values[0] = b;
}
else
{
// off-diagonal for internal row
Indices[0] = MyGlobalElements[i] - 1;
Values[1] = b;
Indices[1] = MyGlobalElements[i] + 1;
Values[0] = c;
NumEntries = 2;
}
Matrix->InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0],
&Indices[0]);
// Put in the diagonal entry
Values[0] = a;
Matrix->InsertGlobalValues(MyGlobalElements[i], 1, &Values[0],
MyGlobalElements + i);
}
// Finish up, trasforming the matrix entries into local numbering,
// to optimize data transfert during matrix-vector products
Matrix->FillComplete();
Matrix->OptimizeStorage();
return(Matrix);
}
int check_graph_sharing(Epetra_Comm& Comm)
{
int numLocalElems = 5;
int localProc = Comm.MyPID();
int firstElem = localProc*numLocalElems;
int err;
Epetra_Map map(-1, numLocalElems, 0, Comm);
Epetra_CrsMatrix* A = new Epetra_CrsMatrix(Copy, map, 1);
for (int i=0; i<numLocalElems; ++i) {
int row = firstElem+i;
int col = row;
double val = 1.0;
err = A->InsertGlobalValues(row, 1, &val, &col);
if (err != 0) {
cerr << "A->InsertGlobalValues("<<row<<") returned err="<<err<<endl;
return(err);
}
}
A->FillComplete(false);
Epetra_CrsMatrix B(Copy, A->Graph());
delete A;
for (int i=0; i<numLocalElems; ++i) {
int row = firstElem+i;
int col = row;
double val = 1.0;
err = B.ReplaceGlobalValues(row, 1, &val, &col);
if (err != 0) {
cerr << "B.InsertGlobalValues("<<row<<") returned err="<<err<<endl;
return(err);
}
}
return(0);
}
int main(int argc, char *argv[])
{
int ierr = 0;
double elapsed_time;
double total_flops;
double MFLOPs;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm comm( MPI_COMM_WORLD );
#else
Epetra_SerialComm comm;
#endif
bool verbose = false;
bool summary = false;
// Check if we should print verbose results to standard out
if (argc>6) if (argv[6][0]=='-' && argv[6][1]=='v') verbose = true;
// Check if we should print verbose results to standard out
if (argc>6) if (argv[6][0]=='-' && argv[6][1]=='s') summary = true;
if(argc < 6) {
cerr << "Usage: " << argv[0]
<< " NumNodesX NumNodesY NumProcX NumProcY NumPoints [-v|-s]" << endl
<< "where:" << endl
<< "NumNodesX - Number of mesh nodes in X direction per processor" << endl
<< "NumNodesY - Number of mesh nodes in Y direction per processor" << endl
<< "NumProcX - Number of processors to use in X direction" << endl
<< "NumProcY - Number of processors to use in Y direction" << endl
<< "NumPoints - Number of points to use in stencil (5, 9 or 25 only)" << endl
<< "-v|-s - (Optional) Run in verbose mode if -v present or summary mode if -s present" << endl
<< " NOTES: NumProcX*NumProcY must equal the number of processors used to run the problem." << endl << endl
<< " Serial example:" << endl
<< argv[0] << " 16 12 1 1 25 -v" << endl
<< " Run this program in verbose mode on 1 processor using a 16 X 12 grid with a 25 point stencil."<< endl <<endl
<< " MPI example:" << endl
<< "mpirun -np 32 " << argv[0] << " 10 12 4 8 9 -v" << endl
<< " Run this program in verbose mode on 32 processors putting a 10 X 12 subgrid on each processor using 4 processors "<< endl
<< " in the X direction and 8 in the Y direction. Total grid size is 40 points in X and 96 in Y with a 9 point stencil."<< endl
<< endl;
return(1);
}
//char tmp;
//if (comm.MyPID()==0) cout << "Press any key to continue..."<< endl;
//if (comm.MyPID()==0) cin >> tmp;
//comm.Barrier();
comm.SetTracebackMode(0); // This should shut down any error traceback reporting
if (verbose && comm.MyPID()==0)
cout << Epetra_Version() << endl << endl;
if (summary && comm.MyPID()==0) {
if (comm.NumProc()==1)
cout << Epetra_Version() << endl << endl;
else
cout << endl << endl; // Print two blank line to keep output columns lined up
}
if (verbose) cout << comm <<endl;
// Redefine verbose to only print on PE 0
if (verbose && comm.MyPID()!=0) verbose = false;
if (summary && comm.MyPID()!=0) summary = false;
int numNodesX = atoi(argv[1]);
int numNodesY = atoi(argv[2]);
int numProcsX = atoi(argv[3]);
int numProcsY = atoi(argv[4]);
int numPoints = atoi(argv[5]);
if (verbose || (summary && comm.NumProc()==1)) {
cout << " Number of local nodes in X direction = " << numNodesX << endl
<< " Number of local nodes in Y direction = " << numNodesY << endl
<< " Number of global nodes in X direction = " << numNodesX*numProcsX << endl
<< " Number of global nodes in Y direction = " << numNodesY*numProcsY << endl
<< " Number of local nonzero entries = " << numNodesX*numNodesY*numPoints << endl
<< " Number of global nonzero entries = " << numNodesX*numNodesY*numPoints*numProcsX*numProcsY << endl
<< " Number of Processors in X direction = " << numProcsX << endl
<< " Number of Processors in Y direction = " << numProcsY << endl
<< " Number of Points in stencil = " << numPoints << endl << endl;
}
// Print blank line to keep output columns lined up
if (summary && comm.NumProc()>1)
cout << endl << endl << endl << endl << endl << endl << endl << endl<< endl << endl;
if (numProcsX*numProcsY!=comm.NumProc()) {
cerr << "Number of processors = " << comm.NumProc() << endl
<< " is not the product of " << numProcsX << " and " << numProcsY << endl << endl;
return(1);
}
if (numPoints!=5 && numPoints!=9 && numPoints!=25) {
cerr << "Number of points specified = " << numPoints << endl
<< " is not 5, 9, 25" << endl << endl;
return(1);
}
if (numNodesX*numNodesY<=0) {
cerr << "Product of number of nodes is <= zero" << endl << endl;
return(1);
}
Epetra_IntSerialDenseVector Xoff, XLoff, XUoff;
Epetra_IntSerialDenseVector Yoff, YLoff, YUoff;
if (numPoints==5) {
// Generate a 5-point 2D Finite Difference matrix
Xoff.Size(5);
Yoff.Size(5);
Xoff[0] = -1; Xoff[1] = 1; Xoff[2] = 0; Xoff[3] = 0; Xoff[4] = 0;
Yoff[0] = 0; Yoff[1] = 0; Yoff[2] = 0; Yoff[3] = -1; Yoff[4] = 1;
// Generate a 2-point 2D Lower triangular Finite Difference matrix
XLoff.Size(2);
YLoff.Size(2);
XLoff[0] = -1; XLoff[1] = 0;
YLoff[0] = 0; YLoff[1] = -1;
// Generate a 3-point 2D upper triangular Finite Difference matrix
XUoff.Size(3);
YUoff.Size(3);
XUoff[0] = 0; XUoff[1] = 1; XUoff[2] = 0;
YUoff[0] = 0; YUoff[1] = 0; YUoff[2] = 1;
}
else if (numPoints==9) {
// Generate a 9-point 2D Finite Difference matrix
Xoff.Size(9);
Yoff.Size(9);
Xoff[0] = -1; Xoff[1] = 0; Xoff[2] = 1;
Yoff[0] = -1; Yoff[1] = -1; Yoff[2] = -1;
Xoff[3] = -1; Xoff[4] = 0; Xoff[5] = 1;
Yoff[3] = 0; Yoff[4] = 0; Yoff[5] = 0;
Xoff[6] = -1; Xoff[7] = 0; Xoff[8] = 1;
Yoff[6] = 1; Yoff[7] = 1; Yoff[8] = 1;
// Generate a 5-point lower triangular 2D Finite Difference matrix
XLoff.Size(5);
YLoff.Size(5);
XLoff[0] = -1; XLoff[1] = 0; Xoff[2] = 1;
YLoff[0] = -1; YLoff[1] = -1; Yoff[2] = -1;
XLoff[3] = -1; XLoff[4] = 0;
YLoff[3] = 0; YLoff[4] = 0;
// Generate a 4-point upper triangular 2D Finite Difference matrix
XUoff.Size(4);
YUoff.Size(4);
XUoff[0] = 1;
YUoff[0] = 0;
XUoff[1] = -1; XUoff[2] = 0; XUoff[3] = 1;
YUoff[1] = 1; YUoff[2] = 1; YUoff[3] = 1;
}
else {
// Generate a 25-point 2D Finite Difference matrix
Xoff.Size(25);
Yoff.Size(25);
int xi = 0, yi = 0;
int xo = -2, yo = -2;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
xo = -2, yo++;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
xo = -2, yo++;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
xo = -2, yo++;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
xo = -2, yo++;
Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++; Xoff[xi++] = xo++;
Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ; Yoff[yi++] = yo ;
// Generate a 13-point lower triangular 2D Finite Difference matrix
XLoff.Size(13);
YLoff.Size(13);
xi = 0, yi = 0;
xo = -2, yo = -2;
XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++;
YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ;
xo = -2, yo++;
XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++;
YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ;
xo = -2, yo++;
XLoff[xi++] = xo++; XLoff[xi++] = xo++; XLoff[xi++] = xo++;
YLoff[yi++] = yo ; YLoff[yi++] = yo ; YLoff[yi++] = yo ;
// Generate a 13-point upper triangular 2D Finite Difference matrix
XUoff.Size(13);
YUoff.Size(13);
xi = 0, yi = 0;
xo = 0, yo = 0;
XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++;
YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ;
xo = -2, yo++;
XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++;
YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ;
xo = -2, yo++;
XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++; XUoff[xi++] = xo++;
YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ; YUoff[yi++] = yo ;
}
Epetra_Map * map;
Epetra_Map * mapL;
Epetra_Map * mapU;
Epetra_CrsMatrix * A;
Epetra_CrsMatrix * L;
Epetra_CrsMatrix * U;
Epetra_MultiVector * b;
Epetra_MultiVector * bt;
Epetra_MultiVector * xexact;
Epetra_MultiVector * bL;
Epetra_MultiVector * btL;
Epetra_MultiVector * xexactL;
Epetra_MultiVector * bU;
Epetra_MultiVector * btU;
Epetra_MultiVector * xexactU;
Epetra_SerialDenseVector resvec(0);
//Timings
Epetra_Flops flopcounter;
Epetra_Time timer(comm);
#ifdef EPETRA_VERY_SHORT_PERFTEST
int jstop = 1;
#elif EPETRA_SHORT_PERFTEST
int jstop = 1;
#else
int jstop = 2;
#endif
for (int j=0; j<jstop; j++) {
for (int k=1; k<17; k++) {
#ifdef EPETRA_VERY_SHORT_PERFTEST
if (k<3 || (k%4==0 && k<9)) {
#elif EPETRA_SHORT_PERFTEST
if (k<6 || k%4==0) {
#else
if (k<7 || k%2==0) {
#endif
int nrhs=k;
if (verbose) cout << "\n*************** Results for " << nrhs << " RHS with ";
bool StaticProfile = (j!=0);
if (verbose) {
if (StaticProfile) cout << " static profile\n";
else cout << " dynamic profile\n";
}
GenerateCrsProblem(numNodesX, numNodesY, numProcsX, numProcsY, numPoints,
Xoff.Values(), Yoff.Values(), nrhs, comm, verbose, summary,
map, A, b, bt, xexact, StaticProfile, false);
#ifdef EPETRA_HAVE_JADMATRIX
timer.ResetStartTime();
Epetra_JadMatrix JA(*A);
elapsed_time = timer.ElapsedTime();
if (verbose) cout << "Time to create Jagged diagonal matrix = " << elapsed_time << endl;
//cout << "A = " << *A << endl;
//cout << "JA = " << JA << endl;
runJadMatrixTests(&JA, b, bt, xexact, StaticProfile, verbose, summary);
#endif
runMatrixTests(A, b, bt, xexact, StaticProfile, verbose, summary);
delete A;
delete b;
delete bt;
delete xexact;
GenerateCrsProblem(numNodesX, numNodesY, numProcsX, numProcsY, XLoff.Length(),
XLoff.Values(), YLoff.Values(), nrhs, comm, verbose, summary,
mapL, L, bL, btL, xexactL, StaticProfile, true);
GenerateCrsProblem(numNodesX, numNodesY, numProcsX, numProcsY, XUoff.Length(),
XUoff.Values(), YUoff.Values(), nrhs, comm, verbose, summary,
mapU, U, bU, btU, xexactU, StaticProfile, true);
runLUMatrixTests(L, bL, btL, xexactL, U, bU, btU, xexactU, StaticProfile, verbose, summary);
delete L;
delete bL;
delete btL;
delete xexactL;
delete mapL;
delete U;
delete bU;
delete btU;
delete xexactU;
delete mapU;
Epetra_MultiVector q(*map, nrhs);
Epetra_MultiVector z(q);
Epetra_MultiVector r(q);
delete map;
q.SetFlopCounter(flopcounter);
z.SetFlopCounter(q);
r.SetFlopCounter(q);
resvec.Resize(nrhs);
flopcounter.ResetFlops();
timer.ResetStartTime();
//10 norms
for( int i = 0; i < 10; ++i )
q.Norm2( resvec.Values() );
elapsed_time = timer.ElapsedTime();
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "\nTotal MFLOPs for 10 Norm2's= " << MFLOPs << endl;
if (summary) {
if (comm.NumProc()==1) cout << "Norm2" << '\t';
cout << MFLOPs << endl;
}
flopcounter.ResetFlops();
timer.ResetStartTime();
//10 dot's
for( int i = 0; i < 10; ++i )
q.Dot(z, resvec.Values());
elapsed_time = timer.ElapsedTime();
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Total MFLOPs for 10 Dot's = " << MFLOPs << endl;
if (summary) {
if (comm.NumProc()==1) cout << "DotProd" << '\t';
cout << MFLOPs << endl;
}
flopcounter.ResetFlops();
timer.ResetStartTime();
//10 dot's
for( int i = 0; i < 10; ++i )
q.Update(1.0, z, 1.0, r, 0.0);
elapsed_time = timer.ElapsedTime();
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Total MFLOPs for 10 Updates= " << MFLOPs << endl;
if (summary) {
if (comm.NumProc()==1) cout << "Update" << '\t';
cout << MFLOPs << endl;
}
}
}
}
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return ierr ;
}
// Constructs a 2D PDE finite difference matrix using the list of x and y offsets.
//
// nx (In) - number of grid points in x direction
// ny (In) - number of grid points in y direction
// The total number of equations will be nx*ny ordered such that the x direction changes
// most rapidly:
// First equation is at point (0,0)
// Second at (1,0)
// ...
// nx equation at (nx-1,0)
// nx+1st equation at (0,1)
// numPoints (In) - number of points in finite difference stencil
// xoff (In) - stencil offsets in x direction (of length numPoints)
// yoff (In) - stencil offsets in y direction (of length numPoints)
// A standard 5-point finite difference stencil would be described as:
// numPoints = 5
// xoff = [-1, 1, 0, 0, 0]
// yoff = [ 0, 0, 0, -1, 1]
// nrhs - Number of rhs to generate. (First interface produces vectors, so nrhs is not needed
// comm (In) - an Epetra_Comm object describing the parallel machine (numProcs and my proc ID)
// map (Out) - Epetra_Map describing distribution of matrix and vectors/multivectors
// A (Out) - Epetra_CrsMatrix constructed for nx by ny grid using prescribed stencil
// Off-diagonal values are random between 0 and 1. If diagonal is part of stencil,
// diagonal will be slightly diag dominant.
// b (Out) - Generated RHS. Values satisfy b = A*xexact
// bt (Out) - Generated RHS. Values satisfy b = A'*xexact
// xexact (Out) - Generated exact solution to Ax = b and b' = A'xexact
// Note: Caller of this function is responsible for deleting all output objects.
void GenerateCrsProblem(int numNodesX, int numNodesY, int numProcsX, int numProcsY, int numPoints,
int * xoff, int * yoff,
const Epetra_Comm &comm, bool verbose, bool summary,
Epetra_Map *& map,
Epetra_CrsMatrix *& A,
Epetra_Vector *& b,
Epetra_Vector *& bt,
Epetra_Vector *&xexact, bool StaticProfile, bool MakeLocalOnly) {
Epetra_MultiVector * b1, * bt1, * xexact1;
GenerateCrsProblem(numNodesX, numNodesY, numProcsX, numProcsY, numPoints,
xoff, yoff, 1, comm, verbose, summary,
map, A, b1, bt1, xexact1, StaticProfile, MakeLocalOnly);
b = dynamic_cast<Epetra_Vector *>(b1);
bt = dynamic_cast<Epetra_Vector *>(bt1);
xexact = dynamic_cast<Epetra_Vector *>(xexact1);
return;
}
void GenerateCrsProblem(int numNodesX, int numNodesY, int numProcsX, int numProcsY, int numPoints,
int * xoff, int * yoff, int nrhs,
const Epetra_Comm &comm, bool verbose, bool summary,
Epetra_Map *& map,
Epetra_CrsMatrix *& A,
Epetra_MultiVector *& b,
Epetra_MultiVector *& bt,
Epetra_MultiVector *&xexact, bool StaticProfile, bool MakeLocalOnly) {
Epetra_Time timer(comm);
// Determine my global IDs
long long * myGlobalElements;
GenerateMyGlobalElements(numNodesX, numNodesY, numProcsX, numProcsY, comm.MyPID(), myGlobalElements);
int numMyEquations = numNodesX*numNodesY;
map = new Epetra_Map((long long)-1, numMyEquations, myGlobalElements, 0, comm); // Create map with 2D block partitioning.
delete [] myGlobalElements;
long long numGlobalEquations = map->NumGlobalElements64();
int profile = 0; if (StaticProfile) profile = numPoints;
#ifdef EPETRA_HAVE_STATICPROFILE
if (MakeLocalOnly)
A = new Epetra_CrsMatrix(Copy, *map, *map, profile, StaticProfile); // Construct matrix with rowmap=colmap
else
A = new Epetra_CrsMatrix(Copy, *map, profile, StaticProfile); // Construct matrix
#else
if (MakeLocalOnly)
A = new Epetra_CrsMatrix(Copy, *map, *map, profile); // Construct matrix with rowmap=colmap
else
A = new Epetra_CrsMatrix(Copy, *map, profile); // Construct matrix
#endif
long long * indices = new long long[numPoints];
double * values = new double[numPoints];
double dnumPoints = (double) numPoints;
int nx = numNodesX*numProcsX;
for (int i=0; i<numMyEquations; i++) {
long long rowID = map->GID64(i);
int numIndices = 0;
for (int j=0; j<numPoints; j++) {
long long colID = rowID + xoff[j] + nx*yoff[j]; // Compute column ID based on stencil offsets
if (colID>-1 && colID<numGlobalEquations) {
indices[numIndices] = colID;
double value = - ((double) rand())/ ((double) RAND_MAX);
if (colID==rowID)
values[numIndices++] = dnumPoints - value; // Make diagonal dominant
else
values[numIndices++] = value;
}
}
//cout << "Building row " << rowID << endl;
A->InsertGlobalValues(rowID, numIndices, values, indices);
}
delete [] indices;
delete [] values;
double insertTime = timer.ElapsedTime();
timer.ResetStartTime();
A->FillComplete(false);
double fillCompleteTime = timer.ElapsedTime();
if (verbose)
cout << "Time to insert matrix values = " << insertTime << endl
<< "Time to complete fill = " << fillCompleteTime << endl;
if (summary) {
if (comm.NumProc()==1) cout << "InsertTime" << '\t';
cout << insertTime << endl;
if (comm.NumProc()==1) cout << "FillCompleteTime" << '\t';
cout << fillCompleteTime << endl;
}
if (nrhs<=1) {
b = new Epetra_Vector(*map);
bt = new Epetra_Vector(*map);
xexact = new Epetra_Vector(*map);
}
else {
b = new Epetra_MultiVector(*map, nrhs);
bt = new Epetra_MultiVector(*map, nrhs);
xexact = new Epetra_MultiVector(*map, nrhs);
}
xexact->Random(); // Fill xexact with random values
A->Multiply(false, *xexact, *b);
A->Multiply(true, *xexact, *bt);
return;
}
int Aztec2Petra(int * proc_config,
AZ_MATRIX * Amat, double * az_x, double * az_b,
Epetra_Comm * & comm,
Epetra_BlockMap * & map,
Epetra_RowMatrix * &A,
Epetra_Vector * & x,
Epetra_Vector * & b,
int ** global_indices) {
bool do_throw = false;
#ifdef EPETRA_NO_32BIT_GLOBAL_INDICES
do_throw = true;
#else
do_throw =
map->GlobalIndicesLongLong() ||
A->RowMatrixRowMap().GlobalIndicesLongLong();
#endif
if(do_throw) {
// We throw rather than let the compiler error out so that the
// rest of the library is available and all possible tests can run.
const char* error = "Aztec2Petra: Not available for 64-bit Maps.";
std::cerr << error << std::endl;
throw error;
}
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES // REMOVE BEGIN
// If no 32 bit indices, remove the code below using the preprocessor
// otherwise VbrMatrix functions cause linker issues.
// Build Epetra_Comm object
#ifdef AZTEC_MPI
MPI_Comm * mpicomm = (MPI_Comm * ) AZ_get_comm(proc_config);
comm = (Epetra_Comm *) new Epetra_MpiComm(*mpicomm);
#else
comm = (Epetra_Comm *) new Epetra_SerialComm();
#endif
int * MyGlobalElements, *global_bindx, *update;
if (!Amat->has_global_indices) {
//create a global bindx
AZ_revert_to_global(proc_config, Amat, &global_bindx, &update);
MyGlobalElements = update;
}
else // Already have global ordering
{
global_bindx = Amat->bindx;
MyGlobalElements = Amat->update;
if (MyGlobalElements==0) EPETRA_CHK_ERR(-1);
}
// Get matrix information
int NumMyElements = 0;
if (Amat->data_org[AZ_matrix_type] == AZ_VBR_MATRIX)
NumMyElements = Amat->data_org[AZ_N_int_blk] + Amat->data_org[AZ_N_bord_blk];
else
NumMyElements = Amat->data_org[AZ_N_internal] + Amat->data_org[AZ_N_border];
// int NumMyElements = Amat->N_update; // Note: This "official" way does not always work
int * bpntr = Amat->bpntr;
int * rpntr = Amat->rpntr;
int * indx = Amat->indx;
double * val = Amat->val;
int NumGlobalElements;
comm->SumAll(&NumMyElements, &NumGlobalElements, 1);
// Make ElementSizeList (if VBR) - number of block entries in each block row
int * ElementSizeList = 0;
if (Amat->data_org[AZ_matrix_type] == AZ_VBR_MATRIX) {
ElementSizeList = new int[NumMyElements];
if (ElementSizeList==0) EPETRA_CHK_ERR(-1); // Ran out of memory
for (int i=0; i<NumMyElements; i++) ElementSizeList[i] = rpntr[i+1] - rpntr[i];
#ifdef EPETRA_NO_32BIT_GLOBAL_INDICES
map = 0;
#else
map = new Epetra_BlockMap(NumGlobalElements, NumMyElements, MyGlobalElements,
ElementSizeList, 0, *comm);
#endif
if (map==0) EPETRA_CHK_ERR(-2); // Ran out of memory
delete [] ElementSizeList;
Epetra_VbrMatrix * AA = new Epetra_VbrMatrix(View, *map, 0);
if (AA==0) EPETRA_CHK_ERR(-3); // Ran out of memory
/* Add block rows one-at-a-time */
{for (int i=0; i<NumMyElements; i++) {
int BlockRow = MyGlobalElements[i];
int NumBlockEntries = bpntr[i+1] - bpntr[i];
int *BlockIndices = global_bindx + bpntr[i];
int ierr = AA->BeginInsertGlobalValues(BlockRow, NumBlockEntries, BlockIndices);
if (ierr!=0) {
cerr << "Error in BeginInsertGlobalValues(GlobalBlockRow = " << BlockRow
<< ") = " << ierr << endl;
EPETRA_CHK_ERR(ierr);
}
int LDA = rpntr[i+1] - rpntr[i];
int NumRows = LDA;
for (int j=bpntr[i]; j<bpntr[i+1]; j++) {
int NumCols = (indx[j+1] - indx[j])/LDA;
double * Values = val + indx[j];
ierr = AA->SubmitBlockEntry(Values, LDA, NumRows, NumCols);
if (ierr!=0) {
cerr << "Error in SubmitBlockEntry, GlobalBlockRow = " << BlockRow
<< "GlobalBlockCol = " << BlockIndices[j] << "Error = " << ierr << endl;
EPETRA_CHK_ERR(ierr);
}
}
ierr = AA->EndSubmitEntries();
if (ierr!=0) {
cerr << "Error in EndSubmitEntries(GlobalBlockRow = " << BlockRow
<< ") = " << ierr << endl;
EPETRA_CHK_ERR(ierr);
}
}}
int ierr=AA->FillComplete();
if (ierr!=0) {
cerr <<"Error in Epetra_VbrMatrix FillComplete" << ierr << endl;
EPETRA_CHK_ERR(ierr);
}
A = dynamic_cast<Epetra_RowMatrix *> (AA); // cast VBR pointer to RowMatrix pointer
}
else if (Amat->data_org[AZ_matrix_type] == AZ_MSR_MATRIX) {
/* Make numNzBlks - number of block entries in each block row */
int * numNz = new int[NumMyElements];
for (int i=0; i<NumMyElements; i++) numNz[i] = global_bindx[i+1] - global_bindx[i] + 1;
#ifdef EPETRA_NO_32BIT_GLOBAL_INDICES
Epetra_Map * map1 = 0;
#else
Epetra_Map * map1 = new Epetra_Map(NumGlobalElements, NumMyElements,
MyGlobalElements, 0, *comm);
#endif
Epetra_CrsMatrix * AA = new Epetra_CrsMatrix(Copy, *map1, numNz);
map = (Epetra_BlockMap *) map1; // cast Epetra_Map to Epetra_BlockMap
/* Add rows one-at-a-time */
for (int row=0; row<NumMyElements; row++) {
double * row_vals = val + global_bindx[row];
int * col_inds = global_bindx + global_bindx[row];
int numEntries = global_bindx[row+1] - global_bindx[row];
#ifdef EPETRA_NO_32BIT_GLOBAL_INDICES
int ierr = 1;
#else
int ierr = AA->InsertGlobalValues(MyGlobalElements[row], numEntries, row_vals, col_inds);
#endif
if (ierr!=0) {
cerr << "Error puting row " << MyGlobalElements[row] << endl;
EPETRA_CHK_ERR(ierr);
}
#ifdef EPETRA_NO_32BIT_GLOBAL_INDICES
ierr = 1;
#else
ierr = AA->InsertGlobalValues(MyGlobalElements[row], 1, val+row, MyGlobalElements+row);
#endif
if (ierr!=0) {
cerr << "Error putting diagonal" << endl;
EPETRA_CHK_ERR(ierr);
}
}
int ierr=AA->FillComplete();
if (ierr!=0) {
cerr << "Error in Epetra_CrsMatrix_FillComplete" << endl;
EPETRA_CHK_ERR(ierr);
}
A = dynamic_cast<Epetra_RowMatrix *> (AA); // cast CRS pointer to RowMatrix pointer
}
else cerr << "Not a supported AZ_MATRIX data type" << endl;
// Create x vector
x = new Epetra_Vector(View, *map,az_x);
// RPP: Can not use the OperatorRangeMap in the ctor of the "b" vector
// below. In MPSalsa, we delete the VbrMatrix yet still use the vector "b".
// Deleting the matrix deletes the OperatorRangeMap that the b vector is
// based on. Losing the map means "b" and all vectors that are created
// with the copy constructor of "b" break. Mike has suggested
// using reference counting (Boost smart pointers) so the map is not
// deleted. For now we will use the "map" variable as the base map for "b".
//b = new Epetra_Vector (View, A->OperatorRangeMap(), az_b);
b = new Epetra_Vector (View, *map, az_b);
*global_indices = 0; // Assume return array will be empty
if (!Amat->has_global_indices) {
AZ_free((void *) update);
if (Amat->data_org[AZ_matrix_type] != AZ_VBR_MATRIX)
AZ_free((void *) global_bindx);
else
global_indices = &global_bindx;
}
#endif // EPETRA_NO_32BIT_GLOBAL_INDICES REMOVE END
return 0;
}
int MyCreateCrsMatrix( char *in_filename, const Epetra_Comm &Comm,
Epetra_Map *& readMap,
const bool transpose, const bool distribute,
bool& symmetric, Epetra_CrsMatrix *& Matrix ) {
Epetra_CrsMatrix * readA = 0;
Epetra_Vector * readx = 0;
Epetra_Vector * readb = 0;
Epetra_Vector * readxexact = 0;
//
// This hack allows TestOptions to be run from either the test/TestOptions/ directory or from
// the test/ directory (as it is in nightly testing and in make "run-tests")
//
FILE *in_file = fopen( in_filename, "r");
char *filename;
if (in_file == NULL )
filename = &in_filename[1] ; // Strip off ithe "." from
// "../" and try again
else {
filename = in_filename ;
fclose( in_file );
}
symmetric = false ;
std::string FileName = filename ;
int FN_Size = FileName.size() ;
std::string LastFiveBytes = FileName.substr( EPETRA_MAX(0,FN_Size-5), FN_Size );
std::string LastFourBytes = FileName.substr( EPETRA_MAX(0,FN_Size-4), FN_Size );
if ( LastFiveBytes == ".triU" ) {
// Call routine to read in unsymmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( filename, false, Comm, readMap, readA, readx,
readb, readxexact) );
symmetric = false;
} else {
if ( LastFiveBytes == ".triS" ) {
// Call routine to read in symmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( filename, true, Comm, readMap, readA, readx,
readb, readxexact) );
symmetric = true;
} else {
if ( LastFourBytes == ".mtx" ) {
EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra( filename, Comm, readMap,
readA, readx, readb, readxexact) );
FILE* in_file = fopen( filename, "r");
assert (in_file != NULL) ; // Checked in Trilinos_Util_CountMatrixMarket()
const int BUFSIZE = 800 ;
char buffer[BUFSIZE] ;
fgets( buffer, BUFSIZE, in_file ) ; // Pick symmetry info off of this string
std::string headerline1 = buffer;
#ifdef TFLOP
if ( headerline1.find("symmetric") < BUFSIZE ) symmetric = true;
#else
if ( headerline1.find("symmetric") != std::string::npos) symmetric = true;
#endif
fclose(in_file);
} else {
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra( filename, Comm, readMap, readA, readx,
readb, readxexact) ;
if ( LastFourBytes == ".rsa" ) symmetric = true ;
}
}
}
if ( readb ) delete readb;
if ( readx ) delete readx;
if ( readxexact ) delete readxexact;
Epetra_CrsMatrix *serialA ;
Epetra_CrsMatrix *transposeA;
if ( transpose ) {
transposeA = new Epetra_CrsMatrix( Copy, *readMap, 0 );
assert( CrsMatrixTranspose( readA, transposeA ) == 0 );
serialA = transposeA ;
delete readA;
readA = 0 ;
} else {
serialA = readA ;
}
assert( (void *) &serialA->Graph() ) ;
assert( (void *) &serialA->RowMap() ) ;
assert( serialA->RowMap().SameAs(*readMap) ) ;
if ( distribute ) {
// Create uniform distributed map
Epetra_Map DistMap(readMap->NumGlobalElements(), 0, Comm);
// Create Exporter to distribute read-in matrix and vectors
Epetra_Export exporter( *readMap, DistMap );
Epetra_CrsMatrix *Amat = new Epetra_CrsMatrix( Copy, DistMap, 0 );
Amat->Export(*serialA, exporter, Add);
assert(Amat->FillComplete()==0);
Matrix = Amat;
//
// Make sure that deleting Amat->RowMap() will delete map
//
// Bug: We can't manage to delete map his way anyway,
// and this fails on tranposes, so for now I just accept
// the memory loss.
// assert( &(Amat->RowMap()) == map ) ;
delete readMap;
readMap = 0 ;
delete serialA;
} else {
Matrix = serialA;
}
return 0;
}
//============================================================================
Epetra_CrsMatrix* Ifpack_CreateOverlappingCrsMatrix(const Epetra_RowMatrix* Matrix,
const int OverlappingLevel)
{
if (OverlappingLevel == 0)
return(0); // All done
if (Matrix->Comm().NumProc() == 1)
return(0); // All done
Epetra_CrsMatrix* OverlappingMatrix;
OverlappingMatrix = 0;
Epetra_Map* OverlappingMap;
OverlappingMap = (Epetra_Map*)&(Matrix->RowMatrixRowMap());
const Epetra_RowMatrix* OldMatrix;
const Epetra_Map* DomainMap = &(Matrix->OperatorDomainMap());
const Epetra_Map* RangeMap = &(Matrix->OperatorRangeMap());
for (int level = 1; level <= OverlappingLevel ; ++level) {
if (OverlappingMatrix)
OldMatrix = OverlappingMatrix;
else
OldMatrix = Matrix;
Epetra_Import* OverlappingImporter;
OverlappingImporter = (Epetra_Import*)OldMatrix->RowMatrixImporter();
int NumMyElements = OverlappingImporter->TargetMap().NumMyElements();
// need to build an Epetra_Map in this way because Epetra_CrsMatrix
// requires Epetra_Map and not Epetra_BlockMap
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
if(OverlappingImporter->TargetMap().GlobalIndicesInt()) {
int* MyGlobalElements = OverlappingImporter->TargetMap().MyGlobalElements();
OverlappingMap = new Epetra_Map(-1,NumMyElements,MyGlobalElements,
0, Matrix->Comm());
}
else
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
if(OverlappingImporter->TargetMap().GlobalIndicesLongLong()) {
long long* MyGlobalElements = OverlappingImporter->TargetMap().MyGlobalElements64();
OverlappingMap = new Epetra_Map((long long) -1,NumMyElements,MyGlobalElements,
0, Matrix->Comm());
}
else
#endif
throw "Ifpack_CreateOverlappingCrsMatrix: GlobalIndices type unknown";
if (level < OverlappingLevel)
OverlappingMatrix = new Epetra_CrsMatrix(Copy, *OverlappingMap, 0);
else
// On last iteration, we want to filter out all columns except
// those that correspond
// to rows in the graph. This assures that our matrix is square
OverlappingMatrix = new Epetra_CrsMatrix(Copy, *OverlappingMap,
*OverlappingMap, 0);
OverlappingMatrix->Import(*OldMatrix, *OverlappingImporter, Insert);
if (level < OverlappingLevel) {
OverlappingMatrix->FillComplete(*DomainMap, *RangeMap);
}
else {
OverlappingMatrix->FillComplete(*DomainMap, *RangeMap);
}
delete OverlappingMap;
if (level > 1) {
delete OldMatrix;
}
OverlappingMatrix->FillComplete();
}
return(OverlappingMatrix);
}
int main(int narg, char *arg[])
{
using std::cout;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&narg,&arg);
Epetra_MpiComm comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm comm;
#endif
int me = comm.MyPID();
int np = comm.NumProc();
ITYPE nGlobalRows = 10;
if (narg > 1)
nGlobalRows = (ITYPE) atol(arg[1]);
bool verbose = (nGlobalRows < 20);
// Linear map similar to Trilinos default,
// but want to allow adding OFFSET_EPETRA64 to the indices.
int nMyRows = (int) (nGlobalRows / np + (nGlobalRows % np > me));
ITYPE myFirstRow = (ITYPE)(me * (nGlobalRows / np) + MIN(nGlobalRows%np, me));
ITYPE *myGlobalRows = new ITYPE[nMyRows];
for (int i = 0; i < nMyRows; i++)
myGlobalRows[i] = (ITYPE)i + myFirstRow + OFFSET_EPETRA64;
Epetra_Map *rowMap = new Epetra_Map(-1, nMyRows, &myGlobalRows[0], 0, comm);
if (verbose) rowMap->Print(std::cout);
// Create an integer vector nnzPerRow that is used to build the Epetra Matrix.
// nnzPerRow[i] is the number of entries for the ith local equation
std::vector<int> nnzPerRow(nMyRows+1, 0);
// Also create lists of the nonzeros to be assigned to processors.
// To save programming time and complexity, these vectors are allocated
// bigger than they may actually be needed.
std::vector<ITYPE> iv(3*nMyRows+1);
std::vector<ITYPE> jv(3*nMyRows+1);
std::vector<double> vv(3*nMyRows+1);
// Generate the nonzeros for the Laplacian matrix.
ITYPE nMyNonzeros = 0;
for (ITYPE i = 0, myrowcnt = 0; i < nGlobalRows; i++) {
if (rowMap->MyGID(i+OFFSET_EPETRA64)) {
// This processor owns this row; add nonzeros.
if (i > 0) {
iv[nMyNonzeros] = i + OFFSET_EPETRA64;
jv[nMyNonzeros] = i-1 + OFFSET_EPETRA64;
vv[nMyNonzeros] = -1;
if (verbose)
std::cout << "(" << iv[nMyNonzeros] << "," << jv[nMyNonzeros] << ")="
<< vv[nMyNonzeros] << " on processor " << me
<< " in " << myrowcnt << std::endl;
nMyNonzeros++;
nnzPerRow[myrowcnt]++;
}
iv[nMyNonzeros] = i + OFFSET_EPETRA64;
jv[nMyNonzeros] = i + OFFSET_EPETRA64;
vv[nMyNonzeros] = ((i == 0 || i == nGlobalRows-1) ? 1. : 2.);
if (verbose)
std::cout << "(" << iv[nMyNonzeros] << "," << jv[nMyNonzeros] << ")="
<< vv[nMyNonzeros] << " on processor " << me
<< " in " << myrowcnt << std::endl;
nMyNonzeros++;
nnzPerRow[myrowcnt]++;
if (i < nGlobalRows - 1) {
iv[nMyNonzeros] = i + OFFSET_EPETRA64;
jv[nMyNonzeros] = i+1 + OFFSET_EPETRA64;
vv[nMyNonzeros] = -1;
if (verbose)
std::cout << "(" << iv[nMyNonzeros] << "," << jv[nMyNonzeros] << ")="
<< vv[nMyNonzeros] << " on processor " << me
<< " in " << myrowcnt << std::endl;
nMyNonzeros++;
nnzPerRow[myrowcnt]++;
}
myrowcnt++;
}
}
// Create an Epetra_Matrix
Epetra_CrsMatrix *A = new Epetra_CrsMatrix(Copy, *rowMap, &nnzPerRow[0], false);
// Insert the nonzeros.
int info;
ITYPE sum = 0;
for (int i=0; i < nMyRows; i++) {
if (nnzPerRow[i]) {
if (verbose) {
std::cout << "InsertGlobalValus row " << iv[sum]
<< " count " << nnzPerRow[i]
<< " cols " << jv[sum] << " " << jv[sum+1] << " ";
if (nnzPerRow[i] == 3) std::cout << jv[sum+2];
std::cout << std::endl;
}
info = A->InsertGlobalValues(iv[sum],nnzPerRow[i],&vv[sum],&jv[sum]);
assert(info==0);
sum += nnzPerRow[i];
}
}
// Finish up
info = A->FillComplete();
assert(info==0);
if (verbose) A->Print(std::cout);
// Sanity test: Product of matrix and vector of ones should have norm == 0
// and max/min/mean values of 0
Epetra_Vector sanity(A->RangeMap());
Epetra_Vector sanityres(A->DomainMap());
sanity.PutScalar(1.);
A->Multiply(false, sanity, sanityres);
double jjone, jjtwo, jjmax;
sanityres.Norm1(&jjone);
sanityres.Norm2(&jjtwo);
sanityres.NormInf(&jjmax);
if (me == 0)
std::cout << "SanityTest norms 1/2/inf: " << jjone << " "
<< jjtwo << " " << jjmax << std::endl;
bool test_failed = (jjone != 0) || (jjtwo != 0) || (jjmax != 0);
sanityres.MinValue(&jjone);
sanityres.MeanValue(&jjtwo);
sanityres.MaxValue(&jjmax);
if (me == 0)
std::cout << "SanityTest values min/max/avg: " << jjone << " "
<< jjmax << " " << jjtwo << std::endl;
test_failed = test_failed || (jjone != 0) || (jjtwo != 0) || (jjmax != 0);
if (me == 0) {
if(test_failed)
std::cout << "Bug_5794_IndexBase_LL tests FAILED" << std::endl;
}
delete A;
delete rowMap;
delete [] myGlobalRows;
FINALIZE;
}
int main(int argc, char *argv[])
{
// Create a communicator for Epetra objects.
#ifdef HAVE_MPI
MPI_Init( &argc, &argv );
RCP<const Epetra_MpiComm> Comm =
rcp<Epetra_MpiComm>(new Epetra_MpiComm(MPI_COMM_WORLD));
#else
RCP<const Epetra_SerialComm> Comm =
rcp<Epetra_SerialComm>(new Epetra_SerialComm());
#endif
const int NumGlobalElements = 10;
// ---------------------------------------------------------------------------
// Construct a Map with NumElements and index base of 0
Epetra_Map Map(NumGlobalElements, 0, *Comm);
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map.NumMyElements();
std::vector<int> MyGlobalElements(NumMyElements);
Map.MyGlobalElements(&MyGlobalElements[0]);
// NumNz[i] is the number of nonzero elements in row i of the sparse
// matrix on this MPI process. Epetra_CrsMatrix uses this to figure
// out how much space to allocate.
std::vector<int> NumNz (NumMyElements);
// We are building a tridiagonal matrix where each row contains the
// nonzero elements (-1 2 -1). Thus, we need 2 off-diagonal terms,
// except for the first and last row of the matrix.
for (int i = 0; i < NumMyElements; ++i)
if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1)
NumNz[i] = 2; // First or last row
else
NumNz[i] = 3; // Not the (first or last row)
// Create the Epetra_CrsMatrix.
Epetra_CrsMatrix A (Copy, Map, &NumNz[0]);
//
// Add rows to the sparse matrix one at a time.
//
std::vector<double> Values(2);
Values[0] = -1.0; Values[1] = -1.0;
std::vector<int> Indices(2);
const double two = 2.0;
int NumEntries;
for (int i = 0; i < NumMyElements; ++i)
{
if (MyGlobalElements[i] == 0)
{ // The first row of the matrix.
Indices[0] = 1;
NumEntries = 1;
}
else if (MyGlobalElements[i] == NumGlobalElements - 1)
{ // The last row of the matrix.
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
}
else
{ // Any row of the matrix other than the first or last.
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
TEUCHOS_ASSERT_EQUALITY(0, A.InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]));
// Insert the diagonal entry.
TEUCHOS_ASSERT_EQUALITY(0, A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i]));
}
// Finish up. We can call FillComplete() with no arguments, because
// the matrix is square.
TEUCHOS_ASSERT_EQUALITY(0, A.FillComplete());
// ---------------------------------------------------------------------------
// Now setup the Belos solver.
Teuchos::ParameterList belosList;
belosList.set("Convergence Tolerance", 1.0e-10);
belosList.set("Maximum Iterations", 1000);
belosList.set("Output Frequency", 1);
belosList.set("Output Style", (int) Belos::Brief);
belosList.set("Verbosity", Belos::Errors+Belos::StatusTestDetails+Belos::Warnings+Belos::TimingDetails+Belos::IterationDetails+Belos::FinalSummary );
RCP<Epetra_Vector> x = rcp(new Epetra_Vector(Map));
TEUCHOS_ASSERT_EQUALITY(0, x->PutScalar(0.0)); // not strictly necessary
RCP<Epetra_Vector> b = rcp(new Epetra_Vector(Map));
TEUCHOS_ASSERT_EQUALITY(0, b->Random());
// Construct an unpreconditioned linear problem instance.
Belos::LinearProblem<double,MV,OP> problem(Teuchos::rcpFromRef(A),
x, b);
// Make sure the problem sets up correctly.
TEUCHOS_ASSERT(problem.setProblem());
// Create an iterative solver manager.
RCP<Belos::SolverManager<double,MV,OP> > newSolver =
rcp(new Belos::PseudoBlockCGSolMgr<double,MV,OP>(rcp(&problem, false), rcp(&belosList, false)));
// Perform solve.
Belos::ReturnType ret = newSolver->solve();
if (ret==Belos::Converged)
std::cout << "Success!" << std::endl;
// ---------------------------------------------------------------------------
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return EXIT_SUCCESS;
}
void
example (const Epetra_Comm& comm)
{
// The global number of rows in the matrix A to create. We scale
// this relative to the number of (MPI) processes, so that no matter
// how many MPI processes you run, every process will have 10 rows.
const global_ordinal_type numGblElts = 10 * comm.NumProc ();
// The global min global index in all the Maps here.
const global_ordinal_type indexBase = 0;
// Local error code for use below.
//
// In the ideal case, we would use this to emulate behavior like
// that of Haskell's Maybe in the context of MPI. That is, if one
// process experiences an error, we don't want to abort early and
// cause the other processes to deadlock on MPI communication
// operators. Rather, we want to chain along the local error state,
// until we reach a point where it's natural to pass along that
// state with other processes. For example, if one is doing an
// MPI_Allreduce anyway, it makes sense to pass along one more bit
// of information: whether the calling process is in a local error
// state. Epetra's interface doesn't let one chain the local error
// state in this way, so we use extra collectives below to propagate
// that state. The code below uses very conservative error checks;
// typical user code would not need to be so conservative and could
// therefore avoid all the all-reduces.
int lclerr = 0;
// Construct a Map that is global (not locally replicated), but puts
// all the equations on MPI Proc 0.
const int procZeroMapNumLclElts = (comm.MyPID () == 0) ?
numGblElts :
static_cast<global_ordinal_type> (0);
Epetra_Map procZeroMap (numGblElts, procZeroMapNumLclElts, indexBase, comm);
// Construct a Map that puts approximately the same number of
// equations on each processor.
Epetra_Map globalMap (numGblElts, indexBase, comm);
// Create a sparse matrix using procZeroMap.
Epetra_CrsMatrix* A = createCrsMatrix (procZeroMap);
if (A == NULL) {
lclerr = 1;
}
// Make sure that sparse matrix creation succeeded. Normally you
// don't have to check this; we are being extra conservative because
// this example is also a test. Even though the matrix's rows live
// entirely on Process 0, the matrix is nonnull on all processes in
// its Map's communicator.
int gblerr = 0;
(void) comm.MaxAll (&lclerr, &gblerr, 1);
if (gblerr != 0) {
throw std::runtime_error ("createCrsMatrix returned NULL on at least one "
"process.");
}
//
// We've created a sparse matrix whose rows live entirely on MPI
// Process 0. Now we want to distribute it over all the processes.
//
// Redistribute the matrix. Since both the source and target Maps
// are one-to-one, we could use either an Import or an Export. If
// only the source Map were one-to-one, we would have to use an
// Import; if only the target Map were one-to-one, we would have to
// use an Export. We do not allow redistribution using Import or
// Export if neither source nor target Map is one-to-one.
// Make an export object with procZeroMap as the source Map, and
// globalMap as the target Map. The Export type has the same
// template parameters as a Map. Note that Export does not depend
// on the Scalar template parameter of the objects it
// redistributes. You can reuse the same Export for different
// Tpetra object types, or for Tpetra objects of the same type but
// different Scalar template parameters (e.g., Scalar=float or
// Scalar=double).
Epetra_Export exporter (procZeroMap, globalMap);
// Make a new sparse matrix whose row map is the global Map.
Epetra_CrsMatrix B (Copy, globalMap, 0);
// Redistribute the data, NOT in place, from matrix A (which lives
// entirely on Proc 0) to matrix B (which is distributed evenly over
// the processes).
//
// Export() has collective semantics, so we must always call it on
// all processes collectively. This is why we don't select on
// lclerr, as we do for the local operations above.
lclerr = B.Export (*A, exporter, Insert);
// Make sure that the Export succeeded. Normally you don't have to
// check this; we are being extra conservative because this example
// example is also a test. We test both min and max, since lclerr
// may be negative, zero, or positive.
gblerr = 0;
(void) comm.MinAll (&lclerr, &gblerr, 1);
if (gblerr != 0) {
throw std::runtime_error ("Export() failed on at least one process.");
}
(void) comm.MaxAll (&lclerr, &gblerr, 1);
if (gblerr != 0) {
throw std::runtime_error ("Export() failed on at least one process.");
}
// FillComplete has collective semantics, so we must always call it
// on all processes collectively. This is why we don't select on
// lclerr, as we do for the local operations above.
lclerr = B.FillComplete ();
// Make sure that FillComplete succeeded. Normally you don't have
// to check this; we are being extra conservative because this
// example is also a test. We test both min and max, since lclerr
// may be negative, zero, or positive.
gblerr = 0;
(void) comm.MinAll (&lclerr, &gblerr, 1);
if (gblerr != 0) {
throw std::runtime_error ("B.FillComplete() failed on at least one process.");
}
(void) comm.MaxAll (&lclerr, &gblerr, 1);
if (gblerr != 0) {
throw std::runtime_error ("B.FillComplete() failed on at least one process.");
}
if (A != NULL) {
delete A;
}
}
Epetra_CrsMatrix*
read_matrix_mm(const std::string& mm_file,
const Epetra_Comm& comm)
{
int my_proc = comm.MyPID();
long long num_global_rows = 0;
int nnz_per_row = 0;
std::ifstream* infile = NULL;
infile = new std::ifstream(mm_file.c_str());
if (infile == NULL || !*infile) {
throw std::runtime_error("Failed to open file "+mm_file);
}
std::ifstream& in = *infile;
//first skip over the file header, which has
//lines beginning with '%'.
std::string line;
do {
getline(in, line);
} while(line[0] == '%');
//now get the matrix dimensions.
int numrows, numcols, nnz;
std::istringstream isstr(line);
isstr >> numrows >> numcols >> nnz;
//make sure we successfully read the three ints from that line.
if (isstr.fail()) {
throw std::runtime_error("Failed to parse matrix-market header.");
}
if (my_proc == 0) {
num_global_rows = numrows;
nnz_per_row = nnz/numrows;
}
comm.Broadcast(&num_global_rows, 1, 0);
comm.Broadcast(&nnz_per_row, 1, 0);
const int indexBase = 0;
Epetra_Map rowmap(num_global_rows, indexBase, comm);
Epetra_CrsMatrix* A = new Epetra_CrsMatrix(Copy, rowmap, nnz_per_row);
Teuchos::Array<long long> col;
Teuchos::Array<double> coef;
int irow=0, icol=0;
int g_row=-1, last_row=-1;
double val=0;
while(!in.eof()) {
getline(in, line);
std::istringstream isstr(line);
isstr >> irow >> icol >> val;
if (isstr.fail()) continue;
if (!rowmap.MyGID(irow-1)) continue;
g_row = irow-1;
if (g_row != last_row) {
if (col.size() > 0) {
A->InsertGlobalValues(last_row, col.size(), &coef[0], &col[0] );
col.clear();
coef.clear();
}
last_row = g_row;
}
col.push_back(icol-1);
coef.push_back(val);
}
if (col.size() > 0) {
A->InsertGlobalValues(g_row, col.size(), &coef[0], &col[0]);
}
A->FillComplete();
return A;
}
int main(int argc, char *argv[]) {
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
// initialize the random number generator
int ml_one = 1;
ML_srandom1(&ml_one);
// ===================== //
// create linear problem //
// ===================== //
Epetra_CrsMatrix *BadMatrix;
int * i_blockids;
int numblocks;
EpetraExt::MatlabFileToCrsMatrix("samplemat.dat",Comm,BadMatrix);
const Epetra_Map *Map=&BadMatrix->RowMap();
// Read in the block GLOBAL ids
Epetra_Vector* d_blockids;
int rv=EpetraExt::MatrixMarketFileToVector("blockids.dat",*Map,d_blockids);
i_blockids=new int[d_blockids->MyLength()];
numblocks=-1;
for(int i=0;i<d_blockids->MyLength();i++){
i_blockids[i]=(int)(*d_blockids)[i]-1;
numblocks=MAX(numblocks,i_blockids[i]);
}
numblocks++;
BadMatrix->FillComplete(); BadMatrix->OptimizeStorage();
int N=BadMatrix->RowMatrixRowMap().NumMyElements();
// Create the trivial blockID list
int * trivial_blockids=new int[N];
for(int i=0;i<N;i++)
trivial_blockids[i]=i;
Epetra_Vector LHS(*Map);
Epetra_Vector RHS(*Map);
Epetra_LinearProblem BadProblem(BadMatrix, &LHS, &RHS);
Teuchos::ParameterList MLList;
double TotalErrorResidual = 0.0, TotalErrorExactSol = 0.0;
char mystring[80];
// ====================== //
// ML Cheby
// ====================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type","Chebyshev");
MLList.set("coarse: type","Amesos-KLU");
MLList.set("max levels",2);
MLList.set("aggregation: threshold",.02);
MLList.set("ML output",10);
MLList.set("smoother: polynomial order",2);
strcpy(mystring,"Cheby");
TestMultiLevelPreconditioner(mystring, MLList, BadProblem,
TotalErrorResidual, TotalErrorExactSol);
// These tests only work in serial due to how the block id's are written.
if(Comm.NumProc()==1){
// ====================== //
// ML Block Cheby (Trivial)
// ====================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type","Block Chebyshev");
MLList.set("smoother: Block Chebyshev number of blocks",N);
MLList.set("smoother: Block Chebyshev block list",trivial_blockids);
MLList.set("coarse: type","Amesos-KLU");
MLList.set("max levels",2);
MLList.set("ML output",10);
MLList.set("smoother: polynomial order",2);
strcpy(mystring,"ML Block Cheby (Trivial)");
TestMultiLevelPreconditioner(mystring, MLList, BadProblem,
TotalErrorResidual, TotalErrorExactSol);
// ====================== //
// ML Block Cheby (Smart)
// ====================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type","Block Chebyshev");
MLList.set("smoother: Block Chebyshev number of blocks",numblocks);
MLList.set("smoother: Block Chebyshev block list",i_blockids);
MLList.set("coarse: type","Amesos-KLU");
MLList.set("max levels",2);
MLList.set("ML output",10);
MLList.set("smoother: polynomial order",2);
strcpy(mystring,"ML Block Cheby (Smart)");
TestMultiLevelPreconditioner(mystring, MLList, BadProblem,
TotalErrorResidual, TotalErrorExactSol);
}
#if defined(HAVE_ML_IFPACK)
// ====================== //
// IFPACK Cheby
// ====================== //
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type","IFPACK-Chebyshev");
MLList.set("coarse: type","Amesos-KLU");
MLList.set("max levels",2);
MLList.set("ML output",10);
MLList.set("smoother: polynomial order",2);
strcpy(mystring,"IFPACK Cheby");
TestMultiLevelPreconditioner(mystring, MLList, BadProblem,
TotalErrorResidual, TotalErrorExactSol);
// ====================== //
// IFPACK Block Cheby (Trivial)
// ====================== //
int NumBlocks=Map->NumMyElements();
int *BlockStarts=new int[NumBlocks+1];
int *Blockids=new int [NumBlocks];
for(int i=0;i<NumBlocks;i++){
BlockStarts[i]=i;
Blockids[i]=Map->GID(i);
}
BlockStarts[NumBlocks]=NumBlocks;
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type","IFPACK-Block Chebyshev");
MLList.set("smoother: Block Chebyshev number of blocks",NumBlocks);
MLList.set("smoother: Block Chebyshev block starts",&BlockStarts[0]);
MLList.set("smoother: Block Chebyshev block list",&Blockids[0]);
MLList.set("coarse: type","Amesos-KLU");
MLList.set("max levels",2);
MLList.set("ML output",10);
MLList.set("smoother: polynomial order",2);
strcpy(mystring,"IFPACK Block Cheby (Trivial)");
TestMultiLevelPreconditioner(mystring, MLList, BadProblem,
TotalErrorResidual, TotalErrorExactSol);
delete [] BlockStarts; delete [] Blockids;
// ====================== //
// IFPACK Block Cheby (Smart)
// ====================== //
// Figure out how many blocks we actually have and build a map...
Epetra_Map* IfpackMap;
int g_NumBlocks=-1,g_MaxSize=-1;
if(Comm.MyPID() == 0){
const int lineLength = 1025;
char line[lineLength];
FILE * f=fopen("localids_in_blocks.dat","r");
assert(f!=0);
// Next, strip off header lines (which start with "%")
do {
if(fgets(line, lineLength,f)==0) return(-4);
} while (line[0] == '%');
// Grab the number we actually care about
sscanf(line, "%d %d", &g_NumBlocks, &g_MaxSize);
fclose(f);
}
Comm.Broadcast(&g_NumBlocks,1,0);
Comm.Broadcast(&g_MaxSize,1,0);
Epetra_Map BlockMap(g_NumBlocks,0,Comm);
Epetra_MultiVector *blockids_disk=0;
rv=EpetraExt::MatrixMarketFileToMultiVector("localids_in_blocks.dat",BlockMap,blockids_disk);
// Put all the block info into the right place
NumBlocks=BlockMap.NumMyElements();
BlockStarts=new int[NumBlocks+1];
Blockids= new int[g_MaxSize*NumBlocks];
// NTS: Blockids_ is overallocated because I don't want to write a counting loop
int i,cidx;
for(i=0,cidx=0;i<NumBlocks;i++){
BlockStarts[i]=cidx;
Blockids[cidx]=(int)(*blockids_disk)[0][i];cidx++;
if((*blockids_disk)[1][i] > 1e-2){
Blockids[cidx]=(int)(*blockids_disk)[1][i];cidx++;
}
}
BlockStarts[NumBlocks]=cidx;
if (Comm.MyPID() == 0) PrintLine();
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type","IFPACK-Block Chebyshev");
MLList.set("smoother: Block Chebyshev number of blocks",NumBlocks);
MLList.set("smoother: Block Chebyshev block starts",&BlockStarts[0]);
MLList.set("smoother: Block Chebyshev block list",&Blockids[0]);
MLList.set("coarse: type","Amesos-KLU");
MLList.set("max levels",2);
MLList.set("ML output",10);
MLList.set("smoother: polynomial order",2);
strcpy(mystring,"IFPACK Block Cheby (Smart)");
TestMultiLevelPreconditioner(mystring, MLList, BadProblem,
TotalErrorResidual, TotalErrorExactSol);
delete blockids_disk; delete [] BlockStarts; delete [] Blockids;
#endif
// ===================== //
// print out total error //
// ===================== //
if (Comm.MyPID() == 0) {
cout << endl;
cout << "......Total error for residual = " << TotalErrorResidual << endl;
cout << "......Total error for exact solution = " << TotalErrorExactSol << endl;
cout << endl;
}
delete [] i_blockids;
delete [] trivial_blockids;
delete BadMatrix;
delete d_blockids;
if (TotalErrorResidual > 1e-5) {
cerr << "Error: `BlockCheby.exe' failed!" << endl;
exit(EXIT_FAILURE);
}
#ifdef HAVE_MPI
MPI_Finalize();
#endif
if (Comm.MyPID() == 0)
cerr << "`BlockCheby.exe' passed!" << endl;
return (EXIT_SUCCESS);
}
/*----------------------------------------------------------------------*
| Create the spd system (public) mwgee 12/05|
| Note that this is collective for ALL procs |
*----------------------------------------------------------------------*/
Epetra_CrsMatrix* MOERTEL::Manager::MakeSPDProblem()
{
// time this process
Epetra_Time time(Comm());
time.ResetStartTime();
// check whether all interfaces are complete and integrated
std::map<int,Teuchos::RCP<MOERTEL::Interface> >::iterator curr;
for (curr=interface_.begin(); curr != interface_.end(); ++curr)
{
if (curr->second->IsComplete() == false)
{
cout << "***ERR*** MOERTEL::Manager::MakeSPDProblem:\n"
<< "***ERR*** interface " << curr->second->Id() << " is not Complete()\n"
<< "***ERR*** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
return NULL;
}
if (curr->second->IsIntegrated() == false)
{
cout << "***ERR*** MOERTEL::Manager::MakeSPDProblem:\n"
<< "***ERR*** interface " << curr->second->Id() << " is not integrated yet\n"
<< "***ERR*** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
return NULL;
}
}
// check whether we have a problemmap_
if (problemmap_==Teuchos::null)
{
cout << "***ERR*** MOERTEL::Manager::MakeSPDProblem:\n"
<< "***ERR*** No problemmap_ set\n"
<< "***ERR*** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
return NULL;
}
// check whether we have a constraintsmap_
if (constraintsmap_==Teuchos::null)
{
cout << "***ERR*** MOERTEL::Manager::MakeSPDProblem:\n"
<< "***ERR*** onstraintsmap is NULL\n"
<< "***ERR*** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
return NULL;
}
// check for saddlemap_
if (saddlemap_==Teuchos::null)
{
cout << "***ERR*** MOERTEL::Manager::MakeSPDProblem:\n"
<< "***ERR*** saddlemap_==NULL\n"
<< "***ERR*** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
return NULL;
}
// check for inputmatrix
if (inputmatrix_==Teuchos::null)
{
cout << "***ERR*** MOERTEL::Manager::MakeSPDProblem:\n"
<< "***ERR*** No inputmatrix set\n"
<< "***ERR*** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
return NULL;
}
// check whether we have M and D matrices
if (D_==Teuchos::null || M_==Teuchos::null)
{
cout << "***ERR*** MOERTEL::Manager::MakeSPDProblem:\n"
<< "***ERR*** Matrix M or D is NULL\n"
<< "***ERR*** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
return NULL;
}
// we need a map from lagrange multiplier dofs to primal dofs on the same node
std::vector<MOERTEL::Node*> nodes(0);
std::map<int,int> lm_to_dof;
for (curr=interface_.begin(); curr!=interface_.end(); ++curr)
{
Teuchos::RCP<MOERTEL::Interface> inter = curr->second;
inter->GetNodeView(nodes);
for (int i=0; i<(int)nodes.size(); ++i)
{
if (!nodes[i]->Nlmdof())
continue;
const int* dof = nodes[i]->Dof();
const int* lmdof = nodes[i]->LMDof();
for (int j=0; j<nodes[i]->Nlmdof(); ++j)
{
//cout << "j " << j << " maps lmdof " << lmdof[j] << " to dof " << dof[j] << endl;
lm_to_dof[lmdof[j]] = dof[j];
}
}
}
lm_to_dof_ = Teuchos::rcp(new std::map<int,int>(lm_to_dof)); // this is a very useful map for the moertel_ml_preconditioner
/*
_ _
| |
| Arr Arn Mr |
S = | |
| Anr Ann D |
|
| MrT D 0 |
|_ _ |
_ _
| |
| Arr Arn |
A = | |
| Anr Ann |
|_ _|
1) Ann is square and we need it's Range/DomainMap annmap
_ _
WT = |_ 0 Dinv _|
2) Build WT (has rowmap/rangemap annmap and domainmap problemmap_)
_ _
| |
| Mr |
B = | |
| D |
|_ _|
3) Build B (has rowmap/rangemap problemmap_ and domainmap annmap)
4) Build I, the identity matrix with maps problemmap_,problemmap_);
*/
int err=0;
//--------------------------------------------------------------------------
// 1) create the rangemap of Ann
std::vector<int> myanngids(problemmap_->NumMyElements());
int count=0;
std::map<int,int>::iterator intintcurr;
for (intintcurr=lm_to_dof.begin(); intintcurr!=lm_to_dof.end(); ++intintcurr)
{
if (problemmap_->MyGID(intintcurr->second)==false)
continue;
if ((int)myanngids.size()<=count)
myanngids.resize(myanngids.size()+50);
myanngids[count] = intintcurr->second;
++count;
}
myanngids.resize(count);
int numglobalelements;
Comm().SumAll(&count,&numglobalelements,1);
Epetra_Map* annmap = new Epetra_Map(numglobalelements,count,&myanngids[0],0,Comm());
annmap_ = Teuchos::rcp(annmap);
myanngids.clear();
//--------------------------------------------------------------------------
// 2) create WT
Epetra_CrsMatrix* WT = new Epetra_CrsMatrix(Copy,*annmap,1,false);
for (intintcurr=lm_to_dof.begin(); intintcurr!=lm_to_dof.end(); ++intintcurr)
{
int lmdof = intintcurr->first;
int dof = intintcurr->second;
if (D_->MyGRID(lmdof)==false)
continue;
int lmlrid = D_->LRID(lmdof);
int numentries;
int* indices;
double* values;
err = D_->ExtractMyRowView(lmlrid,numentries,values,indices);
if (err) cout << "D_->ExtractMyRowView returned err=" << err << endl;
bool foundit = false;
for (int j=0; j<numentries; ++j)
{
int gcid = D_->GCID(indices[j]);
if (gcid<0) cout << "Cannot find gcid for indices[j]\n";
//cout << "Proc " << Comm().MyPID() << " lmdof " << lmdof << " dof " << dof << " gcid " << gcid << " val " << values[j] << endl;
if (gcid==dof)
{
double val = 1./values[j];
err = WT->InsertGlobalValues(dof,1,&val,&dof);
if (err<0) cout << "WT->InsertGlobalValues returned err=" << err << endl;
foundit = true;
break;
}
}
if (!foundit)
{
cout << "***ERR*** MOERTEL::Manager::MakeSPDProblem:\n"
<< "***ERR*** Cannot compute inverse of D_\n"
<< "***ERR*** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
cout << "lmdof " << lmdof << " dof " << dof << endl;
return NULL;
}
}
WT->FillComplete(*problemmap_,*annmap);
WT_ = Teuchos::rcp(WT);
//--------------------------------------------------------------------------
// 3) create B
// create a temporary matrix with rowmap of the Ann block
Epetra_CrsMatrix* tmp = new Epetra_CrsMatrix(Copy,*annmap,120);
std::vector<int> newindices(100);
for (intintcurr=lm_to_dof.begin(); intintcurr!=lm_to_dof.end(); ++intintcurr)
{
int lmdof = intintcurr->first;
int dof = intintcurr->second;
if (D_->MyGRID(lmdof)==false)
continue;
int lmlrid = D_->LRID(lmdof);
if (lmlrid<0) cout << "Cannot find lmlrid for lmdof\n";
int numentries;
int* indices;
double* values;
// extract and add values from D
err = D_->ExtractMyRowView(lmlrid,numentries,values,indices);
if (err) cout << "D_->ExtractMyRowView returned err=" << err << endl;
if (numentries>(int)newindices.size()) newindices.resize(numentries);
for (int j=0; j<numentries; ++j)
{
newindices[j] = D_->GCID(indices[j]);
if (newindices[j]<0) cout << "Cannot find gcid for indices[j]\n";
}
//cout << "Inserting from D in row " << dof << " cols/val ";
//for (int j=0; j<numentries; ++j) cout << newindices[j] << "/" << values[j] << " ";
//cout << endl;
err = tmp->InsertGlobalValues(dof,numentries,values,&newindices[0]);
if (err) cout << "tmp->InsertGlobalValues returned err=" << err << endl;
// extract and add values from M
err = M_->ExtractMyRowView(lmlrid,numentries,values,indices);
if (err) cout << "M_->ExtractMyRowView returned err=" << err << endl;
if (numentries>(int)newindices.size()) newindices.resize(numentries);
for (int j=0; j<numentries; ++j)
{
newindices[j] = M_->GCID(indices[j]);
if (newindices[j]<0) cout << "Cannot find gcid for indices[j]\n";
}
//cout << "Inserting from M in row " << dof << " cols/val ";
//for (int j=0; j<numentries; ++j) cout << newindices[j] << "/" << values[j] << " ";
//cout << endl;
err = tmp->InsertGlobalValues(dof,numentries,values,&newindices[0]);
if (err) cout << "tmp->InsertGlobalValues returned err=" << err << endl;
}
tmp->FillComplete(*(problemmap_.get()),*annmap);
// B is transposed of tmp
EpetraExt::RowMatrix_Transpose* trans = new EpetraExt::RowMatrix_Transpose(false);
Epetra_CrsMatrix* B = &(dynamic_cast<Epetra_CrsMatrix&>(((*trans)(const_cast<Epetra_CrsMatrix&>(*tmp)))));
delete tmp; tmp = NULL;
B_ = Teuchos::rcp(new Epetra_CrsMatrix(*B));
newindices.clear();
//--------------------------------------------------------------------------
// 4) create I
Epetra_CrsMatrix* I = MOERTEL::PaddedMatrix(*problemmap_,1.0,1);
I->FillComplete(*problemmap_,*problemmap_);
I_ = Teuchos::rcp(I);
//--------------------------------------------------------------------------
// 5) Build BWT = B * WT
Epetra_CrsMatrix* BWT = MOERTEL::MatMatMult(*B,false,*WT,false,OutLevel());
//--------------------------------------------------------------------------
// 6) create CBT = (I-BWT)*A*W*BT
Epetra_CrsMatrix* spdrhs = new Epetra_CrsMatrix(Copy,*problemmap_,10,false);
MOERTEL::MatrixMatrixAdd(*BWT,false,-1.0,*spdrhs,0.0);
MOERTEL::MatrixMatrixAdd(*I,false,1.0,*spdrhs,1.0);
spdrhs->FillComplete();
spdrhs->OptimizeStorage();
spdrhs_ = Teuchos::rcp(spdrhs);
Epetra_CrsMatrix* tmp1 = MOERTEL::MatMatMult(*inputmatrix_,false,*WT,true,OutLevel());
Epetra_CrsMatrix* tmp2 = MOERTEL::MatMatMult(*tmp1,false,*B,true,OutLevel());
delete tmp1;
Epetra_CrsMatrix* CBT = MOERTEL::MatMatMult(*spdrhs,false,*tmp2,false,OutLevel());
delete tmp2;
//--------------------------------------------------------------------------
// 7) create spdmatrix_ = A - CBT - (CBT)^T
spdmatrix_ = Teuchos::rcp(new Epetra_CrsMatrix(Copy,*problemmap_,10,false));
MOERTEL::MatrixMatrixAdd(*inputmatrix_,false,1.0,*spdmatrix_,0.0);
MOERTEL::MatrixMatrixAdd(*CBT,false,-1.0,*spdmatrix_,1.0);
MOERTEL::MatrixMatrixAdd(*CBT,true,-1.0,*spdmatrix_,1.0);
delete CBT; CBT = NULL;
spdmatrix_->FillComplete();
spdmatrix_->OptimizeStorage();
//--------------------------------------------------------------------------
// tidy up
lm_to_dof.clear();
delete trans; B = NULL;
// time this process
double t = time.ElapsedTime();
if (OutLevel()>5 && Comm().MyPID()==0)
cout << "MOERTEL (Proc 0): Construct spd system in " << t << " sec\n";
return spdmatrix_.get();
}
// Create and return a pointer to an example CrsMatrix, with row
// distribution over the given Map. The caller is responsible for
// freeing the result.
Epetra_CrsMatrix*
createCrsMatrix (const Epetra_Map& map)
{
const Epetra_Comm& comm = map.Comm ();
// Create an Epetra_CrsMatrix using the Map, with dynamic allocation.
Epetra_CrsMatrix* A = new Epetra_CrsMatrix (Copy, map, 3);
// The list of global indices owned by this MPI process.
const global_ordinal_type* myGblElts = NULL;
global_ordinal_type numGblElts = 0;
#ifdef EXAMPLE_USES_64BIT_GLOBAL_INDICES
myGblElts = map.MyGlobalElements64 ();
numGblElts = map.NumGlobalElements64 ();
#else
myGblElts = map.MyGlobalElements ();
numGblElts = map.NumGlobalElements ();
#endif // EXAMPLE_USES_64BIT_GLOBAL_INDICES
// The number of global indices owned by this MPI process.
const int numMyElts = map.NumMyElements ();
// In general, tests like this really should synchronize across all
// processes. However, the likely cause for this case is a
// misconfiguration of Epetra, so we expect it to happen on all
// processes, if it happens at all.
if (numMyElts > 0 && myGblElts == NULL) {
throw std::logic_error ("Failed to get the list of global indices");
}
// Local error code for use below.
int lclerr = 0;
// Fill the sparse matrix, one row at a time.
double tempVals[3];
global_ordinal_type tempGblInds[3];
for (int i = 0; i < numMyElts; ++i) {
// A(0, 0:1) = [2, -1]
if (myGblElts[i] == 0) {
tempVals[0] = 2.0;
tempVals[1] = -1.0;
tempGblInds[0] = myGblElts[i];
tempGblInds[1] = myGblElts[i] + 1;
if (lclerr == 0) {
lclerr = A->InsertGlobalValues (myGblElts[i], 2, tempVals, tempGblInds);
}
if (lclerr != 0) {
break;
}
}
// A(N-1, N-2:N-1) = [-1, 2]
else if (myGblElts[i] == numGblElts - 1) {
tempVals[0] = -1.0;
tempVals[1] = 2.0;
tempGblInds[0] = myGblElts[i] - 1;
tempGblInds[1] = myGblElts[i];
if (lclerr == 0) {
lclerr = A->InsertGlobalValues (myGblElts[i], 2, tempVals, tempGblInds);
}
if (lclerr != 0) {
break;
}
}
// A(i, i-1:i+1) = [-1, 2, -1]
else {
tempVals[0] = -1.0;
tempVals[1] = 2.0;
tempVals[2] = -1.0;
tempGblInds[0] = myGblElts[i] - 1;
tempGblInds[1] = myGblElts[i];
tempGblInds[2] = myGblElts[i] + 1;
if (lclerr == 0) {
lclerr = A->InsertGlobalValues (myGblElts[i], 3, tempVals, tempGblInds);
}
if (lclerr != 0) {
break;
}
}
}
// If any process failed to insert at least one entry, throw.
int gblerr = 0;
(void) comm.MaxAll (&lclerr, &gblerr, 1);
if (gblerr != 0) {
if (A != NULL) {
delete A;
}
throw std::runtime_error ("Some process failed to insert an entry.");
}
// Tell the sparse matrix that we are done adding entries to it.
gblerr = A->FillComplete ();
if (gblerr != 0) {
if (A != NULL) {
delete A;
}
std::ostringstream os;
os << "A->FillComplete() failed with error code " << gblerr << ".";
throw std::runtime_error (os.str ());
}
return A;
}
int CreateTridi(Epetra_CrsMatrix& A)
{
Epetra_Map Map = A.RowMap();
int NumMyElements = Map.NumMyElements();
int NumGlobalElements = Map.NumGlobalElements();
int * MyGlobalElements = new int[NumMyElements];
Map.MyGlobalElements(MyGlobalElements);
// Add rows one-at-a-time
// Need some vectors to help
// Off diagonal Values will always be -1
double *Values = new double[3];
int *Indices = new int[3];
int NumEntries;
for (int i=0; i<NumMyElements; i++)
{
if (MyGlobalElements[i]==0)
{
Indices[0] = 0;
Indices[1] = 1;
Values[0] = 2.0;
Values[1] = -1.0;
NumEntries = 2;
}
else if (MyGlobalElements[i] == NumGlobalElements-1)
{
Indices[0] = NumGlobalElements-1;
Indices[1] = NumGlobalElements-2;
Values[0] = 2.0;
Values[1] = -1.0;
NumEntries = 2;
}
else
{
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i];
Indices[2] = MyGlobalElements[i]+1;
Values[0] = -1.0;
Values[1] = 2.0;
Values[2] = -1.0;
NumEntries = 3;
}
assert(A.InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices)==0);
// Put in the diagonal entry
// assert(A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i])==0);
}
// Finish up
assert(A.FillComplete()==0);
delete[] MyGlobalElements;
delete[] Values;
delete[] Indices;
return 0;
}
// run tests with "Local" permutation strategy and nDofsPerNode = 3
bool runPermutationTest2(const std::string input_filename, const std::string expected_filename, const Teuchos::RCP<const Teuchos::Comm<int> >& comm) {
#ifndef HAVE_MUELU_INST_COMPLEX_INT_INT
Teuchos::RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
Epetra_CrsMatrix * ptrA = NULL;
Epetra_CrsMatrix * ptrExpected = NULL;
int ret = EpetraExt::MatlabFileToCrsMatrix ( input_filename.c_str(),
*Xpetra::toEpetra(comm),
ptrA
);
if(ret!=0)
std::cout << "failed to read matrix from file" << std::endl;
if(expected_filename.size() > 0)
{
int ret2 = EpetraExt::MatlabFileToCrsMatrix (expected_filename.c_str(),
*Xpetra::toEpetra(comm),
ptrExpected
);
if(ret2!=0)
std::cout << "failed to read matrix from file" << std::endl;
}
Teuchos::RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(ptrA);
Teuchos::RCP<Epetra_CrsMatrix> epExpected = Teuchos::rcp(ptrExpected);
// Epetra_CrsMatrix -> Xpetra::Matrix
Teuchos::RCP<CrsMatrix> exA = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(epA));
Teuchos::RCP<CrsMatrixWrap> crsOp = Teuchos::rcp(new CrsMatrixWrap(exA));
Teuchos::RCP<Matrix> A = Teuchos::rcp_dynamic_cast<Matrix>(crsOp);
A->SetFixedBlockSize(3);
Teuchos::RCP<Level> Finest = Teuchos::rcp(new Level());
Finest->SetLevelID(0); // must be level 0 for NullspaceFactory
Finest->Set("A", A);
// permute full matrix
Teuchos::RCP<PermutationFactory> PermFact = Teuchos::rcp(new MueLu::PermutationFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps>());
PermFact->SetParameter("PermutationStrategy",Teuchos::ParameterEntry(std::string("Local")));
PermFact->SetParameter("PermutationRowMapName",Teuchos::ParameterEntry(std::string("")));
PermFact->SetFactory("PermutationRowMapFactory", Teuchos::null);
// setup main factory manager
Teuchos::RCP<FactoryManager> M = Teuchos::rcp(new FactoryManager());
M->SetFactory("permQT", PermFact);
M->SetFactory("A", MueLu::NoFactory::getRCP()); // this is the input matrix
MueLu::SetFactoryManager SFMFinest(Finest, M);
// prepare building process for permutation operators
Finest->Request("A", PermFact.get());
Finest->Request("permA", PermFact.get());
Finest->Request("permP", PermFact.get());
Finest->Request("permQT", PermFact.get());
Finest->Request("permScaling", PermFact.get());
Finest->Request("#RowPermutations", PermFact.get());
Finest->Request("#ColPermutations", PermFact.get());
Finest->Request("#WideRangeRowPermutations", PermFact.get());
Finest->Request("#WideRangeColPermutations", PermFact.get());
// build permutation operators
PermFact->Build(*Finest);
//std::cout << "P" << *GetEpetraMatrix("permP", Finest, PermFact) << std::endl;
//std::cout << "Q^T" << *GetEpetraMatrix("permQT", Finest, PermFact) << std::endl;
//std::cout << "permA" << *GetEpetraMatrix("A", Finest, PermFact) << std::endl;
Teuchos::RCP<const Epetra_CrsMatrix> epResult = GetEpetraMatrix("A", Finest, PermFact);
//std::cout << *epResult << std::endl;
if(epExpected != Teuchos::null) {
Epetra_CrsMatrix* comparison = NULL;
EpetraExt::MatrixMatrix::Add(*epResult, false, -1.0, *epExpected, false, 1.0, comparison);
comparison->FillComplete();
//std::cout << *comparison << std::endl;
double norm = comparison->NormInf();
delete comparison;
comparison = NULL;
if(norm < 1.0e-14) {
*out << "** PASSED **: " << input_filename << std::endl;
return true;
}
else {
*out << "-- FAILED --: " << input_filename << std::endl;
return false;
}
}
#endif
*out << "-- FAILED --: " << input_filename << " no result file found" << std::endl;
return false; // no result for comparison available
}
int
main (int argc, char *argv[])
{
// These "using" statements make the code a bit more concise.
using std::cout;
using std::endl;
int ierr = 0, i;
// If Trilinos was built with MPI, initialize MPI, otherwise
// initialize the serial "communicator" that stands in for MPI.
#ifdef EPETRA_MPI
MPI_Init (&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
const int MyPID = Comm.MyPID();
const int NumProc = Comm.NumProc();
// We only allow (MPI) Process 0 to write to stdout.
const bool verbose = (MyPID == 0);
const int NumGlobalElements = 100;
if (verbose)
cout << Epetra_Version() << endl << endl;
// Asking the Epetra_Comm to print itself is a good test for whether
// you are running in an MPI environment. However, it will print
// something on all MPI processes, so you should remove it for a
// large-scale parallel run.
cout << Comm << endl;
if (NumGlobalElements < NumProc)
{
if (verbose)
cout << "numGlobalBlocks = " << NumGlobalElements
<< " cannot be < number of processors = " << NumProc << endl;
std::exit (EXIT_FAILURE);
}
// Construct a Map that puts approximately the same number of rows
// of the matrix A on each processor.
Epetra_Map Map (NumGlobalElements, 0, Comm);
// Get update list and number of local equations from newly created Map.
int NumMyElements = Map.NumMyElements();
std::vector<int> MyGlobalElements(NumMyElements);
Map.MyGlobalElements(&MyGlobalElements[0]);
// NumNz[i] is the number of nonzero elements in row i of the sparse
// matrix on this MPI process. Epetra_CrsMatrix uses this to figure
// out how much space to allocate.
std::vector<int> NumNz (NumMyElements);
// We are building a tridiagonal matrix where each row contains the
// nonzero elements (-1 2 -1). Thus, we need 2 off-diagonal terms,
// except for the first and last row of the matrix.
for (int i = 0; i < NumMyElements; ++i)
if (MyGlobalElements[i] == 0 || MyGlobalElements[i] == NumGlobalElements-1)
NumNz[i] = 2; // First or last row
else
NumNz[i] = 3; // Not the (first or last row)
// Create the Epetra_CrsMatrix.
Epetra_CrsMatrix A (Copy, Map, &NumNz[0]);
//
// Add rows to the sparse matrix one at a time.
//
std::vector<double> Values(2);
Values[0] = -1.0; Values[1] = -1.0;
std::vector<int> Indices(2);
const double two = 2.0;
int NumEntries;
for (int i = 0; i < NumMyElements; ++i)
{
if (MyGlobalElements[i] == 0)
{ // The first row of the matrix.
Indices[0] = 1;
NumEntries = 1;
}
else if (MyGlobalElements[i] == NumGlobalElements - 1)
{ // The last row of the matrix.
Indices[0] = NumGlobalElements-2;
NumEntries = 1;
}
else
{ // Any row of the matrix other than the first or last.
Indices[0] = MyGlobalElements[i]-1;
Indices[1] = MyGlobalElements[i]+1;
NumEntries = 2;
}
ierr = A.InsertGlobalValues(MyGlobalElements[i], NumEntries, &Values[0], &Indices[0]);
assert (ierr==0);
// Insert the diagonal entry.
ierr = A.InsertGlobalValues(MyGlobalElements[i], 1, &two, &MyGlobalElements[i]);
assert(ierr==0);
}
// Finish up. We can call FillComplete() with no arguments, because
// the matrix is square.
ierr = A.FillComplete ();
assert (ierr==0);
// Parameters for the power method.
const int niters = NumGlobalElements*10;
const double tolerance = 1.0e-2;
//
// Run the power method. Keep track of the flop count and the total
// elapsed time.
//
Epetra_Flops counter;
A.SetFlopCounter(counter);
Epetra_Time timer(Comm);
double lambda = 0.0;
ierr += powerMethod (lambda, A, niters, tolerance, verbose);
double elapsedTime = timer.ElapsedTime ();
double totalFlops =counter.Flops ();
// Mflop/s: Million floating-point arithmetic operations per second.
double Mflop_per_s = totalFlops / elapsedTime / 1000000.0;
if (verbose)
cout << endl << endl << "Total Mflop/s for first solve = "
<< Mflop_per_s << endl<< endl;
// Increase the first (0,0) diagonal entry of the matrix.
if (verbose)
cout << endl << "Increasing magnitude of first diagonal term, solving again"
<< endl << endl << endl;
if (A.MyGlobalRow (0)) {
int numvals = A.NumGlobalEntries (0);
std::vector<double> Rowvals (numvals);
std::vector<int> Rowinds (numvals);
A.ExtractGlobalRowCopy (0, numvals, numvals, &Rowvals[0], &Rowinds[0]); // Get A(0,0)
for (int i = 0; i < numvals; ++i)
if (Rowinds[i] == 0)
Rowvals[i] *= 10.0;
A.ReplaceGlobalValues (0, numvals, &Rowvals[0], &Rowinds[0]);
}
//
// Run the power method again. Keep track of the flop count and the
// total elapsed time.
//
lambda = 0.0;
timer.ResetStartTime();
counter.ResetFlops();
ierr += powerMethod (lambda, A, niters, tolerance, verbose);
elapsedTime = timer.ElapsedTime();
totalFlops = counter.Flops();
Mflop_per_s = totalFlops / elapsedTime / 1000000.0;
if (verbose)
cout << endl << endl << "Total Mflop/s for second solve = "
<< Mflop_per_s << endl << endl;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return ierr;
}
Epetra_LinearProblem* create_epetra_problem(int numProcs,
int localProc,
int local_n)
{
if (localProc == 0) {
std::cout << " creating Epetra_CrsMatrix with un-even distribution..."
<< std::endl;
}
//create an Epetra_CrsMatrix with rows spread un-evenly over
//processors.
Epetra_MpiComm comm(MPI_COMM_WORLD);
int global_num_rows = numProcs*local_n;
int mid_proc = numProcs/2;
bool num_procs_even = numProcs%2==0 ? true : false;
int adjustment = local_n/2;
//adjust local_n so that it's not equal on all procs.
if (localProc < mid_proc) {
local_n -= adjustment;
}
else {
local_n += adjustment;
}
//if numProcs is not an even number, undo the local_n adjustment
//on one proc so that the total will still be correct.
if (localProc == numProcs-1) {
if (num_procs_even == false) {
local_n -= adjustment;
}
}
//now we're ready to create a row-map.
Epetra_Map rowmap(global_num_rows, local_n, 0, comm);
//create a matrix
int nnz_per_row = 9;
Epetra_CrsMatrix* matrix =
new Epetra_CrsMatrix(Copy, rowmap, nnz_per_row);
// Add rows one-at-a-time
double negOne = -1.0;
double posTwo = 4.0;
for (int i=0; i<local_n; i++) {
int GlobalRow = matrix->GRID(i);
int RowLess1 = GlobalRow - 1;
int RowPlus1 = GlobalRow + 1;
int RowLess2 = GlobalRow - 2;
int RowPlus2 = GlobalRow + 2;
int RowLess3 = GlobalRow - 3;
int RowPlus3 = GlobalRow + 3;
int RowLess4 = GlobalRow - 4;
int RowPlus4 = GlobalRow + 4;
if (RowLess4>=0) {
matrix->InsertGlobalValues(GlobalRow, 1, &negOne, &RowLess4);
}
if (RowLess3>=0) {
matrix->InsertGlobalValues(GlobalRow, 1, &negOne, &RowLess3);
}
if (RowLess2>=0) {
matrix->InsertGlobalValues(GlobalRow, 1, &negOne, &RowLess2);
}
if (RowLess1>=0) {
matrix->InsertGlobalValues(GlobalRow, 1, &negOne, &RowLess1);
}
if (RowPlus1<global_num_rows) {
matrix->InsertGlobalValues(GlobalRow, 1, &negOne, &RowPlus1);
}
if (RowPlus2<global_num_rows) {
matrix->InsertGlobalValues(GlobalRow, 1, &negOne, &RowPlus2);
}
if (RowPlus3<global_num_rows) {
matrix->InsertGlobalValues(GlobalRow, 1, &negOne, &RowPlus3);
}
if (RowPlus4<global_num_rows) {
matrix->InsertGlobalValues(GlobalRow, 1, &negOne, &RowPlus4);
}
matrix->InsertGlobalValues(GlobalRow, 1, &posTwo, &GlobalRow);
}
int err = matrix->FillComplete();
if (err != 0) {
throw Isorropia::Exception("create_epetra_matrix: error in matrix.FillComplete()");
}
Epetra_Vector* x = new Epetra_Vector(rowmap);
Epetra_Vector* b = new Epetra_Vector(rowmap);
return(new Epetra_LinearProblem(matrix, x, b));
}
CrsMatrix_SolverMap::NewTypeRef
CrsMatrix_SolverMap::
operator()( OriginalTypeRef orig )
{
origObj_ = &orig;
assert( !orig.IndicesAreGlobal() );
//test if matrix has missing local columns in its col std::map
const Epetra_Map & RowMap = orig.RowMap();
const Epetra_Map & DomainMap = orig.DomainMap();
const Epetra_Map & ColMap = orig.ColMap();
const Epetra_Comm & Comm = RowMap.Comm();
int NumMyRows = RowMap.NumMyElements();
int NumCols = DomainMap.NumMyElements();
int Match = 0;
for( int i = 0; i < NumCols; ++i )
if( DomainMap.GID(i) != ColMap.GID(i) )
{
Match = 1;
break;
}
int MatchAll = 0;
Comm.SumAll( &Match, &MatchAll, 1 );
if( !MatchAll )
{
newObj_ = origObj_;
}
else
{
//create ColMap with all local rows included
std::vector<int> Cols(NumCols);
//fill Cols list with GIDs of all local columns
for( int i = 0; i < NumCols; ++i )
Cols[i] = DomainMap.GID(i);
//now append to Cols any ghost column entries
int NumMyCols = ColMap.NumMyElements();
for( int i = 0; i < NumMyCols; ++i )
if( !DomainMap.MyGID( ColMap.GID(i) ) ) Cols.push_back( ColMap.GID(i) );
int NewNumMyCols = Cols.size();
int NewNumGlobalCols;
Comm.SumAll( &NewNumMyCols, &NewNumGlobalCols, 1 );
//create new column std::map
NewColMap_ = new Epetra_Map( NewNumGlobalCols, NewNumMyCols,&Cols[0], DomainMap.IndexBase(), Comm );
//New Graph
std::vector<int> NumIndicesPerRow( NumMyRows );
for( int i = 0; i < NumMyRows; ++i )
NumIndicesPerRow[i] = orig.NumMyEntries(i);
NewGraph_ = new Epetra_CrsGraph( Copy, RowMap, *NewColMap_, &NumIndicesPerRow[0] );
int MaxNumEntries = orig.MaxNumEntries();
int NumEntries;
std::vector<int> Indices( MaxNumEntries );
for( int i = 0; i < NumMyRows; ++i )
{
int RowGID = RowMap.GID(i);
orig.Graph().ExtractGlobalRowCopy( RowGID, MaxNumEntries, NumEntries, &Indices[0] );
NewGraph_->InsertGlobalIndices( RowGID, NumEntries, &Indices[0] );
}
const Epetra_Map & RangeMap = orig.RangeMap();
NewGraph_->FillComplete(DomainMap,RangeMap);
//intial construction of matrix
Epetra_CrsMatrix * NewMatrix = new Epetra_CrsMatrix( View, *NewGraph_ );
//insert views of row values
int * myIndices;
double * myValues;
int indicesCnt;
int numMyRows = NewMatrix->NumMyRows();
for( int i = 0; i < numMyRows; ++i )
{
orig.ExtractMyRowView( i, indicesCnt, myValues, myIndices );
NewGraph_->ExtractMyRowView( i, indicesCnt, myIndices );
NewMatrix->InsertMyValues( i, indicesCnt, myValues, myIndices );
}
NewMatrix->FillComplete(DomainMap,RangeMap);
newObj_ = NewMatrix;
}
return *newObj_;
}
//=========================================================================
int Epetra_LinearProblemRedistor::CreateRedistProblem(const bool ConstructTranspose,
const bool MakeDataContiguous,
Epetra_LinearProblem *& RedistProblem) {
if (RedistProblemCreated_) EPETRA_CHK_ERR(-1); // This method can only be called once
Epetra_RowMatrix * OrigMatrix = OrigProblem_->GetMatrix();
Epetra_MultiVector * OrigLHS = OrigProblem_->GetLHS();
Epetra_MultiVector * OrigRHS = OrigProblem_->GetRHS();
if (OrigMatrix==0) EPETRA_CHK_ERR(-2); // There is no matrix associated with this Problem
if (RedistMap_==0) {
EPETRA_CHK_ERR(GenerateRedistMap());
}
RedistExporter_ = new Epetra_Export(OrigProblem_->GetMatrix()->RowMatrixRowMap(), *RedistMap_);
RedistProblem_ = new Epetra_LinearProblem();
Epetra_CrsMatrix * RedistMatrix;
// Check if the tranpose should be create or not
if (ConstructTranspose) {
Transposer_ = new Epetra_RowMatrixTransposer(OrigMatrix);
EPETRA_CHK_ERR(Transposer_->CreateTranspose(MakeDataContiguous, RedistMatrix, RedistMap_));
}
else {
// If not, then just do the redistribution based on the the RedistMap
RedistMatrix = new Epetra_CrsMatrix(Copy, *RedistMap_, 0);
// need to do this next step until we generalize the Import/Export ops for CrsMatrix
Epetra_CrsMatrix * OrigCrsMatrix = dynamic_cast<Epetra_CrsMatrix *>(OrigMatrix);
EPETRA_CHK_ERR(RedistMatrix->Export(*OrigCrsMatrix, *RedistExporter_, Add));
EPETRA_CHK_ERR(RedistMatrix->FillComplete());
}
RedistProblem_->SetOperator(RedistMatrix);
// Now redistribute the RHS and LHS if non-zero
Epetra_MultiVector * RedistLHS = 0;
Epetra_MultiVector * RedistRHS = 0;
int ierr = 0;
if (OrigLHS!=0) {
RedistLHS = new Epetra_MultiVector(*RedistMap_, OrigLHS->NumVectors());
EPETRA_CHK_ERR(RedistLHS->Export(*OrigLHS, *RedistExporter_, Add));
}
else ierr = 1;
if (OrigRHS!=0) {
RedistRHS = new Epetra_MultiVector(*RedistMap_, OrigLHS->NumVectors());
EPETRA_CHK_ERR(RedistRHS->Export(*OrigRHS, *RedistExporter_, Add));
}
else ierr ++;
RedistProblem_->SetLHS(RedistLHS);
RedistProblem_->SetRHS(RedistRHS);
RedistProblemCreated_ = true;
return(ierr);
}
int main(int argc, char *argv[]) {
// standard Epetra MPI/Serial Comm startup
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm comm;
#endif
int MyPID = comm.MyPID();
int ierr = 0;
bool verbose = (0 == MyPID);
bool reportErrors = (0 == MyPID);
// setup MatlabEngine
if (verbose) cout << "going to startup a matlab process...\n";
EpetraExt::EpetraExt_MatlabEngine engine (comm);
if (verbose) cout << "matlab started\n";
// setup an array of doubles to be used for the examples
int M = 20;
int numGlobalElements = M * comm.NumProc();
int N = 3;
int numMyEntries = M * N;
double* A = new double[numMyEntries];
double* Aptr = A;
int startValue = numMyEntries * MyPID;
for(int col=0; col < N; col++) {
for(int row=0; row < M; row++) {
*Aptr++ = startValue++;
}
}
// setup an array of ints to be used for the examples
int* intA = new int[numMyEntries];
int* intAptr = intA;
int intStartValue = numMyEntries * MyPID;
for(int i=0; i < M*N; i++) {
*intAptr++ = intStartValue++;
}
// construct a map to be used by distributed objects
Epetra_Map map (numGlobalElements, 0, comm);
// CrsMatrix example
// constructs a globally distributed CrsMatrix and then puts it into Matlab
if (verbose) cout << " constructing CrsMatrix...\n";
Epetra_CrsMatrix crsMatrix (Copy, map, N);
int* indices = new int[N];
for (int col=0; col < N; col++) {
indices[col] = col;
}
double value = startValue;
double* values = new double[numMyEntries];
int minMyGID = map.MinMyGID();
for (int row=0; row < M; row++) {
for (int col=0; col < N; col++) {
values[col] = value++;
}
crsMatrix.InsertGlobalValues(minMyGID + row, N, values, indices);
}
crsMatrix.FillComplete();
if (verbose) cout << " CrsMatrix constructed\n";
if (verbose) cout << " putting CrsMatrix into Matlab as CRSM\n";
ierr = engine.PutRowMatrix(crsMatrix, "CRSM", false);
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutRowMatrix(crsMatrix, \"CRSM\", false): " << ierr << endl;
}
// BlockMap example
// puts a map into Matlab
if (verbose) cout << " putting Map into Matlab as MAP\n";
ierr = engine.PutBlockMap(map, "MAP", false);
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutBlockMap(map, \"MAP\", false);: " << ierr << endl;
}
// MultiVector example
// constructs a globally distributed MultiVector and then puts it into Matlab
if (verbose) cout << " constructing MultiVector...\n";
Epetra_MultiVector multiVector (Copy, map, A, M, N);
if (verbose) cout << " MultiVector constructed\n";
if (verbose) cout << " putting MultiVector into Matlab as MV\n";
ierr = engine.PutMultiVector(multiVector, "MV");
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutMultiVector(multiVector, \"MV\"): " << ierr << endl;
}
// SerialDenseMatrix example
// constructs a SerialDenseMatrix on every PE
if (verbose) cout << " constructing a SerialDenseMatrix...\n";
Epetra_SerialDenseMatrix sdMatrix (Copy, A, M, M, N);
if (verbose) cout << " SerialDenseMatrix constructed\n";
if (verbose) cout << " putting SerialDenseMatrix from PE0 into Matlab as SDM_PE0\n";
// since the third parameter is left out, the SerialDenseMatrix from PE0 is used by default
ierr = engine.PutSerialDenseMatrix(sdMatrix, "SDM_PE0");
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutSerialDenseMatrix(sdMatrix, \"SDM_PE0\"): " << ierr << endl;
}
if (comm.NumProc() > 1) {
if (verbose) cout << " putting SerialDenseMatrix from PE1 into Matlab as SDM_PE1\n";
// specifying 1 as the third parameter will put the SerialDenseMatrix from PE1 into Matlab
ierr = engine.PutSerialDenseMatrix(sdMatrix, "SDM_PE1", 1);
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutSerialDenseMatrix(sdMatrix, \"SDM_PE1\", 1): " << ierr << endl;
}
}
// SerialDenseVector example
// constructs a SerialDenseVector on every PE
if (verbose) cout << " constructing a SerialDenseVector...\n";
Epetra_SerialDenseVector sdVector (Copy, A, M);
if (verbose) cout << " SerialDenseVector constructed\n";
// since the third parameter is left out, the SerialDenseMatrix from PE0 is used by default
if (verbose) cout << " putting SerialDenseVector from PE0 into Matlab as SDV_PE0\n";
ierr = engine.PutSerialDenseMatrix(sdVector, "SDV_PE0");
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutSerialDenseMatrix(sdVector, \"SDV_PE0\"): " << ierr << endl;
}
if (comm.NumProc() > 1) {
if (verbose) cout << " putting SerialDenseVector from PE1 into Matlab as SDV_PE1\n";
// specifying 1 as the third parameter will put the SerialDenseVector from PE1 into Matlab
ierr = engine.PutSerialDenseMatrix(sdVector, "SDV_PE1", 1);
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutSerialDenseMatrix(sdMatrix, \"SDV_PE1\", 1): " << ierr << endl;
}
}
// IntSerialDenseMatrix example
// constructs a IntSerialDenseMatrix on every PE
if (verbose) cout << " constructing a IntSerialDenseMatrix...\n";
Epetra_IntSerialDenseMatrix isdMatrix (Copy, intA, M, M, N);
if (verbose) cout << " IntSerialDenseMatrix constructed\n";
// since the third parameter is left out, the IntSerialDenseMatrix from PE0 is used by default
if (verbose) cout << " putting IntSerialDenseMatrix from PE0 into Matlab as ISDM_PE0\n";
ierr = engine.PutIntSerialDenseMatrix(isdMatrix, "ISDM_PE0");
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutIntSerialDenseMatrix(isdMatrix, \"ISDM_PE0\"): " << ierr << endl;
}
if (comm.NumProc() > 1) {
if (verbose) cout << " putting IntSerialDenseMatrix from PE1 into Matlab as ISDM_PE1\n";
// specifying 1 as the third parameter will put the IntSerialDenseMatrix from PE1 into Matlab
ierr = engine.PutIntSerialDenseMatrix(isdMatrix, "ISDM_PE1", 1);
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutSerialDenseMatrix(isdMatrix, \"ISDM_PE1\", 1): " << ierr << endl;
}
}
// IntSerialDenseVector example
// constructs a IntSerialDenseVector on every PE
if (verbose) cout << " constructing a IntSerialDenseVector...\n";
Epetra_IntSerialDenseVector isdVector (Copy, intA, M);
if (verbose) cout << " IntSerialDenseVector constructed\n";
// since the third parameter is left out, the IntSerialDenseVector from PE0 is used by default
if (verbose) cout << " putting IntSerialDenseVector from PE0 into Matlab as ISDV_PE0\n";
ierr = engine.PutIntSerialDenseMatrix(isdVector, "ISDV_PE0");
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutIntSerialDenseMatrix(isdVector, \"ISDV_PE0\"): " << ierr << endl;
}
if (comm.NumProc() > 1) {
if (verbose) cout << " putting IntSerialDenseVector from PE1 into Matlab as ISDV_PE1\n";
// specifying 1 as the third parameter will put the IntSerialDenseVector from PE1 into Matlab
ierr = engine.PutIntSerialDenseMatrix(isdVector, "ISDV_PE1", 1);
if (ierr) {
if (reportErrors) cout << "There was an error in engine.PutSerialDenseMatrix(isdVector, \"ISDV_PE1\", 1): " << ierr << endl;
}
}
// entering a while loop on PE0 will keep the Matlab workspace alive
/*
if (MyPID == 0)
while(1) {
// do nothing
}
*/
const int bufSize = 200;
char s [bufSize];
const int matlabBufferSize = 1024 * 16;
char matlabBuffer [matlabBufferSize];
// send some commands to Matlab and output the result to stdout
engine.EvalString("whos", matlabBuffer, matlabBufferSize);
if (verbose) cout << matlabBuffer << endl;
engine.EvalString("SDV_PE0", matlabBuffer, matlabBufferSize);
if (verbose) cout << matlabBuffer << endl;
if (comm.NumProc() > 1) {
engine.EvalString("SDV_PE1", matlabBuffer, matlabBufferSize);
if (verbose) cout << matlabBuffer << endl;
}
// the following allows user interaction with Matlab
if (MyPID == 0)
while(1) {
// Prompt the user and get a string
printf(">> ");
if (fgets(s, bufSize, stdin) == NULL) {
printf("Bye\n");
break ;
}
printf ("command :%s:\n", s) ;
// send the command to MATLAB
// output goes to stdout
ierr = engine.EvalString(s, matlabBuffer, matlabBufferSize);
if (ierr != 0) {
printf("there was an error: %d", ierr);
ierr = 0;
}
else {
printf("Matlab Output:\n%s", matlabBuffer);
}
}
if (verbose) cout << endl << " all done\n";
// standard finalizer for Epetra MPI Comms
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
// we need to delete engine because the MatlabEngine finalizer shuts down the Matlab process associated with this example
// if we don't delete the Matlab engine, then this example application will not shut down properly
delete &engine;
return(0);
}
Epetra_CrsMatrix *
Laplace2D::CreateLaplacian( const int nx, const int ny, const Epetra_Comm * Comm)
{
int NumGlobalElements = nx * ny;
// create a map
Epetra_Map * Map = new Epetra_Map(NumGlobalElements,0,*Comm);
// local number of rows
int NumMyElements = Map->NumMyElements();
// get update list
int * MyGlobalElements = Map->MyGlobalElements();
double hx = 1.0/(nx-1);
double hy = 1.0/(ny-1);
double off_left = -1.0/(hx*hx);
double off_right = -1.0/(hx*hx);
double off_lower = -1.0/(hy*hy);
double off_upper = -1.0/(hy*hy);
double diag = 2.0/(hx*hx) + 2.0/(hy*hy);
int left, right, lower, upper;
// a bit overestimated the nonzero per row
Epetra_CrsMatrix * A = new Epetra_CrsMatrix(Copy,*Map,5);
// Add rows one-at-a-time
double * Values = new double[4];
int * Indices = new int[4];
for( int i = 0; i < NumMyElements; ++i )
{
int NumEntries=0;
get_myNeighbours( MyGlobalElements[i], nx, ny, left, right, lower, upper );
if( left != -1 )
{
Indices[NumEntries] = left;
Values[NumEntries] = off_left;
++NumEntries;
}
if( right != -1 )
{
Indices[NumEntries] = right;
Values[NumEntries] = off_right;
++NumEntries;
}
if( lower != -1 )
{
Indices[NumEntries] = lower;
Values[NumEntries] = off_lower;
++NumEntries;
}
if( upper != -1 )
{
Indices[NumEntries] = upper;
Values[NumEntries] = off_upper;
++NumEntries;
}
// put the off-diagonal entries
A->InsertGlobalValues(MyGlobalElements[i], NumEntries, Values, Indices);
// Put in the diagonal entry
A->InsertGlobalValues(MyGlobalElements[i], 1, &diag, MyGlobalElements+i);
}
// put matrix in local ordering
A->FillComplete();
delete [] Indices;
delete [] Values;
delete Map;
return A;
} /* createJacobian */
