void Trilinos_Util_ReadHpc2Epetra(char *data_file,
const Epetra_Comm &comm,
Epetra_Map *& map,
Epetra_CrsMatrix *& A,
Epetra_Vector *& x,
Epetra_Vector *& b,
Epetra_Vector *&xexact) {
FILE *in_file ;
int l;
int * lp = &l;
double v;
double * vp = &v;
#ifdef DEBUG
bool debug = true;
#else
bool debug = false;
#endif
int size = comm.NumProc();
int rank = comm.MyPID();
printf("Reading matrix info from %s...\n",data_file);
in_file = fopen( data_file, "r");
if (in_file == NULL)
{
printf("Error: Cannot open file: %s\n",data_file);
exit(1);
}
int numGlobalEquations, total_nnz;
fscanf(in_file,"%d",&numGlobalEquations);
fscanf(in_file,"%d",&total_nnz);
map = new Epetra_Map(numGlobalEquations, 0, comm); // Create map with uniform distribution
A = new Epetra_CrsMatrix(Copy, *map, 0); // Construct matrix
x = new Epetra_Vector(*map);
b = new Epetra_Vector(*map);
xexact = new Epetra_Vector(*map);
int numMyEquations = map->NumMyPoints();
// Allocate arrays that are of length numMyEquations
// Find max nnz per row for this processor
int max_nnz = 0;
for (int i=0; i<numGlobalEquations; i++) {
fscanf(in_file, "%d",lp); /* row #, nnz in row */
if (map->MyGID(i)) max_nnz = EPETRA_MAX(max_nnz,l);
}
// Allocate arrays that are of length local_nnz
double * list_of_vals = new double[max_nnz];
int *list_of_inds = new int [max_nnz];
{for (int i=0; i<numGlobalEquations; i++)
{
int cur_nnz;
fscanf(in_file, "%d",&cur_nnz);
if (map->MyGID(i)) // See if nnz for row should be added
{
if (debug) cout << "Process "<<rank
<<" of "<<size<<" getting row "<<i<<endl;
int nnz_kept = 0;
for (int j=0; j<cur_nnz; j++)
{
fscanf(in_file, "%lf %d",vp,lp);
if (v!=0.0) {
list_of_vals[nnz_kept] = v;
list_of_inds[nnz_kept] = l;
nnz_kept++;
}
}
A->InsertGlobalValues(i, nnz_kept, list_of_vals, list_of_inds);
}
else
for (int j=0; j<cur_nnz; j++) fscanf(in_file, "%lf %d",vp,lp); // otherwise read and discard
}}
double xt, bt, xxt;
{for (int i=0; i<numGlobalEquations; i++)
{
if (map->MyGID(i)) // See if entry should be added
{
if (debug) cout << "Process "<<rank<<" of "
<<size<<" getting RHS "<<i<<endl;
fscanf(in_file, "%lf %lf %lf",&xt, &bt, &xxt);
int cur_local_row = map->LID(i);
(*x)[cur_local_row] = xt;
(*b)[cur_local_row] = bt;
(*xexact)[cur_local_row] = xxt;
}
else
fscanf(in_file, "%lf %lf %lf",vp, vp, vp); // or thrown away
}}
fclose(in_file);
if (debug)
cout << "Process "<<rank<<" of "<<size<<" has "<<numMyEquations
<< " rows. Min global row "<< map->MinMyGID()
<<" Max global row "<< map->MaxMyGID() <<endl
<<" and "<<A->NumMyNonzeros()<<" nonzeros."<<endl;
A->FillComplete();
Epetra_Vector bcomp(*map);
A->Multiply(false, *xexact, bcomp);
double residual;
bcomp.Norm2(&residual);
if (comm.MyPID()==0) cout << "Norm of computed b = " << residual << endl;
b->Norm2(&residual);
if (comm.MyPID()==0) cout << "Norm of given b = " << residual << endl;
bcomp.Update(-1.0, *b, 1.0);
bcomp.Norm2(&residual);
if (comm.MyPID()==0) cout << "Norm of difference between computed b and given b for xexact = " << residual << endl;
delete [] list_of_vals;
delete []list_of_inds;
return;
}
int main(int argc, char *argv[])
{
int *update; /* vector elements updated on this node. */
int *bindx;
double *val;
double *xguess, *b, *xexact;
int n_nonzeros;
int N_update; /* # of block unknowns updated on this node */
int numLocalEquations;
/* Number scalar equations on this node */
int numGlobalEquations; /* Total number of equations */
int *numNz, *ColInds;
int row, *col_inds, numEntries;
double *row_vals;
int i;
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm comm;
#endif
cout << comm << endl;
if(argc != 2) cerr << "error: enter name of data file on command line" << endl;
/* Set exact solution to NULL */
xexact = NULL;
/* Read matrix file and distribute among processors.
Returns with this processor's set of rows */
Trilinos_Util_read_hb(argv[1], comm.MyPID(), &numGlobalEquations, &n_nonzeros,
&val, &bindx, &xguess, &b, &xexact);
Trilinos_Util_distrib_msr_matrix(comm, &numGlobalEquations, &n_nonzeros, &N_update,
&update, &val, &bindx, &xguess, &b, &xexact);
numLocalEquations = N_update;
/* Make numNzBlks - number of block entries in each block row */
numNz = new int[numLocalEquations];
for (i=0; i<numLocalEquations; i++) numNz[i] = bindx[i+1] - bindx[i] + 1;
/* Make ColInds - Exactly bindx, offset by diag (just copy pointer) */
ColInds = bindx+numLocalEquations+1;
Epetra_Map map(numGlobalEquations, numLocalEquations, update, 0, comm);
Epetra_CrsMatrix A(Copy, map, numNz);
/* Add rows one-at-a-time */
for (row=0; row<numLocalEquations; row++) {
row_vals = val + bindx[row];
col_inds = bindx + bindx[row];
numEntries = bindx[row+1] - bindx[row];
assert(A.InsertGlobalValues(update[row], numEntries, row_vals, col_inds)==0);
assert(A.InsertGlobalValues(update[row], 1, val+row, update+row)==0);
}
assert(A.FillComplete()==0);
Epetra_Vector xx(Copy, map, xexact);
Epetra_Vector bb(Copy, map, b);
// Construct a Petra Linear Problem
Epetra_Vector x(map);
Epetra_LinearProblem problem(&A, &x, &bb);
// Construct a solver object for this problem
AztecOO solver(problem);
// Assert symmetric
// problem->AssertSymmetric();
// Set Problem Difficulty Level
//problem->SetPDL(easy);
//solver.SetAztecOption(AZ_precond, AZ_none);
solver.SetAztecOption(AZ_precond, AZ_dom_decomp);
//solver.SetAztecOption(AZ_precond, AZ_ls);
//solver.SetAztecOption(AZ_scaling, 8);
solver.SetAztecOption(AZ_subdomain_solve, AZ_ilut);
//solver.SetAztecOption(AZ_subdomain_solve, AZ_bilu_ifp);
bool bilu = false;
//solver.SetAztecOption(AZ_output, 0);
//solver.SetAztecOption(AZ_graph_fill, 2);
solver.SetAztecOption(AZ_overlap, 0);
//solver.SetAztecOption(AZ_reorder, 0);
//solver.SetAztecOption(AZ_poly_ord, 9);
solver.SetAztecParam(AZ_ilut_fill, 1.0);
solver.SetAztecParam(AZ_drop, 0.0);
//double rthresh = 1.01;
//cout << "Rel threshold = " << rthresh << endl;
//solver.SetAztecParam(AZ_rthresh, rthresh);
//double athresh = 1.0e-2;
//cout << "Abs threshold = " << athresh << endl;
//solver.SetAztecParam(AZ_athresh, athresh);
//solver.SetAztecOption(AZ_/conv, AZ_noscaled);
//solver.SetAztecParam(AZ_ill_cond_thresh, 1.0e12);
int Niters = 400;
solver.SetAztecOption(AZ_kspace, Niters);
double norminf = A.NormInf();
double normone = A.NormOne();
if (comm.MyPID()==0)
cout << "\n Inf-norm of A before scaling = " << norminf
<< "\n One-norm of A before scaling = " << normone<< endl << endl;
if (bilu) {
int NumTrials = 3;
double athresholds[] = {0.0, 1.0E-14, 1.0E-3};
double rthresholds[] = {0.0, 1.0E-14, 1.0E-3};
double condestThreshold = 1.0E16;
double maxFill = 4.0;
int maxKspace = 4*Niters;
solver.SetAdaptiveParams(NumTrials, athresholds, rthresholds, condestThreshold, maxFill, maxKspace);
}
else {
int NumTrials = 7;
double athresholds[] = {0.0, 1.0E-12, 1.0E-12, 1.0E-5, 1.0E-5, 1.0E-2, 1.0E-2};
double rthresholds[] = {1.0, 1.0, 1.01, 1.0, 1.01, 1.01, 1.1 };
double condestThreshold = 1.0E16;
double maxFill = 4.0;
int maxKspace = 4*Niters;
solver.SetAdaptiveParams(NumTrials, athresholds, rthresholds, condestThreshold, maxFill, maxKspace);
}
Epetra_Time timer(comm);
solver.AdaptiveIterate(Niters, 1, 5.0e-7);
double atime = timer.ElapsedTime();
if (comm.MyPID()==0) cout << "AdaptiveIterate total time = " << atime << endl;
norminf = A.NormInf();
normone = A.NormOne();
if (comm.MyPID()==0)
cout << "\n Inf-norm of A after scaling = " << norminf
<< "\n One-norm of A after scaling = " << normone << endl << endl;
Epetra_Vector bcomp(map);
assert(A.Multiply(false, x, bcomp)==0);
Epetra_Vector resid(map);
assert(resid.Update(1.0, bb, -1.0, bcomp, 0.0)==0);
double residual;
assert(resid.Norm2(&residual)==0);
if (comm.MyPID()==0) cout << "Residual = " << residual << endl;
assert(resid.Update(1.0, xx, -1.0, x, 0.0)==0);
assert(resid.Norm2(&residual)==0);
if (comm.MyPID()==0)
cout << "2-norm of difference between computed and exact solution = " << residual << endl;
if (residual>1.0e-5) {
cout << "Difference between computed and exact solution is large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
assert(A.Multiply(false, resid, bcomp)==0);
assert(bcomp.Norm2(&residual)==0);
if (comm.MyPID()==0)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
free ((void *) xguess);
free ((void *) b);
free ((void *) xexact);
free ((void *) val);
free ((void *) bindx);
free ((void *) update);
delete [] numNz;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return 0 ;
}
int main(int argc, char *argv[])
{
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm comm;
#endif
if (comm.MyPID()==0)
std::cout << AztecOO_Version() << std::endl << std::endl;
if (argc!=3) {
std::cerr << "Usage: " << argv[0] << " nx ny" << std::endl;
exit(1);
}
bool vb = (comm.MyPID()==0);
int nx = atoi(argv[1]);
int ny = atoi(argv[2]);
if (vb) std::cout << "Solving an " << nx << " by " << ny
<< " grid point Poisson problem. " << std::endl;
// Generate nx by ny Poisson operator
Poisson2dOperator A(nx, ny, comm);
// Generate vectors (xx will be used to generate RHS b)
Epetra_Vector xx(A.OperatorDomainMap());
Epetra_Vector x(A.OperatorDomainMap());
Epetra_Vector b(A.OperatorRangeMap());
xx.Random();
A.Apply(xx, b);
if (vb) std::cout << "Building Tridiagonal Approximation to Poisson" << std::endl;
Epetra_CrsMatrix * PrecMatrix = A.GeneratePrecMatrix();
if (vb) std::cout << "Building Epetra_LinearProblem" << std::endl;
Epetra_LinearProblem problem(&A, &x, &b);
if (vb) std::cout << "Building AztecOO solver" << std::endl;
AztecOO solver(problem);
if (vb) std::cout << "Setting Preconditioner Matrix" << std::endl;
solver.SetPrecMatrix(PrecMatrix);
//solver.SetAztecOption(AZ_precond, AZ_none);
int Niters = nx*ny;
solver.SetAztecOption(AZ_kspace, Niters);
solver.Iterate(Niters, 1.0E-12);
Epetra_Vector bcomp(A.OperatorRangeMap());
A.Apply(x, bcomp);
Epetra_Vector resid(A.OperatorRangeMap());
resid.Update(1.0, b, -1.0, bcomp, 0.0);
double residual;
resid.Norm2(&residual);
if (vb) std::cout << "Residual = " << residual << std::endl;
resid.Update(1.0, xx, -1.0, x, 0.0);
resid.Norm2(&residual);
if (vb)
std::cout << "2-norm of difference between computed and exact solution = "
<< residual << std::endl;
if (residual>1.0e-5) {
if (vb) std::cout << "Difference between computed and exact solution is large."
<< std::endl
<< "Computing norm of A times this difference. "
<< std::endl
<< "If this norm is small, then matrix is singular"
<< std::endl;
A.Apply(resid, bcomp);
bcomp.Norm2(&residual);
if (vb) std::cout << "2-norm of A times difference between computed and exact "
<< "solution = " << residual << std::endl;
}
else {
if (vb) std::cout << "Solver converged" << std::endl;
}
delete PrecMatrix;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return 0;
}
void Trilinos_Util_ReadHb2Epetra(char *data_file,
const Epetra_Comm &comm,
Epetra_Map *& map,
Epetra_CrsMatrix *& A,
Epetra_Vector *& x,
Epetra_Vector *& b,
Epetra_Vector *&xexact) {
FILE *in_file ;
int numGlobalEquations=0, N_columns=0, n_entries=0, Nrhs=0;
char Title[73], Key[9], Rhstype[4];
char Type[4] = "XXX";
char Ptrfmt[17], Indfmt[17], Valfmt[21], Rhsfmt[21];
int Ptrcrd, Indcrd, Valcrd, Rhscrd;
for(int ii=0; ii<73; ++ii) Title[ii] = '\0';
int * bindx, * pntr, * indx1, * pntr1;
double * val, * val1, * hbx, * hbxexact, * hbb;
hbx = 0; hbb = 0; hbxexact = 0; hbb = 0;
if(comm.MyPID() == 0) {
in_file = fopen( data_file, "r");
if (in_file == NULL)
{
printf("Error: Cannot open file: %s\n",data_file);
exit(1);
}
/* Get information about the array stored in the file specified in the */
/* argument list: */
printf("Reading matrix info from %s...\n",data_file);
in_file = fopen( data_file, "r");
if (in_file == NULL)
{
printf("Error: Cannot open file: %s\n",data_file);
exit(1);
}
readHB_header(in_file, Title, Key, Type, &numGlobalEquations, &N_columns,
&n_entries, &Nrhs,
Ptrfmt, Indfmt, Valfmt, Rhsfmt,
&Ptrcrd, &Indcrd, &Valcrd, &Rhscrd, Rhstype);
fclose(in_file);
if (Nrhs < 0 ) Nrhs = 0;
printf("%s", "***************************************************************\n");
printf("Matrix in file %s is %d x %d, \n",data_file, numGlobalEquations, N_columns);
printf("with %d nonzeros with type %3s;\n", n_entries, Type);
printf("%s", "***************************************************************\n");
printf("Title: %72s\n",Title);
printf("%s", "***************************************************************\n");
/*Nrhs = 0; */
printf("%d right-hand-side(s) available.\n",Nrhs);
if (Type[0] != 'R') perror("Can only handle real valued matrices");
int isym = 0;
if (Type[1] == 'S')
{
printf("%s", "Converting symmetric matrix to nonsymmetric storage\n");
n_entries = 2*n_entries - N_columns;
isym = 1;
}
if (Type[2] != 'A') perror("Can only handle assembled matrices");
if (N_columns != numGlobalEquations) perror("Matrix dimensions must be the same");
/* Read the matrix information, generating the associated storage arrays */
printf("Reading the matrix from %s...\n",data_file);
/* Allocate space. Note that we add extra storage in case of zero
diagonals. This is necessary for conversion to MSR format. */
pntr = (int *) calloc(N_columns+1,sizeof(int)) ;
bindx = (int *) calloc(n_entries+N_columns+1,sizeof(int)) ;
val = (double *) calloc(n_entries+N_columns+1,sizeof(double)) ;
readHB_mat_double(data_file, pntr, bindx, val);
/* Translate integer arrays to zero base */
for (int i = 0; i <= numGlobalEquations; i++) pntr[i]--;
{for (int i = 0; i <= n_entries; i++) bindx[i]--;}
/* If a rhs is specified in the file, read one,
generating the associate storage */
if (Nrhs > 0 && Rhstype[2] =='X')
{
printf("Reading right-hand-side vector(s) from %s...\n",data_file);
hbb = (double *) calloc(N_columns,sizeof(double));
readHB_aux_double(data_file, 'F', hbb);
printf("Reading exact solution vector(s) from %s...\n",data_file);
hbxexact = (double *) calloc(N_columns,sizeof(double));
readHB_aux_double(data_file, 'X', hbxexact);
}
else
{
/* Set Xexact to a random vector */
printf("%s", "Setting random exact solution vector\n");
hbxexact = (double *) calloc(N_columns,sizeof(double));
for (int i=0;i<numGlobalEquations;i++) hbxexact[i] =
((double)
rand())/((double) RAND_MAX);
/* Compute b to match xexact */
hbb = (double *) calloc(N_columns,sizeof(double)) ;
if (hbb == NULL) perror("Error: Not enough space to create rhs");
Trilinos_Util_scscmv (isym, N_columns, N_columns, val, bindx, pntr, hbxexact, hbb);
}
/* Compute residual using CSC format */
double res = Trilinos_Util_scscres(isym, numGlobalEquations, numGlobalEquations, val, bindx, pntr,
hbxexact, hbb);
printf(
"The residual using CSC format and exact solution is %12.4g\n",
res);
/* Set initial guess to zero */
hbx = (double *) calloc(numGlobalEquations,sizeof(double)) ;
if (hbx == NULL)
perror("Error: Not enough space to create guess");
/* Set RHS to a random vector, initial guess to zero */
{for (int i=0;i<numGlobalEquations;i++) hbx[i] = 0.0;}
/* Allocate temporary space */
pntr1 = (int *) calloc(N_columns+1,sizeof(int)) ;
indx1 = (int *) calloc(n_entries+N_columns+1,sizeof(int)) ;
val1 = (double *) calloc(n_entries+N_columns+1,sizeof(double)) ;
/* Convert in the following way:
- CSC to CSR
- CSR to MSR
*/
Trilinos_Util_csrcsc(numGlobalEquations, numGlobalEquations, 0, 0, val, bindx, pntr, val1, indx1, pntr1);
if (Type[1] == 'S')
{
int *indu, *iwk;
int ierr;
indu = new int[N_columns];
iwk = new int[N_columns+1];
ierr = Trilinos_Util_ssrcsr(3, 1, N_columns, val1, indx1, pntr1, n_entries,
val1, indx1, pntr1, indu, iwk);
delete [] indu;
delete [] iwk;
if (ierr !=0 )
{
printf(" Error in converting from symmetric form\n IERR = %d\n",ierr);
abort();
}
}
}
comm.Broadcast(&numGlobalEquations, 1, 0);
int nlocal = 0;
if (comm.MyPID()==0) nlocal = numGlobalEquations;
map = new Epetra_Map(numGlobalEquations, nlocal, 0, comm); // Create map with all elements on PE 0
A = new Epetra_CrsMatrix(Copy, *map, 0); // Construct matrix on PE 0
if (comm.MyPID()==0)
for (int i=0; i<numGlobalEquations; i++)
A->InsertGlobalValues(i, pntr1[i+1]-pntr1[i], val1+pntr1[i], indx1+pntr1[i]);
A->FillComplete();
x = new Epetra_Vector(Copy, *map, hbx);
b = new Epetra_Vector(Copy, *map, hbb);
xexact = new Epetra_Vector(Copy, *map, hbxexact);
Epetra_Vector bcomp(*map);
A->Multiply(false, *xexact, bcomp);
double residual;
bcomp.Norm2(&residual);
if (comm.MyPID()==0) cout << "Norm of computed b = " << residual << endl;
b->Norm2(&residual);
if (comm.MyPID()==0) cout << "Norm of given b = " << residual << endl;
bcomp.Update(-1.0, *b, 1.0);
bcomp.Norm2(&residual);
if (comm.MyPID()==0) cout << "Norm of difference between computed b and given b for xexact = " << residual << endl;
/* Release unneeded space */
if (comm.MyPID()==0) {
if (hbb!=0) free((void *) hbb);
if (hbx!=0) free((void *) hbx);
if (hbxexact!=0) free((void *) hbxexact);
free((void *) val);
free((void *) bindx);
free((void *) val1);
free((void *) indx1);
free((void *) pntr1);
free((void *) pntr);
}
return;
}
void nComplement(Pixel *a) {
a->color.r = bcomp(a->color.r);
a->color.g = bcomp(a->color.g);
a->color.b = bcomp(a->color.b);
}
