//=======================================================
int EpetraExt_HypreIJMatrix::InitializeDefaults(){
int my_type;
// Get type of Hypre IJ Matrix
EPETRA_CHK_ERR(HYPRE_IJMatrixGetObjectType(Matrix_, &my_type));
MatType_ = my_type;
// Currently only ParCSR is supported
TEUCHOS_TEST_FOR_EXCEPTION(MatType_ != HYPRE_PARCSR, std::logic_error, "Object is not type PARCSR");
// Get the actual ParCSR object from the IJ matrix
EPETRA_CHK_ERR(HYPRE_IJMatrixGetObject(Matrix_, (void**) &ParMatrix_));
int numRows, numCols;
// Get dimensions of the matrix and store as global variables
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixGetDims(ParMatrix_, &numRows, &numCols));
NumGlobalRows_ = numRows;
NumGlobalCols_ = numCols;
// Get the local dimensions of the matrix
int ColStart, ColEnd;
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixGetLocalRange(ParMatrix_, &MyRowStart_, &MyRowEnd_, &ColStart, &ColEnd));
// Determine number of local rows
NumMyRows_ = MyRowEnd_ - MyRowStart_+1;
return 0;
} //InitializeDefaults()
void HypreSolver2D::build_A()
{
HYPRE_IJMatrixCreate(MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
HYPRE_IJMatrixInitialize(A);
int nnz;
double values[5];
int cols[5];
for(int i=ilower; i<=iupper; i++)
{
nnz = 0;
// North
if ((i-N1)>=0)
{
cols[nnz] = i-N1;
values[nnz] = -1.0;
nnz++;
}
// Left
if (i%N1)
{
cols[nnz] = i-1;
values[nnz] = -1.0;
nnz++;
}
// Center
cols[nnz] = i;
values[nnz] = 4.0;
nnz++;
// Right
if ((i+1)%N1)
{
cols[nnz] = i+1;
values[nnz] = -1.0;
nnz++;
}
// South
if ((i+N1)<N)
{
cols[nnz] = i+N1;
values[nnz] = -1.0;
nnz++;
}
HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
}
HYPRE_IJMatrixAssemble(A);
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
}
int32_t
impl_bHYPRE_IJParCSRMatrix_GetIntValue(
/* in */ bHYPRE_IJParCSRMatrix self,
/* in */ const char* name,
/* out */ int32_t* value,
/* out */ sidl_BaseInterface *_ex)
{
*_ex = 0;
{
/* DO-NOT-DELETE splicer.begin(bHYPRE.IJParCSRMatrix.GetIntValue) */
/* Insert the implementation of the GetIntValue method here... */
int ierr = 0;
int M, N;
void * object;
HYPRE_IJMatrix HijA;
struct bHYPRE_IJParCSRMatrix__data * data;
HYPRE_ParCSRMatrix HA ;
data = bHYPRE_IJParCSRMatrix__get_data( self );
HijA = data->ij_A;
ierr += HYPRE_IJMatrixGetObject( HijA, &object );
HA = (HYPRE_ParCSRMatrix) object;
if ( strcmp(name,"GlobalNumRows")==0 )
{
ierr += HYPRE_ParCSRMatrixGetDims( HA, &M, &N );
*value = M;
}
else if ( strcmp(name,"GlobalNumCols")==0 )
{
ierr += HYPRE_ParCSRMatrixGetDims( HA, &M, &N );
*value = N;
}
else
{
ierr = 1;
}
return ierr;
/* DO-NOT-DELETE splicer.end(bHYPRE.IJParCSRMatrix.GetIntValue) */
}
}
int32_t
impl_bHYPRE_IJParCSRMatrix_GetRow(
/* in */ bHYPRE_IJParCSRMatrix self,
/* in */ int32_t row,
/* out */ int32_t* size,
/* out array<int,column-major> */ struct sidl_int__array** col_ind,
/* out array<double,column-major> */ struct sidl_double__array** values,
/* out */ sidl_BaseInterface *_ex)
{
*_ex = 0;
{
/* DO-NOT-DELETE splicer.begin(bHYPRE.IJParCSRMatrix.GetRow) */
/* Insert the implementation of the GetRow method here... */
int ierr=0;
void * object;
struct bHYPRE_IJParCSRMatrix__data * data;
HYPRE_IJMatrix ij_A;
HYPRE_ParCSRMatrix bHYPREP_A;
int * iindices[1];
double * dvalues[1];
data = bHYPRE_IJParCSRMatrix__get_data( self );
ij_A = data -> ij_A;
ierr += HYPRE_IJMatrixGetObject( ij_A, &object );
bHYPREP_A = (HYPRE_ParCSRMatrix) object;
*col_ind = sidl_int__array_create1d( size[0] );
*values = sidl_double__array_create1d( size[0] );
*iindices = sidlArrayAddr1( *col_ind, 0 );
*dvalues = sidlArrayAddr1( *values, 0 );
/* RestoreRow doesn't do anything but reset a parameter. Its
* function is to make sure the user who calls GetRow is aware that
* the data in the output arrays will be changed. */
HYPRE_ParCSRMatrixRestoreRow( bHYPREP_A, row, size, iindices, dvalues );
ierr += HYPRE_ParCSRMatrixGetRow( bHYPREP_A, row, size, iindices, dvalues );
return( ierr );
/* DO-NOT-DELETE splicer.end(bHYPRE.IJParCSRMatrix.GetRow) */
}
}
inline void solve(K* rhs) {
hypre_Vector* loc = hypre_ParVectorLocalVector(reinterpret_cast<hypre_ParVector*>(hypre_IJVectorObject(reinterpret_cast<hypre_IJVector*>(_b))));
K* b = loc->data;
loc->data = rhs;
HYPRE_ParVector par_b;
HYPRE_IJVectorGetObject(_b, reinterpret_cast<void**>(&par_b));
HYPRE_ParVector par_x;
HYPRE_IJVectorGetObject(_x, reinterpret_cast<void**>(&par_x));
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJMatrixGetObject(_A, reinterpret_cast<void**>(&parcsr_A));
if(_strategy == 1) /* AMG */
HYPRE_BoomerAMGSolve(_solver, parcsr_A, par_b, par_x);
else if(_strategy == 2) /* PCG with AMG */
HYPRE_ParCSRPCGSolve(_solver, parcsr_A, par_b, par_x);
else /* GMRES with AMG */
HYPRE_ParCSRFlexGMRESSolve(_solver, parcsr_A, par_b, par_x);
loc->data = b;
loc = hypre_ParVectorLocalVector(reinterpret_cast<hypre_ParVector*>(hypre_IJVectorObject(reinterpret_cast<hypre_IJVector*>(_x))));
std::copy(loc->data, loc->data + _local, rhs);
}
int32_t
impl_bHYPRE_ParCSRDiagScale_Apply(
/* in */ bHYPRE_ParCSRDiagScale self,
/* in */ bHYPRE_Vector b,
/* inout */ bHYPRE_Vector* x,
/* out */ sidl_BaseInterface *_ex)
{
*_ex = 0;
{
/* DO-NOT-DELETE splicer.begin(bHYPRE.ParCSRDiagScale.Apply) */
/* Insert the implementation of the Apply method here... */
int ierr = 0;
MPI_Comm comm;
HYPRE_Solver dummy;
HYPRE_Solver * solver = &dummy;
struct bHYPRE_ParCSRDiagScale__data * data;
/* not used HYPRE_Matrix HYPRE_A;*/
bHYPRE_IJParCSRMatrix bH_A;
HYPRE_ParCSRMatrix AA;
HYPRE_IJMatrix ij_A;
/* not used HYPRE_Vector HYPRE_x, HYPRE_b;*/
bHYPRE_IJParCSRVector bH_b, bH_x;
HYPRE_ParVector bb, xx;
HYPRE_IJVector ij_b, ij_x;
struct bHYPRE_IJParCSRMatrix__data * dataA;
struct bHYPRE_IJParCSRVector__data * datab, * datax;
void * objectA, * objectb, * objectx;
data = bHYPRE_ParCSRDiagScale__get_data( self );
comm = data->comm;
/* SetCommunicator should have been called earlier */
hypre_assert( comm != MPI_COMM_NULL );
bH_A = data->matrix;
hypre_assert( bH_A != NULL );
if ( *x==NULL )
{ /* If vector not supplied, make one...*/
/* There's no good way to check the size of x. It would be good
* to do something similar if x had zero length. Or
* hypre_assert(x-has-the-right-size) */
bHYPRE_Vector_Clone( b, x, _ex ); SIDL_CHECK(*_ex);
bHYPRE_Vector_Clear( *x, _ex ); SIDL_CHECK(*_ex);
}
bH_b = (bHYPRE_IJParCSRVector) bHYPRE_Vector__cast2( b, "bHYPRE.IJParCSRVector", _ex ); SIDL_CHECK(*_ex);
hypre_assert( bH_b!=NULL );
datab = bHYPRE_IJParCSRVector__get_data( bH_b );
ij_b = datab -> ij_b;
ierr += HYPRE_IJVectorGetObject( ij_b, &objectb );
bb = (HYPRE_ParVector) objectb;
/* not used HYPRE_b = (HYPRE_Vector) bb;*/
bH_x = (bHYPRE_IJParCSRVector) bHYPRE_Vector__cast2( *x, "bHYPRE.IJParCSRVector", _ex ); SIDL_CHECK(*_ex);
hypre_assert( bH_x!=NULL );
datax = bHYPRE_IJParCSRVector__get_data( bH_x );
ij_x = datax -> ij_b;
ierr += HYPRE_IJVectorGetObject( ij_x, &objectx );
xx = (HYPRE_ParVector) objectx;
/* not used HYPRE_b = (HYPRE_Vector) xx;*/
dataA = bHYPRE_IJParCSRMatrix__get_data( bH_A );
ij_A = dataA -> ij_A;
ierr += HYPRE_IJMatrixGetObject( ij_A, &objectA );
AA = (HYPRE_ParCSRMatrix) objectA;
/* not used HYPRE_A = (HYPRE_Matrix) AA;*/
/* does x = y/diagA as approximation to solving Ax=y for x ... */
ierr += HYPRE_ParCSRDiagScale( *solver, AA, bb, xx );
bHYPRE_IJParCSRVector_deleteRef( bH_b, _ex ); SIDL_CHECK(*_ex);
bHYPRE_IJParCSRVector_deleteRef( bH_x, _ex ); SIDL_CHECK(*_ex);
return ierr;
hypre_babel_exception_return_error(_ex);
/* DO-NOT-DELETE splicer.end(bHYPRE.ParCSRDiagScale.Apply) */
}
}
inline void numfact(unsigned int ncol, int* I, int* loc2glob, int* J, K* C) {
static_assert(std::is_same<double, K>::value, "Hypre only supports double-precision floating-point real numbers");
static_assert(S == 'G', "Hypre only supports nonsymmetric matrices");
HYPRE_IJMatrixCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], loc2glob[0], loc2glob[1], &_A);
HYPRE_IJMatrixSetObjectType(_A, HYPRE_PARCSR);
HYPRE_IJMatrixSetRowSizes(_A, I + 1);
_local = ncol;
int* rows = new int[3 * _local]();
int* diag_sizes = rows + _local;
int* offdiag_sizes = diag_sizes + _local;
rows[0] = I[0];
for(unsigned int i = 0; i < _local; ++i) {
std::for_each(J + rows[0], J + rows[0] + I[i + 1], [&](int& j) { (j < loc2glob[0] || loc2glob[1] < j) ? ++offdiag_sizes[i] : ++diag_sizes[i]; });
rows[0] += I[i + 1];
}
HYPRE_IJMatrixSetDiagOffdSizes(_A, diag_sizes, offdiag_sizes);
HYPRE_IJMatrixSetMaxOffProcElmts(_A, 0);
HYPRE_IJMatrixInitialize(_A);
std::iota(rows, rows + _local, loc2glob[0]);
HYPRE_IJMatrixSetValues(_A, _local, I + 1, rows, J, C);
HYPRE_IJMatrixAssemble(_A);
HYPRE_IJVectorCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], &_b);
HYPRE_IJVectorSetObjectType(_b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(_b);
HYPRE_IJVectorCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], &_x);
HYPRE_IJVectorSetObjectType(_x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(_x);
delete [] rows;
delete [] I;
delete [] loc2glob;
HYPRE_BoomerAMGCreate(_strategy == 1 ? &_solver : &_precond);
HYPRE_BoomerAMGSetCoarsenType(_strategy == 1 ? _solver : _precond, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(_strategy == 1 ? _solver : _precond, 6); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(_strategy == 1 ? _solver : _precond, 1); /* sweeps on each level */
HYPRE_BoomerAMGSetMaxLevels(_strategy == 1 ? _solver : _precond, 10); /* maximum number of levels */
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJMatrixGetObject(_A, reinterpret_cast<void**>(&parcsr_A));
HYPRE_ParVector par_b;
HYPRE_IJVectorGetObject(_b, reinterpret_cast<void**>(&par_b));
HYPRE_ParVector par_x;
HYPRE_IJVectorGetObject(_x, reinterpret_cast<void**>(&par_x));
if(_strategy == 1) {
HYPRE_BoomerAMGSetTol(_solver, 1.0e-8);
HYPRE_BoomerAMGSetMaxIter(_solver, 1000);
HYPRE_BoomerAMGSetPrintLevel(_solver, 1);
HYPRE_BoomerAMGSetup(_solver, parcsr_A, nullptr, nullptr);
}
else {
HYPRE_BoomerAMGSetTol(_precond, 0.0);
HYPRE_BoomerAMGSetMaxIter(_precond, 1);
HYPRE_BoomerAMGSetPrintLevel(_precond, 1);
if(_strategy == 2) {
HYPRE_ParCSRPCGCreate(DMatrix::_communicator, &_solver);
HYPRE_PCGSetMaxIter(_solver, 500);
HYPRE_PCGSetTol(_solver, 1.0e-8);
HYPRE_PCGSetTwoNorm(_solver, 1);
HYPRE_PCGSetPrintLevel(_solver, 1);
HYPRE_PCGSetLogging(_solver, 1);
HYPRE_PCGSetPrecond(_solver, reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSolve), reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSetup), _precond);
HYPRE_ParCSRPCGSetup(_solver, parcsr_A, par_b, par_x);
}
else {
HYPRE_ParCSRFlexGMRESCreate(DMatrix::_communicator, &_solver);
HYPRE_FlexGMRESSetKDim(_solver, 50);
HYPRE_FlexGMRESSetMaxIter(_solver, 500);
HYPRE_FlexGMRESSetTol(_solver, 1.0e-8);
HYPRE_FlexGMRESSetPrintLevel(_solver, 1);
HYPRE_FlexGMRESSetLogging(_solver, 1);
HYPRE_FlexGMRESSetPrecond(_solver, reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSolve), reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSetup), _precond);
HYPRE_ParCSRFlexGMRESSetup(_solver, parcsr_A, par_b, par_x);
}
}
}
int main (int argc, char *argv[])
{
HYPRE_Int i;
int myid, num_procs;
int N, n;
HYPRE_Int ilower, iupper;
HYPRE_Int local_size, extra;
int solver_id;
int print_solution, print_system;
double h, h2;
HYPRE_IJMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJVector b;
HYPRE_ParVector par_b;
HYPRE_IJVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver, precond;
/* Initialize MPI */
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
/* Default problem parameters */
n = 33;
solver_id = 0;
print_solution = 0;
print_system = 0;
/* Parse command line */
{
int arg_index = 0;
int print_usage = 0;
while (arg_index < argc)
{
if ( strcmp(argv[arg_index], "-n") == 0 )
{
arg_index++;
n = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-solver") == 0 )
{
arg_index++;
solver_id = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-print_solution") == 0 )
{
arg_index++;
print_solution = 1;
}
else if ( strcmp(argv[arg_index], "-print_system") == 0 )
{
arg_index++;
print_system = 1;
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
break;
}
else
{
arg_index++;
}
}
if ((print_usage) && (myid == 0))
{
printf("\n");
printf("Usage: %s [<options>]\n", argv[0]);
printf("\n");
printf(" -n <n> : problem size in each direction (default: 33)\n");
printf(" -solver <ID> : solver ID\n");
printf(" 0 - AMG (default) \n");
printf(" 1 - AMG-PCG\n");
printf(" 8 - ParaSails-PCG\n");
printf(" 50 - PCG\n");
printf(" 61 - AMG-FlexGMRES\n");
printf(" -print_solution : print the solution vector\n");
printf(" -print_system : print the matrix and rhs\n");
printf("\n");
}
if (print_usage)
{
MPI_Finalize();
return (0);
}
}
/* Preliminaries: want at least one processor per row */
if (n*n < num_procs) n = sqrt(num_procs) + 1;
N = n*n; /* global number of rows */
h = 1.0/(n+1); /* mesh size*/
h2 = h*h;
/* Each processor knows only of its own rows - the range is denoted by ilower
and upper. Here we partition the rows. We account for the fact that
N may not divide evenly by the number of processors. */
local_size = N/num_procs;
extra = N - local_size*num_procs;
ilower = local_size*myid;
ilower += hypre_min(myid, extra);
iupper = local_size*(myid+1);
iupper += hypre_min(myid+1, extra);
iupper = iupper - 1;
/* How many rows do I have? */
local_size = iupper - ilower + 1;
/* Create the matrix.
Note that this is a square matrix, so we indicate the row partition
size twice (since number of rows = number of cols) */
HYPRE_IJMatrixCreate(MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
/* Choose a parallel csr format storage (see the User's Manual) */
HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
/* Initialize before setting coefficients */
HYPRE_IJMatrixInitialize(A);
/* Now go through my local rows and set the matrix entries.
Each row has at most 5 entries. For example, if n=3:
A = [M -I 0; -I M -I; 0 -I M]
M = [4 -1 0; -1 4 -1; 0 -1 4]
Note that here we are setting one row at a time, though
one could set all the rows together (see the User's Manual).
*/
{
HYPRE_Int nnz;
double values[5];
HYPRE_Int cols[5];
for (i = ilower; i <= iupper; i++)
{
nnz = 0;
/* The left identity block:position i-n */
if ((i-n)>=0)
{
cols[nnz] = i-n;
values[nnz] = -1.0;
nnz++;
}
/* The left -1: position i-1 */
if (i%n)
{
cols[nnz] = i-1;
values[nnz] = -1.0;
nnz++;
}
/* Set the diagonal: position i */
cols[nnz] = i;
values[nnz] = 4.0;
nnz++;
/* The right -1: position i+1 */
if ((i+1)%n)
{
cols[nnz] = i+1;
values[nnz] = -1.0;
nnz++;
}
/* The right identity block:position i+n */
if ((i+n)< N)
{
cols[nnz] = i+n;
values[nnz] = -1.0;
nnz++;
}
/* Set the values for row i */
HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
}
}
/* Assemble after setting the coefficients */
HYPRE_IJMatrixAssemble(A);
/* Note: for the testing of small problems, one may wish to read
in a matrix in IJ format (for the format, see the output files
from the -print_system option).
In this case, one would use the following routine:
HYPRE_IJMatrixRead( <filename>, MPI_COMM_WORLD,
HYPRE_PARCSR, &A );
<filename> = IJ.A.out to read in what has been printed out
by -print_system (processor numbers are omitted).
A call to HYPRE_IJMatrixRead is an *alternative* to the
following sequence of HYPRE_IJMatrix calls:
Create, SetObjectType, Initialize, SetValues, and Assemble
*/
/* Get the parcsr matrix object to use */
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
/* Create the rhs and solution */
HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&b);
HYPRE_IJVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(b);
HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&x);
HYPRE_IJVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(x);
/* Set the rhs values to h^2 and the solution to zero */
{
double *rhs_values, *x_values;
HYPRE_Int *rows;
rhs_values = calloc(local_size, sizeof(double));
x_values = calloc(local_size, sizeof(double));
rows = calloc(local_size, sizeof(HYPRE_Int));
for (i=0; i<local_size; i++)
{
rhs_values[i] = h2;
x_values[i] = 0.0;
rows[i] = ilower + i;
}
HYPRE_IJVectorSetValues(b, local_size, rows, rhs_values);
HYPRE_IJVectorSetValues(x, local_size, rows, x_values);
free(x_values);
free(rhs_values);
free(rows);
}
HYPRE_IJVectorAssemble(b);
/* As with the matrix, for testing purposes, one may wish to read in a rhs:
HYPRE_IJVectorRead( <filename>, MPI_COMM_WORLD,
HYPRE_PARCSR, &b );
as an alternative to the
following sequence of HYPRE_IJVectors calls:
Create, SetObjectType, Initialize, SetValues, and Assemble
*/
HYPRE_IJVectorGetObject(b, (void **) &par_b);
HYPRE_IJVectorAssemble(x);
HYPRE_IJVectorGetObject(x, (void **) &par_x);
/* Print out the system - files names will be IJ.out.A.XXXXX
and IJ.out.b.XXXXX, where XXXXX = processor id */
if (print_system)
{
HYPRE_IJMatrixPrint(A, "IJ.out.A");
HYPRE_IJVectorPrint(b, "IJ.out.b");
}
/* Choose a solver and solve the system */
/* AMG */
if (solver_id == 0)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_BoomerAMGCreate(&solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_BoomerAMGSetPrintLevel(solver, 3); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(solver, 3); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(solver, 1); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(solver, 20); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(solver, 1e-7); /* conv. tolerance */
/* Now setup and solve! */
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver */
HYPRE_BoomerAMGDestroy(solver);
}
/* PCG */
else if (solver_id == 50)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* prints out the iteration info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver */
HYPRE_ParCSRPCGDestroy(solver);
}
/* PCG with AMG preconditioner */
else if (solver_id == 1)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the AMG preconditioner and specify any parameters */
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
/* Set the PCG preconditioner */
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver and preconditioner */
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
}
/* PCG with Parasails Preconditioner */
else if (solver_id == 8)
{
HYPRE_Int num_iterations;
double final_res_norm;
int sai_max_levels = 1;
double sai_threshold = 0.1;
double sai_filter = 0.05;
int sai_sym = 1;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the ParaSails preconditioner and specify any parameters */
HYPRE_ParaSailsCreate(MPI_COMM_WORLD, &precond);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_ParaSailsSetParams(precond, sai_threshold, sai_max_levels);
HYPRE_ParaSailsSetFilter(precond, sai_filter);
HYPRE_ParaSailsSetSym(precond, sai_sym);
HYPRE_ParaSailsSetLogging(precond, 3);
/* Set the PCG preconditioner */
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
(HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup, precond);
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destory solver and preconditioner */
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_ParaSailsDestroy(precond);
}
/* Flexible GMRES with AMG Preconditioner */
else if (solver_id == 61)
{
HYPRE_Int num_iterations;
double final_res_norm;
int restart = 30;
int modify = 1;
/* Create solver */
HYPRE_ParCSRFlexGMRESCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_FlexGMRESSetKDim(solver, restart);
HYPRE_FlexGMRESSetMaxIter(solver, 1000); /* max iterations */
HYPRE_FlexGMRESSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_FlexGMRESSetPrintLevel(solver, 2); /* print solve info */
HYPRE_FlexGMRESSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the AMG preconditioner and specify any parameters */
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
/* Set the FlexGMRES preconditioner */
HYPRE_FlexGMRESSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
if (modify)
/* this is an optional call - if you don't call it, hypre_FlexGMRESModifyPCDefault
is used - which does nothing. Otherwise, you can define your own, similar to
the one used here */
HYPRE_FlexGMRESSetModifyPC( solver,
(HYPRE_PtrToModifyPCFcn) hypre_FlexGMRESModifyPCAMGExample);
/* Now setup and solve! */
HYPRE_ParCSRFlexGMRESSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRFlexGMRESSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_FlexGMRESGetNumIterations(solver, &num_iterations);
HYPRE_FlexGMRESGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destory solver and preconditioner */
HYPRE_ParCSRFlexGMRESDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
}
else
{
if (myid ==0) printf("Invalid solver id specified.\n");
}
/* Print the solution */
if (print_solution)
HYPRE_IJVectorPrint(x, "ij.out.x");
/* Clean up */
HYPRE_IJMatrixDestroy(A);
HYPRE_IJVectorDestroy(b);
HYPRE_IJVectorDestroy(x);
/* Finalize MPI*/
MPI_Finalize();
return(0);
}
HYPRE_Int hypre_BoomerAMGCoarsenCGC (hypre_ParCSRMatrix *S,HYPRE_Int numberofgrids,HYPRE_Int coarsen_type,HYPRE_Int *CF_marker)
/* CGC algorithm
* ====================================================================================================
* coupling : the strong couplings
* numberofgrids : the number of grids
* coarsen_type : the coarsening type
* gridpartition : the grid partition
* =====================================================================================================*/
{
HYPRE_Int j,/*p,*/mpisize,mpirank,/*rstart,rend,*/choice,*coarse,ierr=0;
HYPRE_Int *vertexrange = NULL;
HYPRE_Int *vertexrange_all = NULL;
HYPRE_Int *CF_marker_offd = NULL;
HYPRE_Int num_variables = hypre_CSRMatrixNumRows (hypre_ParCSRMatrixDiag(S));
/* HYPRE_Int num_cols_offd = hypre_CSRMatrixNumCols (hypre_ParCSRMatrixOffd (S)); */
/* HYPRE_Int *col_map_offd = hypre_ParCSRMatrixColMapOffd (S); */
/* HYPRE_Real wall_time; */
HYPRE_IJMatrix ijG;
hypre_ParCSRMatrix *G;
hypre_CSRMatrix *Gseq;
MPI_Comm comm = hypre_ParCSRMatrixComm(S);
hypre_MPI_Comm_size (comm,&mpisize);
hypre_MPI_Comm_rank (comm,&mpirank);
#if 0
if (!mpirank) {
wall_time = time_getWallclockSeconds();
hypre_printf ("Starting CGC preparation\n");
}
#endif
AmgCGCPrepare (S,numberofgrids,CF_marker,&CF_marker_offd,coarsen_type,&vertexrange);
#if 0 /* debugging */
if (!mpirank) {
wall_time = time_getWallclockSeconds() - wall_time;
hypre_printf ("Finished CGC preparation, wall_time = %f s\n",wall_time);
wall_time = time_getWallclockSeconds();
hypre_printf ("Starting CGC matrix assembly\n");
}
#endif
AmgCGCGraphAssemble (S,vertexrange,CF_marker,CF_marker_offd,coarsen_type,&ijG);
#if 0
HYPRE_IJMatrixPrint (ijG,"graph.txt");
#endif
HYPRE_IJMatrixGetObject (ijG,(void**)&G);
#if 0 /* debugging */
if (!mpirank) {
wall_time = time_getWallclockSeconds() - wall_time;
hypre_printf ("Finished CGC matrix assembly, wall_time = %f s\n",wall_time);
wall_time = time_getWallclockSeconds();
hypre_printf ("Starting CGC matrix communication\n");
}
#endif
#ifdef HYPRE_NO_GLOBAL_PARTITION
{
/* classical CGC does not really make sense in combination with HYPRE_NO_GLOBAL_PARTITION,
but anyway, here it is:
*/
HYPRE_Int nlocal = vertexrange[1]-vertexrange[0];
vertexrange_all = hypre_CTAlloc (HYPRE_Int,mpisize+1);
hypre_MPI_Allgather (&nlocal,1,HYPRE_MPI_INT,vertexrange_all+1,1,HYPRE_MPI_INT,comm);
vertexrange_all[0]=0;
for (j=2;j<=mpisize;j++) vertexrange_all[j]+=vertexrange_all[j-1];
}
#else
vertexrange_all = vertexrange;
#endif
Gseq = hypre_ParCSRMatrixToCSRMatrixAll (G);
#if 0 /* debugging */
if (!mpirank) {
wall_time = time_getWallclockSeconds() - wall_time;
hypre_printf ("Finished CGC matrix communication, wall_time = %f s\n",wall_time);
}
#endif
if (Gseq) { /* BM Aug 31, 2006: Gseq==NULL if G has no local rows */
#if 0 /* debugging */
if (!mpirank) {
wall_time = time_getWallclockSeconds();
hypre_printf ("Starting CGC election\n");
}
#endif
AmgCGCChoose (Gseq,vertexrange_all,mpisize,&coarse);
#if 0 /* debugging */
if (!mpirank) {
wall_time = time_getWallclockSeconds() - wall_time;
hypre_printf ("Finished CGC election, wall_time = %f s\n",wall_time);
}
#endif
#if 0 /* debugging */
if (!mpirank) {
for (j=0;j<mpisize;j++)
hypre_printf ("Processor %d, choice = %d of range %d - %d\n",j,coarse[j],vertexrange_all[j]+1,vertexrange_all[j+1]);
}
fflush(stdout);
#endif
#if 0 /* debugging */
if (!mpirank) {
wall_time = time_getWallclockSeconds();
hypre_printf ("Starting CGC CF assignment\n");
}
#endif
choice = coarse[mpirank];
for (j=0;j<num_variables;j++) {
if (CF_marker[j]==choice)
CF_marker[j] = C_PT;
else
CF_marker[j] = F_PT;
}
hypre_CSRMatrixDestroy (Gseq);
hypre_TFree (coarse);
}
else
for (j=0;j<num_variables;j++) CF_marker[j] = F_PT;
#if 0
if (!mpirank) {
wall_time = time_getWallclockSeconds() - wall_time;
hypre_printf ("Finished CGC CF assignment, wall_time = %f s\n",wall_time);
}
#endif
#if 0 /* debugging */
if (!mpirank) {
wall_time = time_getWallclockSeconds();
hypre_printf ("Starting CGC cleanup\n");
}
#endif
HYPRE_IJMatrixDestroy (ijG);
hypre_TFree (vertexrange);
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_TFree (vertexrange_all);
#endif
hypre_TFree (CF_marker_offd);
#if 0
if (!mpirank) {
wall_time = time_getWallclockSeconds() - wall_time;
hypre_printf ("Finished CGC cleanup, wall_time = %f s\n",wall_time);
}
#endif
return(ierr);
}
HYPRE_Int main (HYPRE_Int argc, char *argv[])
{
HYPRE_Int i;
HYPRE_Int myid, num_procs;
HYPRE_Int N, n;
HYPRE_Int ilower, iupper;
HYPRE_Int local_size, extra;
HYPRE_Int solver_id;
HYPRE_Int print_solution;
double h, h2;
#ifdef HYPRE_FORTRAN
hypre_F90_Obj A;
hypre_F90_Obj parcsr_A;
hypre_F90_Obj b;
hypre_F90_Obj par_b;
hypre_F90_Obj x;
hypre_F90_Obj par_x;
hypre_F90_Obj solver, precond;
hypre_F90_Obj long_temp_COMM;
HYPRE_Int temp_COMM;
HYPRE_Int precond_id;
HYPRE_Int one = 1;
HYPRE_Int two = 2;
HYPRE_Int three = 3;
HYPRE_Int six = 6;
HYPRE_Int twenty = 20;
HYPRE_Int thousand = 1000;
HYPRE_Int hypre_type = HYPRE_PARCSR;
double oo1 = 1.e-3;
double tol = 1.e-7;
#else
HYPRE_IJMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJVector b;
HYPRE_ParVector par_b;
HYPRE_IJVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver, precond;
#endif
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs);
/* Default problem parameters */
n = 33;
solver_id = 0;
print_solution = 0;
/* Parse command line */
{
HYPRE_Int arg_index = 0;
HYPRE_Int print_usage = 0;
while (arg_index < argc)
{
if ( strcmp(argv[arg_index], "-n") == 0 )
{
arg_index++;
n = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-solver") == 0 )
{
arg_index++;
solver_id = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-print_solution") == 0 )
{
arg_index++;
print_solution = 1;
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
break;
}
else
{
arg_index++;
}
}
if ((print_usage) && (myid == 0))
{
hypre_printf("\n");
hypre_printf("Usage: %s [<options>]\n", argv[0]);
hypre_printf("\n");
hypre_printf(" -n <n> : problem size in each direction (default: 33)\n");
hypre_printf(" -solver <ID> : solver ID\n");
hypre_printf(" 0 - AMG (default) \n");
hypre_printf(" 1 - AMG-PCG\n");
hypre_printf(" 8 - ParaSails-PCG\n");
hypre_printf(" 50 - PCG\n");
hypre_printf(" -print_solution : print the solution vector\n");
hypre_printf("\n");
}
if (print_usage)
{
hypre_MPI_Finalize();
return (0);
}
}
/* Preliminaries: want at least one processor per row */
if (n*n < num_procs) n = sqrt(num_procs) + 1;
N = n*n; /* global number of rows */
h = 1.0/(n+1); /* mesh size*/
h2 = h*h;
/* Each processor knows only of its own rows - the range is denoted by ilower
and upper. Here we partition the rows. We account for the fact that
N may not divide evenly by the number of processors. */
local_size = N/num_procs;
extra = N - local_size*num_procs;
ilower = local_size*myid;
ilower += hypre_min(myid, extra);
iupper = local_size*(myid+1);
iupper += hypre_min(myid+1, extra);
iupper = iupper - 1;
/* How many rows do I have? */
local_size = iupper - ilower + 1;
/* Create the matrix.
Note that this is a square matrix, so we indicate the row partition
size twice (since number of rows = number of cols) */
#ifdef HYPRE_FORTRAN
long_temp_COMM = (hypre_F90_Obj) hypre_MPI_COMM_WORLD;
temp_COMM = (HYPRE_Int) hypre_MPI_COMM_WORLD;
HYPRE_IJMatrixCreate(&long_temp_COMM, &ilower, &iupper, &ilower, &iupper, &A);
#else
HYPRE_IJMatrixCreate(hypre_MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
#endif
/* Choose a parallel csr format storage (see the User's Manual) */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixSetObjectType(&A, &hypre_type);
#else
HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
#endif
/* Initialize before setting coefficients */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixInitialize(&A);
#else
HYPRE_IJMatrixInitialize(A);
#endif
/* Now go through my local rows and set the matrix entries.
Each row has at most 5 entries. For example, if n=3:
A = [M -I 0; -I M -I; 0 -I M]
M = [4 -1 0; -1 4 -1; 0 -1 4]
Note that here we are setting one row at a time, though
one could set all the rows together (see the User's Manual).
*/
{
HYPRE_Int nnz;
double values[5];
HYPRE_Int cols[5];
for (i = ilower; i <= iupper; i++)
{
nnz = 0;
/* The left identity block:position i-n */
if ((i-n)>=0)
{
cols[nnz] = i-n;
values[nnz] = -1.0;
nnz++;
}
/* The left -1: position i-1 */
if (i%n)
{
cols[nnz] = i-1;
values[nnz] = -1.0;
nnz++;
}
/* Set the diagonal: position i */
cols[nnz] = i;
values[nnz] = 4.0;
nnz++;
/* The right -1: position i+1 */
if ((i+1)%n)
{
cols[nnz] = i+1;
values[nnz] = -1.0;
nnz++;
}
/* The right identity block:position i+n */
if ((i+n)< N)
{
cols[nnz] = i+n;
values[nnz] = -1.0;
nnz++;
}
/* Set the values for row i */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixSetValues(&A, &one, &nnz, &i, &cols[0], &values[0]);
#else
HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
#endif
}
}
/* Assemble after setting the coefficients */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixAssemble(&A);
#else
HYPRE_IJMatrixAssemble(A);
#endif
/* Get the parcsr matrix object to use */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixGetObject(&A, &parcsr_A);
HYPRE_IJMatrixGetObject(&A, &parcsr_A);
#else
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
#endif
/* Create the rhs and solution */
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorCreate(&temp_COMM, &ilower, &iupper, &b);
HYPRE_IJVectorSetObjectType(&b, &hypre_type);
HYPRE_IJVectorInitialize(&b);
#else
HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, ilower, iupper,&b);
HYPRE_IJVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(b);
#endif
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorCreate(&temp_COMM, &ilower, &iupper, &x);
HYPRE_IJVectorSetObjectType(&x, &hypre_type);
HYPRE_IJVectorInitialize(&x);
#else
HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, ilower, iupper,&x);
HYPRE_IJVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(x);
#endif
/* Set the rhs values to h^2 and the solution to zero */
{
double *rhs_values, *x_values;
HYPRE_Int *rows;
rhs_values = calloc(local_size, sizeof(double));
x_values = calloc(local_size, sizeof(double));
rows = calloc(local_size, sizeof(HYPRE_Int));
for (i=0; i<local_size; i++)
{
rhs_values[i] = h2;
x_values[i] = 0.0;
rows[i] = ilower + i;
}
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorSetValues(&b, &local_size, &rows[0], &rhs_values[0]);
HYPRE_IJVectorSetValues(&x, &local_size, &rows[0], &x_values[0]);
#else
HYPRE_IJVectorSetValues(b, local_size, rows, rhs_values);
HYPRE_IJVectorSetValues(x, local_size, rows, x_values);
#endif
free(x_values);
free(rhs_values);
free(rows);
}
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorAssemble(&b);
HYPRE_IJVectorGetObject(&b, &par_b);
#else
HYPRE_IJVectorAssemble(b);
HYPRE_IJVectorGetObject(b, (void **) &par_b);
#endif
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorAssemble(&x);
HYPRE_IJVectorGetObject(&x, &par_x);
#else
HYPRE_IJVectorAssemble(x);
HYPRE_IJVectorGetObject(x, (void **) &par_x);
#endif
/* Choose a solver and solve the system */
/* AMG */
if (solver_id == 0)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGCreate(&solver);
#else
HYPRE_BoomerAMGCreate(&solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGSetPrintLevel(&solver, &three); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(&solver, &six); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(&solver, &three); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(&solver, &one); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(&solver, &twenty); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(&solver, &tol); /* conv. tolerance */
#else
HYPRE_BoomerAMGSetPrintLevel(solver, 3); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(solver, 3); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(solver, 1); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(solver, 20); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(solver, 1e-7); /* conv. tolerance */
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_BoomerAMGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGGetNumIterations(&solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGDestroy(&solver);
#else
HYPRE_BoomerAMGDestroy(solver);
#endif
}
/* PCG */
else if (solver_id == 50)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* prints out the iteration info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* prints out the iteration info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
#else
HYPRE_ParCSRPCGDestroy(solver);
#endif
}
/* PCG with AMG preconditioner */
else if (solver_id == 1)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* print solve info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now set up the AMG preconditioner and specify any parameters */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(&precond, &one); /* print amg solution info*/
HYPRE_BoomerAMGSetCoarsenType(&precond, &six);
HYPRE_BoomerAMGSetRelaxType(&precond, &three);
HYPRE_BoomerAMGSetNumSweeps(&precond, &one);
HYPRE_BoomerAMGSetTol(&precond, &oo1);
#else
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info*/
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 3);
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 1e-3);
#endif
/* Set the PCG preconditioner */
#ifdef HYPRE_FORTRAN
precond_id = 2;
HYPRE_ParCSRPCGSetPrecond(&solver, &precond_id, &precond);
#else
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver and preconditioner */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
HYPRE_BoomerAMGDestroy(&precond);
#else
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
#endif
}
/* PCG with Parasails Preconditioner */
else if (solver_id == 8)
{
HYPRE_Int num_iterations;
double final_res_norm;
HYPRE_Int sai_max_levels = 1;
double sai_threshold = 0.1;
double sai_filter = 0.05;
HYPRE_Int sai_sym = 1;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* print solve info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now set up the ParaSails preconditioner and specify any parameters */
#ifdef HYPRE_FORTRAN
HYPRE_ParaSailsCreate(&temp_COMM, &precond);
#else
HYPRE_ParaSailsCreate(hypre_MPI_COMM_WORLD, &precond);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParaSailsSetParams(&precond, &sai_threshold, &sai_max_levels);
HYPRE_ParaSailsSetFilter(&precond, &sai_filter);
HYPRE_ParaSailsSetSym(&precond, &sai_sym);
HYPRE_ParaSailsSetLogging(&precond, &three);
#else
HYPRE_ParaSailsSetParams(precond, sai_threshold, sai_max_levels);
HYPRE_ParaSailsSetFilter(precond, sai_filter);
HYPRE_ParaSailsSetSym(precond, sai_sym);
HYPRE_ParaSailsSetLogging(precond, 3);
#endif
/* Set the PCG preconditioner */
#ifdef HYPRE_FORTRAN
precond_id = 4;
HYPRE_ParCSRPCGSetPrecond(&solver, &precond_id, &precond);
#else
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
(HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup, precond);
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destory solver and preconditioner */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
HYPRE_ParaSailsDestroy(&precond);
#else
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_ParaSailsDestroy(precond);
#endif
}
else
{
if (myid ==0) hypre_printf("Invalid solver id specified.\n");
}
/* Print the solution */
#ifdef HYPRE_FORTRAN
if (print_solution)
HYPRE_IJVectorPrint(&x, "ij.out.x");
#else
if (print_solution)
HYPRE_IJVectorPrint(x, "ij.out.x");
#endif
/* Clean up */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixDestroy(&A);
HYPRE_IJVectorDestroy(&b);
HYPRE_IJVectorDestroy(&x);
#else
HYPRE_IJMatrixDestroy(A);
HYPRE_IJVectorDestroy(b);
HYPRE_IJVectorDestroy(x);
#endif
/* Finalize MPI*/
hypre_MPI_Finalize();
return(0);
}
void HypreSolver3D::build_A()
{
HYPRE_IJMatrixCreate(MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
HYPRE_IJMatrixInitialize(A);
const int N12 = N1*N2;
const int N012 = N0*N1*N2;
int nnz;
double values[7];
int cols[7];
int i;
for(int j=ilower; j<=iupper; j++)
{
nnz = 0;
i = j%N12;//*((int)(j/(N12)));
// North
if ((i-N2)>=0)
{
cols[nnz] = j-N2;
values[nnz] = -1.0;
nnz++;
}
// Left
if (i%N2)
{
cols[nnz] = j-1;
values[nnz] = -1.0;
nnz++;
}
// Center
cols[nnz] = j;
values[nnz] = 6.0;
nnz++;
// Right
if ((i+1)%N2)
{
cols[nnz] = j+1;
values[nnz] = -1.0;
nnz++;
}
// South
if ((i+N2)<N12)
{
cols[nnz] = j+N2;
values[nnz] = -1.0;
nnz++;
}
// Top
if ((j+N12)<N012)
{
cols[nnz] = j+N12;
values[nnz] = -1.0;
nnz++;
}
// Bottom
if ((j-N12)>=0)
{
cols[nnz] = j-N12;
values[nnz] = -1.0;
nnz++;
}
HYPRE_IJMatrixSetValues(A, 1, &nnz, &j, cols, values);
}
HYPRE_IJMatrixAssemble(A);
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
}
int HYPRE_LSI_GetParCSRMatrix(HYPRE_IJMatrix Amat, int nrows, int nnz,
int *ia_ptr, int *ja_ptr, double *a_ptr)
{
int nz, i, j, ierr, rowSize, *colInd, nz_ptr, *colInd2;
int firstNnz;
double *colVal, *colVal2;
HYPRE_ParCSRMatrix A_csr;
nz = 0;
nz_ptr = 0;
ia_ptr[0] = nz_ptr;
/* ---old_IJ----------------------------------------------------------- */
/*A_csr = (HYPRE_ParCSRMatrix) HYPRE_IJMatrixGetLocalStorage(Amat);*/
/* ---new_IJ----------------------------------------------------------- */
HYPRE_IJMatrixGetObject(Amat, (void**) &A_csr);
/* -------------------------------------------------------------------- */
for ( i = 0; i < nrows; i++ )
{
ierr = HYPRE_ParCSRMatrixGetRow(A_csr,i,&rowSize,&colInd,&colVal);
assert(!ierr);
colInd2 = (int *) malloc(rowSize * sizeof(int));
colVal2 = (double *) malloc(rowSize * sizeof(double));
for ( j = 0; j < rowSize; j++ )
{
colInd2[j] = colInd[j];
colVal2[j] = colVal[j];
}
qsort1(colInd2, colVal2, 0, rowSize-1);
for ( j = 0; j < rowSize-1; j++ )
if ( colInd2[j] == colInd2[j+1] )
printf("HYPRE_LSI_GetParCSRMatrix-duplicate colind at row %d \n",i);
firstNnz = 0;
for ( j = 0; j < rowSize; j++ )
{
if ( colVal2[j] != 0.0 )
{
if (nz_ptr > 0 && firstNnz > 0 && colInd2[j] == ja_ptr[nz_ptr-1])
{
a_ptr[nz_ptr-1] += colVal2[j];
printf("HYPRE_LSI_GetParCSRMatrix:: repeated col in row %d\n",i);
}
else
{
ja_ptr[nz_ptr] = colInd2[j];
a_ptr[nz_ptr++] = colVal2[j];
if ( nz_ptr > nnz )
{
printf("HYPRE_LSI_GetParCSRMatrix Error (1) - %d %d.\n",i,
nrows);
exit(1);
}
firstNnz++;
}
} else nz++;
}
free( colInd2 );
free( colVal2 );
ia_ptr[i+1] = nz_ptr;
ierr = HYPRE_ParCSRMatrixRestoreRow(A_csr,i,&rowSize,&colInd,&colVal);
assert(!ierr);
}
/*
if ( nnz != nz_ptr )
{
printf("HYPRE_LSI_GetParCSRMatrix note : matrix sparsity has been \n");
printf(" changed since matConfigure - %d > %d ?\n", nnz, nz_ptr);
printf(" number of zeros = %d \n", nz );
}
*/
return nz_ptr;
}
int
main(int argc, char *argv[])
{
GRID *g;
DOF *u_h;
MAT *A, *A0, *B;
MAP *map;
INT i;
size_t nnz, mem, mem_peak;
VEC *x, *y0, *y1, *y2;
double t0, t1, dnz, dnz1, mflops, mop;
char *fn = "../test/cube.dat";
FLOAT mem_max = 300;
INT refine = 0;
phgOptionsRegisterFilename("-mesh_file", "Mesh file", (char **)&fn);
phgOptionsRegisterInt("-loop_count", "Loop count", &loop_count);
phgOptionsRegisterInt("-refine", "Refinement level", &refine);
phgOptionsRegisterFloat("-mem_max", "Maximum memory", &mem_max);
phgInit(&argc, &argv);
g = phgNewGrid(-1);
if (!phgImport(g, fn, FALSE))
phgError(1, "can't read file \"%s\".\n", fn);
phgRefineAllElements(g, refine);
u_h = phgDofNew(g, DOF_DEFAULT, 1, "u_h", DofNoAction);
while (TRUE) {
phgPrintf("\n");
if (phgBalanceGrid(g, 1.2, 1, NULL, 0.))
phgPrintf("Repartition mesh, %d submeshes, load imbalance: %lg\n",
g->nprocs, (double)g->lif);
map = phgMapCreate(u_h, NULL);
A = phgMapCreateMat(map, map);
A->handle_bdry_eqns = TRUE;
build_matrix(A, u_h);
phgMatAssemble(A);
/* Note: A is unsymmetric (A' != A) if boundary entries not removed */
phgMatRemoveBoundaryEntries(A);
#if 0
/* test block matrix operation */
A0 = phgMatCreateBlockMatrix(g->comm, 1, 1, &A, NULL);
#else
A0 = A;
#endif
phgPrintf("%d DOF, %d elems, %d submeshes, matrix size: %d, LIF: %lg\n",
DofGetDataCountGlobal(u_h), g->nleaf_global,
g->nprocs, A->rmap->nglobal, (double)g->lif);
/* test PHG mat-vec multiply */
x = phgMapCreateVec(A->cmap, 1);
y1 = phgMapCreateVec(A->rmap, 1);
phgVecRandomize(x, 123);
phgMatVec(MAT_OP_N, 1.0, A0, x, 0.0, &y1);
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
phgMatVec(MAT_OP_N, 1.0, A0, x, 0.0, &y1);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
y0 = phgVecCopy(y1, NULL);
nnz = A->nnz_d + A->nnz_o;
#if USE_MPI
dnz1 = nnz;
MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm);
#else
dnz = nnz;
#endif
mop = loop_count * (dnz + dnz - A->rmap->nlocal) * 1e-6;
phgPrintf("\n");
t1 -= t0;
phgPrintf(" PHG: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF)\n",
t1, dnz, mop / (t1 == 0 ? 1. : t1), mflops);
/* test trans(A)*x */
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
phgMatVec(MAT_OP_T, 1.0, A0, x, 0.0, &y1);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
t1 -= t0;
phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1 == 0 ? 1. : t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
/* time A * trans(A) */
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
B = phgMatMat(MAT_OP_N, MAT_OP_N, 1.0, A, A, 0.0, NULL);
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
nnz = B->nnz_d + B->nnz_o;
#if USE_MPI
dnz1 = nnz;
MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm);
#else
dnz = nnz;
#endif
/* compare B*x <--> A*A*x */
y2 = phgMatVec(MAT_OP_N, 1.0, B, x, 0.0, NULL);
phgMatVec(MAT_OP_N, 1.0, A0, y0, 0.0, &y1);
phgMatDestroy(&B);
t1 -= t0;
phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n",
t1, dnz, mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL));
#if USE_PETSC
{
Mat ma, mb;
MatInfo info;
Vec va, vb, vc;
PetscScalar *vec;
ma = phgPetscCreateMatAIJ(A);
MatGetVecs(ma, PETSC_NULL, &va);
VecDuplicate(va, &vb);
VecGetArray(va, &vec);
memcpy(vec, x->data, x->map->nlocal * sizeof(*vec));
VecRestoreArray(va, &vec);
MatMult(ma, va, vb);
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
MatMult(ma, va, vb);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
VecGetArray(vb, &vec);
memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec));
VecRestoreArray(vb, &vec);
MatGetInfo(ma, MAT_GLOBAL_SUM, &info);
/*phgPrintf(" --------------------------------------------"
"-------------------------\n");*/
phgPrintf("\n");
t1 -= t0;
dnz = info.nz_used;
phgPrintf(" PETSc: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
MatMultTranspose(ma, va, vb);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
VecGetArray(vb, &vec);
memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec));
VecRestoreArray(vb, &vec);
t1 -= t0;
phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
MatMatMult(ma, ma, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &mb);
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
t1 -= t0;
MatGetInfo(mb, MAT_GLOBAL_SUM, &info);
dnz = info.nz_used;
VecDuplicate(va, &vc);
/* compare B*x <--> A*A*x */
MatMult(ma, vb, vc);
MatMult(mb, va, vb);
VecGetArray(vb, &vec);
memcpy(y1->data, vec, x->map->nlocal * sizeof(*vec));
VecRestoreArray(vb, &vec);
VecGetArray(vc, &vec);
memcpy(y2->data, vec, x->map->nlocal * sizeof(*vec));
VecRestoreArray(vc, &vec);
phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n",
t1, dnz, mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL));
phgPetscMatDestroy(&mb);
phgPetscMatDestroy(&ma);
phgPetscVecDestroy(&va);
phgPetscVecDestroy(&vb);
phgPetscVecDestroy(&vc);
}
#endif /* USE_PETSC */
#if USE_HYPRE
{
HYPRE_IJMatrix ma;
HYPRE_IJVector va, vb, vc;
HYPRE_ParCSRMatrix par_ma;
hypre_ParCSRMatrix *par_mb;
HYPRE_ParVector par_va, par_vb, par_vc;
HYPRE_Int offset, *ni, start, end;
assert(sizeof(INT)==sizeof(int) && sizeof(FLOAT)==sizeof(double));
setup_hypre_mat(A, &ma);
ni = phgAlloc(2 * A->rmap->nlocal * sizeof(*ni));
offset = A->cmap->partition[A->cmap->rank];
for (i = 0; i < A->rmap->nlocal; i++)
ni[i] = i + offset;
HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1,
&va);
HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1,
&vb);
HYPRE_IJVectorCreate(g->comm, offset, offset + A->rmap->nlocal - 1,
&vc);
HYPRE_IJVectorSetObjectType(va, HYPRE_PARCSR);
HYPRE_IJVectorSetObjectType(vb, HYPRE_PARCSR);
HYPRE_IJVectorSetObjectType(vc, HYPRE_PARCSR);
HYPRE_IJVectorSetMaxOffProcElmts(va, 0);
HYPRE_IJVectorSetMaxOffProcElmts(vb, 0);
HYPRE_IJVectorSetMaxOffProcElmts(vc, 0);
HYPRE_IJVectorInitialize(va);
HYPRE_IJVectorInitialize(vb);
HYPRE_IJVectorInitialize(vc);
HYPRE_IJMatrixGetObject(ma, (void **)(void *)&par_ma);
HYPRE_IJVectorGetObject(va, (void **)(void *)&par_va);
HYPRE_IJVectorGetObject(vb, (void **)(void *)&par_vb);
HYPRE_IJVectorGetObject(vc, (void **)(void *)&par_vc);
HYPRE_IJVectorSetValues(va, A->cmap->nlocal, ni, (double *)x->data);
HYPRE_IJVectorAssemble(va);
HYPRE_IJVectorAssemble(vb);
HYPRE_IJVectorAssemble(vc);
HYPRE_IJMatrixGetRowCounts(ma, A->cmap->nlocal,
ni, ni + A->rmap->nlocal);
for (i = 0, nnz = 0; i < A->rmap->nlocal; i++)
nnz += ni[A->rmap->nlocal + i];
#if USE_MPI
dnz1 = nnz;
MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm);
#else
dnz = nnz;
#endif
HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_va, 0.0, par_vb);
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_va, 0.0, par_vb);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data);
/*phgPrintf(" --------------------------------------------"
"-------------------------\n");*/
phgPrintf("\n");
t1 -= t0;
phgPrintf(" HYPRE: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
for (i = 0; i < loop_count; i++) {
HYPRE_ParCSRMatrixMatvecT(1.0, par_ma, par_va, 0.0, par_vb);
}
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data);
t1 -= t0;
phgPrintf(" A'*x: time %0.4lf, nnz %0.16lg, %0.2lfMF (%0.2lfMF), "
"err: %le\n", t1, dnz, mop / (t1==0 ? 1.:t1), mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y0, 1.0, &y1), 0, NULL));
phgPerfGetMflops(g, NULL, NULL); /* reset flops counter */
t0 = phgGetTime(NULL);
/* Note: 'HYPRE_ParCSRMatrix' is currently typedef'ed to
* 'hypre_ParCSRMatrix *' */
par_mb = hypre_ParMatmul((hypre_ParCSRMatrix *)par_ma,
(hypre_ParCSRMatrix *)par_ma);
t1 = phgGetTime(NULL);
mflops = phgPerfGetMflops(g, NULL, NULL);
start = hypre_ParCSRMatrixFirstRowIndex(par_mb);
end = hypre_ParCSRMatrixLastRowIndex(par_mb) + 1;
for (i = start, nnz = 0; i < end; i++) {
HYPRE_Int ncols;
hypre_ParCSRMatrixGetRow(par_mb, i, &ncols, NULL, NULL);
hypre_ParCSRMatrixRestoreRow(par_mb, i, &ncols, NULL, NULL);
nnz += ncols;
}
#if USE_MPI
dnz1 = nnz;
MPI_Reduce(&dnz1, &dnz, 1, MPI_DOUBLE, MPI_SUM, 0, g->comm);
#else
dnz = nnz;
#endif
/* compare B*x <--> A*A*x */
HYPRE_ParCSRMatrixMatvec(1.0, par_ma, par_vb, 0.0, par_vc);
HYPRE_ParCSRMatrixMatvec(1.0, (void *)par_mb, par_va, 0.0, par_vb);
HYPRE_IJVectorGetValues(vb, A->rmap->nlocal, ni, (double*)y1->data);
HYPRE_IJVectorGetValues(vc, A->rmap->nlocal, ni, (double*)y2->data);
hypre_ParCSRMatrixDestroy((par_mb));
t1 -= t0;
phgPrintf(" A*A: time %0.4lf, nnz %0.16lg, %0.2lfMF, err: %le\n",
t1, dnz, mflops,
(double)phgVecNorm2(phgVecAXPBY(-1.0, y1, 1.0, &y2), 0, NULL));
phgFree(ni);
HYPRE_IJMatrixDestroy(ma);
HYPRE_IJVectorDestroy(va);
HYPRE_IJVectorDestroy(vb);
HYPRE_IJVectorDestroy(vc);
}
#endif /* USE_HYPRE */
if (A0 != A)
phgMatDestroy(&A0);
#if 0
if (A->rmap->nglobal > 1000) {
VEC *v = phgMapCreateVec(A->rmap, 3);
for (i = 0; i < v->map->nlocal; i++) {
v->data[i + 0 * v->map->nlocal] = 1 * (i + v->map->partition[g->rank]);
v->data[i + 1 * v->map->nlocal] = 2 * (i + v->map->partition[g->rank]);
v->data[i + 2 * v->map->nlocal] = 3 * (i + v->map->partition[g->rank]);
}
phgMatDumpMATLAB(A, "A", "A.m");
phgVecDumpMATLAB(v, "v", "v.m");
phgFinalize();
exit(0);
}
#endif
phgMatDestroy(&A);
phgVecDestroy(&x);
phgVecDestroy(&y0);
phgVecDestroy(&y1);
phgVecDestroy(&y2);
phgMapDestroy(&map);
mem = phgMemoryUsage(g, &mem_peak);
dnz = mem / (1024.0 * 1024.0);
dnz1 = mem_peak / (1024.0 * 1024.0);
/*phgPrintf(" --------------------------------------------"
"-------------------------\n");*/
phgPrintf("\n");
phgPrintf(" Memory: current %0.4lgMB, peak %0.4lgMB\n", dnz, dnz1);
#if 0
{
static int loop_count = 0;
if (++loop_count == 4)
break;
}
#endif
if (mem_peak > 1024 * (size_t)1024 * mem_max)
break;
phgRefineAllElements(g, 1);
}
phgDofFree(&u_h);
phgFreeGrid(&g);
phgFinalize();
return 0;
}
int32_t
impl_bHYPRE_IJParCSRMatrix_ApplyAdjoint(
/* in */ bHYPRE_IJParCSRMatrix self,
/* in */ bHYPRE_Vector b,
/* inout */ bHYPRE_Vector* x,
/* out */ sidl_BaseInterface *_ex)
{
*_ex = 0;
{
/* DO-NOT-DELETE splicer.begin(bHYPRE.IJParCSRMatrix.ApplyAdjoint) */
/* Insert-Code-Here {bHYPRE.IJParCSRMatrix.ApplyAdjoint} (ApplyAdjoint method) */
/* ApplyAdjoing means to multiply by a vector, y = A'*x , where A' is the
* adjoint of A (=transpose, this is a real matrix). Here, we call
* the HYPRE Matvec function which performs y = a*A*x + b*y (we set
* a=1 and b=0). */
int ierr=0;
void * object;
struct bHYPRE_IJParCSRMatrix__data * data;
struct bHYPRE_IJParCSRVector__data * data_x, * data_b;
bHYPRE_IJParCSRVector bHYPREP_b, bHYPREP_x;
HYPRE_IJMatrix ij_A;
HYPRE_IJVector ij_x, ij_b;
HYPRE_ParVector xx, bb;
HYPRE_ParCSRMatrix A;
data = bHYPRE_IJParCSRMatrix__get_data( self );
ij_A = data -> ij_A;
ierr += HYPRE_IJMatrixGetObject( ij_A, &object );
A = (HYPRE_ParCSRMatrix) object;
/* A bHYPRE_Vector is just an interface, we have no knowledge of its
* contents. Check whether it's something we know how to handle.
* If not, die. */
bHYPREP_b = (bHYPRE_IJParCSRVector) bHYPRE_Vector__cast2(b, "bHYPRE.IJParCSRVector", _ex );
SIDL_CHECK(*_ex);
hypre_assert( bHYPREP_b!=NULL );
bHYPREP_x = (bHYPRE_IJParCSRVector) bHYPRE_Vector__cast2( *x, "bHYPRE.IJParCSRVector", _ex );
SIDL_CHECK(*_ex);
hypre_assert( bHYPREP_x!=NULL );
data_x = bHYPRE_IJParCSRVector__get_data( bHYPREP_x );
ij_x = data_x -> ij_b;
ierr += HYPRE_IJVectorGetObject( ij_x, &object );
xx = (HYPRE_ParVector) object;
data_b = bHYPRE_IJParCSRVector__get_data( bHYPREP_b );
ij_b = data_b -> ij_b;
ierr += HYPRE_IJVectorGetObject( ij_b, &object );
bb = (HYPRE_ParVector) object;
ierr += HYPRE_ParCSRMatrixMatvecT( 1.0, A, bb, 0.0, xx );
bHYPRE_IJParCSRVector_deleteRef( bHYPREP_b, _ex ); SIDL_CHECK(*_ex);
bHYPRE_IJParCSRVector_deleteRef( bHYPREP_x, _ex ); SIDL_CHECK(*_ex);
return( ierr );
hypre_babel_exception_return_error(_ex);
/* DO-NOT-DELETE splicer.end(bHYPRE.IJParCSRMatrix.ApplyAdjoint) */
}
}
HYPRE_Int
HYPRE_IJMatrixPrint( HYPRE_IJMatrix matrix,
const char *filename )
{
MPI_Comm comm;
HYPRE_Int *row_partitioning;
HYPRE_Int *col_partitioning;
HYPRE_Int ilower, iupper, jlower, jupper;
HYPRE_Int i, j, ii;
HYPRE_Int ncols, *cols;
HYPRE_Complex *values;
HYPRE_Int myid;
char new_filename[255];
FILE *file;
void *object;
if (!matrix)
{
hypre_error_in_arg(1);
return hypre_error_flag;
}
if ( (hypre_IJMatrixObjectType(matrix) != HYPRE_PARCSR) )
{
hypre_error_in_arg(1);
return hypre_error_flag;
}
comm = hypre_IJMatrixComm(matrix);
hypre_MPI_Comm_rank(comm, &myid);
hypre_sprintf(new_filename,"%s.%05d", filename, myid);
if ((file = fopen(new_filename, "w")) == NULL)
{
hypre_error_in_arg(2);
return hypre_error_flag;
}
row_partitioning = hypre_IJMatrixRowPartitioning(matrix);
col_partitioning = hypre_IJMatrixColPartitioning(matrix);
#ifdef HYPRE_NO_GLOBAL_PARTITION
ilower = row_partitioning[0];
iupper = row_partitioning[1] - 1;
jlower = col_partitioning[0];
jupper = col_partitioning[1] - 1;
#else
ilower = row_partitioning[myid];
iupper = row_partitioning[myid+1] - 1;
jlower = col_partitioning[myid];
jupper = col_partitioning[myid+1] - 1;
#endif
hypre_fprintf(file, "%d %d %d %d\n", ilower, iupper, jlower, jupper);
HYPRE_IJMatrixGetObject(matrix, &object);
for (i = ilower; i <= iupper; i++)
{
if ( hypre_IJMatrixObjectType(matrix) == HYPRE_PARCSR )
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
ii = i - hypre_IJMatrixGlobalFirstRow(matrix);
#else
ii = i - row_partitioning[0];
#endif
HYPRE_ParCSRMatrixGetRow((HYPRE_ParCSRMatrix) object,
ii, &ncols, &cols, &values);
for (j = 0; j < ncols; j++)
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
cols[j] += hypre_IJMatrixGlobalFirstCol(matrix);
#else
cols[j] += col_partitioning[0];
#endif
}
}
for (j = 0; j < ncols; j++)
{
hypre_fprintf(file, "%d %d %.14e\n", i, cols[j], values[j]);
}
if ( hypre_IJMatrixObjectType(matrix) == HYPRE_PARCSR )
{
for (j = 0; j < ncols; j++)
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
cols[j] -= hypre_IJMatrixGlobalFirstCol(matrix);
#else
cols[j] -= col_partitioning[0];
#endif
}
HYPRE_ParCSRMatrixRestoreRow((HYPRE_ParCSRMatrix) object,
ii, &ncols, &cols, &values);
}
}
fclose(file);
return hypre_error_flag;
}
