void HypreSolver2D::build_solver()
{
HYPRE_BoomerAMGCreate(&solver);
//HYPRE_BoomerAMGSetPrintLevel(solver, 3); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(solver, 3); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(solver, 1); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(solver, 20); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
}
HYPRE_Int hypre_seqAMGSetup( hypre_ParAMGData *amg_data,
HYPRE_Int p_level,
HYPRE_Int coarse_threshold)
{
/* Par Data Structure variables */
hypre_ParCSRMatrix **Par_A_array = hypre_ParAMGDataAArray(amg_data);
MPI_Comm comm = hypre_ParCSRMatrixComm(Par_A_array[0]);
MPI_Comm new_comm, seq_comm;
hypre_ParCSRMatrix *A_seq = NULL;
hypre_CSRMatrix *A_seq_diag;
hypre_CSRMatrix *A_seq_offd;
hypre_ParVector *F_seq = NULL;
hypre_ParVector *U_seq = NULL;
hypre_ParCSRMatrix *A;
HYPRE_Int **dof_func_array;
HYPRE_Int num_procs, my_id;
HYPRE_Int not_finished_coarsening;
HYPRE_Int level;
HYPRE_Solver coarse_solver;
/* misc */
dof_func_array = hypre_ParAMGDataDofFuncArray(amg_data);
/*MPI Stuff */
hypre_MPI_Comm_size(comm, &num_procs);
hypre_MPI_Comm_rank(comm,&my_id);
/*initial */
level = p_level;
not_finished_coarsening = 1;
/* convert A at this level to sequential */
A = Par_A_array[level];
{
double *A_seq_data = NULL;
HYPRE_Int *A_seq_i = NULL;
HYPRE_Int *A_seq_offd_i = NULL;
HYPRE_Int *A_seq_j = NULL;
double *A_tmp_data = NULL;
HYPRE_Int *A_tmp_i = NULL;
HYPRE_Int *A_tmp_j = NULL;
HYPRE_Int *info, *displs, *displs2;
HYPRE_Int i, j, size, num_nonzeros, total_nnz, cnt;
hypre_CSRMatrix *A_diag = hypre_ParCSRMatrixDiag(A);
hypre_CSRMatrix *A_offd = hypre_ParCSRMatrixOffd(A);
HYPRE_Int *col_map_offd = hypre_ParCSRMatrixColMapOffd(A);
HYPRE_Int *A_diag_i = hypre_CSRMatrixI(A_diag);
HYPRE_Int *A_offd_i = hypre_CSRMatrixI(A_offd);
HYPRE_Int *A_diag_j = hypre_CSRMatrixJ(A_diag);
HYPRE_Int *A_offd_j = hypre_CSRMatrixJ(A_offd);
double *A_diag_data = hypre_CSRMatrixData(A_diag);
double *A_offd_data = hypre_CSRMatrixData(A_offd);
HYPRE_Int num_rows = hypre_CSRMatrixNumRows(A_diag);
HYPRE_Int first_row_index = hypre_ParCSRMatrixFirstRowIndex(A);
hypre_MPI_Group orig_group, new_group;
HYPRE_Int *ranks, new_num_procs, *row_starts;
info = hypre_CTAlloc(HYPRE_Int, num_procs);
hypre_MPI_Allgather(&num_rows, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, comm);
ranks = hypre_CTAlloc(HYPRE_Int, num_procs);
new_num_procs = 0;
for (i=0; i < num_procs; i++)
if (info[i])
{
ranks[new_num_procs] = i;
info[new_num_procs++] = info[i];
}
MPI_Comm_group(comm, &orig_group);
hypre_MPI_Group_incl(orig_group, new_num_procs, ranks, &new_group);
MPI_Comm_create(comm, new_group, &new_comm);
hypre_MPI_Group_free(&new_group);
hypre_MPI_Group_free(&orig_group);
if (num_rows)
{
/* alloc space in seq data structure only for participating procs*/
HYPRE_BoomerAMGCreate(&coarse_solver);
HYPRE_BoomerAMGSetMaxRowSum(coarse_solver,
hypre_ParAMGDataMaxRowSum(amg_data));
HYPRE_BoomerAMGSetStrongThreshold(coarse_solver,
hypre_ParAMGDataStrongThreshold(amg_data));
HYPRE_BoomerAMGSetCoarsenType(coarse_solver,
hypre_ParAMGDataCoarsenType(amg_data));
HYPRE_BoomerAMGSetInterpType(coarse_solver,
hypre_ParAMGDataInterpType(amg_data));
HYPRE_BoomerAMGSetTruncFactor(coarse_solver,
hypre_ParAMGDataTruncFactor(amg_data));
HYPRE_BoomerAMGSetPMaxElmts(coarse_solver,
hypre_ParAMGDataPMaxElmts(amg_data));
if (hypre_ParAMGDataUserRelaxType(amg_data) > -1)
HYPRE_BoomerAMGSetRelaxType(coarse_solver,
hypre_ParAMGDataUserRelaxType(amg_data));
HYPRE_BoomerAMGSetRelaxOrder(coarse_solver,
hypre_ParAMGDataRelaxOrder(amg_data));
HYPRE_BoomerAMGSetRelaxWt(coarse_solver,
hypre_ParAMGDataUserRelaxWeight(amg_data));
if (hypre_ParAMGDataUserNumSweeps(amg_data) > -1)
HYPRE_BoomerAMGSetNumSweeps(coarse_solver,
hypre_ParAMGDataUserNumSweeps(amg_data));
HYPRE_BoomerAMGSetNumFunctions(coarse_solver,
hypre_ParAMGDataNumFunctions(amg_data));
HYPRE_BoomerAMGSetMaxIter(coarse_solver, 1);
HYPRE_BoomerAMGSetTol(coarse_solver, 0);
/* Create CSR Matrix, will be Diag part of new matrix */
A_tmp_i = hypre_CTAlloc(HYPRE_Int, num_rows+1);
A_tmp_i[0] = 0;
for (i=1; i < num_rows+1; i++)
A_tmp_i[i] = A_diag_i[i]-A_diag_i[i-1]+A_offd_i[i]-A_offd_i[i-1];
num_nonzeros = A_offd_i[num_rows]+A_diag_i[num_rows];
A_tmp_j = hypre_CTAlloc(HYPRE_Int, num_nonzeros);
A_tmp_data = hypre_CTAlloc(double, num_nonzeros);
cnt = 0;
for (i=0; i < num_rows; i++)
{
for (j=A_diag_i[i]; j < A_diag_i[i+1]; j++)
{
A_tmp_j[cnt] = A_diag_j[j]+first_row_index;
A_tmp_data[cnt++] = A_diag_data[j];
}
for (j=A_offd_i[i]; j < A_offd_i[i+1]; j++)
{
A_tmp_j[cnt] = col_map_offd[A_offd_j[j]];
A_tmp_data[cnt++] = A_offd_data[j];
}
}
displs = hypre_CTAlloc(HYPRE_Int, new_num_procs+1);
displs[0] = 0;
for (i=1; i < new_num_procs+1; i++)
displs[i] = displs[i-1]+info[i-1];
size = displs[new_num_procs];
A_seq_i = hypre_CTAlloc(HYPRE_Int, size+1);
A_seq_offd_i = hypre_CTAlloc(HYPRE_Int, size+1);
hypre_MPI_Allgatherv ( &A_tmp_i[1], num_rows, HYPRE_MPI_INT, &A_seq_i[1], info,
displs, HYPRE_MPI_INT, new_comm );
displs2 = hypre_CTAlloc(HYPRE_Int, new_num_procs+1);
A_seq_i[0] = 0;
displs2[0] = 0;
for (j=1; j < displs[1]; j++)
A_seq_i[j] = A_seq_i[j]+A_seq_i[j-1];
for (i=1; i < new_num_procs; i++)
{
for (j=displs[i]; j < displs[i+1]; j++)
{
A_seq_i[j] = A_seq_i[j]+A_seq_i[j-1];
}
}
A_seq_i[size] = A_seq_i[size]+A_seq_i[size-1];
displs2[new_num_procs] = A_seq_i[size];
for (i=1; i < new_num_procs+1; i++)
{
displs2[i] = A_seq_i[displs[i]];
info[i-1] = displs2[i] - displs2[i-1];
}
total_nnz = displs2[new_num_procs];
A_seq_j = hypre_CTAlloc(HYPRE_Int, total_nnz);
A_seq_data = hypre_CTAlloc(double, total_nnz);
hypre_MPI_Allgatherv ( A_tmp_j, num_nonzeros, HYPRE_MPI_INT,
A_seq_j, info, displs2,
HYPRE_MPI_INT, new_comm );
hypre_MPI_Allgatherv ( A_tmp_data, num_nonzeros, hypre_MPI_DOUBLE,
A_seq_data, info, displs2,
hypre_MPI_DOUBLE, new_comm );
hypre_TFree(displs);
hypre_TFree(displs2);
hypre_TFree(A_tmp_i);
hypre_TFree(A_tmp_j);
hypre_TFree(A_tmp_data);
row_starts = hypre_CTAlloc(HYPRE_Int,2);
row_starts[0] = 0;
row_starts[1] = size;
/* Create 1 proc communicator */
seq_comm = hypre_MPI_COMM_SELF;
A_seq = hypre_ParCSRMatrixCreate(seq_comm,size,size,
row_starts, row_starts,
0,total_nnz,0);
A_seq_diag = hypre_ParCSRMatrixDiag(A_seq);
A_seq_offd = hypre_ParCSRMatrixOffd(A_seq);
hypre_CSRMatrixData(A_seq_diag) = A_seq_data;
hypre_CSRMatrixI(A_seq_diag) = A_seq_i;
hypre_CSRMatrixJ(A_seq_diag) = A_seq_j;
hypre_CSRMatrixI(A_seq_offd) = A_seq_offd_i;
F_seq = hypre_ParVectorCreate(seq_comm, size, row_starts);
U_seq = hypre_ParVectorCreate(seq_comm, size, row_starts);
hypre_ParVectorOwnsPartitioning(F_seq) = 0;
hypre_ParVectorOwnsPartitioning(U_seq) = 0;
hypre_ParVectorInitialize(F_seq);
hypre_ParVectorInitialize(U_seq);
hypre_BoomerAMGSetup(coarse_solver,A_seq,F_seq,U_seq);
hypre_ParAMGDataCoarseSolver(amg_data) = coarse_solver;
hypre_ParAMGDataACoarse(amg_data) = A_seq;
hypre_ParAMGDataFCoarse(amg_data) = F_seq;
hypre_ParAMGDataUCoarse(amg_data) = U_seq;
hypre_ParAMGDataNewComm(amg_data) = new_comm;
}
hypre_TFree(info);
hypre_TFree(ranks);
}
return 0;
}
HYPRE_Int hypre_BlockTridiagSetup(void *data, hypre_ParCSRMatrix *A,
hypre_ParVector *b, hypre_ParVector *x)
{
HYPRE_Int i, j, *index_set1, print_level, nsweeps, relax_type;
HYPRE_Int nrows, nrows1, nrows2, start1, start2, *index_set2;
HYPRE_Int count, ierr;
double threshold;
hypre_ParCSRMatrix **submatrices;
HYPRE_Solver precon1;
HYPRE_Solver precon2;
HYPRE_IJVector ij_u1, ij_u2, ij_f1, ij_f2;
hypre_ParVector *vector;
MPI_Comm comm;
hypre_BlockTridiagData *b_data = (hypre_BlockTridiagData *) data;
HYPRE_ParCSRMatrixGetComm((HYPRE_ParCSRMatrix) A, &comm);
index_set1 = b_data->index_set1;
nrows1 = index_set1[0];
nrows = hypre_ParCSRMatrixNumRows(A);
nrows2 = nrows - nrows1;
b_data->index_set2 = hypre_CTAlloc(HYPRE_Int, nrows2+1);
index_set2 = b_data->index_set2;
index_set2[0] = nrows2;
count = 1;
for (i = 0; i < index_set1[1]; i++) index_set2[count++] = i;
for (i = 1; i < nrows1; i++)
for (j = index_set1[i]+1; j < index_set1[i+1]; j++)
index_set2[count++] = j;
for (i = index_set1[nrows1]+1; i < nrows; i++) index_set2[count++] = i;
submatrices = hypre_CTAlloc(hypre_ParCSRMatrix *, 4);
hypre_ParCSRMatrixExtractSubmatrices(A, index_set1, &submatrices);
nrows1 = hypre_ParCSRMatrixNumRows(submatrices[0]);
nrows2 = hypre_ParCSRMatrixNumRows(submatrices[3]);
start1 = hypre_ParCSRMatrixFirstRowIndex(submatrices[0]);
start2 = hypre_ParCSRMatrixFirstRowIndex(submatrices[3]);
HYPRE_IJVectorCreate(comm, start1, start1+nrows1-1, &ij_u1);
HYPRE_IJVectorSetObjectType(ij_u1, HYPRE_PARCSR);
ierr = HYPRE_IJVectorInitialize(ij_u1);
ierr += HYPRE_IJVectorAssemble(ij_u1);
hypre_assert(!ierr);
HYPRE_IJVectorCreate(comm, start1, start1+nrows1-1, &ij_f1);
HYPRE_IJVectorSetObjectType(ij_f1, HYPRE_PARCSR);
ierr = HYPRE_IJVectorInitialize(ij_f1);
ierr += HYPRE_IJVectorAssemble(ij_f1);
hypre_assert(!ierr);
HYPRE_IJVectorCreate(comm, start2, start2+nrows2-1, &ij_u2);
HYPRE_IJVectorSetObjectType(ij_u2, HYPRE_PARCSR);
ierr = HYPRE_IJVectorInitialize(ij_u2);
ierr += HYPRE_IJVectorAssemble(ij_u2);
hypre_assert(!ierr);
HYPRE_IJVectorCreate(comm, start2, start2+nrows1-1, &ij_f2);
HYPRE_IJVectorSetObjectType(ij_f2, HYPRE_PARCSR);
ierr = HYPRE_IJVectorInitialize(ij_f2);
ierr += HYPRE_IJVectorAssemble(ij_f2);
hypre_assert(!ierr);
HYPRE_IJVectorGetObject(ij_f1, (void **) &vector);
b_data->F1 = vector;
HYPRE_IJVectorGetObject(ij_u1, (void **) &vector);
b_data->U1 = vector;
HYPRE_IJVectorGetObject(ij_f2, (void **) &vector);
b_data->F2 = vector;
HYPRE_IJVectorGetObject(ij_u2, (void **) &vector);
b_data->U2 = vector;
print_level = b_data->print_level;
threshold = b_data->threshold;
nsweeps = b_data->num_sweeps;
relax_type = b_data->relax_type;
threshold = b_data->threshold;
HYPRE_BoomerAMGCreate(&precon1);
HYPRE_BoomerAMGSetMaxIter(precon1, 1);
HYPRE_BoomerAMGSetCycleType(precon1, 1);
HYPRE_BoomerAMGSetPrintLevel(precon1, print_level);
HYPRE_BoomerAMGSetMaxLevels(precon1, 25);
HYPRE_BoomerAMGSetMeasureType(precon1, 0);
HYPRE_BoomerAMGSetCoarsenType(precon1, 0);
HYPRE_BoomerAMGSetStrongThreshold(precon1, threshold);
HYPRE_BoomerAMGSetNumFunctions(precon1, 1);
HYPRE_BoomerAMGSetNumSweeps(precon1, nsweeps);
HYPRE_BoomerAMGSetRelaxType(precon1, relax_type);
hypre_BoomerAMGSetup(precon1, submatrices[0], b_data->U1, b_data->F1);
HYPRE_BoomerAMGCreate(&precon2);
HYPRE_BoomerAMGSetMaxIter(precon2, 1);
HYPRE_BoomerAMGSetCycleType(precon2, 1);
HYPRE_BoomerAMGSetPrintLevel(precon2, print_level);
HYPRE_BoomerAMGSetMaxLevels(precon2, 25);
HYPRE_BoomerAMGSetMeasureType(precon2, 0);
HYPRE_BoomerAMGSetCoarsenType(precon2, 0);
HYPRE_BoomerAMGSetMeasureType(precon2, 1);
HYPRE_BoomerAMGSetStrongThreshold(precon2, threshold);
HYPRE_BoomerAMGSetNumFunctions(precon2, 1);
HYPRE_BoomerAMGSetNumSweeps(precon2, nsweeps);
HYPRE_BoomerAMGSetRelaxType(precon2, relax_type);
hypre_BoomerAMGSetup(precon2, submatrices[3], NULL, NULL);
b_data->precon1 = precon1;
b_data->precon2 = precon2;
b_data->A11 = submatrices[0];
hypre_ParCSRMatrixDestroy(submatrices[1]);
b_data->A21 = submatrices[2];
b_data->A22 = submatrices[3];
hypre_TFree(submatrices);
return (0);
}
inline void numfact(unsigned int ncol, int* I, int* loc2glob, int* J, K* C) {
static_assert(std::is_same<double, K>::value, "Hypre only supports double-precision floating-point real numbers");
static_assert(S == 'G', "Hypre only supports nonsymmetric matrices");
HYPRE_IJMatrixCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], loc2glob[0], loc2glob[1], &_A);
HYPRE_IJMatrixSetObjectType(_A, HYPRE_PARCSR);
HYPRE_IJMatrixSetRowSizes(_A, I + 1);
_local = ncol;
int* rows = new int[3 * _local]();
int* diag_sizes = rows + _local;
int* offdiag_sizes = diag_sizes + _local;
rows[0] = I[0];
for(unsigned int i = 0; i < _local; ++i) {
std::for_each(J + rows[0], J + rows[0] + I[i + 1], [&](int& j) { (j < loc2glob[0] || loc2glob[1] < j) ? ++offdiag_sizes[i] : ++diag_sizes[i]; });
rows[0] += I[i + 1];
}
HYPRE_IJMatrixSetDiagOffdSizes(_A, diag_sizes, offdiag_sizes);
HYPRE_IJMatrixSetMaxOffProcElmts(_A, 0);
HYPRE_IJMatrixInitialize(_A);
std::iota(rows, rows + _local, loc2glob[0]);
HYPRE_IJMatrixSetValues(_A, _local, I + 1, rows, J, C);
HYPRE_IJMatrixAssemble(_A);
HYPRE_IJVectorCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], &_b);
HYPRE_IJVectorSetObjectType(_b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(_b);
HYPRE_IJVectorCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], &_x);
HYPRE_IJVectorSetObjectType(_x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(_x);
delete [] rows;
delete [] I;
delete [] loc2glob;
HYPRE_BoomerAMGCreate(_strategy == 1 ? &_solver : &_precond);
HYPRE_BoomerAMGSetCoarsenType(_strategy == 1 ? _solver : _precond, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(_strategy == 1 ? _solver : _precond, 6); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(_strategy == 1 ? _solver : _precond, 1); /* sweeps on each level */
HYPRE_BoomerAMGSetMaxLevels(_strategy == 1 ? _solver : _precond, 10); /* maximum number of levels */
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJMatrixGetObject(_A, reinterpret_cast<void**>(&parcsr_A));
HYPRE_ParVector par_b;
HYPRE_IJVectorGetObject(_b, reinterpret_cast<void**>(&par_b));
HYPRE_ParVector par_x;
HYPRE_IJVectorGetObject(_x, reinterpret_cast<void**>(&par_x));
if(_strategy == 1) {
HYPRE_BoomerAMGSetTol(_solver, 1.0e-8);
HYPRE_BoomerAMGSetMaxIter(_solver, 1000);
HYPRE_BoomerAMGSetPrintLevel(_solver, 1);
HYPRE_BoomerAMGSetup(_solver, parcsr_A, nullptr, nullptr);
}
else {
HYPRE_BoomerAMGSetTol(_precond, 0.0);
HYPRE_BoomerAMGSetMaxIter(_precond, 1);
HYPRE_BoomerAMGSetPrintLevel(_precond, 1);
if(_strategy == 2) {
HYPRE_ParCSRPCGCreate(DMatrix::_communicator, &_solver);
HYPRE_PCGSetMaxIter(_solver, 500);
HYPRE_PCGSetTol(_solver, 1.0e-8);
HYPRE_PCGSetTwoNorm(_solver, 1);
HYPRE_PCGSetPrintLevel(_solver, 1);
HYPRE_PCGSetLogging(_solver, 1);
HYPRE_PCGSetPrecond(_solver, reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSolve), reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSetup), _precond);
HYPRE_ParCSRPCGSetup(_solver, parcsr_A, par_b, par_x);
}
else {
HYPRE_ParCSRFlexGMRESCreate(DMatrix::_communicator, &_solver);
HYPRE_FlexGMRESSetKDim(_solver, 50);
HYPRE_FlexGMRESSetMaxIter(_solver, 500);
HYPRE_FlexGMRESSetTol(_solver, 1.0e-8);
HYPRE_FlexGMRESSetPrintLevel(_solver, 1);
HYPRE_FlexGMRESSetLogging(_solver, 1);
HYPRE_FlexGMRESSetPrecond(_solver, reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSolve), reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSetup), _precond);
HYPRE_ParCSRFlexGMRESSetup(_solver, parcsr_A, par_b, par_x);
}
}
}
int main (int argc, char *argv[])
{
HYPRE_Int i;
int myid, num_procs;
int N, n;
HYPRE_Int ilower, iupper;
HYPRE_Int local_size, extra;
int solver_id;
int print_solution, print_system;
double h, h2;
HYPRE_IJMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJVector b;
HYPRE_ParVector par_b;
HYPRE_IJVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver, precond;
/* Initialize MPI */
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
/* Default problem parameters */
n = 33;
solver_id = 0;
print_solution = 0;
print_system = 0;
/* Parse command line */
{
int arg_index = 0;
int print_usage = 0;
while (arg_index < argc)
{
if ( strcmp(argv[arg_index], "-n") == 0 )
{
arg_index++;
n = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-solver") == 0 )
{
arg_index++;
solver_id = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-print_solution") == 0 )
{
arg_index++;
print_solution = 1;
}
else if ( strcmp(argv[arg_index], "-print_system") == 0 )
{
arg_index++;
print_system = 1;
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
break;
}
else
{
arg_index++;
}
}
if ((print_usage) && (myid == 0))
{
printf("\n");
printf("Usage: %s [<options>]\n", argv[0]);
printf("\n");
printf(" -n <n> : problem size in each direction (default: 33)\n");
printf(" -solver <ID> : solver ID\n");
printf(" 0 - AMG (default) \n");
printf(" 1 - AMG-PCG\n");
printf(" 8 - ParaSails-PCG\n");
printf(" 50 - PCG\n");
printf(" 61 - AMG-FlexGMRES\n");
printf(" -print_solution : print the solution vector\n");
printf(" -print_system : print the matrix and rhs\n");
printf("\n");
}
if (print_usage)
{
MPI_Finalize();
return (0);
}
}
/* Preliminaries: want at least one processor per row */
if (n*n < num_procs) n = sqrt(num_procs) + 1;
N = n*n; /* global number of rows */
h = 1.0/(n+1); /* mesh size*/
h2 = h*h;
/* Each processor knows only of its own rows - the range is denoted by ilower
and upper. Here we partition the rows. We account for the fact that
N may not divide evenly by the number of processors. */
local_size = N/num_procs;
extra = N - local_size*num_procs;
ilower = local_size*myid;
ilower += hypre_min(myid, extra);
iupper = local_size*(myid+1);
iupper += hypre_min(myid+1, extra);
iupper = iupper - 1;
/* How many rows do I have? */
local_size = iupper - ilower + 1;
/* Create the matrix.
Note that this is a square matrix, so we indicate the row partition
size twice (since number of rows = number of cols) */
HYPRE_IJMatrixCreate(MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
/* Choose a parallel csr format storage (see the User's Manual) */
HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
/* Initialize before setting coefficients */
HYPRE_IJMatrixInitialize(A);
/* Now go through my local rows and set the matrix entries.
Each row has at most 5 entries. For example, if n=3:
A = [M -I 0; -I M -I; 0 -I M]
M = [4 -1 0; -1 4 -1; 0 -1 4]
Note that here we are setting one row at a time, though
one could set all the rows together (see the User's Manual).
*/
{
HYPRE_Int nnz;
double values[5];
HYPRE_Int cols[5];
for (i = ilower; i <= iupper; i++)
{
nnz = 0;
/* The left identity block:position i-n */
if ((i-n)>=0)
{
cols[nnz] = i-n;
values[nnz] = -1.0;
nnz++;
}
/* The left -1: position i-1 */
if (i%n)
{
cols[nnz] = i-1;
values[nnz] = -1.0;
nnz++;
}
/* Set the diagonal: position i */
cols[nnz] = i;
values[nnz] = 4.0;
nnz++;
/* The right -1: position i+1 */
if ((i+1)%n)
{
cols[nnz] = i+1;
values[nnz] = -1.0;
nnz++;
}
/* The right identity block:position i+n */
if ((i+n)< N)
{
cols[nnz] = i+n;
values[nnz] = -1.0;
nnz++;
}
/* Set the values for row i */
HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
}
}
/* Assemble after setting the coefficients */
HYPRE_IJMatrixAssemble(A);
/* Note: for the testing of small problems, one may wish to read
in a matrix in IJ format (for the format, see the output files
from the -print_system option).
In this case, one would use the following routine:
HYPRE_IJMatrixRead( <filename>, MPI_COMM_WORLD,
HYPRE_PARCSR, &A );
<filename> = IJ.A.out to read in what has been printed out
by -print_system (processor numbers are omitted).
A call to HYPRE_IJMatrixRead is an *alternative* to the
following sequence of HYPRE_IJMatrix calls:
Create, SetObjectType, Initialize, SetValues, and Assemble
*/
/* Get the parcsr matrix object to use */
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
/* Create the rhs and solution */
HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&b);
HYPRE_IJVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(b);
HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&x);
HYPRE_IJVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(x);
/* Set the rhs values to h^2 and the solution to zero */
{
double *rhs_values, *x_values;
HYPRE_Int *rows;
rhs_values = calloc(local_size, sizeof(double));
x_values = calloc(local_size, sizeof(double));
rows = calloc(local_size, sizeof(HYPRE_Int));
for (i=0; i<local_size; i++)
{
rhs_values[i] = h2;
x_values[i] = 0.0;
rows[i] = ilower + i;
}
HYPRE_IJVectorSetValues(b, local_size, rows, rhs_values);
HYPRE_IJVectorSetValues(x, local_size, rows, x_values);
free(x_values);
free(rhs_values);
free(rows);
}
HYPRE_IJVectorAssemble(b);
/* As with the matrix, for testing purposes, one may wish to read in a rhs:
HYPRE_IJVectorRead( <filename>, MPI_COMM_WORLD,
HYPRE_PARCSR, &b );
as an alternative to the
following sequence of HYPRE_IJVectors calls:
Create, SetObjectType, Initialize, SetValues, and Assemble
*/
HYPRE_IJVectorGetObject(b, (void **) &par_b);
HYPRE_IJVectorAssemble(x);
HYPRE_IJVectorGetObject(x, (void **) &par_x);
/* Print out the system - files names will be IJ.out.A.XXXXX
and IJ.out.b.XXXXX, where XXXXX = processor id */
if (print_system)
{
HYPRE_IJMatrixPrint(A, "IJ.out.A");
HYPRE_IJVectorPrint(b, "IJ.out.b");
}
/* Choose a solver and solve the system */
/* AMG */
if (solver_id == 0)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_BoomerAMGCreate(&solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_BoomerAMGSetPrintLevel(solver, 3); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(solver, 3); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(solver, 1); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(solver, 20); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(solver, 1e-7); /* conv. tolerance */
/* Now setup and solve! */
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver */
HYPRE_BoomerAMGDestroy(solver);
}
/* PCG */
else if (solver_id == 50)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* prints out the iteration info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver */
HYPRE_ParCSRPCGDestroy(solver);
}
/* PCG with AMG preconditioner */
else if (solver_id == 1)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the AMG preconditioner and specify any parameters */
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
/* Set the PCG preconditioner */
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver and preconditioner */
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
}
/* PCG with Parasails Preconditioner */
else if (solver_id == 8)
{
HYPRE_Int num_iterations;
double final_res_norm;
int sai_max_levels = 1;
double sai_threshold = 0.1;
double sai_filter = 0.05;
int sai_sym = 1;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the ParaSails preconditioner and specify any parameters */
HYPRE_ParaSailsCreate(MPI_COMM_WORLD, &precond);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_ParaSailsSetParams(precond, sai_threshold, sai_max_levels);
HYPRE_ParaSailsSetFilter(precond, sai_filter);
HYPRE_ParaSailsSetSym(precond, sai_sym);
HYPRE_ParaSailsSetLogging(precond, 3);
/* Set the PCG preconditioner */
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
(HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup, precond);
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destory solver and preconditioner */
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_ParaSailsDestroy(precond);
}
/* Flexible GMRES with AMG Preconditioner */
else if (solver_id == 61)
{
HYPRE_Int num_iterations;
double final_res_norm;
int restart = 30;
int modify = 1;
/* Create solver */
HYPRE_ParCSRFlexGMRESCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_FlexGMRESSetKDim(solver, restart);
HYPRE_FlexGMRESSetMaxIter(solver, 1000); /* max iterations */
HYPRE_FlexGMRESSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_FlexGMRESSetPrintLevel(solver, 2); /* print solve info */
HYPRE_FlexGMRESSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the AMG preconditioner and specify any parameters */
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
/* Set the FlexGMRES preconditioner */
HYPRE_FlexGMRESSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
if (modify)
/* this is an optional call - if you don't call it, hypre_FlexGMRESModifyPCDefault
is used - which does nothing. Otherwise, you can define your own, similar to
the one used here */
HYPRE_FlexGMRESSetModifyPC( solver,
(HYPRE_PtrToModifyPCFcn) hypre_FlexGMRESModifyPCAMGExample);
/* Now setup and solve! */
HYPRE_ParCSRFlexGMRESSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRFlexGMRESSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_FlexGMRESGetNumIterations(solver, &num_iterations);
HYPRE_FlexGMRESGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destory solver and preconditioner */
HYPRE_ParCSRFlexGMRESDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
}
else
{
if (myid ==0) printf("Invalid solver id specified.\n");
}
/* Print the solution */
if (print_solution)
HYPRE_IJVectorPrint(x, "ij.out.x");
/* Clean up */
HYPRE_IJMatrixDestroy(A);
HYPRE_IJVectorDestroy(b);
HYPRE_IJVectorDestroy(x);
/* Finalize MPI*/
MPI_Finalize();
return(0);
}
HYPRE_Int hypre_AMESetup(void *esolver)
{
HYPRE_Int ne, *edge_bc;
hypre_AMEData *ame_data = esolver;
hypre_AMSData *ams_data = ame_data -> precond;
if (ams_data -> beta_is_zero)
{
ame_data -> t1 = hypre_ParVectorInDomainOf(ams_data -> G);
ame_data -> t2 = hypre_ParVectorInDomainOf(ams_data -> G);
}
else
{
ame_data -> t1 = ams_data -> r1;
ame_data -> t2 = ams_data -> g1;
}
ame_data -> t3 = ams_data -> r0;
/* Eliminate boundary conditions in G = [Gii, Gib; 0, Gbb], i.e.,
compute [Gii, 0; 0, 0] */
{
HYPRE_Int i, j, k, nv;
HYPRE_Int *offd_edge_bc;
hypre_ParCSRMatrix *Gt;
nv = hypre_ParCSRMatrixNumCols(ams_data -> G);
ne = hypre_ParCSRMatrixNumRows(ams_data -> G);
edge_bc = hypre_TAlloc(HYPRE_Int, ne);
for (i = 0; i < ne; i++)
edge_bc[i] = 0;
/* Find boundary (eliminated) edges */
{
hypre_CSRMatrix *Ad = hypre_ParCSRMatrixDiag(ams_data -> A);
HYPRE_Int *AdI = hypre_CSRMatrixI(Ad);
HYPRE_Int *AdJ = hypre_CSRMatrixJ(Ad);
HYPRE_Real *AdA = hypre_CSRMatrixData(Ad);
hypre_CSRMatrix *Ao = hypre_ParCSRMatrixOffd(ams_data -> A);
HYPRE_Int *AoI = hypre_CSRMatrixI(Ao);
HYPRE_Real *AoA = hypre_CSRMatrixData(Ao);
HYPRE_Real l1_norm;
/* A row (edge) is boundary if its off-diag l1 norm is less than eps */
HYPRE_Real eps = DBL_EPSILON * 1e+4;
for (i = 0; i < ne; i++)
{
l1_norm = 0.0;
for (j = AdI[i]; j < AdI[i+1]; j++)
if (AdJ[j] != i)
l1_norm += fabs(AdA[j]);
if (AoI)
for (j = AoI[i]; j < AoI[i+1]; j++)
l1_norm += fabs(AoA[j]);
if (l1_norm < eps)
edge_bc[i] = 1;
}
}
hypre_ParCSRMatrixTranspose(ams_data -> G, &Gt, 1);
/* Use a Matvec communication to find which of the edges
connected to local vertices are on the boundary */
{
hypre_ParCSRCommHandle *comm_handle;
hypre_ParCSRCommPkg *comm_pkg;
HYPRE_Int num_sends, *int_buf_data;
HYPRE_Int index, start;
offd_edge_bc = hypre_CTAlloc(HYPRE_Int, hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(Gt)));
hypre_MatvecCommPkgCreate(Gt);
comm_pkg = hypre_ParCSRMatrixCommPkg(Gt);
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
int_buf_data = hypre_CTAlloc(HYPRE_Int,
hypre_ParCSRCommPkgSendMapStart(comm_pkg,
num_sends));
index = 0;
for (i = 0; i < num_sends; i++)
{
start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
{
k = hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j);
int_buf_data[index++] = edge_bc[k];
}
}
comm_handle = hypre_ParCSRCommHandleCreate(11, comm_pkg,
int_buf_data, offd_edge_bc);
hypre_ParCSRCommHandleDestroy(comm_handle);
hypre_TFree(int_buf_data);
}
/* Eliminate boundary vertex entries in G^t */
{
hypre_CSRMatrix *Gtd = hypre_ParCSRMatrixDiag(Gt);
HYPRE_Int *GtdI = hypre_CSRMatrixI(Gtd);
HYPRE_Int *GtdJ = hypre_CSRMatrixJ(Gtd);
HYPRE_Real *GtdA = hypre_CSRMatrixData(Gtd);
hypre_CSRMatrix *Gto = hypre_ParCSRMatrixOffd(Gt);
HYPRE_Int *GtoI = hypre_CSRMatrixI(Gto);
HYPRE_Int *GtoJ = hypre_CSRMatrixJ(Gto);
HYPRE_Real *GtoA = hypre_CSRMatrixData(Gto);
HYPRE_Int bdr;
for (i = 0; i < nv; i++)
{
bdr = 0;
/* A vertex is boundary if it belongs to a boundary edge */
for (j = GtdI[i]; j < GtdI[i+1]; j++)
if (edge_bc[GtdJ[j]]) { bdr = 1; break; }
if (!bdr && GtoI)
for (j = GtoI[i]; j < GtoI[i+1]; j++)
if (offd_edge_bc[GtoJ[j]]) { bdr = 1; break; }
if (bdr)
{
for (j = GtdI[i]; j < GtdI[i+1]; j++)
/* if (!edge_bc[GtdJ[j]]) */
GtdA[j] = 0.0;
if (GtoI)
for (j = GtoI[i]; j < GtoI[i+1]; j++)
/* if (!offd_edge_bc[GtoJ[j]]) */
GtoA[j] = 0.0;
}
}
}
hypre_ParCSRMatrixTranspose(Gt, &ame_data -> G, 1);
hypre_ParCSRMatrixDestroy(Gt);
hypre_TFree(offd_edge_bc);
}
/* Compute G^t M G */
{
if (!hypre_ParCSRMatrixCommPkg(ame_data -> G))
hypre_MatvecCommPkgCreate(ame_data -> G);
if (!hypre_ParCSRMatrixCommPkg(ame_data -> M))
hypre_MatvecCommPkgCreate(ame_data -> M);
hypre_BoomerAMGBuildCoarseOperator(ame_data -> G,
ame_data -> M,
ame_data -> G,
&ame_data -> A_G);
hypre_ParCSRMatrixFixZeroRows(ame_data -> A_G);
}
/* Create AMG preconditioner and PCG-AMG solver for G^tMG */
{
HYPRE_BoomerAMGCreate(&ame_data -> B1_G);
HYPRE_BoomerAMGSetCoarsenType(ame_data -> B1_G, ams_data -> B_G_coarsen_type);
HYPRE_BoomerAMGSetAggNumLevels(ame_data -> B1_G, ams_data -> B_G_agg_levels);
HYPRE_BoomerAMGSetRelaxType(ame_data -> B1_G, ams_data -> B_G_relax_type);
HYPRE_BoomerAMGSetNumSweeps(ame_data -> B1_G, 1);
HYPRE_BoomerAMGSetMaxLevels(ame_data -> B1_G, 25);
HYPRE_BoomerAMGSetTol(ame_data -> B1_G, 0.0);
HYPRE_BoomerAMGSetMaxIter(ame_data -> B1_G, 1);
HYPRE_BoomerAMGSetStrongThreshold(ame_data -> B1_G, ams_data -> B_G_theta);
/* don't use exact solve on the coarsest level (matrix may be singular) */
HYPRE_BoomerAMGSetCycleRelaxType(ame_data -> B1_G,
ams_data -> B_G_relax_type,
3);
HYPRE_ParCSRPCGCreate(hypre_ParCSRMatrixComm(ame_data->A_G),
&ame_data -> B2_G);
HYPRE_PCGSetPrintLevel(ame_data -> B2_G, 0);
HYPRE_PCGSetTol(ame_data -> B2_G, 1e-12);
HYPRE_PCGSetMaxIter(ame_data -> B2_G, 20);
HYPRE_PCGSetPrecond(ame_data -> B2_G,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
ame_data -> B1_G);
HYPRE_ParCSRPCGSetup(ame_data -> B2_G,
(HYPRE_ParCSRMatrix)ame_data->A_G,
(HYPRE_ParVector)ame_data->t1,
(HYPRE_ParVector)ame_data->t2);
}
/* Setup LOBPCG */
{
HYPRE_Int seed = 75;
mv_InterfaceInterpreter* interpreter;
mv_MultiVectorPtr eigenvectors;
ame_data -> interpreter = hypre_CTAlloc(mv_InterfaceInterpreter,1);
interpreter = (mv_InterfaceInterpreter*) ame_data -> interpreter;
HYPRE_ParCSRSetupInterpreter(interpreter);
ame_data -> eigenvalues = hypre_CTAlloc(HYPRE_Real, ame_data -> block_size);
ame_data -> eigenvectors =
mv_MultiVectorCreateFromSampleVector(interpreter,
ame_data -> block_size,
ame_data -> t3);
eigenvectors = (mv_MultiVectorPtr) ame_data -> eigenvectors;
mv_MultiVectorSetRandom (eigenvectors, seed);
/* Make the initial vectors discretely divergence free */
{
HYPRE_Int i, j;
HYPRE_Real *data;
mv_TempMultiVector* tmp = mv_MultiVectorGetData(eigenvectors);
HYPRE_ParVector *v = (HYPRE_ParVector*)(tmp -> vector);
hypre_ParVector *vi;
for (i = 0; i < ame_data -> block_size; i++)
{
vi = (hypre_ParVector*) v[i];
data = hypre_VectorData(hypre_ParVectorLocalVector(vi));
for (j = 0; j < ne; j++)
if (edge_bc[j])
data[j] = 0.0;
hypre_AMEDiscrDivFreeComponent(esolver, vi);
}
}
}
hypre_TFree(edge_bc);
return hypre_error_flag;
}
HYPRE_Int main (HYPRE_Int argc, char *argv[])
{
HYPRE_Int i;
HYPRE_Int myid, num_procs;
HYPRE_Int N, n;
HYPRE_Int ilower, iupper;
HYPRE_Int local_size, extra;
HYPRE_Int solver_id;
HYPRE_Int print_solution;
double h, h2;
#ifdef HYPRE_FORTRAN
hypre_F90_Obj A;
hypre_F90_Obj parcsr_A;
hypre_F90_Obj b;
hypre_F90_Obj par_b;
hypre_F90_Obj x;
hypre_F90_Obj par_x;
hypre_F90_Obj solver, precond;
hypre_F90_Obj long_temp_COMM;
HYPRE_Int temp_COMM;
HYPRE_Int precond_id;
HYPRE_Int one = 1;
HYPRE_Int two = 2;
HYPRE_Int three = 3;
HYPRE_Int six = 6;
HYPRE_Int twenty = 20;
HYPRE_Int thousand = 1000;
HYPRE_Int hypre_type = HYPRE_PARCSR;
double oo1 = 1.e-3;
double tol = 1.e-7;
#else
HYPRE_IJMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJVector b;
HYPRE_ParVector par_b;
HYPRE_IJVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver, precond;
#endif
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs);
/* Default problem parameters */
n = 33;
solver_id = 0;
print_solution = 0;
/* Parse command line */
{
HYPRE_Int arg_index = 0;
HYPRE_Int print_usage = 0;
while (arg_index < argc)
{
if ( strcmp(argv[arg_index], "-n") == 0 )
{
arg_index++;
n = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-solver") == 0 )
{
arg_index++;
solver_id = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-print_solution") == 0 )
{
arg_index++;
print_solution = 1;
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
break;
}
else
{
arg_index++;
}
}
if ((print_usage) && (myid == 0))
{
hypre_printf("\n");
hypre_printf("Usage: %s [<options>]\n", argv[0]);
hypre_printf("\n");
hypre_printf(" -n <n> : problem size in each direction (default: 33)\n");
hypre_printf(" -solver <ID> : solver ID\n");
hypre_printf(" 0 - AMG (default) \n");
hypre_printf(" 1 - AMG-PCG\n");
hypre_printf(" 8 - ParaSails-PCG\n");
hypre_printf(" 50 - PCG\n");
hypre_printf(" -print_solution : print the solution vector\n");
hypre_printf("\n");
}
if (print_usage)
{
hypre_MPI_Finalize();
return (0);
}
}
/* Preliminaries: want at least one processor per row */
if (n*n < num_procs) n = sqrt(num_procs) + 1;
N = n*n; /* global number of rows */
h = 1.0/(n+1); /* mesh size*/
h2 = h*h;
/* Each processor knows only of its own rows - the range is denoted by ilower
and upper. Here we partition the rows. We account for the fact that
N may not divide evenly by the number of processors. */
local_size = N/num_procs;
extra = N - local_size*num_procs;
ilower = local_size*myid;
ilower += hypre_min(myid, extra);
iupper = local_size*(myid+1);
iupper += hypre_min(myid+1, extra);
iupper = iupper - 1;
/* How many rows do I have? */
local_size = iupper - ilower + 1;
/* Create the matrix.
Note that this is a square matrix, so we indicate the row partition
size twice (since number of rows = number of cols) */
#ifdef HYPRE_FORTRAN
long_temp_COMM = (hypre_F90_Obj) hypre_MPI_COMM_WORLD;
temp_COMM = (HYPRE_Int) hypre_MPI_COMM_WORLD;
HYPRE_IJMatrixCreate(&long_temp_COMM, &ilower, &iupper, &ilower, &iupper, &A);
#else
HYPRE_IJMatrixCreate(hypre_MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
#endif
/* Choose a parallel csr format storage (see the User's Manual) */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixSetObjectType(&A, &hypre_type);
#else
HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
#endif
/* Initialize before setting coefficients */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixInitialize(&A);
#else
HYPRE_IJMatrixInitialize(A);
#endif
/* Now go through my local rows and set the matrix entries.
Each row has at most 5 entries. For example, if n=3:
A = [M -I 0; -I M -I; 0 -I M]
M = [4 -1 0; -1 4 -1; 0 -1 4]
Note that here we are setting one row at a time, though
one could set all the rows together (see the User's Manual).
*/
{
HYPRE_Int nnz;
double values[5];
HYPRE_Int cols[5];
for (i = ilower; i <= iupper; i++)
{
nnz = 0;
/* The left identity block:position i-n */
if ((i-n)>=0)
{
cols[nnz] = i-n;
values[nnz] = -1.0;
nnz++;
}
/* The left -1: position i-1 */
if (i%n)
{
cols[nnz] = i-1;
values[nnz] = -1.0;
nnz++;
}
/* Set the diagonal: position i */
cols[nnz] = i;
values[nnz] = 4.0;
nnz++;
/* The right -1: position i+1 */
if ((i+1)%n)
{
cols[nnz] = i+1;
values[nnz] = -1.0;
nnz++;
}
/* The right identity block:position i+n */
if ((i+n)< N)
{
cols[nnz] = i+n;
values[nnz] = -1.0;
nnz++;
}
/* Set the values for row i */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixSetValues(&A, &one, &nnz, &i, &cols[0], &values[0]);
#else
HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
#endif
}
}
/* Assemble after setting the coefficients */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixAssemble(&A);
#else
HYPRE_IJMatrixAssemble(A);
#endif
/* Get the parcsr matrix object to use */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixGetObject(&A, &parcsr_A);
HYPRE_IJMatrixGetObject(&A, &parcsr_A);
#else
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
#endif
/* Create the rhs and solution */
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorCreate(&temp_COMM, &ilower, &iupper, &b);
HYPRE_IJVectorSetObjectType(&b, &hypre_type);
HYPRE_IJVectorInitialize(&b);
#else
HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, ilower, iupper,&b);
HYPRE_IJVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(b);
#endif
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorCreate(&temp_COMM, &ilower, &iupper, &x);
HYPRE_IJVectorSetObjectType(&x, &hypre_type);
HYPRE_IJVectorInitialize(&x);
#else
HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, ilower, iupper,&x);
HYPRE_IJVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(x);
#endif
/* Set the rhs values to h^2 and the solution to zero */
{
double *rhs_values, *x_values;
HYPRE_Int *rows;
rhs_values = calloc(local_size, sizeof(double));
x_values = calloc(local_size, sizeof(double));
rows = calloc(local_size, sizeof(HYPRE_Int));
for (i=0; i<local_size; i++)
{
rhs_values[i] = h2;
x_values[i] = 0.0;
rows[i] = ilower + i;
}
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorSetValues(&b, &local_size, &rows[0], &rhs_values[0]);
HYPRE_IJVectorSetValues(&x, &local_size, &rows[0], &x_values[0]);
#else
HYPRE_IJVectorSetValues(b, local_size, rows, rhs_values);
HYPRE_IJVectorSetValues(x, local_size, rows, x_values);
#endif
free(x_values);
free(rhs_values);
free(rows);
}
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorAssemble(&b);
HYPRE_IJVectorGetObject(&b, &par_b);
#else
HYPRE_IJVectorAssemble(b);
HYPRE_IJVectorGetObject(b, (void **) &par_b);
#endif
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorAssemble(&x);
HYPRE_IJVectorGetObject(&x, &par_x);
#else
HYPRE_IJVectorAssemble(x);
HYPRE_IJVectorGetObject(x, (void **) &par_x);
#endif
/* Choose a solver and solve the system */
/* AMG */
if (solver_id == 0)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGCreate(&solver);
#else
HYPRE_BoomerAMGCreate(&solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGSetPrintLevel(&solver, &three); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(&solver, &six); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(&solver, &three); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(&solver, &one); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(&solver, &twenty); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(&solver, &tol); /* conv. tolerance */
#else
HYPRE_BoomerAMGSetPrintLevel(solver, 3); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(solver, 3); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(solver, 1); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(solver, 20); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(solver, 1e-7); /* conv. tolerance */
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_BoomerAMGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGGetNumIterations(&solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGDestroy(&solver);
#else
HYPRE_BoomerAMGDestroy(solver);
#endif
}
/* PCG */
else if (solver_id == 50)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* prints out the iteration info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* prints out the iteration info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
#else
HYPRE_ParCSRPCGDestroy(solver);
#endif
}
/* PCG with AMG preconditioner */
else if (solver_id == 1)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* print solve info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now set up the AMG preconditioner and specify any parameters */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(&precond, &one); /* print amg solution info*/
HYPRE_BoomerAMGSetCoarsenType(&precond, &six);
HYPRE_BoomerAMGSetRelaxType(&precond, &three);
HYPRE_BoomerAMGSetNumSweeps(&precond, &one);
HYPRE_BoomerAMGSetTol(&precond, &oo1);
#else
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info*/
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 3);
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 1e-3);
#endif
/* Set the PCG preconditioner */
#ifdef HYPRE_FORTRAN
precond_id = 2;
HYPRE_ParCSRPCGSetPrecond(&solver, &precond_id, &precond);
#else
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver and preconditioner */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
HYPRE_BoomerAMGDestroy(&precond);
#else
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
#endif
}
/* PCG with Parasails Preconditioner */
else if (solver_id == 8)
{
HYPRE_Int num_iterations;
double final_res_norm;
HYPRE_Int sai_max_levels = 1;
double sai_threshold = 0.1;
double sai_filter = 0.05;
HYPRE_Int sai_sym = 1;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* print solve info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now set up the ParaSails preconditioner and specify any parameters */
#ifdef HYPRE_FORTRAN
HYPRE_ParaSailsCreate(&temp_COMM, &precond);
#else
HYPRE_ParaSailsCreate(hypre_MPI_COMM_WORLD, &precond);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParaSailsSetParams(&precond, &sai_threshold, &sai_max_levels);
HYPRE_ParaSailsSetFilter(&precond, &sai_filter);
HYPRE_ParaSailsSetSym(&precond, &sai_sym);
HYPRE_ParaSailsSetLogging(&precond, &three);
#else
HYPRE_ParaSailsSetParams(precond, sai_threshold, sai_max_levels);
HYPRE_ParaSailsSetFilter(precond, sai_filter);
HYPRE_ParaSailsSetSym(precond, sai_sym);
HYPRE_ParaSailsSetLogging(precond, 3);
#endif
/* Set the PCG preconditioner */
#ifdef HYPRE_FORTRAN
precond_id = 4;
HYPRE_ParCSRPCGSetPrecond(&solver, &precond_id, &precond);
#else
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
(HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup, precond);
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destory solver and preconditioner */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
HYPRE_ParaSailsDestroy(&precond);
#else
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_ParaSailsDestroy(precond);
#endif
}
else
{
if (myid ==0) hypre_printf("Invalid solver id specified.\n");
}
/* Print the solution */
#ifdef HYPRE_FORTRAN
if (print_solution)
HYPRE_IJVectorPrint(&x, "ij.out.x");
#else
if (print_solution)
HYPRE_IJVectorPrint(x, "ij.out.x");
#endif
/* Clean up */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixDestroy(&A);
HYPRE_IJVectorDestroy(&b);
HYPRE_IJVectorDestroy(&x);
#else
HYPRE_IJMatrixDestroy(A);
HYPRE_IJVectorDestroy(b);
HYPRE_IJVectorDestroy(x);
#endif
/* Finalize MPI*/
hypre_MPI_Finalize();
return(0);
}
int main(int argc, char *argv[]) {
// printf("start\n");
int i, j, pi, pj;
int myid, num_procs;
int Nx, Ny, nx, ny;
//nx and ny are number of mesh cells
int solver_id;
int vis;
int ilower[2], iupper[2];
int nparts = 1;
int part = 0;
double hx, hy;
HYPRE_SStructGraph graph;
HYPRE_SStructGrid grid;
HYPRE_SStructStencil stencil;
HYPRE_SStructMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_SStructVector b;
HYPRE_ParVector par_b;
HYPRE_SStructVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver;
// printf("variables initialized\n");
//Initialize MPI
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
// printf("mpi initialized\n");
// Set default problem parameters
nx = 100;
ny = 100;
Nx = 1;
solver_id = 0;
vis = 0;
// printf("defaults set\n");
// Parse command line
int arg_index = 0;
while (arg_index < argc) {
if (strcmp(argv[arg_index], "-nx") == 0) {
arg_index++;
nx = atoi(argv[arg_index++]);
// printf("nx set\n");
} else if (strcmp(argv[arg_index], "-ny") == 0) {
arg_index++;
ny = atoi(argv[arg_index++]);
// printf("ny set\n");
} else if (strcmp(argv[arg_index], "-Nx") == 0) {
arg_index++;
Nx = atoi(argv[arg_index++]);
// printf("Nx set\n");
} else if (strcmp(argv[arg_index], "-solver") == 0) {
arg_index++;
solver_id = atoi(argv[arg_index++]);
// printf("solver set\n");
} else if (strcmp(argv[arg_index], "-vis") == 0) {
arg_index++;
vis = 1;
// printf("print solution\n");
} else {
arg_index++;
// printf("plus plus\n");
}
}
// printf("command line parsed\n");
// Set up processors in grid
Ny = num_procs / Nx;
hx = 1.0 / (Nx * nx);
hy = 1.0 / (Ny * ny);
if (Ny <= Nx) {
pj = myid / Nx;
pi = myid - pj * Ny;
} else {
pi = myid / Ny;
pj = myid - pi * Nx;
}
// printf("processors assigned\n");
// Determine each processor's piece of the grid
ilower[0] = pi * nx;
ilower[1] = pj * ny;
iupper[0] = ilower[0] + nx - 1;
iupper[1] = ilower[1] + ny - 1;
// printf("upper and lowers assigned\n");
HYPRE_SStructGridCreate(MPI_COMM_WORLD, 2, nparts, &grid);
// printf("grid created\n");
HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
// printf("grid set\n");
int nvars = 1;
HYPRE_SStructVariable vartypes[1] = { HYPRE_SSTRUCT_VARIABLE_CELL };
// printf("variable type set\n");
for (i = 0; i < nparts; i++) {
HYPRE_SStructGridSetVariables(grid, i, nvars, vartypes);
}
// printf("variable type set\n");
HYPRE_SStructGridAssemble(grid);
// printf("grid assembled\n");
// Set up the stencil
HYPRE_SStructStencilCreate(2, 5, &stencil);
int entry;
int offsets[5][2] = { { 0, 0 }, { -1, 0 }, { 1, 0 }, { 0, -1 }, { 0, 1 } };
int var = 0;
for (entry = 0; entry < 5; entry++) {
HYPRE_SStructStencilSetEntry(stencil, entry, offsets[entry], var);
}
// Create the graph
HYPRE_SStructGraphCreate(MPI_COMM_WORLD, grid, &graph);
int object_type = HYPRE_PARCSR;
HYPRE_SStructGraphSetObjectType(graph, object_type);
HYPRE_SStructGraphSetStencil(graph, part, var, stencil);
HYPRE_SStructGraphAssemble(graph);
int nentries = 5;
int nvalues = nentries * nx * ny;
double *values;
int stencil_indices[5];
// Setup the matrix
HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, graph, &A);
HYPRE_SStructMatrixSetObjectType(A, HYPRE_PARCSR);
HYPRE_SStructMatrixInitialize(A);
// Set the matrix coefficients
values = calloc(nvalues, sizeof(double));
for (j = 0; j < nentries; j++) {
stencil_indices[j] = j;
}
// Account for differences in hx and hy
double hx2 = hx * hx;
double hy2 = hy * hy;
double hx2inv = 1 / hx2;
double hy2inv = 1 / hy2;
for (i = 0; i < nvalues; i += nentries) {
values[i] = -2.0 * hx2inv - 2.0 * hy2inv;
values[i + 1] = hx2inv;
values[i + 2] = hx2inv;
values[i + 3] = hy2inv;
values[i + 4] = hy2inv;
}
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper, var, nentries,
stencil_indices, values);
// Set the dirchelet boundary condition
{
int bc_ilower[2];
int bc_iupper[2];
int nentries = 1;
int nvaluesx = nentries * nx; /* number of stencil entries times the length
of one side of my grid box */
int nvaluesy = nentries * ny;
double *valuesx, *valuesy, *center_valuesx, *center_valuesy;
int stencil_indices[1];
int center_index[1];
center_index[0] = 0;
valuesx = calloc(nvaluesx, sizeof(double));
center_valuesx = calloc(nvaluesx, sizeof(double));
for (j = 0; j < nvaluesx; j++){
valuesx[j] = 0.0;
center_valuesx[j] = -hx2inv;
}
valuesy = calloc(nvaluesy, sizeof(double));
center_valuesy = calloc(nvaluesy, sizeof(double));
for (j = 0; j < nvaluesy; j++){
valuesy[j] = 0.0;
center_valuesy[j] = -hy2inv;
}
// Recall: pi and pj describe position in the processor grid
if (pj == 0) {
// Bottom row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0] + nx - 1;
bc_iupper[1] = bc_ilower[1];
stencil_indices[0] = 3;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesy);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesy);
// printf("Bottom row zeroed\n");
}
if (pj == Ny - 1) {
// upper row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny + ny - 1;
bc_iupper[0] = bc_ilower[0] + ny - 1;
bc_iupper[1] = bc_ilower[1];
stencil_indices[0] = 4;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesy);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesy);
// printf("Upper row zeroed\n");
}
if (pi == 0) {
// Left row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0];
bc_iupper[1] = bc_ilower[1] + ny - 1;
stencil_indices[0] = 1;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesx);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesx);
// printf("left collumn zeroed\n");
}
if (pi == Nx - 1) {
// Right row of grid points
bc_ilower[0] = pi * nx + nx - 1;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0];
bc_iupper[1] = bc_ilower[1] + ny - 1;
stencil_indices[0] = 2;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesx);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesx);
// printf("right collumn zeroed\n");
}
free(valuesx);
free(center_valuesx);
free(valuesy);
free(center_valuesy);
}
HYPRE_SStructMatrixAssemble(A);
HYPRE_SStructMatrixGetObject(A, (void **)&parcsr_A);
// Create the right hand side and solution vectors
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &b);
HYPRE_SStructVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_SStructVectorInitialize(b);
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &x);
HYPRE_SStructVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_SStructVectorInitialize(x);
// Set the right hand side, exact solution, and initial guess values
double *rhs_values, *x_values, *exactsolution;
nvalues = nx * ny;
rhs_values = calloc(nvalues, sizeof(double));
x_values = calloc(nvalues, sizeof(double));
exactsolution = calloc(nvalues, sizeof(double));
// printf("RHS calculation begin\n");
for (j = 0; j < ny; j++) {
double y = (ilower[1] + j + .5) * hy;
for (i = 0; i < nx; i++) {
double x = (ilower[0] + i + .5) * hx;
rhs_values[i + j * nx] =
-8. * PI2 * sin(2. * M_PI * x) * sin(2. * M_PI * y);
exactsolution[i + j * nx] = 1.0 * sin(2. * M_PI * x) * sin(2. * M_PI * y);
x_values[i + j * nx] = 0.0;
}
}
// printf("RHS calculation ended\n");
HYPRE_SStructVectorSetBoxValues(b, part, ilower, iupper, var, rhs_values);
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, x_values);
//printf("%d %f\n", (ny*nx-1), exactsolution[ny*nx-1]);
free(x_values);
free(rhs_values);
HYPRE_SStructVectorAssemble(b);
// HYPRE_SStructVectorPrint("ss.initial.b", b, 0);
HYPRE_SStructVectorGetObject(b, (void **)&par_b);
HYPRE_SStructVectorAssemble(x);
HYPRE_SStructVectorGetObject(x, (void **)&par_x);
// Print out initial vectors
HYPRE_SStructMatrixPrint("SStructExact/ss.initial.A", A, 0);
HYPRE_SStructVectorPrint("SStructExact/ss.initial.b", b, 0);
// printf("About to solve\n");
// Select a solver
if (solver_id == 0) {
int num_iterations;
double final_res_norm;
// Create AMG solver
HYPRE_BoomerAMGCreate(&solver);
// Set solver parameters
HYPRE_BoomerAMGSetPrintLevel(solver, 3);
HYPRE_BoomerAMGSetCoarsenType(solver, 6);
HYPRE_BoomerAMGSetRelaxType(solver, 3);
HYPRE_BoomerAMGSetNumSweeps(solver, 1);
HYPRE_BoomerAMGSetMaxLevels(solver, 20);
HYPRE_BoomerAMGSetTol(solver, 1e-12);
HYPRE_BoomerAMGSetMaxIter(solver, 100);
// Set up and solve
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
//printf("solution obtained\n");
// Run information
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0) {
printf("\n");
printf("Iterations = %d\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
// Destroy solver
HYPRE_BoomerAMGDestroy(solver);
// HYPRE_ParVectorPrint(par_x, "SStructExact/ss.final.x");
char solutionfile[255];
sprintf(solutionfile, "%s.%06d", "ss.final.x", myid);
HYPRE_ParVectorPrint(par_x, solutionfile);
}
// Save the solution for visualization and L2 norm calculation
if (vis) {
FILE *file;
FILE *solution;
char filename[255];
char solutionfile[255];
sprintf(solutionfile, "%s.%06d.%d", "ss.final.x", myid, myid);
int nvalues = nx * ny;
double sum = 0;
double diff, diff2, L2;
// get the localally calculated and exact solutions)
double *values = calloc(nvalues+1, sizeof(double));
// Opens the local solution file, reads the solution to the array, values
// and closes the file.
// Opens the solution file
if ((solution = fopen(solutionfile, "rt")) == NULL) {
printf("Error: can't open output file %s\n", solutionfile);
MPI_Finalize();
exit(1);
}
// Reads the file to the array values
for (i = 0; i < nvalues + 1; i++) {
fscanf(solution, "%lf", &values[i]);
}
// Close the file
fflush(solution);
fclose(solution);
MPI_Barrier(MPI_COMM_WORLD);
// Calcluates the sum of the squared differences for the exact and numerical solutions on each processor
for (i = 1; i < nvalues + 1; i++) {
diff = values[i] - exactsolution[i-1];
diff2 = diff * diff;
if (diff2 > 10){
printf("%+6f %+6f %+6d\n", values[i], exactsolution[i-1],
i-1);
}
sum += diff2;
}
//printf("Myid is: %d and mysum is: %f\n", myid, sum);
// Ensure all processors are caught up
MPI_Barrier(MPI_COMM_WORLD);
// Passes the individual sums to processor 0 and sums them.
double *sendbuffer, *recvbuffer;
sendbuffer = calloc(1, sizeof(double));
sendbuffer[0] = sum;
recvbuffer = calloc(1, sizeof(double));
int root = 0;
int count = 1;
MPI_Reduce(sendbuffer, recvbuffer, count, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
// Calculates the L2 norm
if (myid == 0){
//printf("%f\n" , recvbuffer[myid]);
// nvalues = (nx-2)*(ny-2);
double totalvalues = nvalues * num_procs * 1.0;
L2 = sqrt(recvbuffer[0] / totalvalues);
printf("The L2 norm is %e\n", L2);
}
sprintf(filename, "%s.%06d", "vis/ss.sol", myid);
if ((file = fopen(filename, "w")) == NULL) {
printf("Error: can't open output file %s\n", filename);
MPI_Finalize();
exit(1);
}
// Save solution
for (i = 1; i < nvalues+1; i++) {
fprintf(file, "%.14e\n", values[i]);
}
// Vis Cleanup
fflush(file);
fclose(file);
free(values);
free(exactsolution);
free(sendbuffer);
free(recvbuffer);
// save global finite element mesh
if (myid == 0) {
GLVis_PrintGlobalMesh("vis/ss.mesh", Nx, Ny, nx, ny, hx, hy);
}
}
// Clean up
HYPRE_SStructMatrixDestroy(A);
HYPRE_SStructVectorDestroy(b);
HYPRE_SStructVectorDestroy(x);
// Finalize MPI*/
MPI_Finalize();
return (0);
}
