HYPRE_Int main( HYPRE_Int argc, char *argv[] )
{
HYPRE_Int arg_index;
HYPRE_Int print_usage;
HYPRE_Int build_matrix_arg_index;
HYPRE_Int solver_id;
HYPRE_Int ierr,i,j;
HYPRE_Int num_iterations;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_Int num_procs, myid;
HYPRE_Int local_row;
HYPRE_Int time_index;
MPI_Comm comm;
HYPRE_Int M, N;
HYPRE_Int first_local_row, last_local_row;
HYPRE_Int first_local_col, last_local_col;
HYPRE_Int size, *col_ind;
HYPRE_Real *values;
/* parameters for BoomerAMG */
HYPRE_Real strong_threshold;
HYPRE_Int num_grid_sweeps;
HYPRE_Real relax_weight;
/* parameters for GMRES */
HYPRE_Int k_dim;
char *paramString = new char[100];
/*-----------------------------------------------------------
* Initialize some stuff
*-----------------------------------------------------------*/
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
/*-----------------------------------------------------------
* Set defaults
*-----------------------------------------------------------*/
build_matrix_arg_index = argc;
solver_id = 0;
strong_threshold = 0.25;
num_grid_sweeps = 2;
relax_weight = 0.5;
k_dim = 20;
/*-----------------------------------------------------------
* Parse command line
*-----------------------------------------------------------*/
print_usage = 0;
arg_index = 1;
while ( (arg_index < argc) && (!print_usage) )
{
if ( strcmp(argv[arg_index], "-solver") == 0 )
{
arg_index++;
solver_id = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-dbg") == 0 )
{
arg_index++;
atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
}
else
{
arg_index++;
}
}
/*-----------------------------------------------------------
* Print usage info
*-----------------------------------------------------------*/
if ( (print_usage) && (myid == 0) )
{
hypre_printf("\n");
hypre_printf("Usage: %s [<options>]\n", argv[0]);
hypre_printf("\n");
hypre_printf(" -solver <ID> : solver ID\n");
hypre_printf(" 0=DS-PCG 1=ParaSails-PCG \n");
hypre_printf(" 2=AMG-PCG 3=DS-GMRES \n");
hypre_printf(" 4=PILUT-GMRES 5=AMG-GMRES \n");
hypre_printf("\n");
hypre_printf(" -rlx <val> : relaxation type\n");
hypre_printf(" 0=Weighted Jacobi \n");
hypre_printf(" 1=Gauss-Seidel (very slow!) \n");
hypre_printf(" 3=Hybrid Jacobi/Gauss-Seidel \n");
hypre_printf("\n");
exit(1);
}
/*-----------------------------------------------------------
* Print driver parameters
*-----------------------------------------------------------*/
if (myid == 0)
{
hypre_printf("Running with these driver parameters:\n");
hypre_printf(" solver ID = %d\n", solver_id);
}
/*-----------------------------------------------------------
* Set up matrix
*-----------------------------------------------------------*/
strcpy(paramString, "LS Interface");
time_index = hypre_InitializeTiming(paramString);
hypre_BeginTiming(time_index);
BuildParLaplacian27pt(argc, argv, build_matrix_arg_index, &parcsr_A);
/*-----------------------------------------------------------
* Copy the parcsr matrix into the LSI through interface calls
*-----------------------------------------------------------*/
ierr = HYPRE_ParCSRMatrixGetComm( parcsr_A, &comm );
ierr += HYPRE_ParCSRMatrixGetDims( parcsr_A, &M, &N );
ierr = HYPRE_ParCSRMatrixGetLocalRange( parcsr_A,
&first_local_row, &last_local_row ,
&first_local_col, &last_local_col );
HYPRE_LinSysCore H(hypre_MPI_COMM_WORLD);
HYPRE_Int numLocalEqns = last_local_row - first_local_row + 1;
H.createMatricesAndVectors(M,first_local_row+1,numLocalEqns);
HYPRE_Int index;
HYPRE_Int *rowLengths = new HYPRE_Int[numLocalEqns];
HYPRE_Int **colIndices = new HYPRE_Int*[numLocalEqns];
local_row = 0;
for (i=first_local_row; i<= last_local_row; i++)
{
ierr += HYPRE_ParCSRMatrixGetRow(parcsr_A,i,&size,&col_ind,&values );
rowLengths[local_row] = size;
colIndices[local_row] = new HYPRE_Int[size];
for (j=0; j<size; j++) colIndices[local_row][j] = col_ind[j] + 1;
local_row++;
HYPRE_ParCSRMatrixRestoreRow(parcsr_A,i,&size,&col_ind,&values);
}
H.allocateMatrix(colIndices, rowLengths);
delete [] rowLengths;
for (i=0; i< numLocalEqns; i++) delete [] colIndices[i];
delete [] colIndices;
HYPRE_Int *newColInd;
for (i=first_local_row; i<= last_local_row; i++)
{
ierr += HYPRE_ParCSRMatrixGetRow(parcsr_A,i,&size,&col_ind,&values );
newColInd = new HYPRE_Int[size];
for (j=0; j<size; j++) newColInd[j] = col_ind[j] + 1;
H.sumIntoSystemMatrix(i+1,size,(const HYPRE_Real*)values,
(const HYPRE_Int*)newColInd);
delete [] newColInd;
ierr += HYPRE_ParCSRMatrixRestoreRow(parcsr_A,i,&size,&col_ind,&values);
}
H.matrixLoadComplete();
HYPRE_ParCSRMatrixDestroy(parcsr_A);
/*-----------------------------------------------------------
* Set up the RHS and initial guess
*-----------------------------------------------------------*/
HYPRE_Real ddata=1.0;
HYPRE_Int status;
for (i=first_local_row; i<= last_local_row; i++)
{
index = i + 1;
H.sumIntoRHSVector(1,(const HYPRE_Real*) &ddata, (const HYPRE_Int*) &index);
}
hypre_EndTiming(time_index);
strcpy(paramString, "LS Interface");
hypre_PrintTiming(paramString, hypre_MPI_COMM_WORLD);
hypre_FinalizeTiming(time_index);
hypre_ClearTiming();
/*-----------------------------------------------------------
* Solve the system using PCG
*-----------------------------------------------------------*/
if ( solver_id == 0 )
{
strcpy(paramString, "solver cg");
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: DS-PCG\n");
strcpy(paramString, "preconditioner diagonal");
H.parameters(1, ¶mString);
}
else if ( solver_id == 1 )
{
strcpy(paramString, "solver cg");
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: ParaSails-PCG\n");
strcpy(paramString, "preconditioner parasails");
H.parameters(1, ¶mString);
strcpy(paramString, "parasailsNlevels 1");
H.parameters(1, ¶mString);
strcpy(paramString, "parasailsThreshold 0.1");
H.parameters(1, ¶mString);
}
else if ( solver_id == 2 )
{
strcpy(paramString, "solver cg");
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: AMG-PCG\n");
strcpy(paramString, "preconditioner boomeramg");
H.parameters(1, ¶mString);
strcpy(paramString, "amgCoarsenType falgout");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgStrongThreshold %e", strong_threshold);
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgNumSweeps %d", num_grid_sweeps);
H.parameters(1, ¶mString);
strcpy(paramString, "amgRelaxType jacobi");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgRelaxWeight %e", relax_weight);
H.parameters(1, ¶mString);
}
else if ( solver_id == 3 )
{
strcpy(paramString, "solver cg");
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: Poly-PCG\n");
strcpy(paramString, "preconditioner poly");
H.parameters(1, ¶mString);
strcpy(paramString, "polyOrder 9");
H.parameters(1, ¶mString);
}
else if ( solver_id == 4 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: DS-GMRES\n");
strcpy(paramString, "preconditioner diagonal");
H.parameters(1, ¶mString);
}
else if ( solver_id == 5 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: PILUT-GMRES\n");
strcpy(paramString, "preconditioner pilut");
H.parameters(1, ¶mString);
strcpy(paramString, "pilutRowSize 0");
H.parameters(1, ¶mString);
strcpy(paramString, "pilutDropTol 0.0");
H.parameters(1, ¶mString);
}
else if ( solver_id == 6 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: AMG-GMRES\n");
strcpy(paramString, "preconditioner boomeramg");
H.parameters(1, ¶mString);
strcpy(paramString, "amgCoarsenType falgout");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgStrongThreshold %e", strong_threshold);
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgNumSweeps %d", num_grid_sweeps);
H.parameters(1, ¶mString);
strcpy(paramString, "amgRelaxType jacobi");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgRelaxWeight %e", relax_weight);
H.parameters(1, ¶mString);
}
else if ( solver_id == 7 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: DDILUT-GMRES\n");
strcpy(paramString, "preconditioner ddilut");
H.parameters(1, ¶mString);
strcpy(paramString, "ddilutFillin 5.0");
H.parameters(1, ¶mString);
strcpy(paramString, "ddilutDropTol 0.0");
H.parameters(1, ¶mString);
}
else if ( solver_id == 8 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: POLY-GMRES\n");
strcpy(paramString, "preconditioner poly");
H.parameters(1, ¶mString);
strcpy(paramString, "polyOrder 5");
H.parameters(1, ¶mString);
}
strcpy(paramString, "Krylov Solve");
time_index = hypre_InitializeTiming(paramString);
hypre_BeginTiming(time_index);
H.launchSolver(status, num_iterations);
hypre_EndTiming(time_index);
strcpy(paramString, "Solve phase times");
hypre_PrintTiming(paramString, hypre_MPI_COMM_WORLD);
hypre_FinalizeTiming(time_index);
hypre_ClearTiming();
if (myid == 0)
{
hypre_printf("\n Iterations = %d\n", num_iterations);
hypre_printf("\n");
}
/*-----------------------------------------------------------
* Finalize things
*-----------------------------------------------------------*/
delete [] paramString;
hypre_MPI_Finalize();
return (0);
}
HYPRE_Int hypre_BlockTridiagSetup(void *data, hypre_ParCSRMatrix *A,
hypre_ParVector *b, hypre_ParVector *x)
{
HYPRE_Int i, j, *index_set1, print_level, nsweeps, relax_type;
HYPRE_Int nrows, nrows1, nrows2, start1, start2, *index_set2;
HYPRE_Int count, ierr;
double threshold;
hypre_ParCSRMatrix **submatrices;
HYPRE_Solver precon1;
HYPRE_Solver precon2;
HYPRE_IJVector ij_u1, ij_u2, ij_f1, ij_f2;
hypre_ParVector *vector;
MPI_Comm comm;
hypre_BlockTridiagData *b_data = (hypre_BlockTridiagData *) data;
HYPRE_ParCSRMatrixGetComm((HYPRE_ParCSRMatrix) A, &comm);
index_set1 = b_data->index_set1;
nrows1 = index_set1[0];
nrows = hypre_ParCSRMatrixNumRows(A);
nrows2 = nrows - nrows1;
b_data->index_set2 = hypre_CTAlloc(HYPRE_Int, nrows2+1);
index_set2 = b_data->index_set2;
index_set2[0] = nrows2;
count = 1;
for (i = 0; i < index_set1[1]; i++) index_set2[count++] = i;
for (i = 1; i < nrows1; i++)
for (j = index_set1[i]+1; j < index_set1[i+1]; j++)
index_set2[count++] = j;
for (i = index_set1[nrows1]+1; i < nrows; i++) index_set2[count++] = i;
submatrices = hypre_CTAlloc(hypre_ParCSRMatrix *, 4);
hypre_ParCSRMatrixExtractSubmatrices(A, index_set1, &submatrices);
nrows1 = hypre_ParCSRMatrixNumRows(submatrices[0]);
nrows2 = hypre_ParCSRMatrixNumRows(submatrices[3]);
start1 = hypre_ParCSRMatrixFirstRowIndex(submatrices[0]);
start2 = hypre_ParCSRMatrixFirstRowIndex(submatrices[3]);
HYPRE_IJVectorCreate(comm, start1, start1+nrows1-1, &ij_u1);
HYPRE_IJVectorSetObjectType(ij_u1, HYPRE_PARCSR);
ierr = HYPRE_IJVectorInitialize(ij_u1);
ierr += HYPRE_IJVectorAssemble(ij_u1);
hypre_assert(!ierr);
HYPRE_IJVectorCreate(comm, start1, start1+nrows1-1, &ij_f1);
HYPRE_IJVectorSetObjectType(ij_f1, HYPRE_PARCSR);
ierr = HYPRE_IJVectorInitialize(ij_f1);
ierr += HYPRE_IJVectorAssemble(ij_f1);
hypre_assert(!ierr);
HYPRE_IJVectorCreate(comm, start2, start2+nrows2-1, &ij_u2);
HYPRE_IJVectorSetObjectType(ij_u2, HYPRE_PARCSR);
ierr = HYPRE_IJVectorInitialize(ij_u2);
ierr += HYPRE_IJVectorAssemble(ij_u2);
hypre_assert(!ierr);
HYPRE_IJVectorCreate(comm, start2, start2+nrows1-1, &ij_f2);
HYPRE_IJVectorSetObjectType(ij_f2, HYPRE_PARCSR);
ierr = HYPRE_IJVectorInitialize(ij_f2);
ierr += HYPRE_IJVectorAssemble(ij_f2);
hypre_assert(!ierr);
HYPRE_IJVectorGetObject(ij_f1, (void **) &vector);
b_data->F1 = vector;
HYPRE_IJVectorGetObject(ij_u1, (void **) &vector);
b_data->U1 = vector;
HYPRE_IJVectorGetObject(ij_f2, (void **) &vector);
b_data->F2 = vector;
HYPRE_IJVectorGetObject(ij_u2, (void **) &vector);
b_data->U2 = vector;
print_level = b_data->print_level;
threshold = b_data->threshold;
nsweeps = b_data->num_sweeps;
relax_type = b_data->relax_type;
threshold = b_data->threshold;
HYPRE_BoomerAMGCreate(&precon1);
HYPRE_BoomerAMGSetMaxIter(precon1, 1);
HYPRE_BoomerAMGSetCycleType(precon1, 1);
HYPRE_BoomerAMGSetPrintLevel(precon1, print_level);
HYPRE_BoomerAMGSetMaxLevels(precon1, 25);
HYPRE_BoomerAMGSetMeasureType(precon1, 0);
HYPRE_BoomerAMGSetCoarsenType(precon1, 0);
HYPRE_BoomerAMGSetStrongThreshold(precon1, threshold);
HYPRE_BoomerAMGSetNumFunctions(precon1, 1);
HYPRE_BoomerAMGSetNumSweeps(precon1, nsweeps);
HYPRE_BoomerAMGSetRelaxType(precon1, relax_type);
hypre_BoomerAMGSetup(precon1, submatrices[0], b_data->U1, b_data->F1);
HYPRE_BoomerAMGCreate(&precon2);
HYPRE_BoomerAMGSetMaxIter(precon2, 1);
HYPRE_BoomerAMGSetCycleType(precon2, 1);
HYPRE_BoomerAMGSetPrintLevel(precon2, print_level);
HYPRE_BoomerAMGSetMaxLevels(precon2, 25);
HYPRE_BoomerAMGSetMeasureType(precon2, 0);
HYPRE_BoomerAMGSetCoarsenType(precon2, 0);
HYPRE_BoomerAMGSetMeasureType(precon2, 1);
HYPRE_BoomerAMGSetStrongThreshold(precon2, threshold);
HYPRE_BoomerAMGSetNumFunctions(precon2, 1);
HYPRE_BoomerAMGSetNumSweeps(precon2, nsweeps);
HYPRE_BoomerAMGSetRelaxType(precon2, relax_type);
hypre_BoomerAMGSetup(precon2, submatrices[3], NULL, NULL);
b_data->precon1 = precon1;
b_data->precon2 = precon2;
b_data->A11 = submatrices[0];
hypre_ParCSRMatrixDestroy(submatrices[1]);
b_data->A21 = submatrices[2];
b_data->A22 = submatrices[3];
hypre_TFree(submatrices);
return (0);
}
//=======================================================
int EpetraExt_HypreIJMatrix::CreateSolver(){
MPI_Comm comm;
HYPRE_ParCSRMatrixGetComm(ParMatrix_, &comm);
return (this->*SolverCreatePtr_)(comm, &Solver_);
} //CreateSolver()
//=======================================================
int EpetraExt_HypreIJMatrix::CreatePrecond(){
MPI_Comm comm;
HYPRE_ParCSRMatrixGetComm(ParMatrix_, &comm);
return (this->*PrecondCreatePtr_)(comm, &Preconditioner_);
} //CreatePrecond()
//=======================================================
EpetraExt_HypreIJMatrix::EpetraExt_HypreIJMatrix(HYPRE_IJMatrix matrix)
: Epetra_BasicRowMatrix(Epetra_MpiComm(hypre_IJMatrixComm(matrix))),
Matrix_(matrix),
ParMatrix_(0),
NumMyRows_(-1),
NumGlobalRows_(-1),
NumGlobalCols_(-1),
MyRowStart_(-1),
MyRowEnd_(-1),
MatType_(-1),
TransposeSolve_(false),
SolveOrPrec_(Solver)
{
IsSolverSetup_ = new bool[1];
IsPrecondSetup_ = new bool[1];
IsSolverSetup_[0] = false;
IsPrecondSetup_[0] = false;
// Initialize default values for global variables
int ierr = 0;
ierr += InitializeDefaults();
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't initialize default values.");
// Create array of global row ids
Teuchos::Array<int> GlobalRowIDs; GlobalRowIDs.resize(NumMyRows_);
for (int i = MyRowStart_; i <= MyRowEnd_; i++) {
GlobalRowIDs[i-MyRowStart_] = i;
}
// Create array of global column ids
int new_value = 0; int entries = 0;
std::set<int> Columns;
int num_entries;
double *values;
int *indices;
for(int i = 0; i < NumMyRows_; i++){
ierr += HYPRE_ParCSRMatrixGetRow(ParMatrix_, i+MyRowStart_, &num_entries, &indices, &values);
ierr += HYPRE_ParCSRMatrixRestoreRow(ParMatrix_, i+MyRowStart_,&num_entries,&indices,&values);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get row of matrix.");
entries = num_entries;
for(int j = 0; j < num_entries; j++){
// Insert column ids from this row into set
new_value = indices[j];
Columns.insert(new_value);
}
}
int NumMyCols = Columns.size();
Teuchos::Array<int> GlobalColIDs; GlobalColIDs.resize(NumMyCols);
std::set<int>::iterator it;
int counter = 0;
for (it = Columns.begin(); it != Columns.end(); it++) {
// Get column ids in order
GlobalColIDs[counter] = *it;
counter = counter + 1;
}
//printf("Proc[%d] Rows from %d to %d, num = %d\n", Comm().MyPID(), MyRowStart_,MyRowEnd_, NumMyRows_);
Epetra_Map RowMap(-1, NumMyRows_, &GlobalRowIDs[0], 0, Comm());
Epetra_Map ColMap(-1, NumMyCols, &GlobalColIDs[0], 0, Comm());
//Need to call SetMaps()
SetMaps(RowMap, ColMap);
// Get an MPI_Comm to create vectors.
// The vectors will be reused in Multiply(), so that they aren't recreated every time.
MPI_Comm comm;
ierr += HYPRE_ParCSRMatrixGetComm(ParMatrix_, &comm);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get communicator from Hypre Matrix.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &X_hypre);
ierr += HYPRE_IJVectorSetObjectType(X_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(X_hypre);
ierr += HYPRE_IJVectorAssemble(X_hypre);
ierr += HYPRE_IJVectorGetObject(X_hypre, (void**) &par_x);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre X vector.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &Y_hypre);
ierr += HYPRE_IJVectorSetObjectType(Y_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(Y_hypre);
ierr += HYPRE_IJVectorAssemble(Y_hypre);
ierr += HYPRE_IJVectorGetObject(Y_hypre, (void**) &par_y);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre Y vector.");
x_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) X_hypre));
x_local = hypre_ParVectorLocalVector(x_vec);
y_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) Y_hypre));
y_local = hypre_ParVectorLocalVector(y_vec);
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRPCGCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRPCGDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRPCGSetup;
SolverSolvePtr_ = &HYPRE_ParCSRPCGSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRPCGSetPrecond;
CreateSolver();
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_EuclidCreate;
PrecondDestroyPtr_ = &HYPRE_EuclidDestroy;
PrecondSetupPtr_ = &HYPRE_EuclidSetup;
PrecondSolvePtr_ = &HYPRE_EuclidSolve;
CreatePrecond();
ComputeNumericConstants();
ComputeStructureConstants();
} //EpetraExt_HYPREIJMatrix(Hypre_IJMatrix) Constructor