//=======================================================
int EpetraExt_HypreIJMatrix::SetParameter(Hypre_Chooser chooser, Hypre_Solver solver, bool transpose){
if(chooser == Solver){
if(transpose && solver != BoomerAMG){
EPETRA_CHK_ERR(-1);
}
switch(solver) {
case BoomerAMG:
if(IsSolverSetup_[0]){
SolverDestroyPtr_(Solver_);
IsSolverSetup_[0] = false;
}
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_BoomerAMGCreate;
SolverDestroyPtr_ = &HYPRE_BoomerAMGDestroy;
SolverSetupPtr_ = &HYPRE_BoomerAMGSetup;
SolverPrecondPtr_ = NULL;
if(transpose){
TransposeSolve_ = true;
SolverSolvePtr_ = &HYPRE_BoomerAMGSolveT;
} else {
SolverSolvePtr_ = &HYPRE_BoomerAMGSolve;
}
break;
case AMS:
if(IsSolverSetup_[0]){
SolverDestroyPtr_(Solver_);
IsSolverSetup_[0] = false;
}
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_AMSCreate;
SolverDestroyPtr_ = &HYPRE_AMSDestroy;
SolverSetupPtr_ = &HYPRE_AMSSetup;
SolverSolvePtr_ = &HYPRE_AMSSolve;
SolverPrecondPtr_ = NULL;
break;
case Hybrid:
if(IsSolverSetup_[0]){
SolverDestroyPtr_(Solver_);
IsSolverSetup_[0] = false;
}
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRHybridCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRHybridDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRHybridSetup;
SolverSolvePtr_ = &HYPRE_ParCSRHybridSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRHybridSetPrecond;
break;
case PCG:
if(IsSolverSetup_[0]){
SolverDestroyPtr_(Solver_);
IsSolverSetup_[0] = false;
}
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRPCGCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRPCGDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRPCGSetup;
SolverSolvePtr_ = &HYPRE_ParCSRPCGSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRPCGSetPrecond;
break;
case GMRES:
if(IsSolverSetup_[0]){
SolverDestroyPtr_(Solver_);
IsSolverSetup_[0] = false;
}
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRGMRESCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRGMRESDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRGMRESSetup;
SolverSolvePtr_ = &HYPRE_ParCSRGMRESSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRGMRESSetPrecond;
break;
case FlexGMRES:
if(IsSolverSetup_[0]){
SolverDestroyPtr_(Solver_);
IsSolverSetup_[0] = false;
}
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRFlexGMRESCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRFlexGMRESDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRFlexGMRESSetup;
SolverSolvePtr_ = &HYPRE_ParCSRFlexGMRESSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRFlexGMRESSetPrecond;
break;
case LGMRES:
if(IsSolverSetup_[0]){
SolverDestroyPtr_(Solver_);
IsSolverSetup_[0] = false;
}
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRLGMRESCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRLGMRESDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRLGMRESSetup;
SolverSolvePtr_ = &HYPRE_ParCSRLGMRESSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRLGMRESSetPrecond;
break;
case BiCGSTAB:
if(IsSolverSetup_[0]){
SolverDestroyPtr_(Solver_);
IsSolverSetup_[0] = false;
}
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRBiCGSTABCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRBiCGSTABDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRBiCGSTABSetup;
SolverSolvePtr_ = &HYPRE_ParCSRBiCGSTABSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRBiCGSTABSetPrecond;
break;
default:
EPETRA_CHK_ERR(-1);
}
CreateSolver();
} else {
// Preconditioner
switch(solver) {
case BoomerAMG:
if(IsPrecondSetup_[0]){
PrecondDestroyPtr_(Preconditioner_);
IsPrecondSetup_[0] = false;
}
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_BoomerAMGCreate;
PrecondDestroyPtr_ = &HYPRE_BoomerAMGDestroy;
PrecondSetupPtr_ = &HYPRE_BoomerAMGSetup;
PrecondSolvePtr_ = &HYPRE_BoomerAMGSolve;
break;
case ParaSails:
if(IsPrecondSetup_[0]){
PrecondDestroyPtr_(Preconditioner_);
IsPrecondSetup_[0] = false;
}
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParaSailsCreate;
PrecondDestroyPtr_ = &HYPRE_ParaSailsDestroy;
PrecondSetupPtr_ = &HYPRE_ParaSailsSetup;
PrecondSolvePtr_ = &HYPRE_ParaSailsSolve;
break;
case Euclid:
if(IsPrecondSetup_[0]){
PrecondDestroyPtr_(Preconditioner_);
IsPrecondSetup_[0] = false;
}
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_EuclidCreate;
PrecondDestroyPtr_ = &HYPRE_EuclidDestroy;
PrecondSetupPtr_ = &HYPRE_EuclidSetup;
PrecondSolvePtr_ = &HYPRE_EuclidSolve;
break;
case AMS:
if(IsPrecondSetup_[0]){
PrecondDestroyPtr_(Preconditioner_);
IsPrecondSetup_[0] = false;
}
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_AMSCreate;
PrecondDestroyPtr_ = &HYPRE_AMSDestroy;
PrecondSetupPtr_ = &HYPRE_AMSSetup;
PrecondSolvePtr_ = &HYPRE_AMSSolve;
break;
default:
EPETRA_CHK_ERR(-1);
}
CreatePrecond();
}
return 0;
} //SetParameter() - Choose solver or preconditioner type
//=======================================================
EpetraExt_HypreIJMatrix::EpetraExt_HypreIJMatrix(HYPRE_IJMatrix matrix)
: Epetra_BasicRowMatrix(Epetra_MpiComm(hypre_IJMatrixComm(matrix))),
Matrix_(matrix),
ParMatrix_(0),
NumMyRows_(-1),
NumGlobalRows_(-1),
NumGlobalCols_(-1),
MyRowStart_(-1),
MyRowEnd_(-1),
MatType_(-1),
TransposeSolve_(false),
SolveOrPrec_(Solver)
{
IsSolverSetup_ = new bool[1];
IsPrecondSetup_ = new bool[1];
IsSolverSetup_[0] = false;
IsPrecondSetup_[0] = false;
// Initialize default values for global variables
int ierr = 0;
ierr += InitializeDefaults();
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't initialize default values.");
// Create array of global row ids
Teuchos::Array<int> GlobalRowIDs; GlobalRowIDs.resize(NumMyRows_);
for (int i = MyRowStart_; i <= MyRowEnd_; i++) {
GlobalRowIDs[i-MyRowStart_] = i;
}
// Create array of global column ids
int new_value = 0; int entries = 0;
std::set<int> Columns;
int num_entries;
double *values;
int *indices;
for(int i = 0; i < NumMyRows_; i++){
ierr += HYPRE_ParCSRMatrixGetRow(ParMatrix_, i+MyRowStart_, &num_entries, &indices, &values);
ierr += HYPRE_ParCSRMatrixRestoreRow(ParMatrix_, i+MyRowStart_,&num_entries,&indices,&values);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get row of matrix.");
entries = num_entries;
for(int j = 0; j < num_entries; j++){
// Insert column ids from this row into set
new_value = indices[j];
Columns.insert(new_value);
}
}
int NumMyCols = Columns.size();
Teuchos::Array<int> GlobalColIDs; GlobalColIDs.resize(NumMyCols);
std::set<int>::iterator it;
int counter = 0;
for (it = Columns.begin(); it != Columns.end(); it++) {
// Get column ids in order
GlobalColIDs[counter] = *it;
counter = counter + 1;
}
//printf("Proc[%d] Rows from %d to %d, num = %d\n", Comm().MyPID(), MyRowStart_,MyRowEnd_, NumMyRows_);
Epetra_Map RowMap(-1, NumMyRows_, &GlobalRowIDs[0], 0, Comm());
Epetra_Map ColMap(-1, NumMyCols, &GlobalColIDs[0], 0, Comm());
//Need to call SetMaps()
SetMaps(RowMap, ColMap);
// Get an MPI_Comm to create vectors.
// The vectors will be reused in Multiply(), so that they aren't recreated every time.
MPI_Comm comm;
ierr += HYPRE_ParCSRMatrixGetComm(ParMatrix_, &comm);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get communicator from Hypre Matrix.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &X_hypre);
ierr += HYPRE_IJVectorSetObjectType(X_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(X_hypre);
ierr += HYPRE_IJVectorAssemble(X_hypre);
ierr += HYPRE_IJVectorGetObject(X_hypre, (void**) &par_x);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre X vector.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &Y_hypre);
ierr += HYPRE_IJVectorSetObjectType(Y_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(Y_hypre);
ierr += HYPRE_IJVectorAssemble(Y_hypre);
ierr += HYPRE_IJVectorGetObject(Y_hypre, (void**) &par_y);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre Y vector.");
x_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) X_hypre));
x_local = hypre_ParVectorLocalVector(x_vec);
y_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) Y_hypre));
y_local = hypre_ParVectorLocalVector(y_vec);
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRPCGCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRPCGDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRPCGSetup;
SolverSolvePtr_ = &HYPRE_ParCSRPCGSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRPCGSetPrecond;
CreateSolver();
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_EuclidCreate;
PrecondDestroyPtr_ = &HYPRE_EuclidDestroy;
PrecondSetupPtr_ = &HYPRE_EuclidSetup;
PrecondSolvePtr_ = &HYPRE_EuclidSolve;
CreatePrecond();
ComputeNumericConstants();
ComputeStructureConstants();
} //EpetraExt_HYPREIJMatrix(Hypre_IJMatrix) Constructor
void AbstractCardiacProblem<ELEMENT_DIM,SPACE_DIM,PROBLEM_DIM>::Solve()
{
PreSolveChecks();
std::vector<double> additional_stopping_times;
SetUpAdditionalStoppingTimes(additional_stopping_times);
TimeStepper stepper(mCurrentTime,
HeartConfig::Instance()->GetSimulationDuration(),
HeartConfig::Instance()->GetPrintingTimeStep(),
false,
additional_stopping_times);
// Note that SetUpAdditionalStoppingTimes is a method from the BidomainWithBath class it adds
// electrode events into the regular time-stepping
// EXCEPTION("Electrode switch on/off events should coincide with printing time steps.");
if (!mpBoundaryConditionsContainer) // the user didn't supply a bcc
{
// Set up the default bcc
mpDefaultBoundaryConditionsContainer.reset(new BoundaryConditionsContainer<ELEMENT_DIM, SPACE_DIM, PROBLEM_DIM>);
for (unsigned problem_index=0; problem_index<PROBLEM_DIM; problem_index++)
{
mpDefaultBoundaryConditionsContainer->DefineZeroNeumannOnMeshBoundary(mpMesh, problem_index);
}
mpBoundaryConditionsContainer = mpDefaultBoundaryConditionsContainer;
}
assert(mpSolver==NULL);
mpSolver = CreateSolver(); // passes mpBoundaryConditionsContainer to solver
// If we have already run a simulation, use the old solution as initial condition
Vec initial_condition;
if (mSolution)
{
initial_condition = mSolution;
}
else
{
initial_condition = CreateInitialCondition();
}
std::string progress_reporter_dir;
if (mPrintOutput)
{
HeartEventHandler::BeginEvent(HeartEventHandler::WRITE_OUTPUT);
bool extending_file = false;
try
{
extending_file = InitialiseWriter();
}
catch (Exception& e)
{
delete mpWriter;
mpWriter = NULL;
delete mpSolver;
if (mSolution != initial_condition)
{
/*
* A PETSc Vec is a pointer, so we *don't* need to free the memory if it is
* freed somewhere else (e.g. in the destructor). If this is a resumed solution
* we set initial_condition = mSolution earlier. mSolution is going to be
* cleaned up in the constructor. So, only PetscTools::Destroy( initial_condition ) when
* it is not equal to mSolution.
*/
PetscTools::Destroy(initial_condition);
}
throw e;
}
/*
* If we are resuming a simulation (i.e. mSolution already exists) and
* we are extending a .h5 file that already exists then there is no need
* to write the initial condition to file - it is already there as the
* final solution of the previous run.
*/
if (!(mSolution && extending_file))
{
WriteOneStep(stepper.GetTime(), initial_condition);
mpWriter->AdvanceAlongUnlimitedDimension();
}
HeartEventHandler::EndEvent(HeartEventHandler::WRITE_OUTPUT);
progress_reporter_dir = HeartConfig::Instance()->GetOutputDirectory();
}
else
{
progress_reporter_dir = ""; // progress printed to CHASTE_TEST_OUTPUT
}
BOOST_FOREACH(boost::shared_ptr<AbstractOutputModifier> p_output_modifier, mOutputModifiers)
{
p_output_modifier->InitialiseAtStart(this->mpMesh->GetDistributedVectorFactory());
p_output_modifier->ProcessSolutionAtTimeStep(stepper.GetTime(), initial_condition, PROBLEM_DIM);
}
