// -- FACTORY METHODS --
static Epetra_Map epetraMap( DataMap const& dmap )
{
std::vector<int> e( dmap.nMyElements() );
std::copy( dmap.myGlobalElements().begin(),
dmap.myGlobalElements().end(),
e.begin() );
return Epetra_Map( -1, dmap.nMyElements(), e.data(), 0, Epetra_MpiComm( dmap.comm() ) );
}
VectorEpetra<T>::VectorEpetra( )
:
super(),
#ifdef FEELPP_HAS_MPI
M_emap( Epetra_BlockMap( -1, 0, 0, Epetra_MpiComm( super::comm() ) ) ),
M_vec( M_emap ) // false (zerout)?
#else
M_emap( Epetra_BlockMap( -1, 0, 0, Epetra_SerialComm ) ),
M_vec( M_emap )
#endif
{
}
void EpetraMatrix<T>::init (const numeric_index_type m,
const numeric_index_type n,
const numeric_index_type m_l,
const numeric_index_type n_l,
const numeric_index_type nnz,
const numeric_index_type noz,
const numeric_index_type /* blocksize */)
{
if ((m==0) || (n==0))
return;
{
// Clear initialized matrices
if (this->initialized())
this->clear();
libmesh_assert (!this->_mat);
libmesh_assert (!this->_map);
this->_is_initialized = true;
}
// error checking
#ifndef NDEBUG
{
libmesh_assert_equal_to (n, m);
libmesh_assert_equal_to (n_l, m_l);
numeric_index_type
summed_m_l = m_l,
summed_n_l = n_l;
this->comm().sum (summed_m_l);
this->comm().sum (summed_n_l);
libmesh_assert_equal_to (m, summed_m_l);
libmesh_assert_equal_to (n, summed_n_l);
}
#endif
// build a map defining the data distribution
_map = new Epetra_Map (static_cast<int>(m),
m_l,
0,
Epetra_MpiComm (this->comm().get()));
libmesh_assert_equal_to (static_cast<numeric_index_type>(_map->NumGlobalPoints()), m);
libmesh_assert_equal_to (static_cast<numeric_index_type>(_map->MaxAllGID()+1), m);
_mat = new Epetra_FECrsMatrix (Copy, *_map, nnz + noz);
}
void EpetraMatrix<T>::init (const unsigned int m,
const unsigned int n,
const unsigned int m_l,
const unsigned int n_l,
const unsigned int nnz,
const unsigned int noz)
{
if ((m==0) || (n==0))
return;
{
// Clear initialized matrices
if (this->initialized())
this->clear();
libmesh_assert (this->_mat == NULL);
libmesh_assert (this->_map == NULL);
this->_is_initialized = true;
}
// error checking
#ifndef NDEBUG
{
libmesh_assert (n == m);
libmesh_assert (n_l == m_l);
unsigned int
summed_m_l = m_l,
summed_n_l = n_l;
Parallel::sum (summed_m_l);
Parallel::sum (summed_n_l);
libmesh_assert (m == summed_m_l);
libmesh_assert (n == summed_n_l);
}
#endif
// build a map defining the data distribution
_map = new Epetra_Map (m,
m_l,
0,
Epetra_MpiComm (libMesh::COMM_WORLD));
libmesh_assert (static_cast<unsigned int>(_map->NumGlobalPoints()) == m);
libmesh_assert (static_cast<unsigned int>(_map->MaxAllGID()+1) == m);
_mat = new Epetra_FECrsMatrix (Copy, *_map, nnz + noz);
}
static Epetra_Map epetraMapStatic( DataMap const& dmap )
{
return Epetra_Map( dmap.nGlobalElements(), dmap.nMyElements(), 0, Epetra_MpiComm( dmap.comm() ) );
}
void EpetraMatrix<T>::update_sparsity_pattern (const SparsityPattern::Graph &sparsity_pattern)
{
// clear data, start over
this->clear ();
// big trouble if this fails!
libmesh_assert (this->_dof_map != NULL);
const unsigned int n_rows = sparsity_pattern.size();
const unsigned int m = this->_dof_map->n_dofs();
const unsigned int n = m;
const unsigned int n_l = this->_dof_map->n_dofs_on_processor(libMesh::processor_id());
const unsigned int m_l = n_l;
// error checking
#ifndef NDEBUG
{
libmesh_assert (n == m);
libmesh_assert (n_l == m_l);
unsigned int
summed_m_l = m_l,
summed_n_l = n_l;
Parallel::sum (summed_m_l);
Parallel::sum (summed_n_l);
libmesh_assert (m == summed_m_l);
libmesh_assert (n == summed_n_l);
}
#endif
// build a map defining the data distribution
_map = new Epetra_Map (m,
m_l,
0,
Epetra_MpiComm (libMesh::COMM_WORLD));
libmesh_assert (static_cast<unsigned int>(_map->NumGlobalPoints()) == m);
libmesh_assert (static_cast<unsigned int>(_map->MaxAllGID()+1) == m);
const std::vector<unsigned int>& n_nz = this->_dof_map->get_n_nz();
const std::vector<unsigned int>& n_oz = this->_dof_map->get_n_oz();
// Make sure the sparsity pattern isn't empty
libmesh_assert (n_nz.size() == n_l);
libmesh_assert (n_oz.size() == n_l);
// Epetra wants the total number of nonzeros, both local and remote.
std::vector<int> n_nz_tot; /**/ n_nz_tot.reserve(n_nz.size());
for (unsigned int i=0; i<n_nz.size(); i++)
n_nz_tot.push_back(std::min(n_nz[i] + n_oz[i], n));
if (m==0)
return;
_graph = new Epetra_CrsGraph(Copy, *_map, &n_nz_tot[0]);
// Tell the matrix about its structure. Initialize it
// to zero.
for (unsigned int i=0; i<n_rows; i++)
_graph->InsertGlobalIndices(_graph->GRID(i),
sparsity_pattern[i].size(),
const_cast<int *>((const int *)&sparsity_pattern[i][0]));
_graph->FillComplete();
//Initialize the matrix
libmesh_assert (!this->initialized());
this->init ();
libmesh_assert (this->initialized());
}
//=======================================================
EpetraExt_HypreIJMatrix::EpetraExt_HypreIJMatrix(HYPRE_IJMatrix matrix)
: Epetra_BasicRowMatrix(Epetra_MpiComm(hypre_IJMatrixComm(matrix))),
Matrix_(matrix),
ParMatrix_(0),
NumMyRows_(-1),
NumGlobalRows_(-1),
NumGlobalCols_(-1),
MyRowStart_(-1),
MyRowEnd_(-1),
MatType_(-1),
TransposeSolve_(false),
SolveOrPrec_(Solver)
{
IsSolverSetup_ = new bool[1];
IsPrecondSetup_ = new bool[1];
IsSolverSetup_[0] = false;
IsPrecondSetup_[0] = false;
// Initialize default values for global variables
int ierr = 0;
ierr += InitializeDefaults();
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't initialize default values.");
// Create array of global row ids
Teuchos::Array<int> GlobalRowIDs; GlobalRowIDs.resize(NumMyRows_);
for (int i = MyRowStart_; i <= MyRowEnd_; i++) {
GlobalRowIDs[i-MyRowStart_] = i;
}
// Create array of global column ids
int new_value = 0; int entries = 0;
std::set<int> Columns;
int num_entries;
double *values;
int *indices;
for(int i = 0; i < NumMyRows_; i++){
ierr += HYPRE_ParCSRMatrixGetRow(ParMatrix_, i+MyRowStart_, &num_entries, &indices, &values);
ierr += HYPRE_ParCSRMatrixRestoreRow(ParMatrix_, i+MyRowStart_,&num_entries,&indices,&values);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get row of matrix.");
entries = num_entries;
for(int j = 0; j < num_entries; j++){
// Insert column ids from this row into set
new_value = indices[j];
Columns.insert(new_value);
}
}
int NumMyCols = Columns.size();
Teuchos::Array<int> GlobalColIDs; GlobalColIDs.resize(NumMyCols);
std::set<int>::iterator it;
int counter = 0;
for (it = Columns.begin(); it != Columns.end(); it++) {
// Get column ids in order
GlobalColIDs[counter] = *it;
counter = counter + 1;
}
//printf("Proc[%d] Rows from %d to %d, num = %d\n", Comm().MyPID(), MyRowStart_,MyRowEnd_, NumMyRows_);
Epetra_Map RowMap(-1, NumMyRows_, &GlobalRowIDs[0], 0, Comm());
Epetra_Map ColMap(-1, NumMyCols, &GlobalColIDs[0], 0, Comm());
//Need to call SetMaps()
SetMaps(RowMap, ColMap);
// Get an MPI_Comm to create vectors.
// The vectors will be reused in Multiply(), so that they aren't recreated every time.
MPI_Comm comm;
ierr += HYPRE_ParCSRMatrixGetComm(ParMatrix_, &comm);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get communicator from Hypre Matrix.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &X_hypre);
ierr += HYPRE_IJVectorSetObjectType(X_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(X_hypre);
ierr += HYPRE_IJVectorAssemble(X_hypre);
ierr += HYPRE_IJVectorGetObject(X_hypre, (void**) &par_x);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre X vector.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &Y_hypre);
ierr += HYPRE_IJVectorSetObjectType(Y_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(Y_hypre);
ierr += HYPRE_IJVectorAssemble(Y_hypre);
ierr += HYPRE_IJVectorGetObject(Y_hypre, (void**) &par_y);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre Y vector.");
x_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) X_hypre));
x_local = hypre_ParVectorLocalVector(x_vec);
y_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) Y_hypre));
y_local = hypre_ParVectorLocalVector(y_vec);
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRPCGCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRPCGDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRPCGSetup;
SolverSolvePtr_ = &HYPRE_ParCSRPCGSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRPCGSetPrecond;
CreateSolver();
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_EuclidCreate;
PrecondDestroyPtr_ = &HYPRE_EuclidDestroy;
PrecondSetupPtr_ = &HYPRE_EuclidSetup;
PrecondSolvePtr_ = &HYPRE_EuclidSolve;
CreatePrecond();
ComputeNumericConstants();
ComputeStructureConstants();
} //EpetraExt_HYPREIJMatrix(Hypre_IJMatrix) Constructor
void test ()
{
int n_proc;
#ifdef HAVE_MPI
MPI_Comm_size(MPI_COMM_WORLD, &n_proc);
#else
n_proc = 1;
#endif
int my_id;
#ifdef HAVE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &my_id);
#else
my_id = 0;
#endif
//All processes should own 10 entries
const int entries_per_process = 10;
const long long begin_index = ((long long)my_id)*entries_per_process;
const long long end_index = ((long long)(my_id+1))*entries_per_process;
const long long local_begin = std::max(0LL, begin_index-entries_per_process/2);
const long long local_end = entries_per_process*n_proc;
//create Epetra maps
std::vector<unsigned long long> ghosted_indices;
ghosted_indices.reserve(local_end-local_begin);
for (long long i = local_begin; i< local_end; ++i)
ghosted_indices.push_back(i);
Epetra_Map map_ghosted
(-1LL,
local_end-local_begin,
reinterpret_cast<long long*>(&ghosted_indices[0]),
0,
#ifdef HAVE_MPI
Epetra_MpiComm(MPI_COMM_WORLD));
#else
Epetra_SerialComm());
#endif
std::vector<unsigned long long> distributed_indices;
distributed_indices.reserve(entries_per_process*n_proc);
for (long long i = begin_index; i< end_index; ++i)
distributed_indices.push_back(i);
Epetra_Map map_distributed
(entries_per_process*n_proc,
entries_per_process,
reinterpret_cast<long long*>(&distributed_indices[0]),
0,
#ifdef HAVE_MPI
Epetra_MpiComm(MPI_COMM_WORLD));
#else
Epetra_SerialComm());
#endif
Epetra_FEVector v_ghosted(map_ghosted);
Epetra_FEVector v_distributed(map_distributed);
v_distributed.PutScalar(2.);
v_ghosted.PutScalar(1.);
Epetra_Import data_exchange (v_distributed.Map(), v_ghosted.Map());
int ierr = v_distributed.Import(v_ghosted, data_exchange, Epetra_AddLocalAlso);
std::cout << "Distributed:" << std::endl;
for (long long i=begin_index; i<end_index; ++i)
{
int trilinos_i
= v_distributed.Map().LID(i);
double value = v_distributed[0][trilinos_i];
std::cout<<"proc "<<my_id<<" "<< i << ": " << value << std::endl;
if(value != 3)
std::cerr << "tests FAILED: value = " << value << std::endl;
}
}
