void PetscMatrix<T>::_get_submatrix(SparseMatrix<T>& submatrix,
const std::vector<numeric_index_type> &rows,
const std::vector<numeric_index_type> &cols,
const bool reuse_submatrix) const
{
// Can only extract submatrices from closed matrices
this->close();
// Make sure the SparseMatrix passed in is really a PetscMatrix
PetscMatrix<T>* petsc_submatrix = libmesh_cast_ptr<PetscMatrix<T>*>(&submatrix);
// If we're not reusing submatrix and submatrix is already initialized
// then we need to clear it, otherwise we get a memory leak.
if( !reuse_submatrix && submatrix.initialized() )
submatrix.clear();
// Construct row and column index sets.
PetscErrorCode ierr=0;
IS isrow, iscol;
ierr = ISCreateLibMesh(this->comm().get(),
rows.size(),
(PetscInt*) &rows[0],
PETSC_USE_POINTER,
&isrow); LIBMESH_CHKERRABORT(ierr);
ierr = ISCreateLibMesh(this->comm().get(),
cols.size(),
(PetscInt*) &cols[0],
PETSC_USE_POINTER,
&iscol); LIBMESH_CHKERRABORT(ierr);
// Extract submatrix
#if !PETSC_VERSION_LESS_THAN(3,0,1) || !PETSC_VERSION_RELEASE
ierr = MatGetSubMatrix(_mat,
isrow,
iscol,
(reuse_submatrix ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX),
&(petsc_submatrix->_mat)); LIBMESH_CHKERRABORT(ierr);
#else
ierr = MatGetSubMatrix(_mat,
isrow,
iscol,
PETSC_DECIDE,
(reuse_submatrix ? MAT_REUSE_MATRIX : MAT_INITIAL_MATRIX),
&(petsc_submatrix->_mat)); LIBMESH_CHKERRABORT(ierr);
#endif
// Specify that the new submatrix is initialized and close it.
petsc_submatrix->_is_initialized = true;
petsc_submatrix->close();
// Clean up PETSc data structures
ierr = LibMeshISDestroy(&isrow); LIBMESH_CHKERRABORT(ierr);
ierr = LibMeshISDestroy(&iscol); LIBMESH_CHKERRABORT(ierr);
}
void PetscVector<T>::localize (NumericVector<T>& v_local_in) const
{
this->_restore_array();
// Make sure the NumericVector passed in is really a PetscVector
PetscVector<T>* v_local = libmesh_cast_ptr<PetscVector<T>*>(&v_local_in);
libmesh_assert(v_local);
libmesh_assert_equal_to (v_local->size(), this->size());
PetscErrorCode ierr = 0;
const PetscInt n = this->size();
IS is;
VecScatter scatter;
// Create idx, idx[i] = i;
std::vector<PetscInt> idx(n); Utility::iota (idx.begin(), idx.end(), 0);
// Create the index set & scatter object
ierr = ISCreateLibMesh(libMesh::COMM_WORLD, n, &idx[0], PETSC_USE_POINTER, &is);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterCreate(_vec, is,
v_local->_vec, is,
&scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Perform the scatter
#if PETSC_VERSION_LESS_THAN(2,3,3)
ierr = VecScatterBegin(_vec, v_local->_vec, INSERT_VALUES,
SCATTER_FORWARD, scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd (_vec, v_local->_vec, INSERT_VALUES,
SCATTER_FORWARD, scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
// API argument order change in PETSc 2.3.3
ierr = VecScatterBegin(scatter, _vec, v_local->_vec,
INSERT_VALUES, SCATTER_FORWARD);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd (scatter, _vec, v_local->_vec,
INSERT_VALUES, SCATTER_FORWARD);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif
// Clean up
ierr = LibMeshISDestroy (&is);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = LibMeshVecScatterDestroy(&scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Make sure ghost dofs are up to date
if (v_local->type() == GHOSTED)
v_local->close();
}
void PetscVector<T>::localize (const numeric_index_type first_local_idx,
const numeric_index_type last_local_idx,
const std::vector<numeric_index_type>& send_list)
{
this->_restore_array();
libmesh_assert_less_equal (send_list.size(), this->size());
libmesh_assert_less_equal (last_local_idx+1, this->size());
const numeric_index_type size = this->size();
const numeric_index_type local_size = (last_local_idx - first_local_idx + 1);
PetscErrorCode ierr=0;
// Don't bother for serial cases
// if ((first_local_idx == 0) &&
// (local_size == size))
// But we do need to stay in sync for degenerate cases
if (libMesh::n_processors() == 1)
return;
// Build a parallel vector, initialize it with the local
// parts of (*this)
PetscVector<T> parallel_vec;
parallel_vec.init (size, local_size, true, PARALLEL);
// Copy part of *this into the parallel_vec
{
IS is;
VecScatter scatter;
// Create idx, idx[i] = i+first_local_idx;
std::vector<PetscInt> idx(local_size);
Utility::iota (idx.begin(), idx.end(), first_local_idx);
// Create the index set & scatter object
ierr = ISCreateLibMesh(libMesh::COMM_WORLD, local_size,
local_size ? &idx[0] : NULL, PETSC_USE_POINTER, &is);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterCreate(_vec, is,
parallel_vec._vec, is,
&scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Perform the scatter
#if PETSC_VERSION_LESS_THAN(2,3,3)
ierr = VecScatterBegin(_vec, parallel_vec._vec, INSERT_VALUES,
SCATTER_FORWARD, scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd (_vec, parallel_vec._vec, INSERT_VALUES,
SCATTER_FORWARD, scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
// API argument order change in PETSc 2.3.3
ierr = VecScatterBegin(scatter, _vec, parallel_vec._vec,
INSERT_VALUES, SCATTER_FORWARD);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd (scatter, _vec, parallel_vec._vec,
INSERT_VALUES, SCATTER_FORWARD);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif
// Clean up
ierr = LibMeshISDestroy (&is);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = LibMeshVecScatterDestroy(&scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
}
// localize like normal
parallel_vec.close();
parallel_vec.localize (*this, send_list);
this->close();
}
void PetscVector<T>::localize (NumericVector<T>& v_local_in,
const std::vector<numeric_index_type>& send_list) const
{
// FIXME: Workaround for a strange bug at large-scale.
// If we have ghosting, PETSc lets us just copy the solution, and
// doing so avoids a segfault?
if (v_local_in.type() == GHOSTED &&
this->type() == PARALLEL)
{
v_local_in = *this;
return;
}
// Normal code path begins here
this->_restore_array();
// Make sure the NumericVector passed in is really a PetscVector
PetscVector<T>* v_local = libmesh_cast_ptr<PetscVector<T>*>(&v_local_in);
libmesh_assert(v_local);
libmesh_assert_equal_to (v_local->size(), this->size());
libmesh_assert_less_equal (send_list.size(), v_local->size());
PetscErrorCode ierr=0;
const numeric_index_type n_sl = send_list.size();
IS is;
VecScatter scatter;
std::vector<PetscInt> idx(n_sl + this->local_size());
for (numeric_index_type i=0; i<n_sl; i++)
idx[i] = static_cast<PetscInt>(send_list[i]);
for (numeric_index_type i = 0; i != this->local_size(); ++i)
idx[n_sl+i] = i + this->first_local_index();
// Create the index set & scatter object
if (idx.empty())
ierr = ISCreateLibMesh(libMesh::COMM_WORLD,
n_sl+this->local_size(), PETSC_NULL, PETSC_USE_POINTER, &is);
else
ierr = ISCreateLibMesh(libMesh::COMM_WORLD,
n_sl+this->local_size(), &idx[0], PETSC_USE_POINTER, &is);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterCreate(_vec, is,
v_local->_vec, is,
&scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Perform the scatter
#if PETSC_VERSION_LESS_THAN(2,3,3)
ierr = VecScatterBegin(_vec, v_local->_vec, INSERT_VALUES,
SCATTER_FORWARD, scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd (_vec, v_local->_vec, INSERT_VALUES,
SCATTER_FORWARD, scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
// API argument order change in PETSc 2.3.3
ierr = VecScatterBegin(scatter, _vec, v_local->_vec,
INSERT_VALUES, SCATTER_FORWARD);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd (scatter, _vec, v_local->_vec,
INSERT_VALUES, SCATTER_FORWARD);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif
// Clean up
ierr = LibMeshISDestroy (&is);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = LibMeshVecScatterDestroy(&scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Make sure ghost dofs are up to date
if (v_local->type() == GHOSTED)
v_local->close();
}
void PetscVector<T>::create_subvector(NumericVector<T>& subvector,
const std::vector<numeric_index_type>& rows) const
{
this->_restore_array();
// PETSc data structures
IS parent_is, subvector_is;
VecScatter scatter;
PetscErrorCode ierr = 0;
// Make sure the passed in subvector is really a PetscVector
PetscVector<T>* petsc_subvector = libmesh_cast_ptr<PetscVector<T>*>(&subvector);
// If the petsc_subvector is already initialized, we assume that the
// user has already allocated the *correct* amount of space for it.
// If not, we use the appropriate PETSc routines to initialize it.
if (!petsc_subvector->initialized())
{
// Initialize the petsc_subvector to have enough space to hold
// the entries which will be scattered into it. Note: such an
// init() function (where we let PETSc decide the number of local
// entries) is not currently offered by the PetscVector
// class. Should we differentiate here between sequential and
// parallel vector creation based on libMesh::n_processors() ?
ierr = VecCreateMPI(libMesh::COMM_WORLD,
PETSC_DECIDE, // n_local
rows.size(), // n_global
&(petsc_subvector->_vec)); CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecSetFromOptions (petsc_subvector->_vec); CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Mark the subvector as initialized
petsc_subvector->_is_initialized = true;
}
else
{
petsc_subvector->_restore_array();
}
// Use iota to fill an array with entries [0,1,2,3,4,...rows.size()]
std::vector<PetscInt> idx(rows.size());
Utility::iota (idx.begin(), idx.end(), 0);
// Construct index sets
ierr = ISCreateLibMesh(libMesh::COMM_WORLD,
rows.size(),
(PetscInt*) &rows[0],
PETSC_USE_POINTER,
&parent_is); CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = ISCreateLibMesh(libMesh::COMM_WORLD,
rows.size(),
(PetscInt*) &idx[0],
PETSC_USE_POINTER,
&subvector_is); CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Construct the scatter object
ierr = VecScatterCreate(this->_vec,
parent_is,
petsc_subvector->_vec,
subvector_is,
&scatter); CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Actually perform the scatter
#if PETSC_VERSION_LESS_THAN(2,3,3)
ierr = VecScatterBegin(this->_vec,
petsc_subvector->_vec,
INSERT_VALUES,
SCATTER_FORWARD,
scatter); CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd(this->_vec,
petsc_subvector->_vec,
INSERT_VALUES,
SCATTER_FORWARD,
scatter); CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
// API argument order change in PETSc 2.3.3
ierr = VecScatterBegin(scatter,
this->_vec,
petsc_subvector->_vec,
INSERT_VALUES,
SCATTER_FORWARD); CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd(scatter,
this->_vec,
petsc_subvector->_vec,
INSERT_VALUES,
SCATTER_FORWARD); CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif
// Clean up
ierr = LibMeshISDestroy(&parent_is); CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = LibMeshISDestroy(&subvector_is); CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = LibMeshVecScatterDestroy(&scatter); CHKERRABORT(libMesh::COMM_WORLD,ierr);
}
