inline void PetscVector::init (const int n,
const int n_local,
const std::vector<int>& ghost,
const bool fast,
const ParallelType type) {
int ierr=0;
PetscInt petsc_n=static_cast<int>(n);
PetscInt petsc_n_local=static_cast<int>(n_local);
PetscInt petsc_n_ghost=static_cast<int>(ghost.size());
// If the mesh is not disjoint, every processor will either have
// all the dofs, none of the dofs, or some non-zero dofs at the
// boundary between processors.
//
// However we can't assert this, because someone might want to
// construct a GHOSTED vector which doesn't include neighbor element
// dofs. Boyce tried to do so in user code, and we're going to want
// to do so in System::project_vector().
//
// libmesh_assert(n_local == 0 || n_local == n || !ghost.empty());
assert(sizeof(PetscInt) == sizeof(int));
// If the mesh is disjoint, the following assertion will fail.
// If the mesh is not disjoint, every processor will either have
// all the dofs, none of the dofs, or some non-zero dofs at the
// boundary between processors.
//assert(n_local == 0 || n_local == n || !ghost.empty());
PetscInt* petsc_ghost = ghost.empty() ? PETSC_NULL :
const_cast<int*>(reinterpret_cast<const PetscInt*>(&ghost[0]));
// Clear initialized vectors
if (this->initialized()) this->clear();
assert(type == AUTOMATIC || type == GHOSTED);
this->_type = GHOSTED;
/* Make the global-to-local ghost cell map. */
for (int i=0; i<(int)ghost.size(); i++){
_global_to_local_map[ghost[i]] = i;
}
/* Create vector. */
ierr = VecCreateGhost (MPI_COMM_WORLD, petsc_n_local, petsc_n,
petsc_n_ghost, petsc_ghost, &_vec);
CHKERRABORT(MPI_COMM_WORLD,ierr);
ierr = VecSetFromOptions (_vec);
CHKERRABORT(MPI_COMM_WORLD,ierr);
this->_is_initialized = true;
this->_is_closed = true;
if (fast == false)
this->zero ();
}
LData::LData(const std::string& name,
const unsigned int num_local_nodes,
const unsigned int depth,
const std::vector<int>& nonlocal_petsc_indices)
: d_name(name), d_global_node_count(0), d_local_node_count(0), d_ghost_node_count(0), d_depth(depth),
d_nonlocal_petsc_indices(nonlocal_petsc_indices), d_global_vec(NULL), d_managing_petsc_vec(true), d_array(NULL),
d_boost_array(NULL), d_boost_local_array(NULL), d_boost_vec_array(NULL), d_boost_local_vec_array(NULL),
d_ghosted_local_vec(NULL), d_ghosted_local_array(NULL), d_boost_ghosted_local_array(NULL),
d_boost_vec_ghosted_local_array(NULL)
{
// Create the PETSc Vec that provides storage for the Lagrangian data.
int ierr;
if (d_depth == 1)
{
ierr = VecCreateGhost(PETSC_COMM_WORLD,
num_local_nodes,
PETSC_DECIDE,
static_cast<int>(d_nonlocal_petsc_indices.size()),
d_nonlocal_petsc_indices.empty() ? NULL : &d_nonlocal_petsc_indices[0],
&d_global_vec);
IBTK_CHKERRQ(ierr);
}
else
{
ierr = VecCreateGhostBlock(PETSC_COMM_WORLD,
d_depth,
d_depth * num_local_nodes,
PETSC_DECIDE,
static_cast<int>(d_nonlocal_petsc_indices.size()),
d_nonlocal_petsc_indices.empty() ? NULL : &d_nonlocal_petsc_indices[0],
&d_global_vec);
IBTK_CHKERRQ(ierr);
}
int global_node_count;
ierr = VecGetSize(d_global_vec, &global_node_count);
IBTK_CHKERRQ(ierr);
#if !defined(NDEBUG)
TBOX_ASSERT(global_node_count >= 0);
#endif
d_global_node_count = global_node_count;
d_global_node_count /= d_depth;
d_local_node_count = num_local_nodes;
d_ghost_node_count = static_cast<int>(d_nonlocal_petsc_indices.size());
return;
} // LData
Vec CreateGhostedVector(Grid& g) {
auto dist = boost::any_cast<const Decomposition&>(g.userData());
int n;
ISLocalToGlobalMappingGetSize(dist.locToGlobMap, &n);
const PetscInt *idx;
ISLocalToGlobalMappingGetIndices(dist.locToGlobMap, &idx);
Vec v;
PetscInt nLoc = dist.cellEnd - dist.cellStart;
std::cout << n-nLoc << std::endl;
VecCreateGhost(PETSC_COMM_WORLD, g.cells().size(), PETSC_DECIDE,
n - nLoc, idx + nLoc, &v);
ISLocalToGlobalMappingRestoreIndices(dist.locToGlobMap, &idx);
return v;
}
PETScVector::PETScVector(const PetscInt vec_size,
const std::vector<PetscInt>& ghost_ids,
const bool is_global_size)
: _size_ghosts{static_cast<PetscInt>(ghost_ids.size())}
, _has_ghost_id{true}
{
PetscInt nghosts = static_cast<PetscInt>( ghost_ids.size() );
if ( is_global_size )
{
VecCreateGhost(PETSC_COMM_WORLD, PETSC_DECIDE, vec_size, nghosts,
ghost_ids.data(), &_v);
}
else
{
VecCreate(PETSC_COMM_WORLD, &_v);
VecSetType(_v, VECMPI);
VecSetSizes(_v, vec_size, PETSC_DECIDE);
VecMPISetGhost(_v, nghosts, ghost_ids.data());
}
config();
}
PETScVector::PETScVector(const PetscInt vec_size,
const std::vector<PetscInt>& ghost_ids,
const bool is_global_size) :
_size_ghosts(ghost_ids.size()), _has_ghost_id(true)
{
_v.reset(new PETSc_Vec);
PetscInt nghosts = static_cast<PetscInt>( ghost_ids.size() );
if ( is_global_size )
{
VecCreateGhost(PETSC_COMM_WORLD, PETSC_DECIDE, vec_size, nghosts,
ghost_ids.data(), _v.get());
}
else
{
VecCreate(PETSC_COMM_WORLD, _v.get());
VecSetType(*_v, VECMPI);
VecSetSizes(*_v, vec_size, PETSC_DECIDE);
VecMPISetGhost(*_v, nghosts, ghost_ids.data());
}
config();
}
int main(int argc,char **argv)
{
PetscMPIInt rank,size;
PetscInt nlocal = 6,nghost = 2,ifrom[2],i,rstart,rend;
PetscErrorCode ierr;
PetscBool flg,flg2;
PetscScalar value,*array,*tarray=0;
Vec lx,gx,gxs;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 2) SETERRQ(PETSC_COMM_SELF,1,"Must run example with two processors\n");
/*
Construct a two dimensional graph connecting nlocal degrees of
freedom per processor. From this we will generate the global
indices of needed ghost values
For simplicity we generate the entire graph on each processor:
in real application the graph would stored in parallel, but this
example is only to demonstrate the management of ghost padding
with VecCreateGhost().
In this example we consider the vector as representing
degrees of freedom in a one dimensional grid with periodic
boundary conditions.
----Processor 1--------- ----Processor 2 --------
0 1 2 3 4 5 6 7 8 9 10 11
|----|
|-------------------------------------------------|
*/
if (!rank) {
ifrom[0] = 11; ifrom[1] = 6;
} else {
ifrom[0] = 0; ifrom[1] = 5;
}
/*
Create the vector with two slots for ghost points. Note that both
the local vector (lx) and the global vector (gx) share the same
array for storing vector values.
*/
ierr = PetscOptionsHasName(PETSC_NULL,"-allocate",&flg);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-vecmpisetghost",&flg2);CHKERRQ(ierr);
if (flg) {
ierr = PetscMalloc((nlocal+nghost)*sizeof(PetscScalar),&tarray);CHKERRQ(ierr);
ierr = VecCreateGhostWithArray(PETSC_COMM_WORLD,nlocal,PETSC_DECIDE,nghost,ifrom,tarray,&gxs);CHKERRQ(ierr);
} else if (flg2) {
ierr = VecCreate(PETSC_COMM_WORLD,&gxs);CHKERRQ(ierr);
ierr = VecSetType(gxs,VECMPI);CHKERRQ(ierr);
ierr = VecSetSizes(gxs,nlocal,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecMPISetGhost(gxs,nghost,ifrom);CHKERRQ(ierr);
} else {
ierr = VecCreateGhost(PETSC_COMM_WORLD,nlocal,PETSC_DECIDE,nghost,ifrom,&gxs);CHKERRQ(ierr);
}
/*
Test VecDuplicate()
*/
ierr = VecDuplicate(gxs,&gx);CHKERRQ(ierr);
ierr = VecDestroy(&gxs);CHKERRQ(ierr);
/*
Access the local representation
*/
ierr = VecGhostGetLocalForm(gx,&lx);CHKERRQ(ierr);
/*
Set the values from 0 to 12 into the "global" vector
*/
ierr = VecGetOwnershipRange(gx,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
value = (PetscScalar) i;
ierr = VecSetValues(gx,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(gx);CHKERRQ(ierr);
ierr = VecAssemblyEnd(gx);CHKERRQ(ierr);
ierr = VecGhostUpdateBegin(gx,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGhostUpdateEnd(gx,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/*
Print out each vector, including the ghost padding region.
*/
ierr = VecGetArray(lx,&array);CHKERRQ(ierr);
for (i=0; i<nlocal+nghost; i++) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"%D %G\n",i,PetscRealPart(array[i]));CHKERRQ(ierr);
}
ierr = VecRestoreArray(lx,&array);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecGhostRestoreLocalForm(gx,&lx);CHKERRQ(ierr);
ierr = VecDestroy(&gx);CHKERRQ(ierr);
if (flg) {ierr = PetscFree(tarray);CHKERRQ(ierr);}
ierr = PetscFinalize();
return 0;
}
PETSC_EXTERN void PETSC_STDCALL veccreateghost_(MPI_Fint * comm,PetscInt *n,PetscInt *N,PetscInt *nghost, PetscInt ghosts[],Vec *vv, int *__ierr ){
*__ierr = VecCreateGhost(
MPI_Comm_f2c( *(comm) ),*n,*N,*nghost,ghosts,vv);
}
PetscErrorCode MatFDColoringSetUp_MPIXAIJ(Mat mat,ISColoring iscoloring,MatFDColoring c)
{
PetscErrorCode ierr;
PetscMPIInt size,*ncolsonproc,*disp,nn;
PetscInt i,n,nrows,nrows_i,j,k,m,ncols,col,*rowhit,cstart,cend,colb;
const PetscInt *is,*A_ci,*A_cj,*B_ci,*B_cj,*row=NULL,*ltog=NULL;
PetscInt nis=iscoloring->n,nctot,*cols;
IS *isa;
ISLocalToGlobalMapping map=mat->cmap->mapping;
PetscInt ctype=c->ctype,*spidxA,*spidxB,nz,bs,bs2,spidx;
Mat A,B;
PetscScalar *A_val,*B_val,**valaddrhit;
MatEntry *Jentry;
MatEntry2 *Jentry2;
PetscBool isBAIJ;
PetscInt bcols=c->bcols;
#if defined(PETSC_USE_CTABLE)
PetscTable colmap=NULL;
#else
PetscInt *colmap=NULL; /* local col number of off-diag col */
#endif
PetscFunctionBegin;
if (ctype == IS_COLORING_GHOSTED) {
if (!map) SETERRQ(PetscObjectComm((PetscObject)mat),PETSC_ERR_ARG_INCOMP,"When using ghosted differencing matrix must have local to global mapping provided with MatSetLocalToGlobalMapping");
ierr = ISLocalToGlobalMappingGetIndices(map,<og);CHKERRQ(ierr);
}
ierr = MatGetBlockSize(mat,&bs);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIBAIJ,&isBAIJ);CHKERRQ(ierr);
if (isBAIJ) {
Mat_MPIBAIJ *baij=(Mat_MPIBAIJ*)mat->data;
Mat_SeqBAIJ *spA,*spB;
A = baij->A; spA = (Mat_SeqBAIJ*)A->data; A_val = spA->a;
B = baij->B; spB = (Mat_SeqBAIJ*)B->data; B_val = spB->a;
nz = spA->nz + spB->nz; /* total nonzero entries of mat */
if (!baij->colmap) {
ierr = MatCreateColmap_MPIBAIJ_Private(mat);CHKERRQ(ierr);
colmap = baij->colmap;
}
ierr = MatGetColumnIJ_SeqBAIJ_Color(A,0,PETSC_FALSE,PETSC_FALSE,&ncols,&A_ci,&A_cj,&spidxA,NULL);CHKERRQ(ierr);
ierr = MatGetColumnIJ_SeqBAIJ_Color(B,0,PETSC_FALSE,PETSC_FALSE,&ncols,&B_ci,&B_cj,&spidxB,NULL);CHKERRQ(ierr);
if (ctype == IS_COLORING_GLOBAL && c->htype[0] == 'd') { /* create vscale for storing dx */
PetscInt *garray;
ierr = PetscMalloc1(B->cmap->n,&garray);CHKERRQ(ierr);
for (i=0; i<baij->B->cmap->n/bs; i++) {
for (j=0; j<bs; j++) {
garray[i*bs+j] = bs*baij->garray[i]+j;
}
}
ierr = VecCreateGhost(PetscObjectComm((PetscObject)mat),mat->cmap->n,PETSC_DETERMINE,B->cmap->n,garray,&c->vscale);CHKERRQ(ierr);
ierr = PetscFree(garray);CHKERRQ(ierr);
}
} else {
Mat_MPIAIJ *aij=(Mat_MPIAIJ*)mat->data;
Mat_SeqAIJ *spA,*spB;
A = aij->A; spA = (Mat_SeqAIJ*)A->data; A_val = spA->a;
B = aij->B; spB = (Mat_SeqAIJ*)B->data; B_val = spB->a;
nz = spA->nz + spB->nz; /* total nonzero entries of mat */
if (!aij->colmap) {
/* Allow access to data structures of local part of matrix
- creates aij->colmap which maps global column number to local number in part B */
ierr = MatCreateColmap_MPIAIJ_Private(mat);CHKERRQ(ierr);
colmap = aij->colmap;
}
ierr = MatGetColumnIJ_SeqAIJ_Color(A,0,PETSC_FALSE,PETSC_FALSE,&ncols,&A_ci,&A_cj,&spidxA,NULL);CHKERRQ(ierr);
ierr = MatGetColumnIJ_SeqAIJ_Color(B,0,PETSC_FALSE,PETSC_FALSE,&ncols,&B_ci,&B_cj,&spidxB,NULL);CHKERRQ(ierr);
bs = 1; /* only bs=1 is supported for non MPIBAIJ matrix */
if (ctype == IS_COLORING_GLOBAL && c->htype[0] == 'd') { /* create vscale for storing dx */
ierr = VecCreateGhost(PetscObjectComm((PetscObject)mat),mat->cmap->n,PETSC_DETERMINE,B->cmap->n,aij->garray,&c->vscale);CHKERRQ(ierr);
}
}
m = mat->rmap->n/bs;
cstart = mat->cmap->rstart/bs;
cend = mat->cmap->rend/bs;
ierr = PetscMalloc1(nis,&c->ncolumns);CHKERRQ(ierr);
ierr = PetscMalloc1(nis,&c->columns);CHKERRQ(ierr);
ierr = PetscMalloc1(nis,&c->nrows);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)c,3*nis*sizeof(PetscInt));CHKERRQ(ierr);
if (c->htype[0] == 'd') {
ierr = PetscMalloc1(nz,&Jentry);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)c,nz*sizeof(MatEntry));CHKERRQ(ierr);
c->matentry = Jentry;
} else if (c->htype[0] == 'w') {
ierr = PetscMalloc1(nz,&Jentry2);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)c,nz*sizeof(MatEntry2));CHKERRQ(ierr);
c->matentry2 = Jentry2;
} else SETERRQ(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"htype is not supported");
ierr = PetscMalloc2(m+1,&rowhit,m+1,&valaddrhit);CHKERRQ(ierr);
nz = 0;
ierr = ISColoringGetIS(iscoloring,PETSC_IGNORE,&isa);CHKERRQ(ierr);
for (i=0; i<nis; i++) { /* for each local color */
ierr = ISGetLocalSize(isa[i],&n);CHKERRQ(ierr);
ierr = ISGetIndices(isa[i],&is);CHKERRQ(ierr);
c->ncolumns[i] = n; /* local number of columns of this color on this process */
if (n) {
ierr = PetscMalloc1(n,&c->columns[i]);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)c,n*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(c->columns[i],is,n*sizeof(PetscInt));CHKERRQ(ierr);
} else {
c->columns[i] = 0;
}
if (ctype == IS_COLORING_GLOBAL) {
/* Determine nctot, the total (parallel) number of columns of this color */
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&size);CHKERRQ(ierr);
ierr = PetscMalloc2(size,&ncolsonproc,size,&disp);CHKERRQ(ierr);
/* ncolsonproc[j]: local ncolumns on proc[j] of this color */
ierr = PetscMPIIntCast(n,&nn);CHKERRQ(ierr);
ierr = MPI_Allgather(&nn,1,MPI_INT,ncolsonproc,1,MPI_INT,PetscObjectComm((PetscObject)mat));CHKERRQ(ierr);
nctot = 0; for (j=0; j<size; j++) nctot += ncolsonproc[j];
if (!nctot) {
ierr = PetscInfo(mat,"Coloring of matrix has some unneeded colors with no corresponding rows\n");CHKERRQ(ierr);
}
disp[0] = 0;
for (j=1; j<size; j++) {
disp[j] = disp[j-1] + ncolsonproc[j-1];
}
/* Get cols, the complete list of columns for this color on each process */
ierr = PetscMalloc1(nctot+1,&cols);CHKERRQ(ierr);
ierr = MPI_Allgatherv((void*)is,n,MPIU_INT,cols,ncolsonproc,disp,MPIU_INT,PetscObjectComm((PetscObject)mat));CHKERRQ(ierr);
ierr = PetscFree2(ncolsonproc,disp);CHKERRQ(ierr);
} else if (ctype == IS_COLORING_GHOSTED) {
/* Determine local number of columns of this color on this process, including ghost points */
nctot = n;
ierr = PetscMalloc1(nctot+1,&cols);CHKERRQ(ierr);
ierr = PetscMemcpy(cols,is,n*sizeof(PetscInt));CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Not provided for this MatFDColoring type");
/* Mark all rows affect by these columns */
ierr = PetscMemzero(rowhit,m*sizeof(PetscInt));CHKERRQ(ierr);
bs2 = bs*bs;
nrows_i = 0;
for (j=0; j<nctot; j++) { /* loop over columns*/
if (ctype == IS_COLORING_GHOSTED) {
col = ltog[cols[j]];
} else {
col = cols[j];
}
if (col >= cstart && col < cend) { /* column is in A, diagonal block of mat */
row = A_cj + A_ci[col-cstart];
nrows = A_ci[col-cstart+1] - A_ci[col-cstart];
nrows_i += nrows;
/* loop over columns of A marking them in rowhit */
for (k=0; k<nrows; k++) {
/* set valaddrhit for part A */
spidx = bs2*spidxA[A_ci[col-cstart] + k];
valaddrhit[*row] = &A_val[spidx];
rowhit[*row++] = col - cstart + 1; /* local column index */
}
} else { /* column is in B, off-diagonal block of mat */
#if defined(PETSC_USE_CTABLE)
ierr = PetscTableFind(colmap,col+1,&colb);CHKERRQ(ierr);
colb--;
#else
colb = colmap[col] - 1; /* local column index */
#endif
if (colb == -1) {
nrows = 0;
} else {
colb = colb/bs;
row = B_cj + B_ci[colb];
nrows = B_ci[colb+1] - B_ci[colb];
}
nrows_i += nrows;
/* loop over columns of B marking them in rowhit */
for (k=0; k<nrows; k++) {
/* set valaddrhit for part B */
spidx = bs2*spidxB[B_ci[colb] + k];
valaddrhit[*row] = &B_val[spidx];
rowhit[*row++] = colb + 1 + cend - cstart; /* local column index */
}
}
}
c->nrows[i] = nrows_i;
if (c->htype[0] == 'd') {
for (j=0; j<m; j++) {
if (rowhit[j]) {
Jentry[nz].row = j; /* local row index */
Jentry[nz].col = rowhit[j] - 1; /* local column index */
Jentry[nz].valaddr = valaddrhit[j]; /* address of mat value for this entry */
nz++;
}
}
} else { /* c->htype == 'wp' */
for (j=0; j<m; j++) {
if (rowhit[j]) {
Jentry2[nz].row = j; /* local row index */
Jentry2[nz].valaddr = valaddrhit[j]; /* address of mat value for this entry */
nz++;
}
}
}
ierr = PetscFree(cols);CHKERRQ(ierr);
}
if (bcols > 1) { /* reorder Jentry for faster MatFDColoringApply() */
ierr = MatFDColoringSetUpBlocked_AIJ_Private(mat,c,nz);CHKERRQ(ierr);
}
if (isBAIJ) {
ierr = MatRestoreColumnIJ_SeqBAIJ_Color(A,0,PETSC_FALSE,PETSC_FALSE,&ncols,&A_ci,&A_cj,&spidxA,NULL);CHKERRQ(ierr);
ierr = MatRestoreColumnIJ_SeqBAIJ_Color(B,0,PETSC_FALSE,PETSC_FALSE,&ncols,&B_ci,&B_cj,&spidxB,NULL);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*mat->rmap->n,&c->dy);CHKERRQ(ierr);
} else {
ierr = MatRestoreColumnIJ_SeqAIJ_Color(A,0,PETSC_FALSE,PETSC_FALSE,&ncols,&A_ci,&A_cj,&spidxA,NULL);CHKERRQ(ierr);
ierr = MatRestoreColumnIJ_SeqAIJ_Color(B,0,PETSC_FALSE,PETSC_FALSE,&ncols,&B_ci,&B_cj,&spidxB,NULL);CHKERRQ(ierr);
}
ierr = ISColoringRestoreIS(iscoloring,&isa);CHKERRQ(ierr);
ierr = PetscFree2(rowhit,valaddrhit);CHKERRQ(ierr);
if (ctype == IS_COLORING_GHOSTED) {
ierr = ISLocalToGlobalMappingRestoreIndices(map,<og);CHKERRQ(ierr);
}
ierr = PetscInfo3(c,"ncolors %D, brows %D and bcols %D are used.\n",c->ncolors,c->brows,c->bcols);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
LData::LData(Pointer<Database> db)
: d_name(db->getString("d_name")), d_global_node_count(0), d_local_node_count(0), d_ghost_node_count(0),
d_depth(db->getInteger("d_depth")), d_nonlocal_petsc_indices(), d_global_vec(NULL), d_array(NULL),
d_boost_array(NULL), d_boost_local_array(NULL), d_boost_vec_array(NULL), d_boost_local_vec_array(NULL),
d_ghosted_local_vec(NULL), d_ghosted_local_array(NULL), d_boost_ghosted_local_array(NULL),
d_boost_vec_ghosted_local_array(NULL)
{
int num_local_nodes = db->getInteger("num_local_nodes");
int num_ghost_nodes = db->getInteger("num_ghost_nodes");
d_nonlocal_petsc_indices.resize(num_ghost_nodes);
if (num_ghost_nodes > 0)
{
db->getIntegerArray("d_nonlocal_petsc_indices",
d_nonlocal_petsc_indices.empty() ? NULL : &d_nonlocal_petsc_indices[0],
num_ghost_nodes);
}
// Create the PETSc Vec which actually provides the storage for the
// Lagrangian data.
int ierr;
if (d_depth == 1)
{
ierr = VecCreateGhost(PETSC_COMM_WORLD,
num_local_nodes,
PETSC_DECIDE,
static_cast<int>(d_nonlocal_petsc_indices.size()),
d_nonlocal_petsc_indices.empty() ? NULL : &d_nonlocal_petsc_indices[0],
&d_global_vec);
IBTK_CHKERRQ(ierr);
}
else
{
ierr = VecCreateGhostBlock(PETSC_COMM_WORLD,
d_depth,
d_depth * num_local_nodes,
PETSC_DECIDE,
static_cast<int>(d_nonlocal_petsc_indices.size()),
d_nonlocal_petsc_indices.empty() ? NULL : &d_nonlocal_petsc_indices[0],
&d_global_vec);
IBTK_CHKERRQ(ierr);
}
int global_node_count;
ierr = VecGetSize(d_global_vec, &global_node_count);
IBTK_CHKERRQ(ierr);
#if !defined(NDEBUG)
TBOX_ASSERT(global_node_count >= 0);
#endif
d_global_node_count = global_node_count;
d_global_node_count /= d_depth;
d_local_node_count = num_local_nodes;
d_ghost_node_count = static_cast<int>(d_nonlocal_petsc_indices.size());
// Extract the values from the database.
double* ghosted_local_vec_array = getGhostedLocalFormVecArray()->data();
if (num_local_nodes + num_ghost_nodes > 0)
{
db->getDoubleArray("vals", ghosted_local_vec_array, d_depth * (num_local_nodes + num_ghost_nodes));
}
restoreArrays();
return;
} // LData