static PetscErrorCode DMGlobalToLocalBegin_Sliced(DM da,Vec g,InsertMode mode,Vec l)
{
PetscErrorCode ierr;
PetscBool flg;
PetscFunctionBegin;
ierr = VecGhostIsLocalForm(g,l,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(((PetscObject)da)->comm,PETSC_ERR_ARG_WRONG,"Local vector is not local form of global vector");
ierr = VecGhostUpdateEnd(g,mode,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGhostUpdateBegin(g,mode,SCATTER_FORWARD);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscScalar* PETScVector::getLocalVector() const
{
PetscScalar *loc_array;
if (_has_ghost_id)
{
VecGhostUpdateBegin(_v, INSERT_VALUES, SCATTER_FORWARD);
VecGhostUpdateEnd(_v, INSERT_VALUES, SCATTER_FORWARD);
VecGhostGetLocalForm(_v, &_v_loc);
VecGetArray(_v_loc, &loc_array);
}
else
VecGetArray(_v, &loc_array);
return loc_array;
}
inline void PetscVector::close () {
this->_restore_array();
int ierr=0;
ierr = VecAssemblyBegin(_vec); CHKERRABORT(MPI_COMM_WORLD,ierr);
ierr = VecAssemblyEnd(_vec); CHKERRABORT(MPI_COMM_WORLD,ierr);
if (this->type() == GHOSTED) {
ierr = VecGhostUpdateBegin(_vec,INSERT_VALUES,SCATTER_FORWARD); CHKERRABORT(MPI_COMM_WORLD,ierr);
ierr = VecGhostUpdateEnd(_vec,INSERT_VALUES,SCATTER_FORWARD); CHKERRABORT(MPI_COMM_WORLD,ierr);
}
this->_is_closed = true;
}
void CalculateRezidual(Grid& g, Vec u,
Vec r) {
Vec uLoc, rLoc;
const double *ul;
double *rl;
VecGhostGetLocalForm(u, &uLoc);
VecGetArrayRead(uLoc, &ul);
VecGhostGetLocalForm(r, &rLoc);
VecSet(rLoc, 0.0);
VecGetArray(rLoc, &rl);
for (auto f: g.internalFaces()) {
double flux = Flux( ul[f.owner], ul[f.neighbour], f.s);
rl[f.owner] -= flux;
rl[f.neighbour] += flux;
}
for (auto bnd: g.boundaryPatches()) {
double ub = 0;
if (bnd.first == BND_BOTTOM) ub = 1;
for (auto f: bnd.second) {
rl[f.owner] -= Flux( ul[f.owner], ub, f.s);
}
}
VecRestoreArray(rLoc, &rl);
VecGhostRestoreLocalForm(r, &rLoc);
VecRestoreArrayRead(uLoc, &ul);
VecGhostRestoreLocalForm(u, &uLoc);
VecGhostUpdateBegin(r,ADD_VALUES,SCATTER_REVERSE);
VecGhostUpdateEnd(r,ADD_VALUES,SCATTER_REVERSE);
VecGetArray(r, &rl);
for (size_t i=0; i<g.cells().size(); i++) rl[i] /= g.cells()[i].vol;
VecRestoreArray(r, &rl);
}
int main(int argc, char **argv) {
PetscInitialize( &argc, &argv, (char*)0, 0);
Grid g = DecomposeGrid( CartesianGrid(N, N) );
std::cout << g.cells().size() << std::endl;
Vec u,r;
u = CreateGhostedVector(g);
VecDuplicate(u, &r);
double dt = 1.0 / (N*(fabs(a)+fabs(b)));
// Pocatecni podminka
VecSet(u, 0.0);
double t = 0;
while (t<tEnd) {
VecGhostUpdateBegin(u, INSERT_VALUES,SCATTER_FORWARD);
VecGhostUpdateEnd(u, INSERT_VALUES,SCATTER_FORWARD);
CalculateRezidual(g, u, r);
VecAXPY(u, dt, r);
t += dt;
};
Save(u, "u.dat");
VecDestroy(&r);
VecDestroy(&u);
PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank,size;
PetscInt nlocal = 6,nghost = 2,ifrom[2],i,rstart,rend;
PetscErrorCode ierr;
PetscBool flg,flg2;
PetscScalar value,*array,*tarray=0;
Vec lx,gx,gxs;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 2) SETERRQ(PETSC_COMM_SELF,1,"Must run example with two processors\n");
/*
Construct a two dimensional graph connecting nlocal degrees of
freedom per processor. From this we will generate the global
indices of needed ghost values
For simplicity we generate the entire graph on each processor:
in real application the graph would stored in parallel, but this
example is only to demonstrate the management of ghost padding
with VecCreateGhost().
In this example we consider the vector as representing
degrees of freedom in a one dimensional grid with periodic
boundary conditions.
----Processor 1--------- ----Processor 2 --------
0 1 2 3 4 5 6 7 8 9 10 11
|----|
|-------------------------------------------------|
*/
if (!rank) {
ifrom[0] = 11; ifrom[1] = 6;
} else {
ifrom[0] = 0; ifrom[1] = 5;
}
/*
Create the vector with two slots for ghost points. Note that both
the local vector (lx) and the global vector (gx) share the same
array for storing vector values.
*/
ierr = PetscOptionsHasName(PETSC_NULL,"-allocate",&flg);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-vecmpisetghost",&flg2);CHKERRQ(ierr);
if (flg) {
ierr = PetscMalloc((nlocal+nghost)*sizeof(PetscScalar),&tarray);CHKERRQ(ierr);
ierr = VecCreateGhostWithArray(PETSC_COMM_WORLD,nlocal,PETSC_DECIDE,nghost,ifrom,tarray,&gxs);CHKERRQ(ierr);
} else if (flg2) {
ierr = VecCreate(PETSC_COMM_WORLD,&gxs);CHKERRQ(ierr);
ierr = VecSetType(gxs,VECMPI);CHKERRQ(ierr);
ierr = VecSetSizes(gxs,nlocal,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecMPISetGhost(gxs,nghost,ifrom);CHKERRQ(ierr);
} else {
ierr = VecCreateGhost(PETSC_COMM_WORLD,nlocal,PETSC_DECIDE,nghost,ifrom,&gxs);CHKERRQ(ierr);
}
/*
Test VecDuplicate()
*/
ierr = VecDuplicate(gxs,&gx);CHKERRQ(ierr);
ierr = VecDestroy(&gxs);CHKERRQ(ierr);
/*
Access the local representation
*/
ierr = VecGhostGetLocalForm(gx,&lx);CHKERRQ(ierr);
/*
Set the values from 0 to 12 into the "global" vector
*/
ierr = VecGetOwnershipRange(gx,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
value = (PetscScalar) i;
ierr = VecSetValues(gx,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(gx);CHKERRQ(ierr);
ierr = VecAssemblyEnd(gx);CHKERRQ(ierr);
ierr = VecGhostUpdateBegin(gx,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGhostUpdateEnd(gx,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/*
Print out each vector, including the ghost padding region.
*/
ierr = VecGetArray(lx,&array);CHKERRQ(ierr);
for (i=0; i<nlocal+nghost; i++) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"%D %G\n",i,PetscRealPart(array[i]));CHKERRQ(ierr);
}
ierr = VecRestoreArray(lx,&array);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecGhostRestoreLocalForm(gx,&lx);CHKERRQ(ierr);
ierr = VecDestroy(&gx);CHKERRQ(ierr);
if (flg) {ierr = PetscFree(tarray);CHKERRQ(ierr);}
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatFDColoringApply_BAIJ(Mat J,MatFDColoring coloring,Vec x1,void *sctx)
{
PetscErrorCode (*f)(void*,Vec,Vec,void*) = (PetscErrorCode (*)(void*,Vec,Vec,void*))coloring->f;
PetscErrorCode ierr;
PetscInt k,cstart,cend,l,row,col,nz,spidx,i,j;
PetscScalar dx=0.0,*w3_array,*dy_i,*dy=coloring->dy;
PetscScalar *vscale_array;
const PetscScalar *xx;
PetscReal epsilon=coloring->error_rel,umin=coloring->umin,unorm;
Vec w1=coloring->w1,w2=coloring->w2,w3,vscale=coloring->vscale;
void *fctx=coloring->fctx;
PetscInt ctype=coloring->ctype,nxloc,nrows_k;
PetscScalar *valaddr;
MatEntry *Jentry=coloring->matentry;
MatEntry2 *Jentry2=coloring->matentry2;
const PetscInt ncolors=coloring->ncolors,*ncolumns=coloring->ncolumns,*nrows=coloring->nrows;
PetscInt bs=J->rmap->bs;
PetscFunctionBegin;
/* (1) Set w1 = F(x1) */
if (!coloring->fset) {
ierr = PetscLogEventBegin(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = (*f)(sctx,x1,w1,fctx);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
} else {
coloring->fset = PETSC_FALSE;
}
/* (2) Compute vscale = 1./dx - the local scale factors, including ghost points */
ierr = VecGetLocalSize(x1,&nxloc);CHKERRQ(ierr);
if (coloring->htype[0] == 'w') {
/* vscale = dx is a constant scalar */
ierr = VecNorm(x1,NORM_2,&unorm);CHKERRQ(ierr);
dx = 1.0/(PetscSqrtReal(1.0 + unorm)*epsilon);
} else {
ierr = VecGetArrayRead(x1,&xx);CHKERRQ(ierr);
ierr = VecGetArray(vscale,&vscale_array);CHKERRQ(ierr);
for (col=0; col<nxloc; col++) {
dx = xx[col];
if (PetscAbsScalar(dx) < umin) {
if (PetscRealPart(dx) >= 0.0) dx = umin;
else if (PetscRealPart(dx) < 0.0 ) dx = -umin;
}
dx *= epsilon;
vscale_array[col] = 1.0/dx;
}
ierr = VecRestoreArrayRead(x1,&xx);CHKERRQ(ierr);
ierr = VecRestoreArray(vscale,&vscale_array);CHKERRQ(ierr);
}
if (ctype == IS_COLORING_GLOBAL && coloring->htype[0] == 'd') {
ierr = VecGhostUpdateBegin(vscale,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGhostUpdateEnd(vscale,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
/* (3) Loop over each color */
if (!coloring->w3) {
ierr = VecDuplicate(x1,&coloring->w3);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)coloring,(PetscObject)coloring->w3);CHKERRQ(ierr);
}
w3 = coloring->w3;
ierr = VecGetOwnershipRange(x1,&cstart,&cend);CHKERRQ(ierr); /* used by ghosted vscale */
if (vscale) {
ierr = VecGetArray(vscale,&vscale_array);CHKERRQ(ierr);
}
nz = 0;
for (k=0; k<ncolors; k++) {
coloring->currentcolor = k;
/*
(3-1) Loop over each column associated with color
adding the perturbation to the vector w3 = x1 + dx.
*/
ierr = VecCopy(x1,w3);CHKERRQ(ierr);
dy_i = dy;
for (i=0; i<bs; i++) { /* Loop over a block of columns */
ierr = VecGetArray(w3,&w3_array);CHKERRQ(ierr);
if (ctype == IS_COLORING_GLOBAL) w3_array -= cstart; /* shift pointer so global index can be used */
if (coloring->htype[0] == 'w') {
for (l=0; l<ncolumns[k]; l++) {
col = i + bs*coloring->columns[k][l]; /* local column (in global index!) of the matrix we are probing for */
w3_array[col] += 1.0/dx;
if (i) w3_array[col-1] -= 1.0/dx; /* resume original w3[col-1] */
}
} else { /* htype == 'ds' */
vscale_array -= cstart; /* shift pointer so global index can be used */
for (l=0; l<ncolumns[k]; l++) {
col = i + bs*coloring->columns[k][l]; /* local column (in global index!) of the matrix we are probing for */
w3_array[col] += 1.0/vscale_array[col];
if (i) w3_array[col-1] -= 1.0/vscale_array[col-1]; /* resume original w3[col-1] */
}
vscale_array += cstart;
}
if (ctype == IS_COLORING_GLOBAL) w3_array += cstart;
ierr = VecRestoreArray(w3,&w3_array);CHKERRQ(ierr);
/*
(3-2) Evaluate function at w3 = x1 + dx (here dx is a vector of perturbations)
w2 = F(x1 + dx) - F(x1)
*/
ierr = PetscLogEventBegin(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = VecPlaceArray(w2,dy_i);CHKERRQ(ierr); /* place w2 to the array dy_i */
ierr = (*f)(sctx,w3,w2,fctx);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = VecAXPY(w2,-1.0,w1);CHKERRQ(ierr);
ierr = VecResetArray(w2);CHKERRQ(ierr);
dy_i += nxloc; /* points to dy+i*nxloc */
}
/*
(3-3) Loop over rows of vector, putting results into Jacobian matrix
*/
nrows_k = nrows[k];
if (coloring->htype[0] == 'w') {
for (l=0; l<nrows_k; l++) {
row = bs*Jentry2[nz].row; /* local row index */
valaddr = Jentry2[nz++].valaddr;
spidx = 0;
dy_i = dy;
for (i=0; i<bs; i++) { /* column of the block */
for (j=0; j<bs; j++) { /* row of the block */
valaddr[spidx++] = dy_i[row+j]*dx;
}
dy_i += nxloc; /* points to dy+i*nxloc */
}
}
} else { /* htype == 'ds' */
for (l=0; l<nrows_k; l++) {
row = bs*Jentry[nz].row; /* local row index */
col = bs*Jentry[nz].col; /* local column index */
valaddr = Jentry[nz++].valaddr;
spidx = 0;
dy_i = dy;
for (i=0; i<bs; i++) { /* column of the block */
for (j=0; j<bs; j++) { /* row of the block */
valaddr[spidx++] = dy_i[row+j]*vscale_array[col+i];
}
dy_i += nxloc; /* points to dy+i*nxloc */
}
}
}
}
ierr = MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (vscale) {
ierr = VecRestoreArray(vscale,&vscale_array);CHKERRQ(ierr);
}
coloring->currentcolor = -1;
PetscFunctionReturn(0);
}
/* this is declared PETSC_EXTERN because it is used by MatFDColoringUseDM() which is in the DM library */
PetscErrorCode MatFDColoringApply_AIJ(Mat J,MatFDColoring coloring,Vec x1,void *sctx)
{
PetscErrorCode (*f)(void*,Vec,Vec,void*) = (PetscErrorCode (*)(void*,Vec,Vec,void*))coloring->f;
PetscErrorCode ierr;
PetscInt k,cstart,cend,l,row,col,nz;
PetscScalar dx=0.0,*y,*w3_array;
const PetscScalar *xx;
PetscScalar *vscale_array;
PetscReal epsilon=coloring->error_rel,umin=coloring->umin,unorm;
Vec w1=coloring->w1,w2=coloring->w2,w3,vscale=coloring->vscale;
void *fctx=coloring->fctx;
ISColoringType ctype=coloring->ctype;
PetscInt nxloc,nrows_k;
MatEntry *Jentry=coloring->matentry;
MatEntry2 *Jentry2=coloring->matentry2;
const PetscInt ncolors=coloring->ncolors,*ncolumns=coloring->ncolumns,*nrows=coloring->nrows;
PetscFunctionBegin;
ierr = VecPinToCPU(x1,PETSC_TRUE);CHKERRQ(ierr);
if ((ctype == IS_COLORING_LOCAL) && (J->ops->fdcoloringapply == MatFDColoringApply_AIJ)) SETERRQ(PetscObjectComm((PetscObject)J),PETSC_ERR_SUP,"Must call MatColoringUseDM() with IS_COLORING_LOCAL");
/* (1) Set w1 = F(x1) */
if (!coloring->fset) {
ierr = PetscLogEventBegin(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = (*f)(sctx,x1,w1,fctx);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
} else {
coloring->fset = PETSC_FALSE;
}
/* (2) Compute vscale = 1./dx - the local scale factors, including ghost points */
if (coloring->htype[0] == 'w') {
/* vscale = 1./dx is a constant scalar */
ierr = VecNorm(x1,NORM_2,&unorm);CHKERRQ(ierr);
dx = 1.0/(PetscSqrtReal(1.0 + unorm)*epsilon);
} else {
ierr = VecGetLocalSize(x1,&nxloc);CHKERRQ(ierr);
ierr = VecGetArrayRead(x1,&xx);CHKERRQ(ierr);
ierr = VecGetArray(vscale,&vscale_array);CHKERRQ(ierr);
for (col=0; col<nxloc; col++) {
dx = xx[col];
if (PetscAbsScalar(dx) < umin) {
if (PetscRealPart(dx) >= 0.0) dx = umin;
else if (PetscRealPart(dx) < 0.0 ) dx = -umin;
}
dx *= epsilon;
vscale_array[col] = 1.0/dx;
}
ierr = VecRestoreArrayRead(x1,&xx);CHKERRQ(ierr);
ierr = VecRestoreArray(vscale,&vscale_array);CHKERRQ(ierr);
}
if (ctype == IS_COLORING_GLOBAL && coloring->htype[0] == 'd') {
ierr = VecGhostUpdateBegin(vscale,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGhostUpdateEnd(vscale,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
/* (3) Loop over each color */
if (!coloring->w3) {
ierr = VecDuplicate(x1,&coloring->w3);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)coloring,(PetscObject)coloring->w3);CHKERRQ(ierr);
}
w3 = coloring->w3;
ierr = VecGetOwnershipRange(x1,&cstart,&cend);CHKERRQ(ierr); /* used by ghosted vscale */
if (vscale) {
ierr = VecGetArray(vscale,&vscale_array);CHKERRQ(ierr);
}
nz = 0;
if (coloring->bcols > 1) { /* use blocked insertion of Jentry */
PetscInt i,m=J->rmap->n,nbcols,bcols=coloring->bcols;
PetscScalar *dy=coloring->dy,*dy_k;
nbcols = 0;
for (k=0; k<ncolors; k+=bcols) {
/*
(3-1) Loop over each column associated with color
adding the perturbation to the vector w3 = x1 + dx.
*/
dy_k = dy;
if (k + bcols > ncolors) bcols = ncolors - k;
for (i=0; i<bcols; i++) {
coloring->currentcolor = k+i;
ierr = VecCopy(x1,w3);CHKERRQ(ierr);
ierr = VecGetArray(w3,&w3_array);CHKERRQ(ierr);
if (ctype == IS_COLORING_GLOBAL) w3_array -= cstart; /* shift pointer so global index can be used */
if (coloring->htype[0] == 'w') {
for (l=0; l<ncolumns[k+i]; l++) {
col = coloring->columns[k+i][l]; /* local column (in global index!) of the matrix we are probing for */
w3_array[col] += 1.0/dx;
}
} else { /* htype == 'ds' */
vscale_array -= cstart; /* shift pointer so global index can be used */
for (l=0; l<ncolumns[k+i]; l++) {
col = coloring->columns[k+i][l]; /* local column (in global index!) of the matrix we are probing for */
w3_array[col] += 1.0/vscale_array[col];
}
vscale_array += cstart;
}
if (ctype == IS_COLORING_GLOBAL) w3_array += cstart;
ierr = VecRestoreArray(w3,&w3_array);CHKERRQ(ierr);
/*
(3-2) Evaluate function at w3 = x1 + dx (here dx is a vector of perturbations)
w2 = F(x1 + dx) - F(x1)
*/
ierr = PetscLogEventBegin(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = VecPlaceArray(w2,dy_k);CHKERRQ(ierr); /* place w2 to the array dy_i */
ierr = (*f)(sctx,w3,w2,fctx);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = VecAXPY(w2,-1.0,w1);CHKERRQ(ierr);
ierr = VecResetArray(w2);CHKERRQ(ierr);
dy_k += m; /* points to dy+i*nxloc */
}
/*
(3-3) Loop over block rows of vector, putting results into Jacobian matrix
*/
nrows_k = nrows[nbcols++];
if (coloring->htype[0] == 'w') {
for (l=0; l<nrows_k; l++) {
row = Jentry2[nz].row; /* local row index */
*(Jentry2[nz++].valaddr) = dy[row]*dx;
}
} else { /* htype == 'ds' */
for (l=0; l<nrows_k; l++) {
row = Jentry[nz].row; /* local row index */
*(Jentry[nz].valaddr) = dy[row]*vscale_array[Jentry[nz].col];
nz++;
}
}
}
} else { /* bcols == 1 */
for (k=0; k<ncolors; k++) {
coloring->currentcolor = k;
/*
(3-1) Loop over each column associated with color
adding the perturbation to the vector w3 = x1 + dx.
*/
ierr = VecCopy(x1,w3);CHKERRQ(ierr);
ierr = VecGetArray(w3,&w3_array);CHKERRQ(ierr);
if (ctype == IS_COLORING_GLOBAL) w3_array -= cstart; /* shift pointer so global index can be used */
if (coloring->htype[0] == 'w') {
for (l=0; l<ncolumns[k]; l++) {
col = coloring->columns[k][l]; /* local column (in global index!) of the matrix we are probing for */
w3_array[col] += 1.0/dx;
}
} else { /* htype == 'ds' */
vscale_array -= cstart; /* shift pointer so global index can be used */
for (l=0; l<ncolumns[k]; l++) {
col = coloring->columns[k][l]; /* local column (in global index!) of the matrix we are probing for */
w3_array[col] += 1.0/vscale_array[col];
}
vscale_array += cstart;
}
if (ctype == IS_COLORING_GLOBAL) w3_array += cstart;
ierr = VecRestoreArray(w3,&w3_array);CHKERRQ(ierr);
/*
(3-2) Evaluate function at w3 = x1 + dx (here dx is a vector of perturbations)
w2 = F(x1 + dx) - F(x1)
*/
ierr = PetscLogEventBegin(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = (*f)(sctx,w3,w2,fctx);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = VecAXPY(w2,-1.0,w1);CHKERRQ(ierr);
/*
(3-3) Loop over rows of vector, putting results into Jacobian matrix
*/
nrows_k = nrows[k];
ierr = VecGetArray(w2,&y);CHKERRQ(ierr);
if (coloring->htype[0] == 'w') {
for (l=0; l<nrows_k; l++) {
row = Jentry2[nz].row; /* local row index */
*(Jentry2[nz++].valaddr) = y[row]*dx;
}
} else { /* htype == 'ds' */
for (l=0; l<nrows_k; l++) {
row = Jentry[nz].row; /* local row index */
*(Jentry[nz].valaddr) = y[row]*vscale_array[Jentry[nz].col];
nz++;
}
}
ierr = VecRestoreArray(w2,&y);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (vscale) {
ierr = VecRestoreArray(vscale,&vscale_array);CHKERRQ(ierr);
}
coloring->currentcolor = -1;
ierr = VecPinToCPU(x1,PETSC_FALSE);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char *argv[])
{
char mat_type[256] = "aij"; /* default matrix type */
PetscErrorCode ierr;
MPI_Comm comm;
PetscMPIInt rank,size;
DM slice;
PetscInt i,bs=1,N=5,n,m,rstart,ghosts[2],*d_nnz,*o_nnz,dfill[4]={1,0,0,1},ofill[4]={1,1,1,1};
PetscReal alpha =1,K=1,rho0=1,u0=0,sigma=0.2;
PetscBool useblock=PETSC_TRUE;
PetscScalar *xx;
Mat A;
Vec x,b,lf;
ierr = PetscInitialize(&argc,&argv,0,help);CHKERRQ(ierr);
comm = PETSC_COMM_WORLD;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscOptionsBegin(comm,0,"Options for DMSliced test",0);CHKERRQ(ierr);
{
ierr = PetscOptionsInt("-n","Global number of nodes","",N,&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsInt("-bs","Block size (1 or 2)","",bs,&bs,NULL);CHKERRQ(ierr);
if (bs != 1) {
if (bs != 2) SETERRQ(PETSC_COMM_WORLD,1,"Block size must be 1 or 2");
ierr = PetscOptionsReal("-alpha","Inverse time step for wave operator","",alpha,&alpha,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-K","Bulk modulus of compressibility","",K,&K,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-rho0","Reference density","",rho0,&rho0,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-u0","Reference velocity","",u0,&u0,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-sigma","Width of Gaussian density perturbation","",sigma,&sigma,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-block","Use block matrix assembly","",useblock,&useblock,NULL);CHKERRQ(ierr);
}
ierr = PetscOptionsString("-sliced_mat_type","Matrix type to use (aij or baij)","",mat_type,mat_type,sizeof(mat_type),NULL);CHKERRQ(ierr);
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
/* Split ownership, set up periodic grid in 1D */
n = PETSC_DECIDE;
ierr = PetscSplitOwnership(comm,&n,&N);CHKERRQ(ierr);
rstart = 0;
ierr = MPI_Scan(&n,&rstart,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr);
rstart -= n;
ghosts[0] = (N+rstart-1)%N;
ghosts[1] = (rstart+n)%N;
ierr = PetscMalloc2(n,PetscInt,&d_nnz,n,PetscInt,&o_nnz);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (size > 1 && (i==0 || i==n-1)) {
d_nnz[i] = 2;
o_nnz[i] = 1;
} else {
d_nnz[i] = 3;
o_nnz[i] = 0;
}
}
ierr = DMSlicedCreate(comm,bs,n,2,ghosts,d_nnz,o_nnz,&slice);CHKERRQ(ierr); /* Currently does not copy X_nnz so we can't free them until after DMSlicedGetMatrix */
if (!useblock) {ierr = DMSlicedSetBlockFills(slice,dfill,ofill);CHKERRQ(ierr);} /* Irrelevant for baij formats */
ierr = DMSetMatType(slice,mat_type);CHKERRQ(ierr);
ierr = DMCreateMatrix(slice,&A);CHKERRQ(ierr);
ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slice,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecGhostGetLocalForm(x,&lf);CHKERRQ(ierr);
ierr = VecGetSize(lf,&m);CHKERRQ(ierr);
if (m != (n+2)*bs) SETERRQ2(PETSC_COMM_SELF,1,"size of local form %D, expected %D",m,(n+2)*bs);
ierr = VecGetArray(lf,&xx);CHKERRQ(ierr);
for (i=0; i<n; i++) {
PetscInt row[2],col[9],im,ip;
PetscScalar v[12];
const PetscReal xref = 2.0*(rstart+i)/N - 1; /* [-1,1] */
const PetscReal h = 1.0/N; /* grid spacing */
im = (i==0) ? n : i-1;
ip = (i==n-1) ? n+1 : i+1;
switch (bs) {
case 1: /* Laplacian with periodic boundaries */
col[0] = im; col[1] = i; col[2] = ip;
v[0] = -h; v[1] = 2*h; v[2] = -h;
ierr = MatSetValuesLocal(A,1,&i,3,col,v,INSERT_VALUES);CHKERRQ(ierr);
xx[i] = sin(xref*PETSC_PI);
break;
case 2: /* Linear acoustic wave operator in variables [rho, u], central differences, periodic, timestep 1/alpha */
v[0] = -0.5*u0; v[1] = -0.5*K; v[2] = alpha; v[3] = 0; v[4] = 0.5*u0; v[5] = 0.5*K;
v[6] = -0.5/rho0; v[7] = -0.5*u0; v[8] = 0; v[9] = alpha; v[10] = 0.5/rho0; v[11] = 0.5*u0;
if (useblock) {
row[0] = i; col[0] = im; col[1] = i; col[2] = ip;
ierr = MatSetValuesBlockedLocal(A,1,row,3,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else {
row[0] = 2*i; row[1] = 2*i+1;
col[0] = 2*im; col[1] = 2*im+1; col[2] = 2*i; col[3] = 2*ip; col[4] = 2*ip+1;
v[3] = v[4]; v[4] = v[5]; /* pack values in first row */
ierr = MatSetValuesLocal(A,1,row,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
col[2] = 2*i+1;
v[8] = v[9]; v[9] = v[10]; v[10] = v[11]; /* pack values in second row */
ierr = MatSetValuesLocal(A,1,row+1,5,col,v+6,INSERT_VALUES);CHKERRQ(ierr);
}
/* Set current state (gaussian density perturbation) */
xx[2*i] = 0.2*exp(-PetscSqr(xref)/(2*PetscSqr(sigma)));
xx[2*i+1] = 0;
break;
default: SETERRQ1(PETSC_COMM_SELF,1,"not implemented for block size %D",bs);
}
}
ierr = VecRestoreArray(lf,&xx);CHKERRQ(ierr);
ierr = VecGhostRestoreLocalForm(x,&lf);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatMult(A,x,b);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecView(b,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Update the ghosted values, view the result on rank 0. */
ierr = VecGhostUpdateBegin(b,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGhostUpdateEnd(b,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (!rank) {
ierr = VecGhostGetLocalForm(b,&lf);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_SELF,"Local form of b on rank 0, last two nodes are ghost nodes\n");CHKERRQ(ierr);
ierr = VecView(lf,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecGhostRestoreLocalForm(b,&lf);CHKERRQ(ierr);
}
ierr = DMDestroy(&slice);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatFDColoringApply_AIJ(Mat J,MatFDColoring coloring,Vec x1,MatStructure *flag,void *sctx)
{
PetscErrorCode (*f)(void*,Vec,Vec,void*) = (PetscErrorCode (*)(void*,Vec,Vec,void *))coloring->f;
PetscErrorCode ierr;
PetscInt k,start,end,l,row,col,srow,**vscaleforrow;
PetscScalar dx,*y,*xx,*w3_array;
PetscScalar *vscale_array;
PetscReal epsilon = coloring->error_rel,umin = coloring->umin,unorm;
Vec w1=coloring->w1,w2=coloring->w2,w3;
void *fctx = coloring->fctx;
PetscBool flg = PETSC_FALSE;
PetscInt ctype=coloring->ctype,N,col_start=0,col_end=0;
Vec x1_tmp;
PetscFunctionBegin;
PetscValidHeaderSpecific(J,MAT_CLASSID,1);
PetscValidHeaderSpecific(coloring,MAT_FDCOLORING_CLASSID,2);
PetscValidHeaderSpecific(x1,VEC_CLASSID,3);
if (!f) SETERRQ(((PetscObject)J)->comm,PETSC_ERR_ARG_WRONGSTATE,"Must call MatFDColoringSetFunction()");
ierr = PetscLogEventBegin(MAT_FDColoringApply,coloring,J,x1,0);CHKERRQ(ierr);
ierr = MatSetUnfactored(J);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-mat_fd_coloring_dont_rezero",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
ierr = PetscInfo(coloring,"Not calling MatZeroEntries()\n");CHKERRQ(ierr);
} else {
PetscBool assembled;
ierr = MatAssembled(J,&assembled);CHKERRQ(ierr);
if (assembled) {
ierr = MatZeroEntries(J);CHKERRQ(ierr);
}
}
x1_tmp = x1;
if (!coloring->vscale){
ierr = VecDuplicate(x1_tmp,&coloring->vscale);CHKERRQ(ierr);
}
if (coloring->htype[0] == 'w') { /* tacky test; need to make systematic if we add other approaches to computing h*/
ierr = VecNorm(x1_tmp,NORM_2,&unorm);CHKERRQ(ierr);
}
ierr = VecGetOwnershipRange(w1,&start,&end);CHKERRQ(ierr); /* OwnershipRange is used by ghosted x! */
/* Set w1 = F(x1) */
if (!coloring->fset) {
ierr = PetscLogEventBegin(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = (*f)(sctx,x1_tmp,w1,fctx);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
} else {
coloring->fset = PETSC_FALSE;
}
if (!coloring->w3) {
ierr = VecDuplicate(x1_tmp,&coloring->w3);CHKERRQ(ierr);
ierr = PetscLogObjectParent(coloring,coloring->w3);CHKERRQ(ierr);
}
w3 = coloring->w3;
/* Compute all the local scale factors, including ghost points */
ierr = VecGetLocalSize(x1_tmp,&N);CHKERRQ(ierr);
ierr = VecGetArray(x1_tmp,&xx);CHKERRQ(ierr);
ierr = VecGetArray(coloring->vscale,&vscale_array);CHKERRQ(ierr);
if (ctype == IS_COLORING_GHOSTED){
col_start = 0; col_end = N;
} else if (ctype == IS_COLORING_GLOBAL){
xx = xx - start;
vscale_array = vscale_array - start;
col_start = start; col_end = N + start;
}
for (col=col_start; col<col_end; col++){
/* Loop over each local column, vscale[col] = 1./(epsilon*dx[col]) */
if (coloring->htype[0] == 'w') {
dx = 1.0 + unorm;
} else {
dx = xx[col];
}
if (dx == (PetscScalar)0.0) dx = 1.0;
if (PetscAbsScalar(dx) < umin && PetscRealPart(dx) >= 0.0) dx = umin;
else if (PetscRealPart(dx) < 0.0 && PetscAbsScalar(dx) < umin) dx = -umin;
dx *= epsilon;
vscale_array[col] = (PetscScalar)1.0/dx;
}
if (ctype == IS_COLORING_GLOBAL) vscale_array = vscale_array + start;
ierr = VecRestoreArray(coloring->vscale,&vscale_array);CHKERRQ(ierr);
if (ctype == IS_COLORING_GLOBAL){
ierr = VecGhostUpdateBegin(coloring->vscale,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGhostUpdateEnd(coloring->vscale,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
if (coloring->vscaleforrow) {
vscaleforrow = coloring->vscaleforrow;
} else SETERRQ(((PetscObject)J)->comm,PETSC_ERR_ARG_NULL,"Null Object: coloring->vscaleforrow");
/*
Loop over each color
*/
ierr = VecGetArray(coloring->vscale,&vscale_array);CHKERRQ(ierr);
for (k=0; k<coloring->ncolors; k++) {
coloring->currentcolor = k;
ierr = VecCopy(x1_tmp,w3);CHKERRQ(ierr);
ierr = VecGetArray(w3,&w3_array);CHKERRQ(ierr);
if (ctype == IS_COLORING_GLOBAL) w3_array = w3_array - start;
/*
Loop over each column associated with color
adding the perturbation to the vector w3.
*/
for (l=0; l<coloring->ncolumns[k]; l++) {
col = coloring->columns[k][l]; /* local column of the matrix we are probing for */
if (coloring->htype[0] == 'w') {
dx = 1.0 + unorm;
} else {
dx = xx[col];
}
if (dx == (PetscScalar)0.0) dx = 1.0;
if (PetscAbsScalar(dx) < umin && PetscRealPart(dx) >= 0.0) dx = umin;
else if (PetscRealPart(dx) < 0.0 && PetscAbsScalar(dx) < umin) dx = -umin;
dx *= epsilon;
if (!PetscAbsScalar(dx)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Computed 0 differencing parameter");
w3_array[col] += dx;
}
if (ctype == IS_COLORING_GLOBAL) w3_array = w3_array + start;
ierr = VecRestoreArray(w3,&w3_array);CHKERRQ(ierr);
/*
Evaluate function at w3 = x1 + dx (here dx is a vector of perturbations)
w2 = F(x1 + dx) - F(x1)
*/
ierr = PetscLogEventBegin(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = (*f)(sctx,w3,w2,fctx);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_FDColoringFunction,0,0,0,0);CHKERRQ(ierr);
ierr = VecAXPY(w2,-1.0,w1);CHKERRQ(ierr);
/*
Loop over rows of vector, putting results into Jacobian matrix
*/
ierr = VecGetArray(w2,&y);CHKERRQ(ierr);
for (l=0; l<coloring->nrows[k]; l++) {
row = coloring->rows[k][l]; /* local row index */
col = coloring->columnsforrow[k][l]; /* global column index */
y[row] *= vscale_array[vscaleforrow[k][l]];
srow = row + start;
ierr = MatSetValues(J,1,&srow,1,&col,y+row,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecRestoreArray(w2,&y);CHKERRQ(ierr);
} /* endof for each color */
if (ctype == IS_COLORING_GLOBAL) xx = xx + start;
ierr = VecRestoreArray(coloring->vscale,&vscale_array);CHKERRQ(ierr);
ierr = VecRestoreArray(x1_tmp,&xx);CHKERRQ(ierr);
coloring->currentcolor = -1;
ierr = MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MAT_FDColoringApply,coloring,J,x1,0);CHKERRQ(ierr);
ierr = MatFDColoringViewFromOptions(coloring,"-mat_fd_coloring_view");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode CalculateRHS(TS ts, PetscReal t, Vec u, Vec r, void* ctx) {
Grid& g = *( static_cast<MyContext*>(ctx) -> gptr );
VecGhostUpdateBegin(u, INSERT_VALUES,SCATTER_FORWARD);
VecGhostUpdateEnd(u, INSERT_VALUES,SCATTER_FORWARD);
CalculateRezidual(g, u, r);
}
