/*@C
DMDAVecGetArrayDOF - Returns a multiple dimension array that shares data with
the underlying vector and is indexed using the global dimensions.
Logically collective
Input Parameter:
+ da - the distributed array
- vec - the vector, either a vector the same size as one obtained with
DMCreateGlobalVector() or DMCreateLocalVector()
Output Parameter:
. array - the array
Notes:
Call DMDAVecRestoreArrayDOF() once you have finished accessing the vector entries.
In C, the indexing is "backwards" from what expects: array[k][j][i][DOF] NOT array[i][j][k][DOF]!
In Fortran 90 you do not need a version of DMDAVecRestoreArrayDOF() just use DMDAVecRestoreArrayF90() and declare your array with one higher dimension,
see src/dm/examples/tutorials/ex11f90.F
Level: intermediate
.keywords: distributed array, get, corners, nodes, local indices, coordinates
.seealso: DMDAGetGhostCorners(), DMDAGetCorners(), VecGetArray(), VecRestoreArray(), DMDAVecRestoreArray(), DMDAVecGetArray(), DMDAVecRestoreArrayDOF()
@*/
PetscErrorCode DMDAVecGetArrayDOF(DM da,Vec vec,void *array)
{
PetscErrorCode ierr;
PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof;
PetscFunctionBegin;
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gxm,&gym,&gzm);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,&dim,0,0,0,0,0,0,&dof,0,0,0,0,0);CHKERRQ(ierr);
/* Handle case where user passes in global vector as opposed to local */
ierr = VecGetLocalSize(vec,&N);CHKERRQ(ierr);
if (N == xm*ym*zm*dof) {
gxm = xm;
gym = ym;
gzm = zm;
gxs = xs;
gys = ys;
gzs = zs;
} else if (N != gxm*gym*gzm*dof) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Vector local size %D is not compatible with DMDA local sizes %D %D\n",N,xm*ym*zm*dof,gxm*gym*gzm*dof);
if (dim == 1) {
ierr = VecGetArray2d(vec,gxm,dof,gxs,0,(PetscScalar***)array);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = VecGetArray3d(vec,gym,gxm,dof,gys,gxs,0,(PetscScalar****)array);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = VecGetArray4d(vec,gzm,gym,gxm,dof,gzs,gys,gxs,0,(PetscScalar*****)array);CHKERRQ(ierr);
} else SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"DMDA dimension not 1, 2, or 3, it is %D\n",dim);
PetscFunctionReturn(0);
}
int SmoothingRL(Vec *x, Vec *y, PetscScalar *cacheScalar, PetscInt *cacheInt , VecScatter *ctx,PetscInt n, PetscInt Istart, PetscInt Iend){
PetscInt rank,size;
PetscScalar C0,C1,C2;
PetscErrorCode ierr;
PetscInt n2,i, j, k, localsizex;
PetscInt nvec;
Vec xbc,ybc,lvecx,lvecy;
PetscScalar *xbcpt,*ybcpt,**lvecptx,**lvecpty;
PetscScalar **xbcpt2,**ybcpt2;
PetscInt *bcISarray;
PetscInt left,right;
PetscInt *bcindI,*bcindJ;
nvec = 2;
IS ISfrom[nvec],ISto[nvec];
VecScatter ctxt[nvec];
Vec xvec[nvec];
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
localsizex = Iend-Istart;
C0 = 1.0/8;
C1 = 1.0/4;
C2 = 3.0/16;
n2 = (PetscInt)(n*0.5);
xbcpt = cacheScalar;
ybcpt = cacheScalar;
bcindI = cacheInt;
bcindJ = cacheInt+ 4*n2;
LargeVecCreate(x,nvec,xvec);
if(rank==0){left = n-1;}
else{left = Istart-1;}
if(rank==size-1){right = 0;}
else{right = Iend;}
for(i=0;i<n2;i++){ *(bcindI+i) = left-1;
*(bcindJ+i) = i; }
for(i=0;i<n2;i++){ *(bcindI+n2+i) = left;
*(bcindJ+n2+i) = i; }
for(i=0;i<n2;i++){ *(bcindI+n2*2+i) = right;
*(bcindJ+n2*2+i) = i; }
for(i=0;i<n2;i++){ *(bcindI+n2*3+i) = right+1;
*(bcindJ+n2*3+i) = i; }
VecCreateSeqWithArray(MPI_COMM_SELF,4*n2,cacheScalar,&xbc);
ISCreateGeneralWithIJ(MPI_COMM_SELF,x,xvec,nvec,n2,4*n2, bcindI, bcindJ,ISfrom, ISto);
LargeVecScatterCreate(xvec,ISfrom,xbc,ISto ,ctxt,nvec);
ISArrayDestroy(ISfrom,nvec);
ISArrayDestroy(ISto,nvec);
LargeVecScatterBeginEnd(xvec,xbc,INSERT_VALUES,SCATTER_FORWARD,ctxt,nvec);
VecScatterArrayDestroy(ctxt,nvec);
VecGetArray2d(*x,localsizex,n2,0,0,&lvecptx);
VecGetArray2d(*y,localsizex,n2,0,0,&lvecpty);
VecGetArray2d(xbc,4,n2,0,0,&xbcpt2);
for(j=0;j<n2;j++){
for(i=2;i<localsizex-2;i++){
lvecpty[i][j] = C0*lvecptx[i][j]
+C1*lvecptx[i-1][j] +C1*lvecptx[i+1][j]
+C2*lvecptx[i-2][j]+C2*lvecptx[i+2][j];
}
}
for(j=0;j<n2;j++){
lvecpty[0][j] = C0*lvecptx[0][j]
+C1*xbcpt2[1][j] +C1*lvecptx[1][j]
+C2*xbcpt2[0][j]+C2*lvecptx[2][j];
lvecpty[1][j] = C0*lvecptx[1][j]
+C1*lvecptx[0][j] +C1*lvecptx[2][j]
+C2*xbcpt2[1][j]+C2*lvecptx[3][j];
lvecpty[localsizex-1][j] = C0*lvecptx[localsizex-1][j]
+C1*lvecptx[localsizex-2][j] +C1*xbcpt2[2][j]
+C2*lvecptx[localsizex-3][j] +C2*xbcpt2[3][j];
lvecpty[localsizex-2][j] = C0*lvecptx[localsizex-2][j]
+C1*lvecptx[localsizex-3][j] +C1*lvecptx[localsizex-1][j]
+C2*lvecptx[localsizex-4][j] +C2*xbcpt2[2][j];
}
VecRestoreArray2d(xbc,4,n2,0,0,&xbcpt2);
VecDestroy(xbc);
//////////////////////////////////////////////////////////////////////////////////
k = 0;
for(i=Istart;i<Iend;i++){ *(bcindI+k) = n-i-1;
*(bcindJ+k) = 1;
k++; }
for(i=Istart;i<Iend;i++){ *(bcindI+k) = n-i-1;
*(bcindJ+k) = 0;
k++; }
for(i=Istart;i<Iend;i++){ *(bcindI+k) = n-i-1;
*(bcindJ+k) = n2-1;
k++; }
for(i=Istart;i<Iend;i++){ *(bcindI+k) = n-i-1;
*(bcindJ+k) = n2-2;
k++; }
VecCreateSeqWithArray(MPI_COMM_SELF,4*localsizex,cacheScalar,&ybc);
ISCreateGeneralWithIJ(MPI_COMM_SELF,*x,xvec,nvec,n2,4*localsizex, bcindI, bcindJ,ISfrom, ISto);
LargeVecScatterCreate(xvec,ISfrom,ybc,ISto ,ctxt,nvec);
ISArrayDestroy(ISfrom,nvec);
ISArrayDestroy(ISto,nvec);
LargeVecScatterBeginEnd(xvec,ybc,INSERT_VALUES,SCATTER_FORWARD,ctxt,nvec);
VecScatterArrayDestroy(ctxt,nvec);
VecGetArray2d(ybc,4,localsizex,0,0,&ybcpt2);
for(j=2;j<n2-2;j++){
for(i=0;i<localsizex;i++){
lvecptx[i][j] = C0*lvecpty[i][j]
+C1*lvecpty[i][j-1] +C1*lvecpty[i][j+1]
+C2*lvecpty[i][j-2] +C2*lvecpty[i][j+2];
}
}
for(i=0;i<localsizex;i++){
lvecptx[i][0] = C0*lvecpty[i][0]
+C1*ybcpt2[1][i] +C1*lvecpty[i][1]
+C2*ybcpt2[0][i] +C2*lvecpty[i][2];
lvecptx[i][1] = C0*lvecpty[i][1]
+C1*lvecpty[i][0] +C1*lvecpty[i][2]
+C2*ybcpt2[1][i] +C2*lvecpty[i][3];
lvecptx[i][n2-2] = C0*lvecpty[i][n2-2]
+C1*lvecpty[i][n2-3] +C1*lvecpty[i][n2-1]
+C2*lvecpty[i][n2-4] +C2*ybcpt2[2][i];
lvecptx[i][n2-1] = C0*lvecpty[i][n2-1]
+C1*lvecpty[i][n2-2] +C1*ybcpt2[2][i]
+C2*lvecpty[i][n2-3] +C2*ybcpt2[3][i];
}
VecRestoreArray2d(ybc,4,localsizex,0,0,&ybcpt2);
VecDestroy(ybc);
/////////////////////////////////////////////////////////////////////////////////
VecRestoreArray2d(*x,localsizex,n2,0,0,&lvecptx);
VecRestoreArray2d(*y,localsizex,n2,0,0,&lvecpty);
VecArrayDestroy(xvec,nvec);
return 0;
}
int Smoothing(Vec *x, Vec *y, PetscScalar *cacheScalar, PetscInt *cacheInt , VecScatter *ctx,PetscInt n,DA myDA, PetscInt Istart, PetscInt Iend){
PetscScalar C0,C1,C2;
PetscErrorCode ierr;
PetscInt localsizex;
PetscInt n2,i, j, k;
Vec lvecx,lvecy;
PetscScalar **lvecptx,**lvecpty;
Vec bcvec;
IS isbc;
//VecScatter ctx;
PetscScalar *bcpt1,*bcpt2,*bcpt3,*bcpt4;
localsizex = Iend-Istart;
C0 = 1.0/8;
C1 = 1*1.0/4;
C2 = 1*3.0/16;
n2 = (PetscInt)(n*0.5);
DACreateLocalVector(myDA,&lvecx);
DACreateLocalVector(myDA,&lvecy);
DAGlobalToLocalBegin(myDA,*x,INSERT_VALUES,lvecx);
DAGlobalToLocalEnd(myDA,*x,INSERT_VALUES,lvecx);
VecGetArray2d(lvecx,localsizex+4,n2+4,0,0,&lvecptx);
VecGetArray2d(lvecy,localsizex+4,n2+4,0,0,&lvecpty);
// X direction smoothing
for(j=0;j<n2+4;j++){
for(i=2;i<localsizex+2;i++){
lvecpty[i][j] = C0*lvecptx[i][j]
+C1*lvecptx[i-1][j] +C1*lvecptx[i+1][j]
+C2*lvecptx[i-2][j]+C2*lvecptx[i+2][j];
}
}
VecCreateSeqWithArray(MPI_COMM_SELF,localsizex*4,cacheScalar,&bcvec);
VecScatterBegin(*x,bcvec,INSERT_VALUES,SCATTER_FORWARD,*ctx);
VecScatterEnd(*x,bcvec,INSERT_VALUES,SCATTER_FORWARD,*ctx);
bcpt1 = cacheScalar;
bcpt2 = cacheScalar+localsizex;
bcpt3 = cacheScalar+localsizex*2;
bcpt4 = cacheScalar+localsizex*3;
k= 0;
for(i=2;i<localsizex+2;i++){
lvecpty[i][0]= *(bcpt3+k);
lvecpty[i][1]= *(bcpt4+k);
lvecpty[i][n2+3]= *(bcpt2+k);
lvecpty[i][n2+2]= *(bcpt1+k);
k++;
}
// Y direction smoothing
for(j=2;j<n2+2;j++){
for(i=2;i<localsizex+2;i++){
lvecptx[i][j] = C0*lvecpty[i][j]
+C1*lvecpty[i][j-1] +C1*lvecpty[i][j+1]
+C2*lvecpty[i][j-2] +C2*lvecpty[i][j+2];
}
}
VecRestoreArray2d(lvecy,localsizex+4,n2+4,0,0,&lvecpty);
VecRestoreArray2d(lvecx,localsizex+4,n2+4,0,0,&lvecptx);
DALocalToGlobal(myDA,lvecx,INSERT_VALUES,*x);
VecDestroy(bcvec);
VecDestroy(lvecx);
VecDestroy(lvecy);
return 0;
}
int main(int argc,char **args)
{
PetscInt rank,size,npt;
PetscScalar dx,dy,cx,cy;
PetscErrorCode ierr;
Vec x,x0,tempvec, *vinda,*vindb,*vindc;
PetscInt i,j,k,n,n2,pmax,puse,Istart,Iend,localsize,niter;
PetscScalar **x0array, **aarray,**barray;
PetscInt *cacheInt;
PetscScalar *cacheScalar;
DA myDA;
PetscScalar *Mixnorm;
PetscInt iter,*iterind,*nind;
FILE *fidoutput, *fidtimelog;
char fname[50],ftimelog[50];
PetscViewer socketviewer;
PetscInt withMatlab, doFFT, doSmoothing;
PetscTruth Matlabflag, FFTflag, Smoothingflag;
PetscInt timelogcount;
MPI_Status status;
PetscLogDouble v1,v2,elapsed_time;
timelogcount = 0;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
ierr = PetscOptionsGetInt(PETSC_NULL,"-doFFT",&doFFT,&FFTflag);CHKERRQ(ierr);
if (FFTflag == PETSC_FALSE){doFFT = 0;}else{doFFT = 1;}
ierr = PetscOptionsGetInt(PETSC_NULL,"-doSmoothing",&doSmoothing,&Smoothingflag);CHKERRQ(ierr);
if (Smoothingflag == PETSC_FALSE){doSmoothing = 0;}else{doSmoothing = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// Receive n from Matlab
IntReceive(socketviewer, &nind);
n = *nind;
// Receive iter from Matlab
IntReceive(socketviewer, &iterind);
iter = *iterind;
}else{
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr);
}
/////////////////////////////////////////////////////////////////////////////////////
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter);
Mixnorm = malloc(iter*sizeof(PetscScalar));
dx = 1.0/n;
dy = 1.0/n;
n2 = (PetscInt)(n*0.5);
npt = 5;
pmax = 5e5;
puse = pmax;
PetscInt logmax = 1000;
PetscScalar Timelog[logmax];
PetscLogDouble t1,t2;
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*17 );
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*1);
/////////////////////////////////////////////////////////////////////////////////////
// ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
// ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
// localsize = Iend-Istart;
// ierr = VecDestroy(tempvec);CHKERRQ(ierr);
/////////////////////////////////////////////////////////////////////////////////////
if(doSmoothing==1){
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: Now Do DACreate2d \n\n\n\n" );
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: %d %d %d\n\n\n\n",n2,n,size);
DACreate2d(MPI_COMM_WORLD,DA_XYPERIODIC,DA_STENCIL_BOX,n2,n,1,size,1,2,PETSC_NULL,PETSC_NULL,&myDA);
DACreateGlobalVector(myDA,&x0);
DAGetCorners(myDA,PETSC_NULL,&Istart,PETSC_NULL,PETSC_NULL,&localsize,PETSC_NULL);
Iend = Istart+localsize;
}else{
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
localsize = Iend-Istart;
ierr = VecDestroy(tempvec);CHKERRQ(ierr);
VecCreateMPI(PETSC_COMM_WORLD,localsize*n2,PETSC_DETERMINE ,&x0);
}
//ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: So far so good\n\n\n\n");
VecGetArray2d(x0,n2,localsize,0,0,&x0array);
// Create initial vector
for(j=0;j<localsize;j++){
for(i=0;i<n2;i++){
cx = (Istart+j+0.5)*dx;
x0array[i][j] = cos(2*M_PI*cx);
}
}
VecRestoreArray2d(x0,n2,localsize,0,0,&x0array);
ierr = VecDuplicate(x0,&x);CHKERRQ(ierr);
ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n );
vinda = &x0;
vindb = &x;
sprintf(fname, "mixnorm_%d_%d",n,iter);
ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr);
ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr);
///////////////////////////////////////////////////////////////////////////////////////////////////
// Memory allocation for the iteration scheme
// cacheInt = malloc(1*pmax*sizeof(PetscInt));
// cacheScalar = malloc(2*pmax*sizeof(PetscScalar));
cacheInt = malloc(2*pmax*sizeof(PetscInt));
cacheScalar = malloc(2*pmax*sizeof(PetscScalar));
///////////////////////////////////////////////////////////////////////////////////////////////////
// Iteration here!
for(niter=0;niter<iter;niter++){
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
// BackwardAverage(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
// BackwardAverageR(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
BackwardAverageRL(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
vindc = vindb;
vindb = vinda;
vinda = vindc;
// if(doSmoothing==1){Smoothing(vinda, vindb,n, myDA, Istart,Iend);}
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
//vindc = vindb;
//vindb = vinda;
//vinda = vindc;
ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr);
*(Mixnorm+niter) = *(Mixnorm+niter)/n;
elapsed_time = v2 - v1;
PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n"
,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//Change oremtation of vector
VecGetArray2d(*vinda,n2,localsize,0,0,&aarray);
VecGetArray2d(*vindb,localsize,n2,0,0,&barray);
for(j=0;j<localsize;j++){
for(i=0;i<n2;i++){
barray[j][i] = aarray[i][j];
}
}
VecRestoreArray2d(*vinda,n2,localsize,0,0,&aarray);
VecRestoreArray2d(*vindb,localsize,n2,0,0,&barray);
vindc = vindb;
vindb = vinda;
vinda = vindc;
////////////////////////////////////////////////////////////////////////////////////////////////////
// FFT part
if(doFFT==1){FFT2D(*vinda,*vindb, localsize, n, Istart,Iend, iter,doSmoothing);}
////////////////////////////////////////////////////////////////////////////////////////////////////
/*
if(rank==0){
sprintf(ftimelog, "timelog_%d_%d",n,iter);
fidtimelog = fopen(ftimelog,"w");
for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); }
fprintf(fidtimelog,"\n ");
for(j = 1;j<size;j++){
MPI_Recv(Timelog,timelogcount,MPI_DOUBLE,j,j,PETSC_COMM_WORLD,&status);
for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); }
fprintf(fidtimelog,"\n ");
}
fclose(fidtimelog);
}else{
MPI_Send(Timelog ,timelogcount,MPI_DOUBLE,0,rank,PETSC_COMM_WORLD);
}
PetscFClose(PETSC_COMM_WORLD,fidoutput);
*/
///////////////////////////////////////////////////////////////////////////
if(withMatlab==1){
VecView(*vinda,socketviewer);
PetscScalarView(iter,Mixnorm,socketviewer);
}
// free(x0array);
free(Mixnorm);
free(cacheInt);
free(cacheScalar);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Done!");
//////////////////////////////////////////////////////////////////////////////////////
ierr = PetscFinalize();CHKERRQ(ierr);