int main(int argc,char **args)
{
PetscInt rank,size,npt;
PetscErrorCode ierr;
Vec x,y0,tempvec, *vinda,*vindb,*vindc;
PetscInt i,j,k,l,n,p,m,m2,pmax,puse,Istart,Iend,localsize,niter;
PetscScalar dx,dy,dx2,dy2;
PetscScalar *Mixnorm;
PetscInt iter,*iterind,*nind;
FILE *fidoutput;
char fname[50];
PetscViewer socketviewer;
PetscInt withMatlab;
PetscTruth Matlabflag;
PetscLogDouble v1,v2,elapsed_time;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// Receive n from Matlab
IntReceive(socketviewer, &nind);
n = *nind;
// Receive iter from Matlab
IntReceive(socketviewer, &iterind);
iter = *iterind;
}else{
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr);
}
/////////////////////////////////////////////////////////////////////////////////////
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter);
Mixnorm = malloc(iter*sizeof(PetscScalar));
dx = 1.0/n;
dy = 1.0/n;
dx2 = dx/2-dx/1e6;
dy2 = dy/2-dy/1e6;
npt = 5;
pmax = 4e6;
puse = pmax;
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*16 );
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*2);
/////////////////////////////////////////////////////////////////////////////////////
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
localsize = Iend-Istart;
ierr = VecDestroy(tempvec);CHKERRQ(ierr);
/////////////////////////////////////////////////////////////////////////////////////
// Create initial vector
Vec x0;
PetscScalar *x0array;
x0array = malloc((localsize)*n*sizeof(PetscScalar));
k = 0;
for(i=Istart;i<Iend;i++){
for(j=0;j<n;j++){
*(x0array+k) = cos(2*M_PI*(dx/2+i*dx));
//*(x0array+k) = cos(2*M_PI*(dy/2+j*dy));
k++;
}
}
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,n*localsize,PETSC_DECIDE,x0array,&x0);CHKERRQ(ierr);
ierr = VecDuplicate(x0,&x);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pmax*npt,&y0);CHKERRQ(ierr);
ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n );
///////////////////////////////////////////////////////////////////////////
// Map Center Points
PetscInt *NzindJ,*idx,*idy,*idp;
PetscScalar *CenterX,*CenterY,*VecVal,*pty;
PetscScalar *ShiftX,*ShiftY,CX,CY, *yarray;
IS isx,isy;
VecScatter ctx;
CenterX = malloc(npt*sizeof(PetscScalar));
CenterY = malloc(npt*sizeof(PetscScalar));
ShiftX = malloc(npt*sizeof(PetscScalar));
ShiftY = malloc(npt*sizeof(PetscScalar));
VecVal = malloc(npt*sizeof(PetscScalar));
yarray = malloc(pmax*sizeof(PetscScalar));
NzindJ = malloc(pmax*npt*sizeof(PetscInt));
idx = malloc(pmax*npt*sizeof(PetscInt));
idy = malloc(pmax*npt*sizeof(PetscInt));
idp = malloc(pmax*sizeof(PetscInt));
*(ShiftX+0) = 0;
*(ShiftY+0) = 0;
*(ShiftX+1) = -dx2;
*(ShiftY+1) = -dy2;
*(ShiftX+2) = dx2;
*(ShiftY+2) = -dy2;
*(ShiftX+3) = -dx2;
*(ShiftY+3) = dy2;
*(ShiftX+4) = dy2;
*(ShiftY+4) = dx2;
//*(ShiftX+5) = 0;
//*(ShiftY+5) = -dy2;
//*(ShiftX+6) = -dx2;
//*(ShiftY+6) = 0;
//*(ShiftX+7) = dx2;
//*(ShiftY+7) = 0;
//*(ShiftX+8) = 0;
//*(ShiftY+9) = dy2;
for(i=0;i<npt*pmax;i++){ *(idy+i)=i; }
ISCreateGeneralWithArray(PETSC_COMM_SELF,npt*pmax,idy,&isy);
vinda = &x0;
vindb = &x;
sprintf(fname, "mixnorm_%d_%d",n,iter);
ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr);
ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr);
for(niter=0;niter<iter;niter++){
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
l = 0; p = 0;
if (n*localsize-l<=pmax){puse = n*localsize-l;}else{puse=pmax;}
for(i=Istart;i<Iend;i++){
for(j=0;j<n;j++){
CX = dx2+i*dx;
CY = dy2+j*dy;
for(k=0;k<npt;k++){
*(CenterX+k) = CX + *(ShiftX+k);
*(CenterY+k) = CY + *(ShiftY+k);
InverseStandardMap((CenterX+k),(CenterY+k));
*(NzindJ+p*npt +k) = floor(*(CenterX+k)*n)*n + floor(*(CenterY+k)*n);
}
*(idp+p) = Istart*n+ l;
if(p>=puse-1){
ierr = ISCreateGeneralWithArray(PETSC_COMM_WORLD,npt*puse,NzindJ,&isx);CHKERRQ(ierr);
for(m=0;m<npt*puse;m++){ *(idy+m)=m; }
ierr = ISCreateGeneralWithArray(PETSC_COMM_SELF,npt*puse,idy,&isy);CHKERRQ(ierr);
ierr = VecScatterCreate(*vinda,isx,y0,isy,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(*vinda,y0,INSERT_VALUES,SCATTER_FORWARD,ctx);CHKERRQ(ierr);
ierr = VecScatterEnd(*vinda,y0,INSERT_VALUES,SCATTER_FORWARD,ctx);CHKERRQ(ierr);
ierr = VecScatterDestroy(ctx);
ierr = VecGetArray(y0,&pty);CHKERRQ(ierr);
for(m=0;m<puse;m++){
for(m2=0;m2<npt;m2++){
*(yarray+m) = *(yarray+m)+*(pty+m*npt+m2);
}
*(yarray+m) = *(yarray+m)/npt;
}
VecRestoreArray(y0,&pty);
VecSetValues(*vindb,puse,idp,yarray,INSERT_VALUES);
for(m=0;m<pmax;m++){*(yarray+m) = 0; }
p = 0;
if (n*localsize-l<=pmax){puse = n*localsize-l-1;}else{puse=pmax;}
}else{p++;}
l++;
}
}
VecAssemblyBegin(*vindb);
VecAssemblyEnd(*vindb);
vindc = vindb;
vindb = vinda;
vinda = vindc;
//ierr = VecCopy(x,x0);CHKERRQ(ierr);
ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr);
*(Mixnorm+niter) = *(Mixnorm+niter)/n;
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
elapsed_time = v2 - v1;
PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n"
,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
}
PetscFClose(PETSC_COMM_WORLD,fidoutput);
///////////////////////////////////////////////////////////////////////////
if(withMatlab==1){
VecView(x0,socketviewer);
PetscScalarView(iter,Mixnorm,socketviewer);
}
free(CenterX);
free(CenterY);
free(ShiftX);
free(ShiftY);
free(x0array);
free(idx);
free(idy);
free(idp);
free(yarray);
free(NzindJ);
free(Mixnorm);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(y0);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Done!");
//////////////////////////////////////////////////////////////////////////////////////
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt nredundant1 = 5,nredundant2 = 2,i,*ridx1,*ridx2,*lidx1,*lidx2,nlocal;
PetscMPIInt rank;
PetscScalar *redundant1,*redundant2;
DMComposite packer;
Vec global,local1,local2;
PF pf;
DA da1,da2;
PetscViewer sviewer;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = DMCompositeCreate(PETSC_COMM_WORLD,&packer);CHKERRQ(ierr);
ierr = PetscMalloc(nredundant1*sizeof(PetscScalar),&redundant1);CHKERRQ(ierr);
ierr = DMCompositeAddArray(packer,0,nredundant1);CHKERRQ(ierr);
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,8,1,1,PETSC_NULL,&da1);CHKERRQ(ierr);
ierr = DACreateLocalVector(da1,&local1);CHKERRQ(ierr);
ierr = DMCompositeAddDM(packer,(DM)da1);CHKERRQ(ierr);
ierr = PetscMalloc(nredundant2*sizeof(PetscScalar),&redundant2);CHKERRQ(ierr);
ierr = DMCompositeAddArray(packer,0,nredundant2);CHKERRQ(ierr);
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,6,1,1,PETSC_NULL,&da2);CHKERRQ(ierr);
ierr = DACreateLocalVector(da2,&local2);CHKERRQ(ierr);
ierr = DMCompositeAddDM(packer,(DM)da2);CHKERRQ(ierr);
ierr = DMCompositeCreateGlobalVector(packer,&global);CHKERRQ(ierr);
ierr = PFCreate(PETSC_COMM_WORLD,1,1,&pf);CHKERRQ(ierr);
ierr = PFSetType(pf,PFIDENTITY,PETSC_NULL);CHKERRQ(ierr);
ierr = PFApplyVec(pf,PETSC_NULL,global);CHKERRQ(ierr);
ierr = PFDestroy(pf);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMCompositeScatter(packer,global,redundant1,local1,redundant2,local2);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"[%d] My part of redundant1 array\n",rank);CHKERRQ(ierr);
ierr = PetscScalarView(nredundant1,redundant1,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"[%d] My part of da1 vector\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local1,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"[%d] My part of redundant2 array\n",rank);CHKERRQ(ierr);
ierr = PetscScalarView(nredundant2,redundant2,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"[%d] My part of da2 vector\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local2,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
for (i=0; i<nredundant1; i++) redundant1[i] = (rank+2)*i;
for (i=0; i<nredundant2; i++) redundant2[i] = (rank+10)*i;
ierr = DMCompositeGather(packer,global,redundant1,local1,redundant2,local2);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* get the global numbering for each subvector/array element */
ierr = DMCompositeGetGlobalIndices(packer,&ridx1,&lidx1,&ridx2,&lidx2);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"[%d] Global numbering of redundant1 array\n",rank);CHKERRQ(ierr);
ierr = PetscIntView(nredundant1,ridx1,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"[%d] Global numbering of local1 vector\n",rank);CHKERRQ(ierr);
ierr = VecGetSize(local1,&nlocal);CHKERRQ(ierr);
ierr = PetscIntView(nlocal,lidx1,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"[%d] Global numbering of redundant2 array\n",rank);CHKERRQ(ierr);
ierr = PetscIntView(nredundant2,ridx2,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"[%d] Global numbering of local2 vector\n",rank);CHKERRQ(ierr);
ierr = VecGetSize(local2,&nlocal);CHKERRQ(ierr);
ierr = PetscIntView(nlocal,lidx2,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscFree(ridx1);CHKERRQ(ierr);
ierr = PetscFree(lidx1);CHKERRQ(ierr);
ierr = PetscFree(ridx2);CHKERRQ(ierr);
ierr = PetscFree(lidx2);CHKERRQ(ierr);
ierr = DADestroy(da1);CHKERRQ(ierr);
ierr = DADestroy(da2);CHKERRQ(ierr);
ierr = VecDestroy(local1);CHKERRQ(ierr);
ierr = VecDestroy(local2);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = DMCompositeDestroy(packer);CHKERRQ(ierr);
ierr = PetscFree(redundant1);CHKERRQ(ierr);
ierr = PetscFree(redundant2);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **args)
{
PetscInt rank,size,npt;
PetscScalar dx,dy,cx,cy;
PetscErrorCode ierr;
Vec x,x0,tempvec, *vinda,*vindb,*vindc;
PetscInt i,j,k,n,n2,pmax,puse,Istart,Iend,localsize,niter;
PetscScalar **x0array, **aarray,**barray;
PetscInt *cacheInt;
PetscScalar *cacheScalar;
DA myDA;
PetscScalar *Mixnorm;
PetscInt iter,*iterind,*nind;
FILE *fidoutput, *fidtimelog;
char fname[50],ftimelog[50];
PetscViewer socketviewer;
PetscInt withMatlab, doFFT, doSmoothing;
PetscTruth Matlabflag, FFTflag, Smoothingflag;
PetscInt timelogcount;
MPI_Status status;
PetscLogDouble v1,v2,elapsed_time;
timelogcount = 0;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
ierr = PetscOptionsGetInt(PETSC_NULL,"-doFFT",&doFFT,&FFTflag);CHKERRQ(ierr);
if (FFTflag == PETSC_FALSE){doFFT = 0;}else{doFFT = 1;}
ierr = PetscOptionsGetInt(PETSC_NULL,"-doSmoothing",&doSmoothing,&Smoothingflag);CHKERRQ(ierr);
if (Smoothingflag == PETSC_FALSE){doSmoothing = 0;}else{doSmoothing = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// Receive n from Matlab
IntReceive(socketviewer, &nind);
n = *nind;
// Receive iter from Matlab
IntReceive(socketviewer, &iterind);
iter = *iterind;
}else{
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr);
}
/////////////////////////////////////////////////////////////////////////////////////
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter);
Mixnorm = malloc(iter*sizeof(PetscScalar));
dx = 1.0/n;
dy = 1.0/n;
n2 = (PetscInt)(n*0.5);
npt = 5;
pmax = 5e5;
puse = pmax;
PetscInt logmax = 1000;
PetscScalar Timelog[logmax];
PetscLogDouble t1,t2;
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*17 );
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*1);
/////////////////////////////////////////////////////////////////////////////////////
// ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
// ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
// localsize = Iend-Istart;
// ierr = VecDestroy(tempvec);CHKERRQ(ierr);
/////////////////////////////////////////////////////////////////////////////////////
if(doSmoothing==1){
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: Now Do DACreate2d \n\n\n\n" );
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: %d %d %d\n\n\n\n",n2,n,size);
DACreate2d(MPI_COMM_WORLD,DA_XYPERIODIC,DA_STENCIL_BOX,n2,n,1,size,1,2,PETSC_NULL,PETSC_NULL,&myDA);
DACreateGlobalVector(myDA,&x0);
DAGetCorners(myDA,PETSC_NULL,&Istart,PETSC_NULL,PETSC_NULL,&localsize,PETSC_NULL);
Iend = Istart+localsize;
}else{
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
localsize = Iend-Istart;
ierr = VecDestroy(tempvec);CHKERRQ(ierr);
VecCreateMPI(PETSC_COMM_WORLD,localsize*n2,PETSC_DETERMINE ,&x0);
}
//ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: So far so good\n\n\n\n");
VecGetArray2d(x0,n2,localsize,0,0,&x0array);
// Create initial vector
for(j=0;j<localsize;j++){
for(i=0;i<n2;i++){
cx = (Istart+j+0.5)*dx;
x0array[i][j] = cos(2*M_PI*cx);
}
}
VecRestoreArray2d(x0,n2,localsize,0,0,&x0array);
ierr = VecDuplicate(x0,&x);CHKERRQ(ierr);
ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n );
vinda = &x0;
vindb = &x;
sprintf(fname, "mixnorm_%d_%d",n,iter);
ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr);
ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr);
///////////////////////////////////////////////////////////////////////////////////////////////////
// Memory allocation for the iteration scheme
// cacheInt = malloc(1*pmax*sizeof(PetscInt));
// cacheScalar = malloc(2*pmax*sizeof(PetscScalar));
cacheInt = malloc(2*pmax*sizeof(PetscInt));
cacheScalar = malloc(2*pmax*sizeof(PetscScalar));
///////////////////////////////////////////////////////////////////////////////////////////////////
// Iteration here!
for(niter=0;niter<iter;niter++){
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
// BackwardAverage(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
// BackwardAverageR(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
BackwardAverageRL(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
vindc = vindb;
vindb = vinda;
vinda = vindc;
// if(doSmoothing==1){Smoothing(vinda, vindb,n, myDA, Istart,Iend);}
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
//vindc = vindb;
//vindb = vinda;
//vinda = vindc;
ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr);
*(Mixnorm+niter) = *(Mixnorm+niter)/n;
elapsed_time = v2 - v1;
PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n"
,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//Change oremtation of vector
VecGetArray2d(*vinda,n2,localsize,0,0,&aarray);
VecGetArray2d(*vindb,localsize,n2,0,0,&barray);
for(j=0;j<localsize;j++){
for(i=0;i<n2;i++){
barray[j][i] = aarray[i][j];
}
}
VecRestoreArray2d(*vinda,n2,localsize,0,0,&aarray);
VecRestoreArray2d(*vindb,localsize,n2,0,0,&barray);
vindc = vindb;
vindb = vinda;
vinda = vindc;
////////////////////////////////////////////////////////////////////////////////////////////////////
// FFT part
if(doFFT==1){FFT2D(*vinda,*vindb, localsize, n, Istart,Iend, iter,doSmoothing);}
////////////////////////////////////////////////////////////////////////////////////////////////////
/*
if(rank==0){
sprintf(ftimelog, "timelog_%d_%d",n,iter);
fidtimelog = fopen(ftimelog,"w");
for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); }
fprintf(fidtimelog,"\n ");
for(j = 1;j<size;j++){
MPI_Recv(Timelog,timelogcount,MPI_DOUBLE,j,j,PETSC_COMM_WORLD,&status);
for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); }
fprintf(fidtimelog,"\n ");
}
fclose(fidtimelog);
}else{
MPI_Send(Timelog ,timelogcount,MPI_DOUBLE,0,rank,PETSC_COMM_WORLD);
}
PetscFClose(PETSC_COMM_WORLD,fidoutput);
*/
///////////////////////////////////////////////////////////////////////////
if(withMatlab==1){
VecView(*vinda,socketviewer);
PetscScalarView(iter,Mixnorm,socketviewer);
}
// free(x0array);
free(Mixnorm);
free(cacheInt);
free(cacheScalar);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Done!");
//////////////////////////////////////////////////////////////////////////////////////
ierr = PetscFinalize();CHKERRQ(ierr);
int main(int argc,char **args)
{
Vec sol,x0,y0,z0;
PetscInt rank,size,Istart,Iend,Lsize;
Mat Dy,Dz,MarkovA,dAdy,dAdz;
PetscErrorCode ierr;
PetscViewer socketviewer;
CoordInfo *cdinfo;
// for streamline calculation
PetscScalar *xarray, *yarray,*zarray;
PetscScalar *x,*y,*z;
PetscInt n, withMatlab;
PetscTruth Matlabflag,flg;
PetscInt ny,nz;
PetscInt solmaxp;
PetscScalar solmax;
n = 5500;
withMatlab = 0;
ny = 2;
nz = 2;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
//////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////
// Use Matlab as output
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngridy",&ny,&flg);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngridz",&nz,&flg);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// receive cdinfo from Matlab
ierr = CdinfoReceive(socketviewer, &cdinfo);
// receive the velocity field from matlab.
ierr = VecReceive(socketviewer,VECSEQ,&sol);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: sol is received from Matlab \n");CHKERRQ(ierr);
//test
ierr = VecLoadfromFile(PETSC_COMM_WORLD,"sol",VECSEQ,&sol);CHKERRQ(ierr);
ierr = CdinfoLoad(PETSC_COMM_WORLD,"cdinfofiled", &cdinfo);CHKERRQ(ierr);
ierr = VecReceive(socketviewer,VECMPI,&x0);CHKERRQ(ierr);
ierr = VecReceive(socketviewer,VECMPI,&y0);CHKERRQ(ierr);
ierr = VecReceive(socketviewer,VECMPI,&z0);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: x0,y0 and z0 are received from Matlab \n");CHKERRQ(ierr);
}else{
// Load cdinfo
ierr = CdinfoLoad(PETSC_COMM_WORLD,"cdinfofiled", &cdinfo);CHKERRQ(ierr);
ierr = VecLoadfromFile(PETSC_COMM_WORLD,"sol",VECSEQ,&sol);CHKERRQ(ierr);
// Load x0, y0, z0 as initial points for streamlines
ierr = VecLoadfromFile(PETSC_COMM_WORLD,"x0filed",VECMPI,&x0);CHKERRQ(ierr);
ierr = VecLoadfromFile(PETSC_COMM_WORLD,"y0filed",VECMPI,&y0);CHKERRQ(ierr);
ierr = VecLoadfromFile(PETSC_COMM_WORLD,"z0filed",VECMPI,&z0);CHKERRQ(ierr);
}
ierr = VecMax(sol,&solmaxp,&solmax);CHKERRQ(ierr);
StreamlineMap(x0, y0, z0, sol, cdinfo, &Dy, &Dz,0.01/solmax, n ,10,&x,&y,&z);
MarkovMap2d(sol, ny, nz, cdinfo,&MarkovA,&dAdy,&dAdz);
ierr = VecGetOwnershipRange(x0,&Istart,&Iend);CHKERRQ(ierr);
Lsize = Iend - Istart;
if(withMatlab==1){
VecView(x0,socketviewer);
VecView(y0,socketviewer);
VecView(z0,socketviewer);
VecSave(x0,"xefilew");
VecSave(y0,"yefilew");
VecSave(z0,"zefilew");
PetscScalarView(Lsize*n,x,socketviewer);
PetscScalarView(Lsize*n,y,socketviewer);
PetscScalarView(Lsize*n,z,socketviewer);
SparseView(Dy,socketviewer);
SparseView(Dz,socketviewer);
}else{
SparseSave(MarkovA,"A");
VecSave(x0,"xefilew");
VecSave(y0,"yefilew");
VecSave(z0,"zefilew");
Vec vx,vy,vz;
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,Lsize*n,PETSC_DECIDE,x,&vx);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,Lsize*n,PETSC_DECIDE,y,&vy);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,Lsize*n,PETSC_DECIDE,z,&vz);CHKERRQ(ierr);
VecSave(vx,"xeallfilew");
VecSave(vy,"yeallfilew");
VecSave(vz,"zeallfilew");
ierr = VecDestroy(vx);CHKERRQ(ierr);
ierr = VecDestroy(vy);CHKERRQ(ierr);
ierr = VecDestroy(vz);CHKERRQ(ierr);
}
ierr = VecRestoreArray(x0,&xarray);CHKERRQ(ierr);
ierr = VecRestoreArray(y0,&yarray);CHKERRQ(ierr);
ierr = VecRestoreArray(z0,&zarray);CHKERRQ(ierr);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(y0);CHKERRQ(ierr);
ierr = VecDestroy(z0);CHKERRQ(ierr);
ierr = MatDestroy(Dy);CHKERRQ(ierr);
ierr = MatDestroy(Dz);CHKERRQ(ierr);
ierr = MatDestroy(dAdy);CHKERRQ(ierr);
ierr = MatDestroy(dAdz);CHKERRQ(ierr);
ierr = MatDestroy(MarkovA);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
