int main(int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start %s\n", __FILE__); CHKERRQ(ierr);
MPI_Comm comm = PETSC_COMM_WORLD;
DA da;
Mat mat;
int MX = 100;
int dof = 3;
ierr = PetscPrintf(comm,"Started Assembly\n"); CHKERRQ(ierr);
ierr = MatMake(MX,dof,&da, &mat); CHKERRQ(ierr);
ierr = MatWrite("J",mat); CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Finished Assembly\n"); CHKERRQ(ierr);
Vec rhs, sol;
ierr = DACreateGlobalVector(da,&rhs); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&sol); CHKERRQ(ierr);
int i,j;
int xs,ys,xm,ym;
Field **array;
ierr = DAVecGetArray(da,rhs,&array); CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL); CHKERRQ(ierr);
for (j = xs; j < xm; ++j) {
for (i = ys; i < ym; ++i) {
if( 3*MX/4 > i && i > MX/4 &&
3*MX/4 > j && j > MX/4 )
{
array[j][i].u = 100.;
array[j][i].v = 100.;
}
}
}
ierr = DAVecRestoreArray(da,rhs,&array); CHKERRQ(ierr);
KSP ksp;
ierr = KSPCreate(comm,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY); CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,mat,mat,DIFFERENT_NONZERO_PATTERN); CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
//Split pressure from velocity
PC pc;
ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr);
ierr = PCFieldSplitSetFields(pc,2,(int[]){0,1}); CHKERRQ(ierr);
int main( int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start\n"); CHKERRQ(ierr);
DA da;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
64,64,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
printf("[%d] rank\n", rank);
Vec p;
ierr = DACreateGlobalVector(da,&p); CHKERRQ(ierr);
ierr = VecSet( p, 1.);
char file[256] = "vec";
sprintf( file, "vec-%d", rank);
PetscViewer view;
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_WRITE,&view
); CHKERRQ(ierr);
ierr = VecView(p, view); CHKERRQ(ierr);
ierr = PetscViewerDestroy(view); CHKERRQ(ierr);
ierr = VecDestroy( p );
ierr = DADestroy( da );
ierr = PetscPrintf(PETSC_COMM_WORLD, "End\n"); CHKERRQ(ierr);
ierr = PetscFinalize(); CHKERRQ(ierr);
}
int main(int argc,char **argv)
{
PetscInt dof = 2,M = 3,N = 3,m = PETSC_DECIDE,n = PETSC_DECIDE;
PetscErrorCode ierr;
DA da;
Vec global,local;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(0,"-dof",&dof,0);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_BOX,
M,N,m,n,dof,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = doit(da,global);CHKERRQ(ierr);
ierr = VecView(global,0);CHKERRQ(ierr);
/* Free memory */
ierr = VecDestroy(local);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
/*
Given a DA generates a VecScatter context that will deliver a slice
of the global vector to each processor. In this example, each processor
receives the values i=*, j=*, k=rank, i.e. one z plane.
Note: This code is written assuming only one degree of freedom per node.
For multiple degrees of freedom per node use ISCreateBlock()
instead of ISCreateGeneral().
*/
PetscErrorCode GenerateSliceScatter(DA da,VecScatter *scatter,Vec *vslice)
{
AO ao;
PetscInt M,N,P,nslice,*sliceindices,count,i,j;
PetscMPIInt rank;
PetscErrorCode ierr;
MPI_Comm comm;
Vec vglobal;
IS isfrom,isto;
ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = DAGetAO(da,&ao);CHKERRQ(ierr);
ierr = DAGetInfo(da,0,&M,&N,&P,0,0,0,0,0,0,0);CHKERRQ(ierr);
/*
nslice is number of degrees of freedom in this processors slice
if there are more processors then z plans the extra processors get 0
elements in their slice.
*/
if (rank < P) {nslice = M*N;} else nslice = 0;
/*
Generate the local vector to hold this processors slice
*/
ierr = VecCreateSeq(PETSC_COMM_SELF,nslice,vslice);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&vglobal);CHKERRQ(ierr);
/*
Generate the indices for the slice in the "natural" global ordering
Note: this is just an example, one could select any subset of nodes
on each processor. Just list them in the global natural ordering.
*/
ierr = PetscMalloc((nslice+1)*sizeof(PetscInt),&sliceindices);CHKERRQ(ierr);
count = 0;
if (rank < P) {
for (j=0; j<N; j++) {
for (i=0; i<M; i++) {
sliceindices[count++] = rank*M*N + j*M + i;
}
}
}
/*
Convert the indices to the "PETSc" global ordering
*/
ierr = AOApplicationToPetsc(ao,nslice,sliceindices);CHKERRQ(ierr);
/* Create the "from" and "to" index set */
/* This is to scatter from the global vector */
ierr = ISCreateGeneral(PETSC_COMM_SELF,nslice,sliceindices,&isfrom);CHKERRQ(ierr);
/* This is to gather into the local vector */
ierr = ISCreateStride(PETSC_COMM_SELF,nslice,0,1,&isto);CHKERRQ(ierr);
ierr = VecScatterCreate(vglobal,isfrom,*vslice,isto,scatter);CHKERRQ(ierr);
ierr = ISDestroy(isfrom);CHKERRQ(ierr);
ierr = ISDestroy(isto);CHKERRQ(ierr);
ierr = PetscFree(sliceindices);CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
Vec u,xy;
DA da;
PetscErrorCode ierr;
PetscInt m = 10, n = 10, dof = 2;
PF pf;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_BOX,m,n,PETSC_DECIDE,PETSC_DECIDE,dof,1,0,0,&da);CHKERRQ(ierr);
ierr = DASetUniformCoordinates(da,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&u);CHKERRQ(ierr);
ierr = DAGetCoordinates(da,&xy);CHKERRQ(ierr);
ierr = DACreatePF(da,&pf);CHKERRQ(ierr);
ierr = PFSet(pf,myfunction,0,0,0,0);CHKERRQ(ierr);
ierr = PFSetFromOptions(pf);CHKERRQ(ierr);
ierr = PFApplyVec(pf,xy,u);CHKERRQ(ierr);
ierr = VecView(u,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = VecDestroy(xy);CHKERRQ(ierr);
ierr = PFDestroy(pf);CHKERRQ(ierr);
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscViewer viewer;
DA da;
Vec global,local,global2;
PetscMPIInt rank;
PetscTruth flg;
/*
Every PETSc routine should begin with the PetscInitialize() routine.
argc, argv - These command line arguments are taken to extract the options
supplied to PETSc and options supplied to MPI.
help - When PETSc executable is invoked with the option -help,
it prints the various options that can be applied at
runtime. The user can use the "help" variable place
additional help messages in this printout.
*/
ierr = PetscInitialize(&argc,&argv,(char *)0,help);CHKERRQ(ierr);
/* Create a DA and an associated vector */
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_BOX,100,90,PETSC_DECIDE,PETSC_DECIDE,2,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = VecSet(global,-1.0);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = VecScale(local,rank+1);CHKERRQ(ierr);
ierr = DALocalToGlobal(da,local,ADD_VALUES,global);CHKERRQ(ierr);
/*
Write output file with PetscViewerHDF5 viewer.
*/
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"hdf5output",FILE_MODE_WRITE,&viewer); CHKERRQ(ierr);
ierr = VecView(global,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
ierr = VecDuplicate(global,&global2);CHKERRQ(ierr);
ierr = VecCopy(global,global2);CHKERRQ(ierr);
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"hdf5output",FILE_MODE_READ,&viewer); CHKERRQ(ierr);
ierr = VecLoadIntoVector(viewer,global);CHKERRQ(ierr);
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
ierr = VecEqual(global,global2,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Vectors are equal\n");CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Vectors are not equal\n");CHKERRQ(ierr);
}
/* clean up and exit */
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = VecDestroy(local);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = VecDestroy(global2);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE,M = 3,N = 5,P=3,s=1;
PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL,*lz = PETSC_NULL;
PetscErrorCode ierr;
PetscTruth flg = PETSC_FALSE;
DA da;
Vec local,global,vslice;
PetscScalar value;
DAPeriodicType wrap = DA_XYPERIODIC;
DAStencilType stencil_type = DA_STENCIL_BOX;
VecScatter scatter;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-P",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetTruth(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DA_STENCIL_STAR;
/* Create distributed array and get vectors */
ierr = DACreate3d(PETSC_COMM_WORLD,wrap,stencil_type,M,N,P,m,n,p,1,s,
lx,ly,lz,&da);CHKERRQ(ierr);
ierr = DAView(da,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = GenerateSliceScatter(da,&scatter,&vslice);CHKERRQ(ierr);
/* Put the value rank+1 into all locations of vslice and transfer back to global vector */
value = 1.0 + rank;
ierr = VecSet(vslice,value);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,vslice,global,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,vslice,global,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(local);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv) {
SNES snes; Vec x,f; Mat J; DA da;
PetscInitialize(&argc,&argv,(char *)0,help);
DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,8,1,1,PETSC_NULL,&da);
DACreateGlobalVector(da,&x); VecDuplicate(x,&f);
DAGetMatrix(da,MATAIJ,&J);
SNESCreate(PETSC_COMM_WORLD,&snes);
SNESSetFunction(snes,f,ComputeFunction,da);
SNESSetJacobian(snes,J,J,ComputeJacobian,da);
SNESSetFromOptions(snes);
SNESSolve(snes,PETSC_NULL,x);
MatDestroy(J); VecDestroy(x); VecDestroy(f); SNESDestroy(snes); DADestroy(da);
PetscFinalize();
return 0;}
/* For now, we assume that only the bottom boundary is changing! No I have upgraded it to a cooler sexier version! :-) */
Immersed *Immersed_create(MAC_grid *grid, char which_quantity) {
int N;
Immersed *new_immersed;
int NZ, NY, NX;
int ierr;
NX = grid->NX;
NY = grid->NY;
NZ = grid->NZ;
new_immersed = (Immersed *)malloc(sizeof(Immersed));
Memory_check_allocation(new_immersed);
/* Total number of the immersed nodes. It should be more or equal to NX */
/* TBC */
N = Immersed_count_immersed_nodes(grid, which_quantity);
PetscPrintf(PCW, "Found %d immeresed points in the domain on the current processor for the %c-quantity\n", N, which_quantity);
/* Maximum number of immersed nodes in the whole domain */
new_immersed->N = N;
/* Allocate the memory for each individual immersed node */
new_immersed->ib_nodes = (ImmersedNode *)calloc(N, sizeof(ImmersedNode));
Memory_check_allocation(new_immersed->ib_nodes);
/* Allocate memory for the global index of the current ib_node in the entire domain. If not an ib_node, the value is -1 */
/* This will help finding the (i,j,k) location of the immersed node */
ierr = DACreateGlobalVector(grid->DA_3D, &new_immersed->G_global_index); PETScErrAct(ierr);
ierr = DACreateLocalVector(grid->DA_3D, &new_immersed->L_global_index); PETScErrAct(ierr);
/* Pass the 3d int-pointer to the L_global_index. This will help save a huge amount of time */
ierr = DAVecGetArray(grid->DA_3D, new_immersed->L_global_index, (void ***)&new_immersed->global_index); PETScErrAct(ierr);
/* u, v, c (or p) */
new_immersed->quantity = which_quantity;
return new_immersed;
}
int main(int argc,char **args)
{
PetscInt rank,size,npt;
PetscScalar dx,dy,cx,cy;
PetscErrorCode ierr;
Vec x,x0,tempvec, *vinda,*vindb,*vindc;
PetscInt i,j,k,n,n2,pmax,puse,Istart,Iend,localsize,niter;
PetscScalar **x0array, **aarray,**barray;
PetscInt *cacheInt;
PetscScalar *cacheScalar;
DA myDA;
PetscScalar *Mixnorm;
PetscInt iter,*iterind,*nind;
FILE *fidoutput, *fidtimelog;
char fname[50],ftimelog[50];
PetscViewer socketviewer;
PetscInt withMatlab, doFFT, doSmoothing;
PetscTruth Matlabflag, FFTflag, Smoothingflag;
PetscInt timelogcount;
MPI_Status status;
PetscLogDouble v1,v2,elapsed_time;
timelogcount = 0;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
ierr = PetscOptionsGetInt(PETSC_NULL,"-doFFT",&doFFT,&FFTflag);CHKERRQ(ierr);
if (FFTflag == PETSC_FALSE){doFFT = 0;}else{doFFT = 1;}
ierr = PetscOptionsGetInt(PETSC_NULL,"-doSmoothing",&doSmoothing,&Smoothingflag);CHKERRQ(ierr);
if (Smoothingflag == PETSC_FALSE){doSmoothing = 0;}else{doSmoothing = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// Receive n from Matlab
IntReceive(socketviewer, &nind);
n = *nind;
// Receive iter from Matlab
IntReceive(socketviewer, &iterind);
iter = *iterind;
}else{
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr);
}
/////////////////////////////////////////////////////////////////////////////////////
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter);
Mixnorm = malloc(iter*sizeof(PetscScalar));
dx = 1.0/n;
dy = 1.0/n;
n2 = (PetscInt)(n*0.5);
npt = 5;
pmax = 5e5;
puse = pmax;
PetscInt logmax = 1000;
PetscScalar Timelog[logmax];
PetscLogDouble t1,t2;
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*17 );
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*1);
/////////////////////////////////////////////////////////////////////////////////////
// ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
// ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
// localsize = Iend-Istart;
// ierr = VecDestroy(tempvec);CHKERRQ(ierr);
/////////////////////////////////////////////////////////////////////////////////////
if(doSmoothing==1){
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: Now Do DACreate2d \n\n\n\n" );
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: %d %d %d\n\n\n\n",n2,n,size);
DACreate2d(MPI_COMM_WORLD,DA_XYPERIODIC,DA_STENCIL_BOX,n2,n,1,size,1,2,PETSC_NULL,PETSC_NULL,&myDA);
DACreateGlobalVector(myDA,&x0);
DAGetCorners(myDA,PETSC_NULL,&Istart,PETSC_NULL,PETSC_NULL,&localsize,PETSC_NULL);
Iend = Istart+localsize;
}else{
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
localsize = Iend-Istart;
ierr = VecDestroy(tempvec);CHKERRQ(ierr);
VecCreateMPI(PETSC_COMM_WORLD,localsize*n2,PETSC_DETERMINE ,&x0);
}
//ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: So far so good\n\n\n\n");
VecGetArray2d(x0,n2,localsize,0,0,&x0array);
// Create initial vector
for(j=0;j<localsize;j++){
for(i=0;i<n2;i++){
cx = (Istart+j+0.5)*dx;
x0array[i][j] = cos(2*M_PI*cx);
}
}
VecRestoreArray2d(x0,n2,localsize,0,0,&x0array);
ierr = VecDuplicate(x0,&x);CHKERRQ(ierr);
ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n );
vinda = &x0;
vindb = &x;
sprintf(fname, "mixnorm_%d_%d",n,iter);
ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr);
ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr);
///////////////////////////////////////////////////////////////////////////////////////////////////
// Memory allocation for the iteration scheme
// cacheInt = malloc(1*pmax*sizeof(PetscInt));
// cacheScalar = malloc(2*pmax*sizeof(PetscScalar));
cacheInt = malloc(2*pmax*sizeof(PetscInt));
cacheScalar = malloc(2*pmax*sizeof(PetscScalar));
///////////////////////////////////////////////////////////////////////////////////////////////////
// Iteration here!
for(niter=0;niter<iter;niter++){
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
// BackwardAverage(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
// BackwardAverageR(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
BackwardAverageRL(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend);
vindc = vindb;
vindb = vinda;
vinda = vindc;
// if(doSmoothing==1){Smoothing(vinda, vindb,n, myDA, Istart,Iend);}
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
//vindc = vindb;
//vindb = vinda;
//vinda = vindc;
ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr);
*(Mixnorm+niter) = *(Mixnorm+niter)/n;
elapsed_time = v2 - v1;
PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n"
,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//Change oremtation of vector
VecGetArray2d(*vinda,n2,localsize,0,0,&aarray);
VecGetArray2d(*vindb,localsize,n2,0,0,&barray);
for(j=0;j<localsize;j++){
for(i=0;i<n2;i++){
barray[j][i] = aarray[i][j];
}
}
VecRestoreArray2d(*vinda,n2,localsize,0,0,&aarray);
VecRestoreArray2d(*vindb,localsize,n2,0,0,&barray);
vindc = vindb;
vindb = vinda;
vinda = vindc;
////////////////////////////////////////////////////////////////////////////////////////////////////
// FFT part
if(doFFT==1){FFT2D(*vinda,*vindb, localsize, n, Istart,Iend, iter,doSmoothing);}
////////////////////////////////////////////////////////////////////////////////////////////////////
/*
if(rank==0){
sprintf(ftimelog, "timelog_%d_%d",n,iter);
fidtimelog = fopen(ftimelog,"w");
for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); }
fprintf(fidtimelog,"\n ");
for(j = 1;j<size;j++){
MPI_Recv(Timelog,timelogcount,MPI_DOUBLE,j,j,PETSC_COMM_WORLD,&status);
for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); }
fprintf(fidtimelog,"\n ");
}
fclose(fidtimelog);
}else{
MPI_Send(Timelog ,timelogcount,MPI_DOUBLE,0,rank,PETSC_COMM_WORLD);
}
PetscFClose(PETSC_COMM_WORLD,fidoutput);
*/
///////////////////////////////////////////////////////////////////////////
if(withMatlab==1){
VecView(*vinda,socketviewer);
PetscScalarView(iter,Mixnorm,socketviewer);
}
// free(x0array);
free(Mixnorm);
free(cacheInt);
free(cacheScalar);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Done!");
//////////////////////////////////////////////////////////////////////////////////////
ierr = PetscFinalize();CHKERRQ(ierr);
int main(int argc,char **argv)
{
PetscErrorCode ierr;
SNES snes; /* nonlinear solver */
Vec Hu,r; /* solution, residual vectors */
Mat J; /* Jacobian matrix */
AppCtx user; /* user-defined work context */
PetscInt its, i, tmpxs, tmpxm; /* iteration count, index, etc. */
PetscReal tmp1, tmp2, tmp3, tmp4, tmp5,
errnorms[2], descaleNode[2];
PetscTruth eps_set = PETSC_FALSE, dump = PETSC_FALSE, exactinitial = PETSC_FALSE,
snes_mf_set, snes_fd_set;
MatFDColoring matfdcoloring = 0;
ISColoring iscoloring;
SNESConvergedReason reason; /* Check convergence */
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &user.rank); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"BODVARDSSON solves for thickness and velocity in 1D, steady ice stream\n"
" [run with -help for info and options]\n");CHKERRQ(ierr);
user.n = 3.0; /* Glen flow law exponent */
user.secpera = 31556926.0;
user.rho = 910.0; /* kg m^-3 */
user.rhow = 1028.0; /* kg m^-3 */
user.g = 9.81; /* m s^-2 */
/* ask Test N for its parameters, but only those we need to solve */
ierr = params_exactN(&(user.H0), &tmp1, &(user.xc), &tmp2, &tmp3, &tmp4, &tmp5,
&(user.Txc)); CHKERRQ(ierr);
/* regularize using strain rate of 1/xc per year */
user.epsilon = (1.0 / user.secpera) / user.xc;
/* tools for non-dimensionalizing to improve equation scaling */
user.scaleNode[0] = 1000.0; user.scaleNode[1] = 100.0 / user.secpera;
ierr = PetscOptionsTruth("-snes_mf","","",PETSC_FALSE,&snes_mf_set,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-snes_fd","","",PETSC_FALSE,&snes_fd_set,NULL);CHKERRQ(ierr);
if (!snes_mf_set && !snes_fd_set) {
PetscPrintf(PETSC_COMM_WORLD,
"\n***ERROR: bodvardsson needs one or zero of '-snes_mf', '-snes_fd'***\n\n"
"USAGE FOLLOWS ...\n\n%s",help);
PetscEnd();
}
if (snes_fd_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" using approximate Jacobian; finite-differencing using coloring\n");
CHKERRQ(ierr);
} else if (snes_mf_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" matrix free; no preconditioner\n"); CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" true Jacobian\n"); CHKERRQ(ierr);
}
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,
"bodvardsson program options",__FILE__);CHKERRQ(ierr);
{
ierr = PetscOptionsTruth("-bod_up_one","","",PETSC_FALSE,&user.upwind1,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-bod_exact_init","","",PETSC_FALSE,&exactinitial,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-bod_dump",
"dump out exact and approximate solution and residual, as asci","",
dump,&dump,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-bod_epsilon","regularization (a strain rate in units of 1/a)","",
user.epsilon * user.secpera,&user.epsilon,&eps_set);CHKERRQ(ierr);
if (eps_set) user.epsilon *= 1.0 / user.secpera;
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
/* Create machinery for parallel grid management (DA), nonlinear solver (SNES),
and Vecs for fields (solution, RHS). Note default Mx=46 grid points means
dx=10 km. Also degrees of freedom = 2 (thickness and velocity
at each point) and stencil radius = ghost width = 2 for 2nd-order upwinding. */
user.solnghostwidth = 2;
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,-46,2,user.solnghostwidth,PETSC_NULL,&user.da);
CHKERRQ(ierr);
ierr = DASetUniformCoordinates(user.da,0.0,user.xc,
PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DASetFieldName(user.da,0,"ice thickness [non-dimensional]"); CHKERRQ(ierr);
ierr = DASetFieldName(user.da,1,"ice velocity [non-dimensional]"); CHKERRQ(ierr);
ierr = DAGetInfo(user.da,PETSC_IGNORE,&user.Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
ierr = DAGetCorners(user.da,&user.xs,PETSC_NULL,PETSC_NULL,&user.xm,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
user.dx = user.xc / (PetscReal)(user.Mx-1);
/* another DA for scalar parameters, with same length */
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,user.Mx,1,1,PETSC_NULL,&user.scalarda);CHKERRQ(ierr);
ierr = DASetUniformCoordinates(user.scalarda,0.0,user.xc,
PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
/* check that parallel layout of scalar DA is same as dof=2 DA */
ierr = DAGetCorners(user.scalarda,&tmpxs,PETSC_NULL,PETSC_NULL,&tmpxm,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
if ((tmpxs != user.xs) || (tmpxm != user.xm)) {
PetscPrintf(PETSC_COMM_SELF,
"\n***ERROR: rank %d gets different ownership range for the two DAs! ENDING ...***\n\n",
user.rank);
PetscEnd();
}
ierr = PetscPrintf(PETSC_COMM_WORLD,
" Mx = %D points, dx = %.3f m\n H0 = %.2f m, xc = %.2f km, Txc = %.5e Pa m\n",
user.Mx, user.dx, user.H0, user.xc/1000.0, user.Txc);CHKERRQ(ierr);
/* Extract/allocate global vectors from DAs and duplicate for remaining same types */
ierr = DACreateGlobalVector(user.da,&Hu);CHKERRQ(ierr);
ierr = VecSetBlockSize(Hu,2);CHKERRQ(ierr);
ierr = VecDuplicate(Hu,&r);CHKERRQ(ierr); /* inherits block size */
ierr = VecDuplicate(Hu,&user.Huexact);CHKERRQ(ierr); /* ditto */
ierr = DACreateGlobalVector(user.scalarda,&user.M);CHKERRQ(ierr);
ierr = VecDuplicate(user.M,&user.Bstag);CHKERRQ(ierr);
ierr = VecDuplicate(user.M,&user.beta);CHKERRQ(ierr);
ierr = DASetLocalFunction(user.da,(DALocalFunction1)scshell);CHKERRQ(ierr);
ierr = DASetLocalJacobian(user.da,(DALocalFunction1)BodJacobianMatrixLocal);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
ierr = SNESSetFunction(snes,r,SNESDAFormFunction,&user);CHKERRQ(ierr);
/* setting up a matrix is only actually needed for -snes_fd case */
ierr = DAGetMatrix(user.da,MATAIJ,&J);CHKERRQ(ierr);
if (snes_fd_set) {
/* tools needed so DA can use sparse matrix for its F.D. Jacobian approx */
ierr = DAGetColoring(user.da,IS_COLORING_GLOBAL,MATAIJ,&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringCreate(J,iscoloring,&matfdcoloring);CHKERRQ(ierr);
ierr = ISColoringDestroy(iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringSetFunction(matfdcoloring,
(PetscErrorCode (*)(void))SNESDAFormFunction,&user);CHKERRQ(ierr);
ierr = MatFDColoringSetFromOptions(matfdcoloring);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes,J,J,SNESDefaultComputeJacobianColor,matfdcoloring);CHKERRQ(ierr);
} else {
ierr = SNESSetJacobian(snes,J,J,SNESDAComputeJacobian,&user);CHKERRQ(ierr);
}
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
/* the the Bodvardsson (1955) exact solution allows setting M(x), B(x), beta(x), T(xc) */
ierr = FillDistributedParams(&user);CHKERRQ(ierr);
/* the exact thickness and exact ice velocity (user.uHexact) are known from Bodvardsson (1955) */
ierr = FillExactSoln(&user); CHKERRQ(ierr);
if (exactinitial) {
ierr = PetscPrintf(PETSC_COMM_WORLD," using exact solution as initial guess\n");
CHKERRQ(ierr);
/* the initial guess is the exact continuum solution */
ierr = VecCopy(user.Huexact,Hu); CHKERRQ(ierr);
} else {
ierr = FillInitial(&user, &Hu); CHKERRQ(ierr);
}
/************ SOLVE NONLINEAR SYSTEM ************/
/* recall that RHS r is used internally by KSP, and is set by the SNES */
for (i = 0; i < 2; i++) descaleNode[i] = 1.0 / user.scaleNode[i];
ierr = VecStrideScaleAll(Hu,descaleNode); CHKERRQ(ierr); /* de-dimensionalize initial guess */
ierr = SNESSolve(snes,PETSC_NULL,Hu);CHKERRQ(ierr);
ierr = VecStrideScaleAll(Hu,user.scaleNode); CHKERRQ(ierr); /* put back in "real" scale */
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = SNESGetConvergedReason(snes,&reason);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" %s Number of Newton iterations = %D\n",
SNESConvergedReasons[reason],its);CHKERRQ(ierr);
if (dump) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing combined result Hu\n");CHKERRQ(ierr);
ierr = VecView(Hu,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing combined exact result Huexact\n");CHKERRQ(ierr);
ierr = VecView(user.Huexact,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing final combined residual at Hu\n");CHKERRQ(ierr);
ierr = VecView(r,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
}
/* evaluate error relative to exact solution */
ierr = VecAXPY(Hu,-1.0,user.Huexact); CHKERRQ(ierr); /* Hu = - Huexact + Hu */
ierr = VecStrideNormAll(Hu,NORM_INFINITY,errnorms); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"(dx,errHinf,erruinf) %.3f %.4e %.4e\n",
user.dx,errnorms[0],errnorms[1]*user.secpera);CHKERRQ(ierr);
ierr = VecDestroy(Hu);CHKERRQ(ierr);
ierr = VecDestroy(r);CHKERRQ(ierr);
ierr = VecDestroy(user.Huexact);CHKERRQ(ierr);
ierr = VecDestroy(user.M);CHKERRQ(ierr);
ierr = VecDestroy(user.Bstag);CHKERRQ(ierr);
ierr = VecDestroy(user.beta);CHKERRQ(ierr);
ierr = MatDestroy(J); CHKERRQ(ierr);
ierr = SNESDestroy(snes);CHKERRQ(ierr);
ierr = DADestroy(user.da);CHKERRQ(ierr);
ierr = DADestroy(user.scalarda);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc, char **argv)
{
PetscInitialize(&argc, &argv, "elas.opt", help);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
int Ns = 32;
unsigned int dof = 3;
char problemName[PETSC_MAX_PATH_LEN];
char filename[PETSC_MAX_PATH_LEN];
double t0 = 0.0;
double dt = 0.1;
double t1 = 1.0;
double beta = 0.000001;
double gamma = 0.0; // percent of noise added ...
// double dtratio = 1.0;
DA da; // Underlying scalar DA - for scalar properties
DA da3d; // Underlying vector DA - for vector properties
Vec rho; // density - elemental scalar
Vec lambda; // Lame parameter - lambda - elemental scalar
Vec mu; // Lame parameter - mu - elemental scalar
Vec fibers; // Fiber orientations - nodal vector (3-dof)
Vec fibersElemental; // for IO. will be destroyed. - elemental vector (3-dof)
std::vector<Vec> tau; // the scalar activation - nodal scalar
// Initial conditions
Vec initialDisplacement;
Vec initialVelocity;
timeInfo ti;
// get Ns
CHKERRQ ( PetscOptionsGetInt(0,"-Ns",&Ns,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-t0",&t0,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-t1",&t1,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-dt",&dt,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-beta",&beta,0) );
CHKERRQ ( PetscOptionsGetString(PETSC_NULL,"-pn",problemName,PETSC_MAX_PATH_LEN-1,PETSC_NULL));
if (!rank) {
std::cout << "Problem size is " << Ns+1 << " spatially and NT = " << (int)ceil(1.0/dt) << std::endl;
}
// Time info for timestepping
ti.start = t0;
ti.stop = t1;
ti.step = dt;
// create DA
CHKERRQ ( DACreate3d ( PETSC_COMM_WORLD, DA_NONPERIODIC, DA_STENCIL_BOX,
Ns+1, Ns+1, Ns+1, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE,
1, 1, 0, 0, 0, &da) );
CHKERRQ ( DACreate3d ( PETSC_COMM_WORLD, DA_NONPERIODIC, DA_STENCIL_BOX,
Ns+1, Ns+1, Ns+1, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE,
dof, 1, 0, 0, 0, &da3d) );
elasMass *Mass = new elasMass(feMat::PETSC); // Mass Matrix
elasStiffness *Stiffness = new elasStiffness(feMat::PETSC); // Stiffness matrix
raleighDamping *Damping = new raleighDamping(feMat::PETSC); // Damping Matrix
cardiacDynamic *Force = new cardiacDynamic(feVec::PETSC); // Force Vector
// create vectors
CHKERRQ( DACreateGlobalVector(da, &rho) );
CHKERRQ( DACreateGlobalVector(da, &mu) );
CHKERRQ( DACreateGlobalVector(da, &lambda) );
CHKERRQ( DACreateGlobalVector(da3d, &initialDisplacement) );
CHKERRQ( DACreateGlobalVector(da3d, &initialVelocity) );
// Set initial conditions
CHKERRQ( VecSet ( initialDisplacement, 0.0) );
CHKERRQ( VecSet ( initialVelocity, 0.0) );
VecZeroEntries( mu );
VecZeroEntries( lambda );
VecZeroEntries( rho );
int x, y, z, m, n, p;
int mx,my,mz, xne, yne, zne;
CHKERRQ( DAGetCorners(da, &x, &y, &z, &m, &n, &p) );
CHKERRQ( DAGetInfo(da,0, &mx, &my, &mz, 0,0,0,0,0,0,0) );
if (x+m == mx) {
xne=m-1;
} else {
xne=m;
}
if (y+n == my) {
yne=n-1;
} else {
yne=n;
}
if (z+p == mz) {
zne=p-1;
} else {
zne=p;
}
double acx,acy,acz;
double hx = 1.0/((double)Ns);
// SET MATERIAL PROPERTIES ...
// @todo - Write routines to read/write in Parallel
// allocate for temporary buffers ...
unsigned int elemSize = Ns*Ns*Ns;
// std::cout << "Elem size is " << elemSize << std::endl;
unsigned int nodeSize = (Ns+1)*(Ns+1)*(Ns+1);
unsigned char *tmp_mat = new unsigned char[elemSize];
double *tmp_tau = new double[dof*elemSize];
// generate filenames & read in the raw arrays first ...
std::ifstream fin;
sprintf(filename, "%s.%d.img", problemName, Ns);
fin.open(filename, std::ios::binary); fin.read((char *)tmp_mat, elemSize); fin.close();
// Set Elemental material properties
PetscScalar ***muArray, ***lambdaArray, ***rhoArray;
CHKERRQ(DAVecGetArray(da, mu, &muArray));
CHKERRQ(DAVecGetArray(da, lambda, &lambdaArray));
CHKERRQ(DAVecGetArray(da, rho, &rhoArray));
// assign material properties ...
// myo, tissue
// nu = 0.49, 0.45
// E = 10000, 1000
// rho = 1.0, 0.1
// std::cout << "Setting Elemental properties." << std::endl;
// loop through all elements ...
for (int k=z; k<z+zne; k++) {
for (int j=y; j<y+yne; j++) {
for (int i=x; i<x+xne; i++) {
int indx = k*Ns*Ns + j*Ns + i;
if ( tmp_mat[indx] ) {
muArray[k][j][i] = 344.82; //3355.7;
lambdaArray[k][j][i] = 3103.448;// 164429.53;
rhoArray[k][j][i] = 1.0;
} else {
muArray[k][j][i] = 344.82;
lambdaArray[k][j][i] = 3103.448;
rhoArray[k][j][i] = 1.0;
}
} // end i
} // end j
} // end k
// std::cout << "Finished Elemental loop." << std::endl;
CHKERRQ( DAVecRestoreArray ( da, mu, &muArray ) );
CHKERRQ( DAVecRestoreArray ( da, lambda, &lambdaArray ) );
CHKERRQ( DAVecRestoreArray ( da, rho, &rhoArray ) );
// std::cout << "Finished restoring arrays" << std::endl;
// delete temporary buffers
delete [] tmp_mat;
// Now set the activation ...
unsigned int numSteps = (unsigned int)(ceil(( ti.stop - ti.start)/ti.step));
// tau = (Vec *) new char*[numSteps+1];
// std::cout << "Numsteps is " << numSteps << std::endl;
Vec tauVec, tmpTau;
CHKERRQ( DACreateGlobalVector(da3d, &tmpTau) );
#ifdef __DEBUG__
if (!rank) {
std::cout << x << ", " << y << ", " << z << " + " << xne << ", " << yne << ", " << zne << std::endl;
}
#endif
PetscScalar ***tauArray;
double tauNorm;
for (unsigned int t=0; t<numSteps+1; t++) {
CHKERRQ( DACreateGlobalVector(da3d, &tauVec) );
CHKERRQ( VecSet( tmpTau, 0.0));
CHKERRQ(DAVecGetArray(da3d, tmpTau, &tauArray));
// std::cout << "Setting force vectors" << std::endl;
sprintf(filename, "%s.%d.%.3d.fld", problemName, Ns, t);
// std::cout << "Reading force file " << filename << std::endl;
fin.open(filename); fin.read((char *)tmp_tau, dof*elemSize*sizeof(double)); fin.close();
for (int k = z; k < z + zne ; k++) {
for (int j = y; j < y + yne; j++) {
for (int i = x; i < x + xne; i++) {
int indx = dof*(k*Ns*Ns + j*Ns + i);
tauArray[k][j][dof*i] = tmp_tau[indx];
tauArray[k][j][dof*i+1] = tmp_tau[indx+1];
tauArray[k][j][dof*i+2] = tmp_tau[indx+2];
}
}
}
// std::cout << CYN"\tFinished elemental loop"NRM << std::endl;
CHKERRQ( DAVecRestoreArray ( da3d, tmpTau, &tauArray ) );
// std::cout << "Converting to Nodal Vector" << std::endl;
// VecNorm(tmpTau, NORM_2, &tauNorm);
// tauNorm = tauNorm/pow(Ns,1.5);
// std::cout << "Elemental Norm is " << tauNorm << std::endl;
// std::cout << rank << " Converting to Nodal" << std::endl;
elementToNode(da3d, tmpTau, tauVec);
/*
VecNorm(tauVec, NORM_2, &tauNorm);
tauNorm = tauNorm/pow(Ns,1.5);
std::cout << "Nodal Norm is " << tauNorm << std::endl;
*/
// std::cout << rank << " Done converting to Nodal Vector" << std::endl;
tau.push_back(tauVec);
}
//if (!rank) {
// std::cout << "Finished setting forces" << std::endl;
// }
// CHKERRQ( VecDestroy( tmpTau ) );
delete [] tmp_tau;
// DONE - SET MATERIAL PROPERTIES ...
// Setup Matrices and Force Vector ...
Mass->setProblemDimensions(1.0, 1.0, 1.0);
Mass->setDA(da3d);
Mass->setDof(dof);
Mass->setDensity(rho);
Stiffness->setProblemDimensions(1.0, 1.0, 1.0);
Stiffness->setDA(da3d);
Stiffness->setDof(dof);
Stiffness->setLame(lambda, mu);
Damping->setAlpha(0.0);
Damping->setBeta(0.00075);
Damping->setMassMatrix(Mass);
Damping->setStiffnessMatrix(Stiffness);
Damping->setDA(da3d);
Damping->setDof(dof);
// Force Vector
Force->setProblemDimensions(1.0,1.0,1.0);
Force->setDA(da3d);
// Force->setActivationVec(tau);
// Force->setFiberOrientations(fibers);
Force->setFDynamic(tau);
Force->setTimeInfo(&ti);
// Newmark time stepper ...
newmark *ts = new newmark;
ts->setMassMatrix(Mass);
ts->setDampingMatrix(Damping);
ts->setStiffnessMatrix(Stiffness);
ts->damp(false);
ts->setTimeFrames(1);
ts->setForceVector(Force);
ts->setInitialDisplacement(initialDisplacement);
ts->setInitialVelocity(initialVelocity);
ts->storeVec(true);
ts->setTimeInfo(&ti);
ts->setAdjoint(false); // set if adjoint or forward
ts->init(); // initialize IMPORTANT
if (!rank)
std::cout << RED"Starting Newmark Solve"NRM << std::endl;
ts->solve();// solve
if (!rank)
std::cout << GRN"Done Newmark"NRM << std::endl;
std::vector<Vec> solvec = ts->getSolution();
/* Set very initial guess for the inverse problem*/
/*
PetscRandom rctx;
PetscRandomCreate(PETSC_COMM_WORLD,&rctx);
PetscRandomSetFromOptions(rctx);
VecSetRandom(guess,rctx);
VecNorm(guess,NORM_INFINITY,&norm);
PetscPrintf(0,"guess norm = %g\n",norm);
*/
double errnorm;
double exsolnorm;
Vec guess;
Vec truth;
Vec Err;
concatenateVecs(solvec, guess);
concatenateVecs(tau, truth);
iC(VecNorm(truth, NORM_2, &exsolnorm));
/*
std::cout << "Forward solver solution size is " << solvec.size() << std::endl;
std::cout << "Forward solver solution norm is " << exsolnorm << std::endl;
*/
VecZeroEntries(guess);
// Inverse solver set up
hyperbolicInverse *hyperInv = new hyperbolicInverse;
hyperInv->setForwardInitialConditions(initialDisplacement, initialVelocity);
hyperInv->setTimeStepper(ts); // set the timestepper
hyperInv->setInitialGuess(guess);// set the initial guess
// hyperInv->setInitialGuess(truth);// set the initial guess
hyperInv->setRegularizationParameter(beta); // set the regularization paramter
hyperInv->setObservations(solvec); // set the data for the problem
hyperInv->init(); // initialize the inverse solver
hyperInv->solve(); // solve
hyperInv->getCurrentControl(guess); // get the solution
// see the error in the solution relative to the actual solution
VecDuplicate(truth, &Err);
iC(VecZeroEntries(Err));
iC(VecWAXPY(Err, -1.0, guess, truth));
iC(VecNorm(Err, NORM_2, &errnorm));
PetscPrintf(0,"errr in inverse = %g\n", errnorm/exsolnorm);
PetscFinalize();
}
int main(int argc,char **args)
{
PetscErrorCode ierr;
PetscInitialize(&argc,&args,(char *)0,help);
PetscInt m = 10; /* default number of rows and columns IN GRID,
but matrix is (m^2) x (m^2) */
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
MPI_Comm com = PETSC_COMM_WORLD;
PetscMPIInt rank, size;
ierr = MPI_Comm_rank(com, &rank); CHKERRQ(ierr);
ierr = MPI_Comm_size(com, &size); CHKERRQ(ierr);
/* create m x m two-dimensional grid for periodic boundary condition problem */
DA da2;
PetscInt dof=1, stencilwidth=1;
ierr = DACreate2d(com, DA_XYPERIODIC, DA_STENCIL_STAR,
m,m,PETSC_DECIDE,PETSC_DECIDE,
dof,stencilwidth,PETSC_NULL,PETSC_NULL,&da2); CHKERRQ(ierr);
/* get da2-managed Vecs */
Vec x,b,u;
ierr = DACreateGlobalVector(da2,&x); CHKERRQ(ierr);
ierr = VecDuplicate(x,&b); CHKERRQ(ierr);
ierr = VecDuplicate(x,&u); CHKERRQ(ierr);
Mat A;
ierr = DAGetMatrix(da2, MATMPIAIJ, &A); CHKERRQ(ierr);
/* alternative call below is not quite same as result from DAGetMatrix(),
because of nonzero allocation; the Mat ownership ranges are same */
/* ierr = MatCreateMPIAIJ(com, mlocal, mlocal, m*m, m*m,
5, PETSC_NULL, 4, PETSC_NULL,
&A); CHKERRQ(ierr) */
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
/* report on ownership range */
PetscInt rstart,rend,mlocal;
ierr = VecGetOwnershipRange(x,&rstart,&rend);CHKERRQ(ierr);
ierr = VecGetLocalSize(x,&mlocal);CHKERRQ(ierr);
PetscInt A_rstart,A_rend;
ierr = MatGetOwnershipRange(A,&A_rstart,&A_rend);CHKERRQ(ierr);
if ((rstart != A_rstart) || (rend != A_rend)) {
ierr = PetscPrintf(com,
"Vec and Mat ownership ranges different!!! ending ...\n"); CHKERRQ(ierr);
PetscEnd();
} else {
ierr = PetscSynchronizedPrintf(com,
"rank=%d has Vec and Mat ownership: mlocal=%d, rstart=%d, rend=%d\n",
rank,mlocal,rstart,rend); CHKERRQ(ierr);
}
PetscSynchronizedFlush(com);
/* get local part of grid */
PetscInt xm,ym,xs,ys;
DAGetCorners(da2,&xs,&ys,0,&xm,&ym,0);
/* report on local part of grid */
ierr = PetscSynchronizedPrintf(com,
"rank=%d has da2-managed-Vec local ranges: xs=%d, xm=%d, ys=%d, ym=%d\n",
rank,xs,xm,ys,ym); CHKERRQ(ierr);
PetscSynchronizedFlush(com);
/* set up linear system */
PetscScalar **barr, **uarr; /* RHS and exact soln, resp. */
DAVecGetArray(da2, b, &barr);
DAVecGetArray(da2, u, &uarr);
PetscScalar dx = 1.0/(double)m, dy = dx, pi = 3.14159265358979;
PetscScalar xi,yj;
PetscInt diag=0,north=1,east=2,south=3,west=4;
PetscScalar vals[5] = {-4.0 + dx * dx, 1.0, 1.0, 1.0, 1.0};
MatStencil row, col[5]; /* these are not "stencils" at all, but local grid
to global indices helpers */
PetscInt i,j,num;
for (j=ys; j<ys+ym; j++) {
for(i=xs; i<xs+xm; i++) {
/* entries of matrix A */
row.i = i; row.j = j; row.c = 0; /* dof = 1 so first component;
note row.k is for 3d DAs */
for (num=0; num<5; num++) col[num].c = 0;
/* set diag first, then go through stencil neighbors */
col[diag].i = i; col[diag].j = j;
col[north].i = i; col[north].j = j+1;
col[east].i = i+1; col[east].j = j;
col[south].i = i; col[south].j = j-1;
col[west].i = i-1; col[west].j = j;
ierr = MatSetValuesStencil(A,1,&row,5,col,vals,INSERT_VALUES); CHKERRQ(ierr);
/* entries of vectors: exact solution u and right-hand-side b */
xi = (double)i * dx;
yj = (double)j * dy;
uarr[j][i] = sin(2.0 * pi * xi) * cos(4.0 * pi * yj);
barr[j][i] = (1.0 - 20.0 * pi * pi) * uarr[j][i];
barr[j][i] *= dx * dx;
}
}
DAVecRestoreArray(da2, b, &barr);
DAVecRestoreArray(da2, u, &uarr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = VecAssemblyBegin(b); CHKERRQ(ierr);
ierr = VecAssemblyEnd(b); CHKERRQ(ierr);
ierr = VecAssemblyBegin(u); CHKERRQ(ierr);
ierr = VecAssemblyEnd(u); CHKERRQ(ierr);
/* uncomment for dense view; -mat_view default format is good enough for most purposes
PetscViewer viewer;
PetscViewerCreate(com, &viewer);
PetscViewerSetType(viewer, PETSC_VIEWER_ASCII);
PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_DENSE);
MatView(A,viewer);
PetscViewerDestroy(viewer);
*/
/* setup solver context now that Mat and Vec are assembled */
KSP ksp;
ierr = KSPCreate(com,&ksp);CHKERRQ(ierr);
/* Set "operators". Here the matrix that defines the linear system
also serves as the preconditioning matrix. But we do not assert a
relationship between their nonzero patterns.(???) */
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
/* Following is optional; parameters could be set at runtime. */
ierr = KSPSetTolerances(ksp,1.e-7,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
CHKERRQ(ierr);
/* Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* Solve linear system */
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* Compute and report the error (and the iteration count and reason). */
PetscScalar norminf, normtwo, neg_one=-1.0;
PetscInt its;
KSPConvergedReason reason;
ierr = VecAXPY(x,neg_one,u);CHKERRQ(ierr); // x = x - u
ierr = VecNorm(x,NORM_INFINITY,&norminf);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&normtwo);CHKERRQ(ierr); // discrete norm
normtwo *= dx * dy; // integral norm
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = KSPGetConvergedReason(ksp,&reason); CHKERRQ(ierr);
ierr = PetscPrintf(com,
"Error norms ||err||_inf = %.3e, ||err||_2 = %.3e;\n"
"Iterations = %d; Reason = %d\n",
norminf, normtwo, its, (int) reason);CHKERRQ(ierr);
/* destroy */
ierr = KSPDestroy(ksp);CHKERRQ(ierr);
ierr = MatDestroy(A);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(u);CHKERRQ(ierr);
ierr = VecDestroy(b);CHKERRQ(ierr);
ierr = DADestroy(da2);CHKERRQ(ierr);
/* Always call PetscFinalize() before exiting a program. */
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
PetscErrorCode vizGA2DA()
{
PetscErrorCode ierr;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
int d1 = 40, d2 = 50;
DA da;
Vec vec;
const PetscInt *lx, *ly, *lz;
PetscInt m,n,p;
DALocalInfo info;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr);
ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr);
/**/
ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
for (int i = 0; i < m; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]);
}
for (int i = 0; i < n; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]);
}
/**/
int ga = GA_Create_handle();
int ndim = 2;
int dims[2] = {d2,d1};
GA_Set_data(ga,2,dims,MT_DBL);
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < n; i++ )
{
map[i] = ly[i-1] + map[i-1];
}
map[n] = 0;
for( int i = n+1; i < m+n; i++ )
{
map[i] = lx[i-n-1] + map[i-1];
}
/* correct ordering, but nodeid's dont line up with mpi rank for petsc's da
* DA: +---+---+ GA: +---+---+
* +-2-+-3-+ +-1-+-3-+
* +---+---+ +---+---+
* +-0-+-1-+ +-0-+-2-+
* +---+---+ +---+---+
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < m; i++ )
{
map[i] = lx[i] + map[i-1];
}
map[m] = 0;
for( int i = m+1; i < m+n; i++ )
{
map[i] = ly[i-m] + map[i-1];
}
*/
int block[2] = {n,m};
GA_Set_irreg_distr(ga,map,block);
ierr = GA_Allocate( ga );
if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr);
if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr);
if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1);
GA_Print_distribution(ga);
int lo[2], hi[2];
NGA_Distribution(ga,rank,lo,hi);
if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 ||
lo[0] != info.ys || hi[0] != info.ys+info.ym-1 )
{
PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n",
rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1);
}
PetscBarrier(0);
PetscSynchronizedFlush(PETSC_COMM_WORLD);
AO ao;
DAGetAO(da,&ao);
if( rank == 0 )
{
int *idx, len = d1*d2;
PetscReal *val;
PetscMalloc(sizeof(PetscReal)*len, &val);
PetscMalloc(sizeof(int)*len, &idx);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
idx[i + d1*j] = i + d1*j;
val[i + d1*j] = i + d1*j;
}
}
AOApplicationToPetsc(ao,len,idx);
VecSetValues(vec,len,idx,val,INSERT_VALUES);
int a[2], b[2],ld[1]={0};
double c = 0;
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
a[0] = j;
a[1] = i;
// printf("%5.0f ",c);
NGA_Put(ga,a,a,&c,ld);
c++;
}
}
}
// GA_Print(ga);
VecAssemblyBegin(vec);
VecAssemblyEnd(vec);
int ld;
double *ptr;
NGA_Access(ga,lo,hi,&ptr,&ld);
PetscReal **d;
int c=0;
ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr);
for (int j = info.ys; j < info.ys+info.ym; ++j)
{
for (int i = info.xs; i < info.xs+info.xm; ++i)
{
if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] )
GA_Error("DA array is not equal to GA array",1);
// printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]);
c++;
}
}
ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr);
c=0;
PetscReal *v;
int start, end;
VecGetOwnershipRange(vec, &start, &end);
VecGetArray( vec, &v );
for( int i = start; i < end; i++)
{
// printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") );
}
VecRestoreArray( vec, &v );
NGA_Release_update(ga,lo,hi);
Vec gada;
VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada);
VecView(gada,PETSC_VIEWER_STDOUT_SELF);
GA_Destroy(ga);
ierr = VecDestroy(vec); CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Vec u;
Mat A;
PetscErrorCode ierr;
mv_MultiVectorPtr eigenvectors;
PetscScalar * eigs;
PetscScalar * eigs_hist;
double * resid;
double * resid_hist;
int iterations;
PetscMPIInt rank;
int n_eigs = 1;
int seed = 1;
int i,j;
PetscLogDouble t1,t2,elapsed_time;
DA da;
double tol=1e-08;
PetscTruth option_present;
PetscTruth freepart=PETSC_FALSE;
PetscTruth full_output=PETSC_FALSE;
PetscInt m,n,p;
KSP ksp;
lobpcg_Tolerance lobpcg_tol;
int maxIt = 100;
mv_InterfaceInterpreter ii;
lobpcg_BLASLAPACKFunctions blap_fn;
aux_data_struct aux_data;
/*
PetscViewer viewer;
*/
PetscInt tmp_int;
mv_TempMultiVector * xe;
PetscInt N;
PetscScalar * xx;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n_eigs",&tmp_int,&option_present);CHKERRQ(ierr);
if (option_present)
n_eigs = tmp_int;
ierr = PetscOptionsGetReal(PETSC_NULL,"-tol", &tol,PETSC_NULL); CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-freepart",&freepart); CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-full_out",&full_output); CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-seed",&tmp_int,&option_present);CHKERRQ(ierr);
if (option_present)
seed = tmp_int;
ierr = PetscOptionsGetInt(PETSC_NULL,"-itr",&tmp_int,&option_present);CHKERRQ(ierr);
if (option_present)
maxIt = tmp_int;
if (seed<1)
seed=1;
/* we actually run our code twice: first time we solve small problem just to make sure
that all program code is actually loaded into memory; then we solve the problem
we are interested in; this trick is done for accurate timing
*/
PreLoadBegin(PETSC_TRUE,"grid and matrix assembly");
/* "create" the grid and stencil data; on first run we form small problem */
if (PreLoadIt==0)
{
/* small problem */
ierr=DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,10,10,10,
1,PETSC_DECIDE,1,1,1,0,0,0,&da); CHKERRQ(ierr);
}
else
{
/* actual problem */
if (freepart) /* petsc determines partitioning */
{
ierr=DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,-10,-10,-10,
PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0,0,&da); CHKERRQ(ierr);
}
else /* (1,NP,1) partitioning */
{
ierr=DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,-10,-10,-10,
1,PETSC_DECIDE,1,1,1,0,0,0,&da); CHKERRQ(ierr);
}
/* now we print what partitioning is chosen */
ierr=DAGetInfo(da,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,&m,
&n,&p,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Partitioning: %u %u %u\n",m,n,p);
}
/* create matrix, whose nonzero structure and probably partitioning corresponds to
grid and stencil structure */
ierr=DAGetMatrix(da,MATMPIAIJ,&A); CHKERRQ(ierr);
/* fill the matrix with values. I intend to build 7-pt Laplas */
/* this procedure includes matrix assembly */
ierr=FillMatrix(da,A); CHKERRQ(ierr);
/*
PetscViewerBinaryOpen(PETSC_COMM_WORLD,"matrix.dat",FILE_MODE_WRITE,&viewer);
MatView(A,PETSC_VIEWER_STDOUT_WORLD);
PetscViewerDestroy(viewer);
*/
/*
Create parallel vectors.
- We form 1 vector from scratch and then duplicate as needed.
*/
ierr = DACreateGlobalVector(da,&u); CHKERRQ(ierr);
/* ierr = VecSetFromOptions(u);CHKERRQ(ierr); */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the linear solver and set various options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* Here we START measuring time for preconditioner setup */
PreLoadStage("preconditioner setup");
ierr = PetscGetTime(&t1);CHKERRQ(ierr);
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
/*
Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
These options will override those specified above as long as
KSPSetFromOptions() is called _after_ any other customization
routines.
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* probably this call actually builds the preconditioner */
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
/* Here we STOP measuring time for preconditioner setup */
PreLoadStage("solution");
ierr = PetscGetTime(&t2);CHKERRQ(ierr);
elapsed_time=t2-t1;
if (PreLoadIt==1)
PetscPrintf(PETSC_COMM_WORLD,"Preconditioner setup, seconds: %f\n",elapsed_time);
/* request memory for eig-vals */
ierr = PetscMalloc(sizeof(PetscScalar)*n_eigs,&eigs); CHKERRQ(ierr);
/* request memory for eig-vals history */
ierr = PetscMalloc(sizeof(PetscScalar)*n_eigs*(maxIt+1),&eigs_hist); CHKERRQ(ierr);
/* request memory for resid. norms */
ierr = PetscMalloc(sizeof(double)*n_eigs,&resid); CHKERRQ(ierr);
/* request memory for resid. norms hist. */
ierr = PetscMalloc(sizeof(double)*n_eigs*(maxIt+1),&resid_hist); CHKERRQ(ierr);
LOBPCG_InitRandomContext();
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
PETSCSetupInterpreter( &ii );
eigenvectors = mv_MultiVectorCreateFromSampleVector(&ii, n_eigs,u);
xe = (mv_TempMultiVector *) mv_MultiVectorGetData( eigenvectors );
/*
VecView( (Vec)xe->vector[0],PETSC_VIEWER_STDOUT_WORLD);
*/
for (i=0; i<seed; i++) /* this cycle is to imitate changing random seed */
mv_MultiVectorSetRandom (eigenvectors, 1234);
/*
VecView( (Vec)xe->vector[0],PETSC_VIEWER_STDOUT_WORLD);
*/
VecGetSize( (Vec)xe->vector[0], &N );
N=mv_TempMultiVectorHeight( xe );
VecGetArray( (Vec)xe->vector[0],&xx);
lobpcg_tol.absolute = tol;
lobpcg_tol.relative = 1e-50;
#ifdef PETSC_USE_COMPLEX
blap_fn.zpotrf = PETSC_zpotrf_interface;
blap_fn.zhegv = PETSC_zsygv_interface;
#else
blap_fn.dpotrf = PETSC_dpotrf_interface;
blap_fn.dsygv = PETSC_dsygv_interface;
#endif
aux_data.A = A;
aux_data.ksp = ksp;
aux_data.ii = ii;
/* Here we START measuring time for solution process */
ierr = PetscGetTime(&t1);CHKERRQ(ierr);
#ifdef PETSC_USE_COMPLEX
lobpcg_solve_complex(
eigenvectors, /*input-initial guess of e-vectors */
&aux_data, /*input-matrix A */
OperatorAMultiVector, /*input-operator A */
NULL, /*input-matrix B */
NULL, /*input-operator B */
&aux_data, /*input-matrix T */
Precond_FnMultiVector, /*input-operator T */
NULL, /*input-matrix Y */
blap_fn, /*input-lapack functions */
lobpcg_tol, /*input-tolerances */
PreLoadIt? maxIt:1, /*input-max iterations */
!rank && PreLoadIt, /*input-verbosity level */
&iterations, /*output-actual iterations */
(komplex *) eigs, /*output-eigenvalues */
(komplex *) eigs_hist, /*output-eigenvalues history */
n_eigs, /*output-history global height */
resid, /*output-residual norms */
resid_hist , /*output-residual norms history */
n_eigs /*output-history global height */
);
#else
lobpcg_solve_double( eigenvectors,
&aux_data,
OperatorAMultiVector,
NULL,
NULL,
&aux_data,
Precond_FnMultiVector,
NULL,
blap_fn,
lobpcg_tol,
PreLoadIt? maxIt:1,
!rank && PreLoadIt,
&iterations,
eigs, /* eigenvalues; "lambda_values" should point to array
containing <blocksize> doubles where <blocksize> is the
width of multivector "blockVectorX" */
eigs_hist,
/* eigenvalues history; a pointer to the entries of the
<blocksize>-by-(<maxIterations>+1) matrix stored in
fortran-style. (i.e. column-wise) The matrix may be
a submatrix of a larger matrix, see next argument */
n_eigs, /* global height of the matrix (stored in fotran-style)
specified by previous argument */
resid,
/* residual norms; argument should point to
array of <blocksize> doubles */
resid_hist ,
/* residual norms history; a pointer to the entries of the
<blocksize>-by-(<maxIterations>+1) matrix stored in
fortran-style. (i.e. column-wise) The matrix may be
a submatrix of a larger matrix, see next argument */
n_eigs
/* global height of the matrix (stored in fotran-style)
specified by previous argument */
);
#endif
/* Here we STOP measuring time for solution process */
ierr = PetscGetTime(&t2);CHKERRQ(ierr);
elapsed_time=t2-t1;
if (PreLoadIt)
PetscPrintf(PETSC_COMM_WORLD,"Solution process, seconds: %e\n",elapsed_time);
if (PreLoadIt && full_output)
{
PetscPrintf(PETSC_COMM_WORLD,"Output from LOBPCG, eigenvalues:\n");
for (i=0;i<n_eigs;i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD,"%e\n",PetscRealPart(eigs[i]));
CHKERRQ(ierr);
}
PetscPrintf(PETSC_COMM_WORLD,"Output from LOBPCG, eigenvalues history:\n");
for (j=0; j<iterations+1; j++)
for (i=0;i<n_eigs;i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD,"%e\n",PetscRealPart(*(eigs_hist+j*n_eigs+i)));
CHKERRQ(ierr);
}
PetscPrintf(PETSC_COMM_WORLD,"Output from LOBPCG, residual norms:\n");
for (i=0;i<n_eigs;i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD,"%e\n",resid[i]);
CHKERRQ(ierr);
}
PetscPrintf(PETSC_COMM_WORLD,"Output from LOBPCG, residual norms history:\n");
for (j=0; j<iterations+1; j++)
for (i=0;i<n_eigs;i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD,"%e\n",*(resid_hist+j*n_eigs+i));
CHKERRQ(ierr);
}
}
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = VecDestroy(u);CHKERRQ(ierr);
ierr = MatDestroy(A);CHKERRQ(ierr);
ierr = KSPDestroy(ksp);CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
LOBPCG_DestroyRandomContext();
mv_MultiVectorDestroy(eigenvectors);
/* free memory used for eig-vals */
ierr = PetscFree(eigs); CHKERRQ(ierr);
ierr = PetscFree(eigs_hist); CHKERRQ(ierr);
ierr = PetscFree(resid); CHKERRQ(ierr);
ierr = PetscFree(resid_hist); CHKERRQ(ierr);
/*
Always call PetscFinalize() before exiting a program. This routine
- finalizes the PETSc libraries as well as MPI
- provides summary and diagnostic information if certain runtime
options are chosen (e.g., -log_summary).
*/
PreLoadEnd();
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt its,n,Nx=PETSC_DECIDE,Ny=PETSC_DECIDE,nlocal,i;
PetscMPIInt size;
PC pc;
PetscInt mx,my;
Mat A;
GridCtx fine_ctx;
KSP ksp;
PetscTruth flg;
PetscInitialize(&argc,&argv,PETSC_NULL,help);
/* set up discretization matrix for fine grid */
/* ML requires input of fine-grid matrix. It determines nlevels. */
fine_ctx.mx = 9; fine_ctx.my = 9;
ierr = PetscOptionsGetInt(PETSC_NULL,"-mx",&mx,&flg);CHKERRQ(ierr);
if (flg) fine_ctx.mx = mx;
ierr = PetscOptionsGetInt(PETSC_NULL,"-my",&my,&flg);CHKERRQ(ierr);
if (flg) fine_ctx.my = my;
ierr = PetscPrintf(PETSC_COMM_WORLD,"Fine grid size %D by %D\n",fine_ctx.mx,fine_ctx.my);CHKERRQ(ierr);
n = fine_ctx.mx*fine_ctx.my;
MPI_Comm_size(PETSC_COMM_WORLD,&size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-Nx",&Nx,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-Ny",&Ny,PETSC_NULL);CHKERRQ(ierr);
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,fine_ctx.mx,
fine_ctx.my,Nx,Ny,1,1,PETSC_NULL,PETSC_NULL,&fine_ctx.da);CHKERRQ(ierr);
ierr = DACreateGlobalVector(fine_ctx.da,&fine_ctx.x);CHKERRQ(ierr);
ierr = VecDuplicate(fine_ctx.x,&fine_ctx.b);CHKERRQ(ierr);
ierr = VecGetLocalSize(fine_ctx.x,&nlocal);CHKERRQ(ierr);
ierr = DACreateLocalVector(fine_ctx.da,&fine_ctx.localX);CHKERRQ(ierr);
ierr = VecDuplicate(fine_ctx.localX,&fine_ctx.localF);CHKERRQ(ierr);
ierr = MatCreateMPIAIJ(PETSC_COMM_WORLD,nlocal,nlocal,n,n,5,PETSC_NULL,3,PETSC_NULL,&A);CHKERRQ(ierr);
ierr = FormJacobian_Grid(&fine_ctx,&A);CHKERRQ(ierr);
/* create linear solver */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCML);CHKERRQ(ierr);
/* set options, then solve system */
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* calls PCSetFromOptions_MG/ML */
for (i=0; i<3; i++){
if (i<2){ /* test DIFFERENT_NONZERO_PATTERN */
/* set values for rhs vector */
ierr = VecSet(fine_ctx.b,i+1.0);CHKERRQ(ierr);
/* modify A */
ierr = MatShift(A,1.0);CHKERRQ(ierr);
ierr = MatScale(A,2.0);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
} else { /* test SAME_NONZERO_PATTERN */
ierr = KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
ierr = KSPSolve(ksp,fine_ctx.b,fine_ctx.x);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Number of iterations = %D\n",its);CHKERRQ(ierr);
}
/* free data structures */
ierr = VecDestroy(fine_ctx.x);CHKERRQ(ierr);
ierr = VecDestroy(fine_ctx.b);CHKERRQ(ierr);
ierr = DADestroy(fine_ctx.da);CHKERRQ(ierr);
ierr = VecDestroy(fine_ctx.localX);CHKERRQ(ierr);
ierr = VecDestroy(fine_ctx.localF);CHKERRQ(ierr);
ierr = MatDestroy(A);CHKERRQ(ierr);
ierr = KSPDestroy(ksp);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
void getForces(Vec params, std::vector<Vec> &forces, DA da, timeInfo ti, int numParams) {
#ifdef __DEBUG__
std::cout << RED"Entering "NRM << __func__ << std::endl;
#endif
// Generate the force vector based on the current parameters ...
// F = Sum { B_i a_i}
PetscScalar * pVec;
VecGetArray(params, &pVec);
// Clear the Forces
for (unsigned int i=0; i<forces.size(); i++) {
if (forces[i] != NULL) {
VecDestroy(forces[i]);
}
}
forces.clear();
unsigned int numSteps = (unsigned int)(ceil(( ti.stop - ti.start)/ti.step));
// create and initialize to 0
for (unsigned int i=0; i<numSteps+1; i++) {
Vec tmp;
DACreateGlobalVector(da, &tmp);
VecZeroEntries(tmp);
forces.push_back(tmp);
}
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
PetscScalar ***tauArray;
unsigned int paramTimeSteps = (unsigned int)(ceil(( (double)(numSteps))/ ((double)(2*numParams)) ));
double acx,acy,acz;
int x, y, z, m, n, p;
int mx,my,mz;
DAGetCorners(da, &x, &y, &z, &m, &n, &p);
DAGetInfo(da,0, &mx, &my, &mz, 0,0,0,0,0,0,0);
double hx = 1.0/(mx-1.0);
for (int b=0; b<numParams; b++) {
std::vector<Vec> tau;
unsigned int tBegin = paramTimeSteps*b;
unsigned int tEnd = tBegin + numSteps/2; // paramTimeSteps*(b+2);
// std::cout << "For param " << b << ": Time step range is " << tBegin << " -> " << tEnd << std::endl;
for (unsigned int t=0; t<numSteps+1; t++) {
double newTime = (ti.step*(t-tBegin)*numSteps)/((double)(paramTimeSteps));
if ( (t>=tBegin) && (t<=tEnd)) {
DAVecGetArray(da, forces[t], &tauArray);
for (int k = z; k < z + p ; k++) {
for (int j = y; j < y + n; j++) {
for (int i = x; i < x + m; i++) {
acx = (i)*hx; acy = (j)*hx; acz = (k)*hx;
tauArray[k][j][i] += pVec[b]*sin(M_PI*newTime)*cos(2*M_PI*acx)*cos(2*M_PI*acy)*cos(2*M_PI*acz);
}
}
}
DAVecRestoreArray ( da, forces[t], &tauArray ) ;
}
}
}
VecRestoreArray(params, &pVec);
#ifdef __DEBUG__
// Get the norms of the forces ... just to be safe ..
double fNorm1, fNorm2;
for (unsigned int i=0; i<forces.size(); i++) {
VecNorm(forces[i], NORM_INFINITY, &fNorm1);
VecNorm(forces[i], NORM_2, &fNorm2);
PetscPrintf(0, "Force Norms at timestep %d are %g and %g\n", i, fNorm1, fNorm2);
}
#endif
#ifdef __DEBUG__
std::cout << GRN"Leaving "NRM << __func__ << std::endl;
#endif
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
KSP ksp;
PC pc;
Vec x,b;
DA da;
Mat A,Atrans;
PetscInt dof=1,M=-8;
PetscTruth flg,trans=PETSC_FALSE;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetTruth(PETSC_NULL,"-trans",&trans,PETSC_NULL);CHKERRQ(ierr);
ierr = DACreate(PETSC_COMM_WORLD,&da);CHKERRQ(ierr);
ierr = DASetDim(da,3);CHKERRQ(ierr);
ierr = DASetPeriodicity(da,DA_NONPERIODIC);CHKERRQ(ierr);
ierr = DASetStencilType(da,DA_STENCIL_STAR);CHKERRQ(ierr);
ierr = DASetSizes(da,M,M,M);CHKERRQ(ierr);
ierr = DASetNumProcs(da,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = DASetDof(da,dof);CHKERRQ(ierr);
ierr = DASetStencilWidth(da,1);CHKERRQ(ierr);
ierr = DASetVertexDivision(da,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DASetFromOptions(da);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&b);CHKERRQ(ierr);
ierr = ComputeRHS(da,b);CHKERRQ(ierr);
ierr = DAGetMatrix(da,MATBAIJ,&A);CHKERRQ(ierr);
ierr = ComputeMatrix(da,A);CHKERRQ(ierr);
/* A is non-symmetric. Make A = 0.5*(A + Atrans) symmetric for testing icc and cholesky */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX,&Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,1.0,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatScale(A,0.5);CHKERRQ(ierr);
ierr = MatDestroy(Atrans);CHKERRQ(ierr);
/* Test sbaij matrix */
flg = PETSC_FALSE;
ierr = PetscOptionsGetTruth(PETSC_NULL, "-test_sbaij1", &flg,PETSC_NULL);CHKERRQ(ierr);
if (flg){
Mat sA;
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
ierr = MatDestroy(A);CHKERRQ(ierr);
A = sA;
}
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetDA(pc,da);CHKERRQ(ierr);
if (trans) {
ierr = KSPSolveTranspose(ksp,b,x);CHKERRQ(ierr);
} else {
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
}
/* check final residual */
flg = PETSC_FALSE;
ierr = PetscOptionsGetTruth(PETSC_NULL, "-check_final_residual", &flg,PETSC_NULL);CHKERRQ(ierr);
if (flg){
Vec b1;
PetscReal norm;
ierr = KSPGetSolution(ksp,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&b1);CHKERRQ(ierr);
ierr = MatMult(A,x,b1);CHKERRQ(ierr);
ierr = VecAXPY(b1,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(b1,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Final residual %g\n",norm);CHKERRQ(ierr);
ierr = VecDestroy(b1);CHKERRQ(ierr);
}
ierr = KSPDestroy(ksp);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(b);CHKERRQ(ierr);
ierr = MatDestroy(A);CHKERRQ(ierr);
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
//int BenchmarkBoundaryChecks( int n, PetscLogDouble t_bulk, PetscLogDouble )
int PetscMain()
{
int i,j,k, n=128;
PetscLogDouble t1,t2,s1,s2;
Vec U,V,W,DIV1,DIV2;
PetscReal ***u,***v,***w,***div1, ***div2;
DA da;
DALocalInfo info;
PetscErrorCode ierr;
ierr = DACreate3d(PETSC_COMM_SELF,//MPI Communicator
DA_NONPERIODIC, //DA_NONPERIODIC, DA_XPERIODIC, DA_YPERIODIC, DA_XYPERIODIC
DA_STENCIL_STAR, //DA_STENCIL_BOX or DA_STENCIL_STAR
n,n,n, //Global array dimension
1,1,1,//Number procs per dim
1, //Number of chemical species
1, //stencil width
0,0,0, //specific array of nodes
&da); CHKERRQ(ierr);
DACreateGlobalVector(da,&U);
DACreateGlobalVector(da,&V);
DACreateGlobalVector(da,&W);
DACreateGlobalVector(da,&DIV1);
DACreateGlobalVector(da,&DIV2);
VecSet(DIV1,0);
VecSet(DIV2,0);
DAVecGetArray(da,U,&u);
DAVecGetArray(da,V,&v);
DAVecGetArray(da,W,&w);
DAVecGetArray(da,DIV1,&div1);
DAVecGetArray(da,DIV2,&div2);
DAGetLocalInfo(da,&info);
PetscBarrier(0);
for( k = 0; k < n; ++k)
{
for( j = 0; j < n; ++j)
{
for( i = 0; i < n; ++i)
{
u[k][j][i] = i * (i-n) * j * (j-n) * k * (k-n);
v[k][j][i] = 1 - u[k][j][i];
w[k][j][i] = u[k][j][i] * v[k][j][i];
}
}
}
PetscBarrier(0);
PetscGetTime(&t1);
for( k = 1; k < n-1; ++k)
{
for( j = 1; j < n-1; ++j)
{
for( i = 1; i < n-1; ++i)
{
div1[k][j][i] = u[k][j][i+1] - u[k][j][i-1] +
v[k][j+1][i] - v[k][j-1][i] +
w[k+1][j][i] - w[k-1][j][i];
div1[k][j][i] /= 2;
}
}
}
PetscGetTime(&t2);
PetscReal uE,uW,vN,vS,wF,wB;
PetscReal hx,hy,hz;
PetscBarrier(0);
PetscGetTime(&s1);
for( k = 1; k < n-1; ++k)
{
for( j = 1; j < n-1; ++j)
{
for( i = 1; i < n-1; ++i)
{
/* uE = i == info.mx-1 ? u[k][j][i] : u[k][j][i+1];
uW = i == 0 ? u[k][j][i] : u[k][j][i-1];
vN = j == info.my-1 ? v[k][j][i] : v[k][j+1][i];
vS = j == 0 ? v[k][j][i] : v[k][j-1][i];
wB = k == info.mz-1 ? w[k][j][i] : w[k+1][j][i];
wF = k == 0 ? w[k][j][i] : w[k-1][j][i];
*/
if( i == info.mx-1 ) { uE = u[k][j][i]; hx= 1; }else{ uE = u[k][j][i+1];hx= 2;}
if( i == info.mx-1 ) { uE = u[k][j][i]; hx= 1; }else{ uE = u[k][j][i+1];hx= 2;}
if( j == info.mx-1 ) { uE = u[k][j][i]; hy= 1; }else{ uE = u[k][j][i+1];hy= 2;}
if( j == info.mx-1 ) { uE = u[k][j][i]; hy= 1; }else{ uE = u[k][j][i+1];hy= 2;}
if( k == info.mx-1 ) { uE = u[k][j][i]; hz= 1; }else{ uE = u[k][j][i+1];hz= 2;}
if( k == info.mx-1 ) { uE = u[k][j][i]; hz= 1; }else{ uE = u[k][j][i+1];hz= 2;}
div2[k][j][i] = uE - uW + vN - vS + wB - wF;
div2[k][j][i] /= 2;
// printf("%f\t%f\t%f\n",div1[k][j][i], div2[k][j][i], div2[k][j][i] - div1[k][j][i]);
}
}
}
PetscGetTime(&s2);
DAVecRestoreArray(da,DIV1,&div1);
DAVecRestoreArray(da,DIV2,&div2);
VecAXPY(DIV1,-1,DIV2);
PetscReal norm;
VecNorm(DIV1,NORM_2,&norm);
printf("BULK: %f\nIF's: %f\nDIFF:\t%f\nRATIO:\t%f\nnorm: %f\n", (t2-t1), (s2-s1), (s2-s1)-(t2-t1),(s2-s1)/(t2-t1),norm);
}
int main(int argc, char **argv)
{
PetscInitialize(&argc, &argv, "wave.opt", help);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
double startTime, endTime;
int Ns = 32;
unsigned int dof = 1;
// double dtratio = 1.0;
DA da; // Underlying DA
Vec rho; // density - elemental scalar
Vec nu; // Lame parameter - lambda - elemental scalar
std::vector < std::vector<Vec> > fBasis; // the scalar activation - nodal scalar
// std::vector<Vec> truth; // the ground truth.
// Initial conditions
Vec initialDisplacement;
Vec initialVelocity;
timeInfo ti;
// get Ns
CHKERRQ ( PetscOptionsGetInt(0,"-Ns", &Ns,0) );
double t0 = 0.0;
double dt = 1.0/(Ns);
double t1 = 1.0;
double nuVal = 1.0;
double beta = 0.0001;
int numParams = 5;
CHKERRQ ( PetscOptionsGetInt(0,"-nump",&numParams,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-t0",&t0,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-t1",&t1,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-dt",&dt,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-nu",&nuVal,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-beta",&beta,0) );
// CHKERRQ ( PetscOptionsGetString(PETSC_NULL, "-pn", problemName, PETSC_MAX_PATH_LEN-1, PETSC_NULL));
// Time info for timestepping
ti.start = t0;
ti.stop = t1;
ti.step = dt;
if (!rank) {
std::cout << "Problem size is " << Ns+1 << " spatially and NT = " << (int)ceil(1.0/dt) << std::endl << std::endl;
std::cout << "Number of parameters is " << numParams << std::endl;
}
// create DA
CHKERRQ ( DACreate3d ( PETSC_COMM_WORLD, DA_NONPERIODIC, DA_STENCIL_BOX,
Ns+1, Ns+1, Ns+1, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE,
1, 1, 0, 0, 0, &da) );
massMatrix *Mass = new massMatrix(feMat::PETSC); // Mass Matrix
stiffnessMatrix *Stiffness = new stiffnessMatrix(feMat::PETSC); // Stiffness matrix
waveDamping *Damping = new waveDamping(feMat::PETSC); // Damping Matrix
fdynamicVector *Force = new fdynamicVector(feVec::PETSC); // Force Vector
// create vectors
CHKERRQ( DACreateGlobalVector(da, &rho) );
CHKERRQ( DACreateGlobalVector(da, &nu) );
CHKERRQ( DACreateGlobalVector(da, &initialDisplacement) );
CHKERRQ( DACreateGlobalVector(da, &initialVelocity) );
// Set initial conditions
CHKERRQ( VecSet ( initialDisplacement, 0.0) );
CHKERRQ( VecSet ( initialVelocity, 0.0) );
VecZeroEntries( nu );
VecZeroEntries( rho );
CHKERRQ( VecSet ( nu, nuVal) );
CHKERRQ( VecSet ( rho, 1.0) );
int x, y, z, m, n, p;
int mx,my,mz;
CHKERRQ( DAGetCorners(da, &x, &y, &z, &m, &n, &p) );
CHKERRQ( DAGetInfo(da,0, &mx, &my, &mz, 0,0,0,0,0,0,0) );
double acx,acy,acz;
double hx = 1.0/((double)Ns);
// allocate for temporary buffers ...
// unsigned int elemSize = Ns*Ns*Ns;
// std::cout << "Elem size is " << elemSize << std::endl;
// unsigned int nodeSize = (Ns+1)*(Ns+1)*(Ns+1);
// Now set the activation ...
unsigned int numSteps = (unsigned int)(ceil(( ti.stop - ti.start)/ti.step));
// Vec tauVec;
// PetscScalar ***tauArray;
unsigned int paramTimeSteps = (unsigned int)(ceil(( (double)(numSteps))/ ((double)(2*numParams)) ));
/*
for (int b=0; b<numParams; b++) {
std::vector<Vec> tau;
unsigned int tBegin = paramTimeSteps*b;
unsigned int tEnd = tBegin + numSteps/2; // paramTimeSteps*(b+2);
// std::cout << "For param " << b << ": Time step range is " << tBegin << " -> " << tEnd << std::endl;
for (unsigned int t=0; t<numSteps+1; t++) {
double newTime = (dt*(t-tBegin)*numSteps)/((double)(paramTimeSteps));
// double fff = 0.0;
CHKERRQ( DACreateGlobalVector(da, &tauVec) );
CHKERRQ( VecSet( tauVec, 0.0));
if ( (t>=tBegin) && (t<=tEnd)) {
CHKERRQ(DAVecGetArray(da, tauVec, &tauArray));
for (int k = z; k < z + p ; k++) {
for (int j = y; j < y + n; j++) {
for (int i = x; i < x + m; i++) {
acx = (i)*hx; acy = (j)*hx; acz = (k)*hx;
tauArray[k][j][i] = sin(M_PI*newTime)*cos(2*M_PI*acx)*cos(2*M_PI*acy)*cos(2*M_PI*acz);
}
}
}
CHKERRQ( DAVecRestoreArray ( da, tauVec, &tauArray ) );
}
tau.push_back(tauVec);
}
fBasis.push_back(tau);
}
*/
// std::cout << "Finished setting basis" << std::endl;
/*
// Set initial velocity ...
CHKERRQ(DAVecGetArray(da, initialVelocity, &solArray));
for (int k = z; k < z + p ; k++) {
for (int j = y; j < y + n; j++) {
for (int i = x; i < x + m; i++) {
acx = (i)*hx; acy = (j)*hx; acz = (k)*hx;
solArray[k][j][i] = M_PI*cos(2*M_PI*acx)*cos(2*M_PI*acy)*cos(2*M_PI*acz);
}
}
}
CHKERRQ( DAVecRestoreArray ( da, initialVelocity, &solArray ) );
*/
std::vector<Vec> newF;
Vec alpha;
PetscScalar *avec;
VecCreateSeq(PETSC_COMM_SELF, numParams, &alpha);
/*
VecCreate(PETSC_COMM_WORLD, &alpha);
VecSetSizes(alpha, numParams, PETSC_DECIDE);
VecSetFromOptions(alpha);
*/
VecGetArray(alpha, &avec);
for (int j=0; j<numParams; j++)
avec[j] = 0.5 + 0.5*j;
VecRestoreArray(alpha, &avec);
// getForces(alpha, fBasis, newF);
getForces(alpha, newF, da, ti, numParams);
// Setup Matrices and Force Vector ...
Mass->setProblemDimensions(1.0, 1.0, 1.0);
Mass->setDA(da);
Mass->setDof(dof);
Mass->setNuVec(rho);
Stiffness->setProblemDimensions(1.0, 1.0, 1.0);
Stiffness->setDA(da);
Stiffness->setDof(dof);
Stiffness->setNuVec(nu);
Damping->setAlpha(0.0);
Damping->setBeta(0.00075);
Damping->setMassMatrix(Mass);
Damping->setStiffnessMatrix(Stiffness);
Damping->setDA(da);
Damping->setDof(dof);
// Force Vector
Force->setProblemDimensions(1.0,1.0,1.0);
Force->setDA(da);
Force->setFDynamic(newF);
Force->setTimeInfo(&ti);
// Newmark time stepper ...
newmark *ts = new newmark;
ts->setMassMatrix(Mass);
ts->setDampingMatrix(Damping);
ts->setStiffnessMatrix(Stiffness);
ts->damp(false);
ts->setTimeFrames(1);
ts->storeVec(true);
ts->setAdjoint(false);
ts->setForceVector(Force);
ts->setInitialDisplacement(initialDisplacement);
ts->setInitialVelocity(initialVelocity);
ts->setTimeInfo(&ti);
ts->setAdjoint(false); // set if adjoint or forward
ts->init(); // initialize IMPORTANT
// if (!rank)
// std::cout << RED"Starting initial forward solve"NRM << std::endl;
ts->solve();// solve
// if (!rank)
// std::cout << GRN"Finished with initial forward solve"NRM << std::endl;
std::vector<Vec> solvec = ts->getSolution();
// Now lets check the error ...
// Vec nr;
// concatenateVecs(solvec, nr);
// VecDestroy(nr);
// VecDestroy(gt);
// std::cout << std::endl;
/*************
* INVERSE *
*************/
// True solution is tau ... we want to recover it.
// The observations in this case are, solvec
/* Set very initial guess for the inverse problem*/
// Now can clear memory ...
/*
for (int i=0; i<newF.size(); i++) {
if (newF[i] != NULL) {
VecDestroy(newF[i]);
}
}
newF.clear();
for (int i=0; i<solvec.size(); i++) {
if (solvec[i] != NULL) {
VecDestroy(solvec[i]);
}
}
solvec.clear();
ts->destroy();
VecDestroy(rho);
VecDestroy(nu);
VecDestroy(initialDisplacement);
VecDestroy(initialVelocity);
VecDestroy(alpha);
DADestroy(da);
PetscFinalize();
return 0;
*/
Vec gt, nr;
concatenateVecs(solvec, gt);
Vec guess;
VecDuplicate(alpha, &guess);
VecZeroEntries(guess);
// VecDuplicate(guess, &Out);
// VecZeroEntries(Out);
// double norm;
/*
PetscRandom rctx;
PetscRandomCreate(PETSC_COMM_WORLD, &rctx);
PetscRandomSetFromOptions(rctx);
VecSetRandom(guess, rctx);
VecNorm(guess, NORM_2, &norm);
PetscPrintf(0, "guess norm = %g\n", norm);
*/
// double errnorm;
// double exsolnorm;
// Inverse solver set up
// std::cout << RED"Setting up Inverse Solver"NRM << std::endl;
parametricWaveInverse* hyperInv = new parametricWaveInverse;
// std::cout << GRN"Finished setting up Inverse Solver"NRM << std::endl;
hyperInv->setTimeStepper(ts); // set the timestepper
hyperInv->setForwardInitialConditions(initialDisplacement, initialVelocity);
// std::cout << RED"Setting initial guess"NRM << std::endl;
// hyperInv->setInitialGuess(truth);// set the initial guess
hyperInv->setInitialGuess(guess);// set the initial guess
// std::cout << GRN"Done setting initial guess"NRM << std::endl;
hyperInv->setRegularizationParameter(beta); // set the regularization paramter
hyperInv->setAdjoints(solvec); // set the data for the problem
// hyperInv->setForceBasis(fBasis);
hyperInv->setNumberOfParameter(numParams);
// std::cout << RED"Initializing Inverse Solver"NRM << std::endl;
hyperInv->init(); // initialize the inverse solver
// if (!rank)
// std::cout << RED"Starting Inverse Solve"NRM << std::endl;
startTime = MPI_Wtime();
hyperInv->solve(); // solve
endTime = MPI_Wtime();
// if (!rank)
// std::cout << GRN"FINISHED HESSIAN SOLVE"NRM << std::endl;
hyperInv->getCurrentControl(guess); // get the solution
hyperInv->destroy();
/*
for (int i=0; i<solvec.size(); i++) {
if (solvec[i] != NULL) {
VecDestroy(solvec[i]);
}
}
solvec.clear();
*/
// VecView(guess, 0);
if (!rank)
std::cout << std::endl << "Error Norms " << std::endl;
Vec Err;
double gtNorm, solNorm, errNorm;
VecDuplicate(guess, &Err);
VecWAXPY(Err, -1.0, guess, alpha);
VecNorm(alpha, NORM_2, >Norm);
VecNorm(guess, NORM_2, &solNorm);
VecNorm(Err, NORM_2, &errNorm);
if (!rank) {
std::cout << "The norms are " << gtNorm << ", " << solNorm << ", " << errNorm << std::endl;
std::cout << "Relative error is " << errNorm/gtNorm << std::endl;
}
// Now we shall do another forward solve ...
getForces(guess, newF, da, ti, numParams);
Force->setFDynamic(newF);
ts->setInitialDisplacement(initialDisplacement);
ts->setInitialVelocity(initialVelocity);
ts->setAdjoint(false);
ts->clearMonitor();
ts->solve();
std::vector<Vec> solvec2 = ts->getSolution();
ts->destroy();
concatenateVecs(solvec2, nr);
// Now can clear memory ...
for (int i=0; i<solvec2.size(); i++) {
if (solvec2[i] != NULL) {
VecDestroy(solvec2[i]);
}
}
solvec2.clear();
// Now can clear memory ...
for (int i=0; i<newF.size(); i++) {
if (newF[i] != NULL) {
VecDestroy(newF[i]);
}
}
newF.clear();
/*
for (unsigned int i=0; i<truth.size(); i++) {
VecNorm(truth[i], NORM_2, >Norm);
VecNorm(solvec[i], NORM_2, &solNorm);
VecAXPY(solvec[i], -1.0, truth[i]);
VecNorm(solvec[i], NORM_2, &errNorm);
PetscPrintf(0, "Ground truth at timestep %d is %g, %g, %g\n", i, gtNorm, solNorm, errNorm);
// PetscPrintf(0, "Relative Error at timestep %d is %g\n", i, errNorm/gtNorm);
}
*/
VecNorm(gt, NORM_2, >Norm);
VecAXPY(nr, -1.0, gt);
VecNorm(nr, NORM_2, &errNorm);
if (!rank)
std::cout << "Total Relative error on state is " << errNorm/gtNorm << std::endl;
if (!rank)
std::cout << "Wall time is " << endTime - startTime << std::endl;
VecDestroy(gt);
VecDestroy(nr);
VecDestroy(Err);
VecDestroy(alpha);
VecDestroy(guess);
VecDestroy(rho);
VecDestroy(nu);
VecDestroy(initialDisplacement);
VecDestroy(initialVelocity);
DADestroy(da);
PetscFinalize();
}
int main(int argc,char **argv)
{
PetscMPIInt size,rank;
PetscInt M=8,dof=1,stencil_width=1,i,start,end,P=5,N = 6,m=PETSC_DECIDE,n=PETSC_DECIDE,p=PETSC_DECIDE,pt = 0,st = 0;
PetscErrorCode ierr;
PetscTruth flg2,flg3,flg;
DAPeriodicType periodic = DA_NONPERIODIC;
DAStencilType stencil_type = DA_STENCIL_STAR;
DA da;
SDA sda;
Vec local,global,local_copy;
PetscScalar value,*in,*out;
PetscReal norm,work;
PetscViewer viewer;
char filename[PETSC_MAX_PATH_LEN];
FILE *file;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-P",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-stencil_width",&stencil_width,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-periodic",&pt,PETSC_NULL);CHKERRQ(ierr);
periodic = (DAPeriodicType) pt;
ierr = PetscOptionsGetInt(PETSC_NULL,"-stencil_type",&st,PETSC_NULL);CHKERRQ(ierr);
stencil_type = (DAStencilType) st;
ierr = PetscOptionsHasName(PETSC_NULL,"-1d",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-2d",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-3d",&flg3);CHKERRQ(ierr);
if (flg2) {
ierr = DACreate2d(PETSC_COMM_WORLD,periodic,stencil_type,M,N,m,n,dof,stencil_width,0,0,&da);CHKERRQ(ierr);
ierr = SDACreate2d(PETSC_COMM_WORLD,periodic,stencil_type,M,N,m,n,dof,stencil_width,0,0,&sda);CHKERRQ(ierr);
} else if (flg3) {
ierr = DACreate3d(PETSC_COMM_WORLD,periodic,stencil_type,M,N,P,m,n,p,dof,stencil_width,0,0,0,&da);CHKERRQ(ierr);
ierr = SDACreate3d(PETSC_COMM_WORLD,periodic,stencil_type,M,N,P,m,n,p,dof,stencil_width,0,0,0,&sda);CHKERRQ(ierr);
}
else {
ierr = DACreate1d(PETSC_COMM_WORLD,periodic,M,dof,stencil_width,PETSC_NULL,&da);CHKERRQ(ierr);
ierr = SDACreate1d(PETSC_COMM_WORLD,periodic,M,dof,stencil_width,PETSC_NULL,&sda);CHKERRQ(ierr);
}
ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = VecDuplicate(local,&local_copy);CHKERRQ(ierr);
/* zero out vectors so that ghostpoints are zero */
value = 0;
ierr = VecSet(local,value);CHKERRQ(ierr);
ierr = VecSet(local_copy,value);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(global,&start,&end);CHKERRQ(ierr);
for (i=start; i<end; i++) {
value = i + 1;
ierr = VecSetValues(global,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(global);CHKERRQ(ierr);
ierr = VecAssemblyEnd(global);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetTruth(PETSC_NULL,"-same_array",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
/* test the case where the input and output array is the same */
ierr = VecCopy(local,local_copy);CHKERRQ(ierr);
ierr = VecGetArray(local_copy,&in);CHKERRQ(ierr);
ierr = VecRestoreArray(local_copy,PETSC_NULL);CHKERRQ(ierr);
ierr = SDALocalToLocalBegin(sda,in,INSERT_VALUES,in);CHKERRQ(ierr);
ierr = SDALocalToLocalEnd(sda,in,INSERT_VALUES,in);CHKERRQ(ierr);
} else {
ierr = VecGetArray(local,&out);CHKERRQ(ierr);
ierr = VecRestoreArray(local,PETSC_NULL);CHKERRQ(ierr);
ierr = VecGetArray(local_copy,&in);CHKERRQ(ierr);
ierr = VecRestoreArray(local_copy,PETSC_NULL);CHKERRQ(ierr);
ierr = SDALocalToLocalBegin(sda,out,INSERT_VALUES,in);CHKERRQ(ierr);
ierr = SDALocalToLocalEnd(sda,out,INSERT_VALUES,in);CHKERRQ(ierr);
}
flg = PETSC_FALSE;
ierr = PetscOptionsGetTruth(PETSC_NULL,"-save",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
sprintf(filename,"local.%d",rank);
ierr = PetscViewerASCIIOpen(PETSC_COMM_SELF,filename,&viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIGetPointer(viewer,&file);CHKERRQ(ierr);
ierr = VecView(local,viewer);CHKERRQ(ierr);
fprintf(file,"Vector with correct ghost points\n");
ierr = VecView(local_copy,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
}
ierr = VecAXPY(local_copy,-1.0,local);CHKERRQ(ierr);
ierr = VecNorm(local_copy,NORM_MAX,&work);CHKERRQ(ierr);
ierr = MPI_Allreduce(&work,&norm,1,MPIU_REAL,MPI_MAX,PETSC_COMM_WORLD);CHKERRQ(ierr);
if (norm != 0.0) {
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of difference %G should be zero\n",norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Number of processors %d\n",size);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," M,N,P,dof %D %D %D %D\n",M,N,P,dof);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," stencil_width %D stencil_type %d periodic %d\n",stencil_width,(int)stencil_type,(int)periodic);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," dimension %d\n",1 + (int) flg2 + (int) flg3);CHKERRQ(ierr);
}
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = SDADestroy(sda);CHKERRQ(ierr);
ierr = VecDestroy(local_copy);CHKERRQ(ierr);
ierr = VecDestroy(local);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = -10,N = -8;
PetscErrorCode ierr;
PetscTruth flg = PETSC_FALSE;
DA da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DAPeriodicType ptype = DA_NONPERIODIC;
DAStencilType stype = DA_STENCIL_BOX;
#if defined(PETSC_HAVE_MATLAB_ENGINE)
PetscViewer mviewer;
#endif
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,300,300,&viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MATLAB_ENGINE)
ierr = PetscViewerMatlabOpen(PETSC_COMM_WORLD,"tmp.mat",FILE_MODE_WRITE,&mviewer);CHKERRQ(ierr);
#endif
ierr = PetscOptionsGetTruth(PETSC_NULL,"-star_stencil",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stype = DA_STENCIL_STAR;
/* Create distributed array and get vectors */
ierr = DACreate2d(PETSC_COMM_WORLD,ptype,stype,M,N,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr);
value = -3.0;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank+1;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DALocalToGlobal(da,local,ADD_VALUES,global);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetTruth(PETSC_NULL, "-view_global", &flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) { /* view global vector in natural ordering */
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = DAView(da,viewer);CHKERRQ(ierr);
ierr = VecView(global,viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MATLAB_ENGINE)
ierr = DAView(da,mviewer);CHKERRQ(ierr);
ierr = VecView(global,mviewer);CHKERRQ(ierr);
#endif
/* Free memory */
#if defined(PETSC_HAVE_MATLAB_ENGINE)
ierr = PetscViewerDestroy(mviewer);CHKERRQ(ierr);
#endif
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
ierr = VecDestroy(local);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscInt i,j,M = 10,N = 8,m = PETSC_DECIDE,n = PETSC_DECIDE;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscTruth flg = PETSC_FALSE;
DA da;
PetscViewer viewer;
Vec localall,global;
PetscScalar value,*vlocal;
DAPeriodicType ptype = DA_NONPERIODIC;
DAStencilType stype = DA_STENCIL_BOX;
VecScatter tolocalall,fromlocalall;
PetscInt start,end;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,300,300,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetTruth(PETSC_NULL,"-star_stencil",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stype = DA_STENCIL_STAR;
/* Create distributed array and get vectors */
ierr = DACreate2d(PETSC_COMM_WORLD,ptype,stype,
M,N,m,n,1,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,M*N,&localall);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(global,&start,&end);CHKERRQ(ierr);
for (i=start; i<end; i++) {
value = 5.0*rank;
ierr = VecSetValues(global,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecView(global,viewer);CHKERRQ(ierr);
/*
Create Scatter from global DA parallel vector to local vector that
contains all entries
*/
ierr = DAGlobalToNaturalAllCreate(da,&tolocalall);CHKERRQ(ierr);
ierr = DANaturalAllToGlobalCreate(da,&fromlocalall);CHKERRQ(ierr);
ierr = VecScatterBegin(tolocalall,global,localall,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(tolocalall,global,localall,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(localall,&vlocal);CHKERRQ(ierr);
for (j=0; j<N; j++) {
for (i=0; i<M; i++) {
*vlocal++ += i + j*M;
}
}
ierr = VecRestoreArray(localall,&vlocal);CHKERRQ(ierr);
/* scatter back to global vector */
ierr = VecScatterBegin(fromlocalall,localall,global,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(fromlocalall,localall,global,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecView(global,viewer);CHKERRQ(ierr);
/* Free memory */
ierr = VecScatterDestroy(tolocalall);CHKERRQ(ierr);
ierr = VecScatterDestroy(fromlocalall);CHKERRQ(ierr);
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
ierr = VecDestroy(localall);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscInt M = 13,dof=1,s=1,wrap=0,i,n,j;
PetscErrorCode ierr;
DA da;
PetscViewer viewer;
Vec local,locala,global,coors;
PetscScalar *x,*alocal;
PetscDraw draw;
char fname[16];
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Create viewers */
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",PETSC_DECIDE,PETSC_DECIDE,600,200,&viewer);CHKERRQ(ierr);
ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr);
ierr = PetscDrawSetDoubleBuffer(draw);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-periodic",&wrap,PETSC_NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = DACreate1d(PETSC_COMM_WORLD,(DAPeriodicType)wrap,M,dof,s,PETSC_NULL,&da);CHKERRQ(ierr);
ierr = DASetUniformCoordinates(da,0.0,1.0,0.0,0.0,0.0,0.0);CHKERRQ(ierr);
for (i=0; i<dof; i++) {
sprintf(fname,"Field %d",(int)i);
ierr = DASetFieldName(da,i,fname);
}
ierr = DAView(da,viewer);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = DACreateLocalVector(da,&locala);CHKERRQ(ierr);
ierr = DAGetCoordinates(da,&coors);CHKERRQ(ierr);
ierr = VecGetArray(coors,&x);CHKERRQ(ierr);
/* Set values into global vectors */
ierr = VecGetArray(global,&alocal);CHKERRQ(ierr);
ierr = VecGetLocalSize(global,&n);CHKERRQ(ierr);
n = n/dof;
for (j=0; j<dof; j++) {
for (i=0; i<n; i++) {
alocal[j+dof*i] = PetscSinScalar(2*PETSC_PI*(j+1)*x[i]);
}
}
ierr = VecRestoreArray(global,&alocal);CHKERRQ(ierr);
ierr = VecRestoreArray(coors,&x);CHKERRQ(ierr);
ierr = VecDestroy(coords);CHKERRQ(ierr);
ierr = VecView(global,viewer);CHKERRQ(ierr);
/* Send ghost points to local vectors */
ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,locala);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,locala);CHKERRQ(ierr);
/* Free memory */
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = VecDestroy(local);CHKERRQ(ierr);
ierr = VecDestroy(locala);CHKERRQ(ierr);
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank,size;
PetscErrorCode ierr;
PetscInt M = 14,time_steps = 1000,w=1,s=1,localsize,j,i,mybase,myend;
DA da;
PetscViewer viewer;
PetscDraw draw;
Vec local,global,copy;
PetscScalar *localptr,*copyptr;
PetscReal h,k;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-time",&time_steps,PETSC_NULL);CHKERRQ(ierr);
/* Set up the array */
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,M,w,s,PETSC_NULL,&da);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/* Make copy of local array for doing updates */
ierr = VecDuplicate(local,©);CHKERRQ(ierr);
/* Set Up Display to Show Heat Graph */
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",80,480,500,160,&viewer);CHKERRQ(ierr);
ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr);
ierr = PetscDrawSetDoubleBuffer(draw);CHKERRQ(ierr);
/* determine starting point of each processor */
ierr = VecGetOwnershipRange(global,&mybase,&myend);CHKERRQ(ierr);
/* Initialize the Array */
ierr = VecGetLocalSize (local,&localsize);CHKERRQ(ierr);
ierr = VecGetArray (local,&localptr);CHKERRQ(ierr);
ierr = VecGetArray (copy,©ptr);CHKERRQ(ierr);
localptr[0] = copyptr[0] = 0.0;
localptr[localsize-1] = copyptr[localsize-1] = 1.0;
for (i=1; i<localsize-1; i++) {
j=(i-1)+mybase;
localptr[i] = sin((PETSC_PI*j*6)/((PetscReal)M)
+ 1.2 * sin((PETSC_PI*j*2)/((PetscReal)M))) * 4+4;
}
ierr = VecRestoreArray(local,&localptr);CHKERRQ(ierr);
ierr = VecRestoreArray(copy,©ptr);CHKERRQ(ierr);
ierr = DALocalToGlobal(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
/* Assign Parameters */
h= 1.0/M;
k= h*h/2.2;
for (j=0; j<time_steps; j++) {
/* Global to Local */
ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/*Extract local array */
ierr = VecGetArray(local,&localptr);CHKERRQ(ierr);
ierr = VecGetArray (copy,©ptr);CHKERRQ(ierr);
/* Update Locally - Make array of new values */
/* Note: I don't do anything for the first and last entry */
for (i=1; i< localsize-1; i++) {
copyptr[i] = localptr[i] + (k/(h*h)) *
(localptr[i+1]-2.0*localptr[i]+localptr[i-1]);
}
ierr = VecRestoreArray(copy,©ptr);CHKERRQ(ierr);
ierr = VecRestoreArray(local,&localptr);CHKERRQ(ierr);
/* Local to Global */
ierr = DALocalToGlobal(da,copy,INSERT_VALUES,global);CHKERRQ(ierr);
/* View Wave */
ierr = VecView(global,viewer);CHKERRQ(ierr);
}
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
ierr = VecDestroy(copy);CHKERRQ(ierr);
ierr = VecDestroy(local);CHKERRQ(ierr);
ierr = VecDestroy(global);CHKERRQ(ierr);
ierr = DADestroy(da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
