/*
* === FUNCTION ======================================================================
* Name: makeNewGrid
* Arguments: int gridDim - Dimensions dimXdim of new grid.
* Returns: Pointer to new grid object.
* Description: Makes new grid object and sets all data points to 0.
* =====================================================================================
*/
Grid * makeNewGrid(int gridDimX, int gridDimY) {
fftw_mpi_init() ;
Grid * newGrid = malloc(sizeof(Grid)) ;
ptrdiff_t localDimY, globalOffset, localDimX ;
newGrid->allocScheme = fftw_mpi_local_size_2d(gridDimY, gridDimX, MPI_COMM_WORLD, &localDimY, &globalOffset);
newGrid->localGridDimY = localDimY ;
newGrid->localGridDimX = gridDimX ;
newGrid->globalGridDimY = gridDimY ;
newGrid->globalGridDimX = gridDimX ;
newGrid->globalOffset = globalOffset ;
newGrid->gridPoints = fftw_alloc_real(newGrid->allocScheme) ;
return newGrid ;
} /* ----- end of function makeNewGrid ----- */
PetscInt main(PetscInt argc,char **args)
{
typedef enum {RANDOM, CONSTANT, TANH, NUM_FUNCS} FuncType;
const char *funcNames[NUM_FUNCS] = {"random", "constant", "tanh"};
PetscMPIInt size;
PetscInt n = 10,N,Ny,ndim=4,dim[4],DIM,i;
Vec x,y,z;
PetscScalar s;
PetscRandom rdm;
PetscReal enorm;
PetscInt func=RANDOM;
FuncType function = RANDOM;
PetscBool view = PETSC_FALSE;
PetscErrorCode ierr;
PetscScalar *x_array,*y_array,*z_array;
fftw_plan fplan,bplan;
const ptrdiff_t N0 = 20, N1 = 20;
ptrdiff_t alloc_local, local_n0, local_0_start;
ierr = PetscInitialize(&argc,&args,(char *)0,help);CHKERRQ(ierr);
#if defined(PETSC_USE_COMPLEX)
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This example requires real numbers");
#endif
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
alloc_local=fftw_mpi_local_size_2d(N0, N1, PETSC_COMM_WORLD,
&local_n0, &local_0_start);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This is a uniprocessor example only!");
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, PETSC_NULL, "FFTW Options", "ex142");CHKERRQ(ierr);
ierr = PetscOptionsEList("-function", "Function type", "ex142", funcNames, NUM_FUNCS, funcNames[function], &func, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-vec_view draw", "View the functions", "ex112", view, &view, PETSC_NULL);CHKERRQ(ierr);
function = (FuncType) func;
ierr = PetscOptionsEnd();CHKERRQ(ierr);
for (DIM = 0; DIM < ndim; DIM++){
dim[DIM] = n; /* size of real space vector in DIM-dimension */
}
ierr = PetscRandomCreate(PETSC_COMM_SELF, &rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
for (DIM = 1; DIM < 5; DIM++){
/* create vectors of length N=dim[0]*dim[1]* ...*dim[DIM-1] */
/*----------------------------------------------------------*/
N = Ny = 1;
for (i = 0; i < DIM-1; i++) {
N *= dim[i];
}
Ny = N; Ny *= 2*(dim[DIM-1]/2 + 1); /* add padding elements to output vector y */
N *= dim[DIM-1];
ierr = PetscPrintf(PETSC_COMM_SELF, "\n %d-D: FFTW on vector of size %d \n",DIM,N);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,N,&x);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Real space vector");CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,Ny,&y);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, "Frequency space vector");CHKERRQ(ierr);
ierr = VecDuplicate(x,&z);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) z, "Reconstructed vector");CHKERRQ(ierr);
/* Set fftw plan */
/*----------------------------------*/
ierr = VecGetArray(x,&x_array);CHKERRQ(ierr);
ierr = VecGetArray(y,&y_array);CHKERRQ(ierr);
ierr = VecGetArray(z,&z_array);CHKERRQ(ierr);
unsigned int flags = FFTW_ESTIMATE; //or FFTW_MEASURE
/* The data in the in/out arrays is overwritten during FFTW_MEASURE planning, so such planning
should be done before the input is initialized by the user. */
printf("DIM: %d, N %d, Ny %d\n",DIM,N,Ny);
switch (DIM){
case 1:
fplan = fftw_plan_dft_r2c_1d(dim[0], (double *)x_array, (fftw_complex*)y_array, flags);
bplan = fftw_plan_dft_c2r_1d(dim[0], (fftw_complex*)y_array, (double *)z_array, flags);
break;
case 2:
fplan = fftw_plan_dft_r2c_2d(dim[0],dim[1],(double *)x_array, (fftw_complex*)y_array,flags);
bplan = fftw_plan_dft_c2r_2d(dim[0],dim[1],(fftw_complex*)y_array,(double *)z_array,flags);
break;
case 3:
fplan = fftw_plan_dft_r2c_3d(dim[0],dim[1],dim[2],(double *)x_array, (fftw_complex*)y_array,flags);
bplan = fftw_plan_dft_c2r_3d(dim[0],dim[1],dim[2],(fftw_complex*)y_array,(double *)z_array,flags);
break;
default:
fplan = fftw_plan_dft_r2c(DIM,dim,(double *)x_array, (fftw_complex*)y_array,flags);
bplan = fftw_plan_dft_c2r(DIM,dim,(fftw_complex*)y_array,(double *)z_array,flags);
break;
}
ierr = VecRestoreArray(x,&x_array);CHKERRQ(ierr);
ierr = VecRestoreArray(y,&y_array);CHKERRQ(ierr);
ierr = VecRestoreArray(z,&z_array);CHKERRQ(ierr);
/* Initialize Real space vector x:
The data in the in/out arrays is overwritten during FFTW_MEASURE planning, so planning
should be done before the input is initialized by the user.
--------------------------------------------------------*/
if (function == RANDOM) {
ierr = VecSetRandom(x, rdm);CHKERRQ(ierr);
} else if (function == CONSTANT) {
ierr = VecSet(x, 1.0);CHKERRQ(ierr);
} else if (function == TANH) {
ierr = VecGetArray(x, &x_array);CHKERRQ(ierr);
for (i = 0; i < N; ++i) {
x_array[i] = tanh((i - N/2.0)*(10.0/N));
}
ierr = VecRestoreArray(x, &x_array);CHKERRQ(ierr);
}
if (view) {
ierr = VecView(x, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* FFT - also test repeated transformation */
/*-------------------------------------------*/
ierr = VecGetArray(x,&x_array);CHKERRQ(ierr);
ierr = VecGetArray(y,&y_array);CHKERRQ(ierr);
ierr = VecGetArray(z,&z_array);CHKERRQ(ierr);
for (i=0; i<3; i++){
/* FFTW_FORWARD */
fftw_execute(fplan);
//printf("\n fout:\n");
//fftw_complex* fout = (fftw_complex*)y_array;
//for (i=0; i<N/2+1; i++) printf("%d (%g %g)\n",i,fout[i][0],fout[i][1]);
/* FFTW_BACKWARD: destroys its input array 'y_array' even for out-of-place transforms! */
fftw_execute(bplan);
}
ierr = VecRestoreArray(x,&x_array);CHKERRQ(ierr);
ierr = VecRestoreArray(y,&y_array);CHKERRQ(ierr);
ierr = VecRestoreArray(z,&z_array);CHKERRQ(ierr);
/* Compare x and z. FFTW computes an unnormalized DFT, thus z = N*x */
/*------------------------------------------------------------------*/
s = 1.0/(PetscReal)N;
ierr = VecScale(z,s);CHKERRQ(ierr);
if (view) {ierr = VecView(x, PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);}
if (view) {ierr = VecView(z, PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);}
ierr = VecAXPY(z,-1.0,x);CHKERRQ(ierr);
ierr = VecNorm(z,NORM_1,&enorm);CHKERRQ(ierr);
if (enorm > 1.e-11){
ierr = PetscPrintf(PETSC_COMM_SELF," Error norm of |x - z| %G\n",enorm);CHKERRQ(ierr);
}
/* free spaces */
fftw_destroy_plan(fplan);
fftw_destroy_plan(bplan);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&z);CHKERRQ(ierr);
}
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInt N0=50,N1=20,N=N0*N1,DIM;
PetscRandom rdm;
PetscScalar a;
PetscReal enorm;
Vec x,y,z;
PetscBool view=PETSC_FALSE,use_interface=PETSC_TRUE;
ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr);
#if !defined(PETSC_USE_COMPLEX)
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This example requires complex numbers");
#endif
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "FFTW Options", "ex143");CHKERRQ(ierr);
ierr = PetscOptionsBool("-vec_view draw", "View the vectors", "ex143", view, &view, NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-use_FFTW_interface", "Use PETSc-FFTW interface", "ex143",use_interface, &use_interface, NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-use_FFTW_interface",&use_interface,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD, &rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
if (!use_interface) {
/* Use mpi FFTW without PETSc-FFTW interface, 2D case only */
/*---------------------------------------------------------*/
fftw_plan fplan,bplan;
fftw_complex *data_in,*data_out,*data_out2;
ptrdiff_t alloc_local,local_n0,local_0_start;
DIM = 2;
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Use FFTW without PETSc-FFTW interface, DIM %D\n",DIM);CHKERRQ(ierr);
}
fftw_mpi_init();
N = N0*N1;
alloc_local = fftw_mpi_local_size_2d(N0,N1,PETSC_COMM_WORLD,&local_n0,&local_0_start);
data_in = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
data_out = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
data_out2 = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_in,&x);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Real Space vector");CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_out,&y);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, "Frequency space vector");CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_out2,&z);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) z, "Reconstructed vector");CHKERRQ(ierr);
fplan = fftw_mpi_plan_dft_2d(N0,N1,data_in,data_out,PETSC_COMM_WORLD,FFTW_FORWARD,FFTW_ESTIMATE);
bplan = fftw_mpi_plan_dft_2d(N0,N1,data_out,data_out2,PETSC_COMM_WORLD,FFTW_BACKWARD,FFTW_ESTIMATE);
ierr = VecSetRandom(x, rdm);CHKERRQ(ierr);
if (view) {ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
fftw_execute(fplan);
if (view) {ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
fftw_execute(bplan);
/* Compare x and z. FFTW computes an unnormalized DFT, thus z = N*x */
a = 1.0/(PetscReal)N;
ierr = VecScale(z,a);CHKERRQ(ierr);
if (view) {ierr = VecView(z, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
ierr = VecAXPY(z,-1.0,x);CHKERRQ(ierr);
ierr = VecNorm(z,NORM_1,&enorm);CHKERRQ(ierr);
if (enorm > 1.e-11 && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF," Error norm of |x - z| %g\n",(double)enorm);CHKERRQ(ierr);
}
/* Free spaces */
fftw_destroy_plan(fplan);
fftw_destroy_plan(bplan);
fftw_free(data_in); ierr = VecDestroy(&x);CHKERRQ(ierr);
fftw_free(data_out); ierr = VecDestroy(&y);CHKERRQ(ierr);
fftw_free(data_out2);ierr = VecDestroy(&z);CHKERRQ(ierr);
} else {
/* Use PETSc-FFTW interface */
/*-------------------------------------------*/
PetscInt i,*dim,k;
Mat A;
N=1;
for (i=1; i<5; i++) {
DIM = i;
ierr = PetscMalloc1(i,&dim);CHKERRQ(ierr);
for (k=0; k<i; k++) {
dim[k]=30;
}
N *= dim[i-1];
/* Create FFTW object */
if (!rank) printf("Use PETSc-FFTW interface...%d-DIM: %d\n",(int)DIM,(int)N);
ierr = MatCreateFFT(PETSC_COMM_WORLD,DIM,dim,MATFFTW,&A);CHKERRQ(ierr);
/* Create vectors that are compatible with parallel layout of A - must call MatCreateVecs()! */
ierr = MatCreateVecsFFTW(A,&x,&y,&z);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Real space vector");CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, "Frequency space vector");CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) z, "Reconstructed vector");CHKERRQ(ierr);
/* Set values of space vector x */
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
if (view) {ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
/* Apply FFTW_FORWARD and FFTW_BACKWARD */
ierr = MatMult(A,x,y);CHKERRQ(ierr);
if (view) {ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
ierr = MatMultTranspose(A,y,z);CHKERRQ(ierr);
/* Compare x and z. FFTW computes an unnormalized DFT, thus z = N*x */
a = 1.0/(PetscReal)N;
ierr = VecScale(z,a);CHKERRQ(ierr);
if (view) {ierr = VecView(z,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
ierr = VecAXPY(z,-1.0,x);CHKERRQ(ierr);
ierr = VecNorm(z,NORM_1,&enorm);CHKERRQ(ierr);
if (enorm > 1.e-9 && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF," Error norm of |x - z| %e\n",enorm);CHKERRQ(ierr);
}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&z);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFree(dim);CHKERRQ(ierr);
}
}
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void init_fft2d_(void )
{
int i,j,k;
double vm2,vm1,v,vp1,vp2,vb;
commx = MPI_Comm_f2c(topo_.commxc);
MPI_Comm_rank(commx, &irankx);
MPI_Comm_size(commx, &isizex);
commyz = MPI_Comm_f2c(topo_.commyzc);
MPI_Comm_rank(commyz, &irankyz);
MPI_Comm_size(commyz, &isizeyz);
fftw_mpi_init();
howmany = topo_.mxlc;
//***********
alloc_ly = fftw_mpi_local_size_2d(my, mz, commx, &ly, &lys);
/* alloc_ly=fftw_mpi_local_size_many(rnk, myz, howmany,
FFTW_MPI_DEFAULT_BLOCK, commx, &ly, &lys);
*/
//***********
if(((ly-topo_.mylc)!=0) || topo_.npzc>1) {
printf("Error,npz should equal to 1, or %d\t%d\n",irankx,ly-topo_.mylc);
MPI_Abort(commx,1);
}
minp = fftw_alloc_complex(alloc_ly);
mout = fftw_alloc_complex(alloc_ly);
if( !(freq = r3tensor(topo_.mxlc, topo_.mylc*topo_.mzlc, 2)) )
printf("Malloc error!\n");
if( !(data = r3tensor(topo_.mxlc, topo_.mylc*topo_.mzlc, 2)) )
printf("Malloc error!\n");
/* if( !(dar = r3tensor(topo_.mxlc, topo_.mylc, topo_.mzlc)) )
printf("Malloc error!\n");
if( !(dai = r3tensor(topo_.mxlc, topo_.mylc, topo_.mzlc)) )
printf("Malloc error!\n");
*/
//***********
mplanF = fftw_mpi_plan_dft_2d(my, mz, minp, mout, commx, FFTW_FORWARD, FFTW_MEASURE);
mplanR = fftw_mpi_plan_dft_2d(my, mz, minp, mout, commx, FFTW_BACKWARD, FFTW_MEASURE);
/* mplanF = fftw_mpi_plan_many_dft(rnk, myz, howmany,
FFTW_MPI_DEFAULT_BLOCK,FFTW_MPI_DEFAULT_BLOCK,
minp, mout, commx, FFTW_FORWARD, FFTW_MEASURE);
mplanR = fftw_mpi_plan_many_dft(rnk, myz, howmany,
FFTW_MPI_DEFAULT_BLOCK,FFTW_MPI_DEFAULT_BLOCK,
minp, mout, commx, FFTW_BACKWARD, FFTW_MEASURE);
*/
//***********
//***** Solver part ******
dxs = topo_.dx0*topo_.dx0;
dys = topo_.dy0*topo_.dy0;
dzs = topo_.dz0*topo_.dz0;
vm2=-1.0/12.0;
vm1=16.0/12.0;
v =-30.0/12.0;
vp1=16.0/12.0;
vp2=-1.0/12.0;
MatCreateMPIAIJ(commyz, PETSC_DECIDE, PETSC_DECIDE, mx, mx, 5, PETSC_NULL, 5, PETSC_NULL, &A);
ierr = MatGetOwnershipRange(A,&Istart,&Iend);
for (Ii=Istart; Ii<Iend; Ii++) {
i = Ii; j = Ii;
if ((i>1)&&(i<mx-2)) {
J = Ii - 2; MatSetValues(A,1,&Ii,1,&J,&vm2,INSERT_VALUES);
J = Ii - 1; ierr = MatSetValues(A,1,&Ii,1,&J,&vm1,INSERT_VALUES);CHKERRQ(ierr);
J = Ii; ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);
J = Ii + 1; ierr = MatSetValues(A,1,&Ii,1,&J,&vp1,INSERT_VALUES);CHKERRQ(ierr);
J = Ii + 2; ierr = MatSetValues(A,1,&Ii,1,&J,&vp2,INSERT_VALUES);CHKERRQ(ierr);
}
if (i==0) {
J = Ii; ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (i==1) {
J = Ii - 1; vb = 11.0/12.0; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);CHKERRQ(ierr);
J = Ii ; vb = -5.0/3.0; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);
J = Ii + 1; vb = 0.5; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);
J = Ii + 2; vb = 1.0/3.0; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);
J = Ii + 3; vb = -1.0/12.0; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);
}
if (i==mx-2) {
J = Ii + 1; vb = 11.0/12.0; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);CHKERRQ(ierr);
J = Ii ; vb = -5.0/3.0; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);
J = Ii - 1; vb = 0.5; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);
J = Ii - 2; vb = 1.0/3.0; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);
J = Ii - 3; vb = -1.0/12.0; ierr = MatSetValues(A,1,&Ii,1,&J,&vb,INSERT_VALUES);
}
if (i==mx-1) {J = Ii; ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecCreate(commyz,&br);CHKERRQ(ierr);
ierr = VecSetSizes(br,PETSC_DECIDE,mx);CHKERRQ(ierr);
ierr = VecSetFromOptions(br);CHKERRQ(ierr);
ierr = VecDuplicate(br,&xr);CHKERRQ(ierr);
ierr = VecDuplicate(br,&bi);CHKERRQ(ierr);
ierr = VecDuplicate(br,&xi);CHKERRQ(ierr);
ierr = KSPCreate(commyz,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);
PCSetType(pc,PCJACOBI);
ierr = KSPSetTolerances(ksp,1.e-7,1.e-50,PETSC_DEFAULT,
PETSC_DEFAULT);CHKERRQ(ierr);
//******* End *******
}
