/*
#undef __FUNCT__
#define __FUNCT__ "phi2eval"
PetscErrorCode phi2eval(Vec *X, Vec *Phi, PetscInt n) {
PetscInt i,j,k,lo,hi;
PetscErrorCode ierr;
PetscReal sqrt2;
PetscFunctionBegin;
sqrt = PetscSqrtReal(2.0);
ierr = VecGetOwnershipRange(X,&lo,&hi);CHKERRQ(ierr);
PetscErrorCode phi2eval(PetscReal *x, PetscInt n, PetscReal *phi) {
// Phi = .5*[x(1)^2 sqrt(2)*x(1)*x(2) ... sqrt(2)*x(1)*x(n) ... x(2)^2 sqrt(2)*x(2)*x(3) .. x(n)^2]
PetscInt i,j,k;
PetscReal sqrt2 = PetscSqrtReal(2.0);
PetscFunctionBegin;
j=0;
for (i=0;i<n;i++) {
phi[j] = 0.5 * x[i]*x[i];
j++;
for (k=i+1;k<n;k++) {
phi[j] = x[i]*x[k]/sqrt2;
j++;
}
}
PetscFunctionReturn(0);
}*/
PetscErrorCode PoundersGramSchmidtReset(TAO_POUNDERS *mfqP, Vec *Q, PetscInt n)
{
PetscInt i;
for (i=0;i<n;i++) {
ierr = VecSet(Q[i],0.0);CHKERRQ(ierr);
ierr = VecSetValue(Q[i],i,1.0,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(Q[i]);CHKERRQ(ierr);
ierr = VecAssemblyEnd(Q[i]);CHKERRQ(ierr);
}
}
int EBFV1_elliptic::wells_1(pMesh mesh, Vec &RHS){
int node_ID, row;
double Vt, Vi, Qi, Qt;
if (!pSimPar->mapNodesOnWell.size()){
return 0;
throw Exception(__LINE__,__FILE__,"No wells found!");
}
// for each well flag
map<int,set<int> >::iterator mit = pSimPar->mapNodesOnWell.begin();
for (; mit!=pSimPar->mapNodesOnWell.end(); mit++){
int well_flag = mit->first;
// source/sink term
Qt = pSimPar->getFlowrateValue(well_flag);
if ( fabs(Qt)<=1e-7 ){
//throw Exception(__LINE__,__FILE__,"Flow rate NULL!");
}
// get all flagged node IDs for that well
if (!mit->second.size()){
throw Exception(__LINE__,__FILE__,"No wells found!");
}
SIter sit = mit->second.begin();
for (; sit!=mit->second.end(); sit++){
Vt = pSimPar->getWellVolume(well_flag);
#ifdef _SEEKFORBUGS_
if ( Vt<1e-12 ){
char msg[256]; sprintf(msg,"Well with null volume V = %.6f. Vertex (%d)",Vt,node_ID);
throw Exception(__LINE__,__FILE__,msg);
}
#endif //_SEEKFORBUGS_
Vi = .0;
node_ID = *sit;
for (SIter_const dom=pSimPar->setDomain_begin(); dom!=pSimPar->setDomain_end(); dom++){
pVertex node = (mEntity*)mesh->getVertex( node_ID );
Vi += pGCData->getVolume(node,*dom);
}
// for node i, Q is a fraction of total well flow rate
Qi = Qt*(Vi/Vt);
// FPArray ('F'ree 'P'rescribed array) maps node id: node_ID -> row
// row: position in Petsc GlobalMatrix/RHSVBector where node must be assembled
node_ID = pMData->get_AppToPETSc_Ordering(node_ID);
row = pMData->FPArray(node_ID-1);
// Do not include well flux on nodes with prescribed pressure
if (pSimPar->isNodeFree(well_flag)){
VecSetValue(RHS,row,Qi,ADD_VALUES);
}
}
}
return 0;
}
PetscErrorCode SolveInit(FEMInf fem, int L, PetscScalar *e0, Vec *x) {
PetscErrorCode ierr;
Mat H, S;
ierr = CalcMat(fem, L, &H, &S); CHKERRQ(ierr);
EPS eps;
ierr = PrintTimeStamp(fem->comm, "EPS", NULL); CHKERRQ(ierr);
ierr = EPSCreate(fem->comm, &eps); CHKERRQ(ierr);
ierr = EPSSetTarget(eps, -0.6); CHKERRQ(ierr);
ierr = EPSSetWhichEigenpairs(eps, EPS_TARGET_MAGNITUDE); CHKERRQ(ierr);
ierr = EPSSetOperators(eps, H, S); CHKERRQ(ierr);
if(S == NULL) {
ierr = EPSSetProblemType(eps, EPS_NHEP); CHKERRQ(ierr);
} else {
ierr = EPSSetProblemType(eps, EPS_GNHEP); CHKERRQ(ierr);
}
Vec x0[1]; MatCreateVecs(H, &x0[0], NULL);
int num; FEMInfGetSize(fem, &num);
for(int i = 0; i < num; i++) {
VecSetValue(x0[0], i, 0.5, INSERT_VALUES);
}
VecAssemblyBegin(x0[0]); VecAssemblyEnd(x0[0]);
EPSSetInitialSpace(eps, 1, x0);
ierr = EPSSetFromOptions(eps); CHKERRQ(ierr);
ierr = EPSSolve(eps); CHKERRQ(ierr);
PetscInt nconv;
EPSGetConverged(eps, &nconv);
if(nconv == 0)
SETERRQ(fem->comm, 1, "Failed to digonalize in init state\n");
Vec x_ans;
MatCreateVecs(H, &x_ans, NULL);
EPSGetEigenpair(eps, 0, e0, NULL, x_ans, NULL);
EPSDestroy(&eps);
PetscScalar v[1]; PetscInt idx[1] = {1};
VecGetValues(x_ans, 1, idx, v);
PetscScalar scale_factor = v[0] / cabs(v[0]);
VecScale( x_ans, 1.0/scale_factor);
PetscScalar norm0;
Vec Sx; MatCreateVecs(S, &Sx, NULL);
MatMult(S, x_ans, Sx); VecDot(x_ans, Sx, &norm0);
VecScale(x_ans, 1.0/sqrt(norm0));
*x = x_ans;
return 0;
}
PetscErrorCode GetRealPartVec(Vec vR, int N)
{
PetscErrorCode ierr;
int i, ns, ne;
ierr = VecGetOwnershipRange(vR,&ns,&ne); CHKERRQ(ierr);
for(i=ns; i<ne; i++)
{
if (i<N/2)
VecSetValue(vR,i,1.0,INSERT_VALUES);
else
VecSetValue(vR,i,0.0,INSERT_VALUES);
}
ierr = VecAssemblyBegin(vR); CHKERRQ(ierr);
ierr = VecAssemblyEnd(vR); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
-n <length> : vector length\n\n";
#include <petscvec.h>
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt n = 5,idx;
PetscReal value;
Vec x;
PetscScalar one = 1.0;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
/* create vector */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecSet(x,one);CHKERRQ(ierr);
ierr = VecSetValue(x,0,0.0,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecSetValue(x,n-1,2.0,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecMax(x,&idx,&value);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Maximum value %g index %D\n",(double)value,idx);CHKERRQ(ierr);
ierr = VecMin(x,&idx,&value);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Minimum value %g index %D\n",(double)value,idx);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@C
TaoDefaultComputeGradient - computes the gradient using finite differences.
Collective on Tao
Input Parameters:
+ tao - the Tao context
. X - compute gradient at this point
- dummy - not used
Output Parameters:
. G - Gradient Vector
Options Database Key:
+ -tao_fd_gradient - activates TaoDefaultComputeGradient()
- -tao_fd_delta <delta> - change in X used to calculate finite differences
Level: advanced
Notes:
This routine is slow and expensive, and is not currently optimized
to take advantage of sparsity in the problem. Although
TaoDefaultComputeGradient is not recommended for general use
in large-scale applications, It can be useful in checking the
correctness of a user-provided gradient. Use the tao method TAOTEST
to get an indication of whether your gradient is correct.
This finite difference gradient evaluation can be set using the routine TaoSetGradientRoutine() or by using the command line option -tao_fd_gradient
.seealso: TaoSetGradientRoutine()
@*/
PetscErrorCode TaoDefaultComputeGradient(Tao tao,Vec Xin,Vec G,void *dummy)
{
Vec X;
PetscScalar *g;
PetscReal f, f2;
PetscErrorCode ierr;
PetscInt low,high,N,i;
PetscBool flg;
PetscReal h=.5*PETSC_SQRT_MACHINE_EPSILON;
PetscFunctionBegin;
ierr = PetscOptionsGetReal(((PetscObject)tao)->options,((PetscObject)tao)->prefix,"-tao_fd_delta",&h,&flg);CHKERRQ(ierr);
ierr = VecDuplicate(Xin,&X);CHKERRQ(ierr);
ierr = VecCopy(Xin,X);CHKERRQ(ierr);
ierr = VecGetSize(X,&N);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(X,&low,&high);CHKERRQ(ierr);
ierr = VecSetOption(X,VEC_IGNORE_OFF_PROC_ENTRIES,PETSC_TRUE);CHKERRQ(ierr);
ierr = VecGetArray(G,&g);CHKERRQ(ierr);
for (i=0;i<N;i++) {
ierr = VecSetValue(X,i,-h,ADD_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao,X,&f);CHKERRQ(ierr);
ierr = VecSetValue(X,i,2.0*h,ADD_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao,X,&f2);CHKERRQ(ierr);
ierr = VecSetValue(X,i,-h,ADD_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
if (i>=low && i<high) {
g[i-low]=(f2-f)/(2.0*h);
}
}
ierr = VecRestoreArray(G,&g);CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void Field_solver::init_rhs_vector_in_full_domain( Spatial_mesh &spat_mesh )
{
PetscErrorCode ierr;
int nx = spat_mesh.x_n_nodes;
int ny = spat_mesh.y_n_nodes;
int nz = spat_mesh.z_n_nodes;
//int nrow = (nx-2)*(ny-2);
double dx = spat_mesh.x_cell_size;
double dy = spat_mesh.y_cell_size;
double dz = spat_mesh.z_cell_size;
double rhs_at_node;
// todo: split into separate functions
for ( int k = 1; k <= nz-2; k++ ) {
for ( int j = 1; j <= ny-2; j++ ) {
for ( int i = 1; i <= nx-2; i++ ) {
// - 4 * pi * rho * dx^2 * dy^2
rhs_at_node = -4.0 * M_PI * spat_mesh.charge_density[i][j][k];
rhs_at_node = rhs_at_node * dx * dx * dy * dy * dz * dz;
// left and right boundary
rhs_at_node = rhs_at_node
- dy * dy * dz * dz *
( kronecker_delta(i,1) * spat_mesh.potential[0][j][k] +
kronecker_delta(i,nx-2) * spat_mesh.potential[nx-1][j][k] );
// top and bottom boundary
rhs_at_node = rhs_at_node
- dx * dx * dz * dz *
( kronecker_delta(j,1) * spat_mesh.potential[i][0][k] +
kronecker_delta(j,ny-2) * spat_mesh.potential[i][ny-1][k] );
// near and far boundary
rhs_at_node = rhs_at_node
- dx * dx * dy * dy *
( kronecker_delta(k,1) * spat_mesh.potential[i][j][0] +
kronecker_delta(k,nz-2) * spat_mesh.potential[i][j][nz-1] );
// set rhs vector values
ierr = VecSetValue( rhs,
node_ijk_to_global_index_in_matrix( i, j, k, nx, ny, nz ),
rhs_at_node, INSERT_VALUES );
CHKERRXX( ierr );
}
}
}
ierr = VecAssemblyBegin( rhs ); CHKERRXX( ierr );
ierr = VecAssemblyEnd( rhs ); CHKERRXX( ierr );
return;
}
void init(const mpi::communicator& comm)
{
PetscErrorCode ierr;
PetscInt lo, hi;
ierr = VecCreate(comm,&x); // CHKERRQ(ierr);
ierr = VecSetSizes(x, PETSC_DECIDE, 5*comm.size()); // CHKERRQ(ierr);
ierr = VecSetFromOptions(x); // CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x, &lo, &hi);
for (PetscInt i = lo; i <= hi; ++i) {
std::complex<double> v(i, 5*comm.size() - i - 1);
ierr = VecSetValue(x, i, v, INSERT_VALUES);
}
ierr = VecAssemblyBegin(x); // CHKERRQ(ierr);
ierr = VecAssemblyEnd(x); // CHKERRQ(ierr);
}
PetscErrorCode BSSPsiOne(BSS self, Vec c, PetscReal x, PetscScalar *y) {
PetscErrorCode ierr;
ierr = BSSCheck(self); CHKERRQ(ierr);
int n; BSSGetSize(self, &n);
Vec us; BSSCreateR1Vec(self, &us);
for(int i = 0; i < n; i++) {
PetscScalar u;
ierr = BSSBasisPsi(self, i, x, &u); CHKERRQ(ierr);
VecSetValue(us, i, u, INSERT_VALUES);
}
VecAssemblyBegin(us); VecAssemblyEnd(us);
PetscScalar yy;
VecTDot(us, c, &yy);
*y = yy;
VecDestroy(&us);
return 0;
}
int main(int argc,char **argv)
{
PetscInt N = 3,M=2,P=4,dof=1,rstart,rend,i;
PetscInt stencil_width = 2;
PetscErrorCode ierr;
PetscMPIInt rank;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_STAR;
DM da;
Vec global,local;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-P",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-stencil_width",&stencil_width,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-stencil_type",(PetscInt*)&stencil_type,NULL);CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,0,&da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(global,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {ierr = VecSetValue(global,i,(PetscReal)(i + 100*rank),INSERT_VALUES);CHKERRQ(ierr);}
ierr = VecAssemblyBegin(global);CHKERRQ(ierr);
ierr = VecAssemblyEnd(global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = VecSet(local,-1);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
if (!rank) {ierr = VecView(local,0);CHKERRQ(ierr);}
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode GetWeightVec(Vec weight,int Nx, int Ny, int Nz)
{
PetscErrorCode ierr;
int i, j, ns, ne, ix, iy, iz, ic;
double value, tmp;
int Nc = 3;
ierr = VecGetOwnershipRange(weight,&ns,&ne); CHKERRQ(ierr);
if (Nx == 1 || Ny == 1 || Nz == 1)
{
tmp = 4.0;
PetscPrintf(PETSC_COMM_WORLD,"---Caution! Treat as a 2D problem and Weight is divieded by 2 \n");
}
else
tmp = 8.0;
for(i=ns; i<ne; i++)
{
iz = (j = i) % Nz;
iy = (j /= Nz) % Ny;
ix = (j /= Ny) % Nx;
ic = (j /= Nx) % Nc;
value = tmp; // tmp = 8.0 for 3D and 4.0 for 2D
if(ic==0)
value = value/(((iy==0)+1.0)*((iz==0)+1.0));
if(ic==1)
value = value/(((ix==0)+1.0)*((iz==0)+1.0));
if(ic==2)
value = value/(((ix==0)+1.0)*((iy==0)+1.0));
VecSetValue(weight, i, value, INSERT_VALUES);
}
ierr = VecAssemblyBegin(weight); CHKERRQ(ierr);
ierr = VecAssemblyEnd(weight); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode
ComputeSnesResidual(SNES snes, Vec terminal_flux_solution, Vec terminal_pressure_difference, void* pContext)
{
// Gain access to the ventilation problem
VentilationProblem* p_original_ventilation_problem = (VentilationProblem*) pContext;
/* Copy the fluxes given in the initial guess into the problem */
double* p_terminal_flux_vector;
#if (PETSC_VERSION_MAJOR == 3 && PETSC_VERSION_MINOR >= 2) //PETSc 3.2 or later
// Request read-only access properly
VecGetArrayRead(terminal_flux_solution, (const PetscScalar**) &p_terminal_flux_vector);
#else
VecGetArray(terminal_flux_solution, &p_terminal_flux_vector);
#endif
unsigned num_terminals = p_original_ventilation_problem->mMesh.GetNumBoundaryNodes()-1u;
for (unsigned terminal=0; terminal<num_terminals; terminal++)
{
unsigned edge_index = p_original_ventilation_problem->mTerminalToEdgeIndex[terminal];
p_original_ventilation_problem->mFlux[edge_index] = p_terminal_flux_vector[terminal];
//if (terminal == 0) PRINT_2_VARIABLES(edge_index, p_terminal_flux_vector[terminal]);
}
/* Solve the direct problem */
p_original_ventilation_problem->SolveDirectFromFlux();
/* Form a residual from the new pressures */
for (unsigned terminal=0; terminal<num_terminals; terminal++)
{
unsigned node_index = p_original_ventilation_problem->mTerminalToNodeIndex[terminal];
// How far we are away from matching this boundary condition.
double delta_pressure = p_original_ventilation_problem->mPressureCondition[node_index] - p_original_ventilation_problem->mPressure[node_index];
VecSetValue(terminal_pressure_difference, terminal, delta_pressure, INSERT_VALUES);
//if (terminal == 0) PRINT_4_VARIABLES(node_index, p_original_ventilation_problem->mPressureCondition[node_index], p_original_ventilation_problem->mPressure[node_index], delta_pressure);
}
return 0;
}
Ensure(FFT, PSD_of_delta_function_is_flat)
{
Vec v;
PetscInt dim = 10;
DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,dim,2,1,NULL,&da);
DMCreateGlobalVector(da, &v);
VecZeroEntries(v);
PetscScalar val = 2.8;
VecSetValue(v, 0, val, INSERT_VALUES);
VecAssemblyBegin(v);
VecAssemblyEnd(v);
scFftCreate(da, &fft);
Vec x, y, z;
scFftCreateVecsFFTW(fft, &x, &y, &z);
Vec psd;
scFftCreateVecPSD(fft, &psd);
scFftComputePSD(fft, v, 0, y, psd, PETSC_FALSE);
const PetscScalar *arr;
VecGetArrayRead(psd, &arr);
PetscInt i;
for (i = 0; i < dim / 2; ++i) {
assert_that(fabs(arr[i] - val * val) < 1.0e-6, is_true);
}
VecRestoreArrayRead(psd, &arr);
VecDestroy(&x);
VecDestroy(&y);
VecDestroy(&z);
VecDestroy(&v);
VecDestroy(&psd);
scFftDestroy(&fft);
DMDestroy(&da);
}
PetscErrorCode GetMediumVecwithSub(Vec epsmedium,int Nz, int Mz, double epsair, double epssub)
{
PetscErrorCode ierr;
int i, iz, ns, ne;
double value;
ierr = VecGetOwnershipRange(epsmedium,&ns,&ne); CHKERRQ(ierr);
for(i=ns;i<ne; i++)
{
iz = i%Nz;
if (iz<Nz/2 + Mz/2)
value = epsair;
else
value = epssub;
VecSetValue(epsmedium, i, value, INSERT_VALUES);
}
ierr = VecAssemblyBegin(epsmedium); CHKERRQ(ierr);
ierr = VecAssemblyEnd(epsmedium); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void InterpolateVec(Geometry geo, Vec vM, Vec vN){
VecSet(vN, 0.0);
const double *vals;
VecGetArrayRead(vM, &vals);
int ms, me;
VecGetOwnershipRange(vM, &ms, &me);
int i;
for(i=0; i< Nxyzcr(geo); i++){
Point p;
CreatePoint_i(&p, i, &geo->gN);
if( projectmedium(&p, &geo->gM, geo->LowerPML) ){
int ixyz = xyz(&p);
if( ms <= ixyz && ixyz < me && vals[ixyz-ms] != 0.0)
VecSetValue(vN, i, vals[ixyz-ms], ADD_VALUES);
}
}
VecRestoreArrayRead(vM, &vals);
AssembleVec(vN);
}
/*---------------------------------------------------------
Function to fill the rhs vector with
By + g values ****
---------------------------------------------------------*/
void rhs(AppCtx *obj,PetscScalar *y, PetscInt nz, PetscScalar *z, PetscReal t)
{
PetscInt i,j,js,je,jj;
PetscScalar val,g[num_z],btri[num_z][3],add_term;
PetscErrorCode ierr;
for(i=0; i < nz-2; i++){
for(j=0; j <= 2; j++){
btri[i][j]=0.0;
}
g[i] = 0.0;
}
/* call femBg to set the tri-diagonal b matrix and vector g */
femBg(btri,g,nz,z,t);
/* setting the entries of the right hand side vector */
for(i=0; i < nz-2; i++){
val = 0.0;
js = 0;
if(i == 0) js = 1;
je = 2;
if(i == nz-2) je = 1;
for(jj=js; jj <= je; jj++){
j = i+jj-1;
val += (btri[i][jj])*(y[j]);
}
add_term = val + g[i];
ierr = VecSetValue(obj->ksp_rhs,(PetscInt)i,(PetscScalar)add_term,INSERT_VALUES);
}
VecAssemblyBegin(obj->ksp_rhs);
VecAssemblyEnd(obj->ksp_rhs);
/* return to main driver function */
return;
}
PetscErrorCode GetMediumVecwithSub(Vec epsmedium,int Nr, int Nz, int Mr, int Mz, double epsair, double epssub, int Mzslab, int mr0, int mz0)
{
PetscErrorCode ierr;
int i, j, ir, iz, ns, ne;
double value;
ierr = VecGetOwnershipRange(epsmedium,&ns,&ne); CHKERRQ(ierr);
for(i=ns;i<ne; i++)
{
iz = (j = i) % Nz;
ir = (j /= Nz) % Nr;
if (Mzslab==2){
if (ir<mr0+Mr)
value = epssub;
else
value = epsair;
}else{
if (iz<mz0)
value = epssub;
else
value = epsair;
}
VecSetValue(epsmedium, i, value, INSERT_VALUES);
}
ierr = VecAssemblyBegin(epsmedium); CHKERRQ(ierr);
ierr = VecAssemblyEnd(epsmedium); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Unlike most finite element applications, IBAMR does assembly on many cells
that are not locally owned; in some cases the processor may own zero finite
element cells but still do assembly on a small number of cells anyway. To
simulate this, this code assembles a PETSc vector by adding contributions
to every entry in the vector on every processor. This causes a deadlock
when we save the communication pattern via
VecSetOption(vec, VEC_SUBSET_OFF_PROC_ENTRIES, PETSC_TRUE).
Contributed-by: David Wells <*****@*****.**>
Petsc developers' notes: this test tests how Petsc knows it can reuse existing communication
pattern. All processes must come to the same conclusion, otherwise deadlock may happen due
to mismatched MPI_Send/Recv. It also tests changing VEC_SUBSET_OFF_PROC_ENTRIES back and forth.
*/
int main(int argc, char **argv)
{
Vec v;
PetscInt i, j, k, *ln, n, rstart;
PetscBool saveCommunicationPattern = PETSC_FALSE;
PetscMPIInt size, rank, p;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, NULL, help);if (ierr) return ierr;
ierr = PetscOptionsGetBool(NULL, NULL, "-save_comm", &saveCommunicationPattern, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &ln);CHKERRQ(ierr);
/* This bug is triggered when one of the local lengths is small. Sometimes in IBAMR this value is actually zero. */
for (p=0; p<size; ++p) ln[p] = 10;
ln[0] = 2;
ierr = PetscPrintf(PETSC_COMM_WORLD, "local lengths are:\n");CHKERRQ(ierr);
ierr = PetscIntView(1, &ln[rank], PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
n = ln[rank];
ierr = VecCreateMPI(MPI_COMM_WORLD, n, PETSC_DECIDE, &v);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(v, &rstart, NULL);CHKERRQ(ierr);
for (k=0; k<5; ++k) { /* 5 iterations of VecAssembly */
PetscReal norm = 0.0;
PetscBool flag = (k == 2) ? PETSC_FALSE : PETSC_TRUE;
PetscInt shift = (k < 2) ? 0 : (k == 2) ? 1 : 0; /* Used to change patterns */
/* If saveCommunicationPattern, let's see what should happen in the 5 iterations:
iter 0: flag is true, and this is the first assebmly, so petsc should keep the
communication pattern built during this assembly.
iter 1: flag is true, reuse the pattern.
iter 2: flag is false, discard/free the pattern built in iter 0; rebuild a new
pattern, but do not keep it after VecAssemblyEnd since the flag is false.
iter 3: flag is true again, this is the new first assembly with a true flag. So
petsc should keep the communication pattern built during this assembly.
iter 4: flag is true, reuse the pattern built in iter 3.
When the vector is destroyed, memory used by the pattern is freed. One can also do it early with a call
VecSetOption(v, VEC_SUBSET_OFF_PROC_ENTRIES, PETSC_FALSE);
*/
if (saveCommunicationPattern) {ierr = VecSetOption(v, VEC_SUBSET_OFF_PROC_ENTRIES, flag);CHKERRQ(ierr);}
ierr = VecSet(v, 0.0);CHKERRQ(ierr);
for (i=0; i<n; ++i) {
PetscScalar val = 1.0;
PetscInt r = rstart + i;
ierr = VecSetValue(v, r, val, ADD_VALUES);CHKERRQ(ierr);
/* do assembly on all other processors too (the 'neighbors') */
{
const PetscMPIInt neighbor = (i+shift) % size; /* Adjust communication patterns between iterations */
const PetscInt nn = ln[neighbor];
PetscInt nrstart = 0;
for (p=0; p<neighbor; ++p) nrstart += ln[p];
for (j=0; j<nn/4; j+= 3) {
PetscScalar val = 0.01;
PetscInt nr = nrstart + j;
ierr = VecSetValue(v, nr, val, ADD_VALUES);CHKERRQ(ierr);
}
}
}
ierr = VecAssemblyBegin(v);CHKERRQ(ierr);
ierr = VecAssemblyEnd(v);CHKERRQ(ierr);
ierr = VecNorm(v, NORM_1, &norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "norm is %g\n", (double)norm);CHKERRQ(ierr);
}
ierr = PetscFree(ln);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*
* steady_state solves for the steady_state of the system
* that was previously setup using the add_to_ham and add_lin
* routines. Solver selection and parameterscan be controlled via PETSc
* command line options.
*/
void steady_state(Vec x){
PetscViewer mat_view;
PC pc;
Vec b;
KSP ksp; /* linear solver context */
PetscInt row,col,its,j,i,Istart,Iend;
PetscScalar mat_tmp;
long dim;
int num_pop;
double *populations;
Mat solve_A;
if (_lindblad_terms) {
dim = total_levels*total_levels;
solve_A = full_A;
if (nid==0) {
printf("Lindblad terms found, using Lindblad solver.");
}
} else {
if (nid==0) {
printf("Warning! Steady state not supported for Schrodinger.\n");
printf(" Defaulting to (less efficient) Lindblad Solver\n");
exit(0);
}
dim = total_levels*total_levels;
solve_A = ham_A;
}
if (!stab_added){
if (nid==0) printf("Adding stabilization...\n");
/*
* Add elements to the matrix to make the normalization work
* I have no idea why this works, I am copying it from qutip
* We add 1.0 in the 0th spot and every n+1 after
*/
if (nid==0) {
row = 0;
for (i=0;i<total_levels;i++){
col = i*(total_levels+1);
mat_tmp = 1.0 + 0.*PETSC_i;
MatSetValue(full_A,row,col,mat_tmp,ADD_VALUES);
}
/* Print dense ham, if it was asked for */
if (_print_dense_ham){
FILE *fp_ham;
fp_ham = fopen("ham","w");
if (nid==0){
for (i=0;i<total_levels;i++){
for (j=0;j<total_levels;j++){
fprintf(fp_ham,"%e %e ",PetscRealPart(_hamiltonian[i][j]),PetscImaginaryPart(_hamiltonian[i][j]));
}
fprintf(fp_ham,"\n");
}
}
fclose(fp_ham);
for (i=0;i<total_levels;i++){
free(_hamiltonian[i]);
}
free(_hamiltonian);
_print_dense_ham = 0;
}
}
stab_added = 1;
}
// if (!matrix_assembled) {
MatGetOwnershipRange(full_A,&Istart,&Iend);
/*
* Explicitly add 0.0 to all diagonal elements;
* this fixes a 'matrix in wrong state' message that PETSc
* gives if the diagonal was never initialized.
*/
if (nid==0) printf("Adding 0 to diagonal elements...\n");
for (i=Istart;i<Iend;i++){
mat_tmp = 0 + 0.*PETSC_i;
MatSetValue(full_A,i,i,mat_tmp,ADD_VALUES);
}
/* Tell PETSc to assemble the matrix */
MatAssemblyBegin(full_A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(full_A,MAT_FINAL_ASSEMBLY);
if (nid==0) printf("Matrix Assembled.\n");
matrix_assembled = 1;
// }
/* Print information about the matrix. */
PetscViewerASCIIOpen(PETSC_COMM_WORLD,NULL,&mat_view);
PetscViewerPushFormat(mat_view,PETSC_VIEWER_ASCII_INFO);
MatView(full_A,mat_view);
PetscViewerPopFormat(mat_view);
PetscViewerDestroy(&mat_view);
/*
* Create parallel vectors.
* - When using VecCreate(), VecSetSizes() and VecSetFromOptions(),
* we specify only the vector's global
* dimension; the parallel partitioning is determined at runtime.
* - Note: We form 1 vector from scratch and then duplicate as needed.
*/
VecCreate(PETSC_COMM_WORLD,&b);
VecSetSizes(b,PETSC_DECIDE,dim);
VecSetFromOptions(b);
// VecDuplicate(b,&x); Assume x is passed in
/*
* Set rhs, b, and solution, x to 1.0 in the first
* element, 0.0 elsewhere.
*/
VecSet(b,0.0);
VecSet(x,0.0);
if(nid==0) {
row = 0;
mat_tmp = 1.0 + 0.0*PETSC_i;
VecSetValue(x,row,mat_tmp,INSERT_VALUES);
VecSetValue(b,row,mat_tmp,INSERT_VALUES);
}
/* Assemble x and b */
VecAssemblyBegin(x);
VecAssemblyEnd(x);
VecAssemblyBegin(b);
VecAssemblyEnd(b);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*
* Create the linear solver and set various options *
*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
* Create linear solver context
*/
KSPCreate(PETSC_COMM_WORLD,&ksp);
/*
* Set operators. Here the matrix that defines the linear system
* also serves as the preconditioning matrix.
*/
KSPSetOperators(ksp,full_A,full_A);
/*
* Set good default options for solver
*/
/* relative tolerance */
KSPSetTolerances(ksp,default_rtol,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
/* bjacobi preconditioner */
KSPGetPC(ksp,&pc);
PCSetType(pc,PCASM);
/* gmres solver with 100 restart*/
KSPSetType(ksp,KSPGMRES);
KSPGMRESSetRestart(ksp,default_restart);
/*
* Set runtime options, e.g.,
* -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
*/
KSPSetFromOptions(ksp);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
if (nid==0) printf("KSP set. Solving for steady state...\n");
KSPSolve(ksp,b,x);
num_pop = get_num_populations();
populations = malloc(num_pop*sizeof(double));
get_populations(x,&populations);
if(nid==0){
printf("Final populations: ");
for(i=0;i<num_pop;i++){
printf(" %e ",populations[i]);
}
printf("\n");
}
KSPGetIterationNumber(ksp,&its);
PetscPrintf(PETSC_COMM_WORLD,"Iterations %D\n",its);
/* Free work space */
KSPDestroy(&ksp);
// VecDestroy(&x);
VecDestroy(&b);
return;
}
PetscErrorCode EpsPMLFull(MPI_Comm comm, Vec epspml, int Nx, int Ny, int Nz, int Npmlx, int Npmly, int Npmlz, double sigmax, double sigmay, double sigmaz, double omega, int LowerPML, int Nctrue)
{
PetscErrorCode ierr;
int Nc = 3;
int i, ns, ne;
double sigma[3]={sigmax,sigmay,sigmaz};
double Npml[3]={Npmlx,Npmly,Npmlz};
int Nxyz[3]={Nx,Ny,Nz};
int NxyzProduct = Nx*Ny*Nz; // Nxyz already taken
Vec epsdummy;
VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, 6*NxyzProduct, &epsdummy);
VecGetOwnershipRange(epsdummy, &ns, &ne);
VecDestroy(&epsdummy);
for (i = ns; i < ne; ++i) {
if (ConvertIndex(i, NxyzProduct, Nctrue) == -1) continue; // skip calculations; row doesn't belong
int ixyz[3], ic, ir;
int itmp;
int cp1, cp2;
double dic, dcp1, dcp2, epsijk;
ixyz[2] = (itmp = i) % Nz;
ixyz[1] = (itmp /= Nz) % Ny;
ixyz[0] = (itmp /= Ny) % Nx;
ic = (itmp /= Nx) % Nc;
ir = itmp / Nc;
cp1 = (ic + 1) % Nc;
cp2 = (ic + 2) % Nc;
int npmlic, npmlcp1, npmlcp2;
npmlic = Npml[ic];
npmlcp1 = Npml[cp1];
npmlcp2 = Npml[cp2];
if (npmlic==0)
dic = 0;
else
dic = (LowerPML)* (ixyz[ic] < Npml[ic]) * (Npml[ic]-ixyz[ic]-0.5)/npmlic
+ ( ixyz[ic] > (Nxyz[ic]-Npml[ic]-1-1))*(ixyz[ic]-(Nxyz[ic]-Npml[ic]-1-1) -0.5 )/npmlic ;
/* first -1 is for the postion of E, second -1 is for C index from 0 */
/* I had same double -1 in the implementation for mu, but did not have it here; added it Apr 20 */
//ierr=PetscPrintf(comm,"the dic value is %f \n",dic);
if (npmlcp1==0)
dcp1=0;
else
dcp1 = (LowerPML)* (ixyz[cp1] < Npml[cp1] ) * (Npml[cp1]-ixyz[cp1])/npmlcp1 + ( ixyz[cp1] > (Nxyz[cp1]-Npml[cp1]-1))* (ixyz[cp1] - (Nxyz[cp1]-Npml[cp1]-1))/npmlcp1;
if (npmlcp2==0)
dcp2=0;
else
dcp2 = (LowerPML)* (ixyz[cp2] < Npml[cp2] ) * (Npml[cp2]-ixyz[cp2])/npmlcp2 + (ixyz[cp2] > (Nxyz[cp2]-Npml[cp2]-1))* (ixyz[cp2] - (Nxyz[cp2]-Npml[cp2]-1))/npmlcp2;
if(ir==0) // real part of epspml[ijk], actually,same as muinv[ijk]
{
epsijk = 1 + 1/sqr(omega)*
( sigma[ic]*sigma[cp1] * sqr(dic*dcp1)
+ sigma[ic]*sigma[cp2] * sqr(dic*dcp2)
- sigma[cp1]*sigma[cp2] * sqr(dcp1*dcp2)
);
}
else //imaginary part of epspml[ijk], just sign difference with muinv[ijk]
{
epsijk =(-sigma[ic]*sqr(dic) + sigma[cp1]*sqr(dcp1) + sigma[cp2]*sqr(dcp2)
+ sigma[ic]*sigma[cp1]*sigma[cp2]/sqr(omega) * sqr(dic*dcp1*dcp2))/omega;
}
/* taking care of the denominator from multipling complex conjugate */
epsijk /= (1 + sqr(sigma[ic]/omega*sqr(dic))) ;
//ierr=PetscPrintf(comm,"the epsijk value is %f \n", epsijk);
int itrue = ConvertIndex(i, NxyzProduct, Nctrue);
VecSetValue(epspml,itrue,epsijk,INSERT_VALUES);
}
/*---------------------------*/
ierr = VecAssemblyBegin(epspml); CHKERRQ(ierr);
ierr = VecAssemblyEnd(epspml); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MuinvPMLFull(MPI_Comm comm, Vec *muinvout, int Nx, int Ny, int Nz, int Npmlx, int Npmly, int Npmlz, double sigmax, double sigmay, double sigmaz, double omega, int LowerPML)
{
Vec muinv;
PetscErrorCode ierr;
int Nc = 3, Nr = 2;
int i, ns, ne;
double sigma[3]={sigmax,sigmay,sigmaz};
double Npml[3]={Npmlx,Npmly,Npmlz};
int Nxyz[3]={Nx,Ny,Nz};
int Nxyzc = Nx*Ny*Nz*Nc;
int Nxyzcr = Nxyzc*Nr;
ierr = VecCreate(comm,&muinv);CHKERRQ(ierr);
ierr = VecSetSizes(muinv, PETSC_DECIDE, Nxyzcr);CHKERRQ(ierr);
ierr = VecSetFromOptions(muinv);CHKERRQ(ierr);
VecGetOwnershipRange(muinv, &ns, &ne);
for (i = ns; i < ne; ++i) {
int ixyz[3], ic, ir;
int itmp;
int cp1, cp2;
double dic, dcp1, dcp2, muinvijk;
ixyz[2] = (itmp = i) % Nz;
ixyz[1] = (itmp /= Nz) % Ny;
ixyz[0] = (itmp /= Ny) % Nx;
ic = (itmp /= Nx) % Nc;
ir = itmp / Nc;
cp1 = (ic + 1) % Nc;
cp2 = (ic + 2) % Nc;
double npmlic, npmlcp1, npmlcp2;
npmlic = Npml[ic];
npmlcp1 = Npml[cp1];
npmlcp2 = Npml[cp2];
if (npmlic==0)
dic = 0;
else
dic = (LowerPML)* (ixyz[ic] < Npml[ic] ) * (Npml[ic]-ixyz[ic])/npmlic
+ (ixyz[ic] > (Nxyz[ic]-Npml[ic]-1))* (ixyz[ic] - (Nxyz[ic]-Npml[ic]-1))/npmlic;
//ierr=PetscPrintf(comm,"the dic value in mu is %f \n", dic);
if (npmlcp1==0)
dcp1 = 0;
else
dcp1 = (LowerPML)* (ixyz[cp1] < Npml[cp1] ) * (Npml[cp1]-ixyz[cp1]-0.5)/npmlcp1 + ( ixyz[cp1] > (Nxyz[cp1]-Npml[cp1]-1-1))* (ixyz[cp1] - (Nxyz[cp1]-Npml[cp1]-1-1) -0.5)/npmlcp1;
/* first -1 is for the postion of H, second -1 is for C index from 0 */
if (npmlcp2==0)
dcp2 = 0;
else
dcp2 = (LowerPML)* (ixyz[cp2] < Npml[cp2] ) * (Npml[cp2]-ixyz[cp2]-0.5)/npmlcp2 + (ixyz[cp2] > (Nxyz[cp2]-Npml[cp2]-1-1))* (ixyz[cp2] - (Nxyz[cp2]-Npml[cp2]-1-1) -0.5)/npmlcp2;
if(ir==0) // real part of mu[ijk]
{
muinvijk = 1 + 1/sqr(omega)*
( sigma[ic]*sigma[cp1] * sqr(dic*dcp1)
+ sigma[ic]*sigma[cp2] * sqr(dic*dcp2)
- sigma[cp1]*sigma[cp2] * sqr(dcp1*dcp2)
);
}
else //imaginary part of mu[ijk]
{
muinvijk =(sigma[ic]*sqr(dic) - sigma[cp1]*sqr(dcp1) - sigma[cp2]*sqr(dcp2)
-sigma[ic]*sigma[cp1]*sigma[cp2]/sqr(omega) * sqr(dic*dcp1*dcp2))/omega;
}
/* taking care of the denominator from multipling complex conjugate */
muinvijk /= (1 + sqr(sigma[cp1]/omega*sqr(dcp1))) * (1 + sqr(sigma[cp2]/omega*sqr(dcp2)));
VecSetValue(muinv,i,muinvijk,INSERT_VALUES);
}
/*---------------------------*/
ierr = VecAssemblyBegin(muinv); CHKERRQ(ierr);
ierr = VecAssemblyEnd(muinv); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) muinv,"MuinvPMLFull"); CHKERRQ(ierr);
*muinvout = muinv;
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
/* -------Initialize and Get the parameters from command line ------*/
PetscInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
PetscPrintf(PETSC_COMM_WORLD,"--------Initializing------ \n");
PetscErrorCode ierr;
PetscBool flg;
int myrank;
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
if(myrank==0)
mma_verbose=1;
/*-------------------------------------------------*/
int Mx,My,Mz,Mzslab, Npmlx,Npmly,Npmlz,DegFree, anisotropic;
PetscOptionsGetInt(PETSC_NULL,"-Nx",&Nx,&flg); MyCheckAndOutputInt(flg,Nx,"Nx","Nx");
PetscOptionsGetInt(PETSC_NULL,"-Ny",&Ny,&flg); MyCheckAndOutputInt(flg,Ny,"Ny","Nx");
PetscOptionsGetInt(PETSC_NULL,"-Nz",&Nz,&flg); MyCheckAndOutputInt(flg,Nz,"Nz","Nz");
PetscOptionsGetInt(PETSC_NULL,"-Mx",&Mx,&flg); MyCheckAndOutputInt(flg,Mx,"Mx","Mx");
PetscOptionsGetInt(PETSC_NULL,"-My",&My,&flg); MyCheckAndOutputInt(flg,My,"My","My");
PetscOptionsGetInt(PETSC_NULL,"-Mz",&Mz,&flg); MyCheckAndOutputInt(flg,Mz,"Mz","Mz");
PetscOptionsGetInt(PETSC_NULL,"-Mzslab",&Mzslab,&flg); MyCheckAndOutputInt(flg,Mzslab,"Mzslab","Mzslab");
PetscOptionsGetInt(PETSC_NULL,"-Npmlx",&Npmlx,&flg); MyCheckAndOutputInt(flg,Npmlx,"Npmlx","Npmlx");
PetscOptionsGetInt(PETSC_NULL,"-Npmly",&Npmly,&flg); MyCheckAndOutputInt(flg,Npmly,"Npmly","Npmly");
PetscOptionsGetInt(PETSC_NULL,"-Npmlz",&Npmlz,&flg); MyCheckAndOutputInt(flg,Npmlz,"Npmlz","Npmlz");
Nxyz = Nx*Ny*Nz;
// if anisotropic !=0, Degree of Freedom = 3*Mx*My*Mz; else DegFree = Mx*My*Mz;
PetscOptionsGetInt(PETSC_NULL,"-anisotropic",&anisotropic,&flg);
if(!flg) anisotropic = 0; // by default, it is isotropc.
DegFree = (anisotropic ? 3 : 1 )*Mx*My*((Mzslab==0)?Mz:1);
PetscPrintf(PETSC_COMM_WORLD," the Degree of Freedoms is %d \n ", DegFree);
int DegFreeAll=DegFree+1;
PetscPrintf(PETSC_COMM_WORLD," the Degree of Freedoms ALL is %d \n ", DegFreeAll);
int BCPeriod, Jdirection, Jdirectiontwo, LowerPML;
int bx[2], by[2], bz[2];
PetscOptionsGetInt(PETSC_NULL,"-BCPeriod",&BCPeriod,&flg); MyCheckAndOutputInt(flg,BCPeriod,"BCPeriod","BCPeriod given");
PetscOptionsGetInt(PETSC_NULL,"-Jdirection",&Jdirection,&flg); MyCheckAndOutputInt(flg,Jdirection,"Jdirection","Diapole current direction");
PetscOptionsGetInt(PETSC_NULL,"-Jdirectiontwo",&Jdirectiontwo,&flg); MyCheckAndOutputInt(flg,Jdirectiontwo,"Jdirectiontwo","Diapole current direction for source two");
PetscOptionsGetInt(PETSC_NULL,"-LowerPML",&LowerPML,&flg); MyCheckAndOutputInt(flg,LowerPML,"LowerPML","PML in the lower xyz boundary");
PetscOptionsGetInt(PETSC_NULL,"-bxl",bx,&flg); MyCheckAndOutputInt(flg,bx[0],"bxl","BC at x lower");
PetscOptionsGetInt(PETSC_NULL,"-bxu",bx+1,&flg); MyCheckAndOutputInt(flg,bx[1],"bxu","BC at x upper");
PetscOptionsGetInt(PETSC_NULL,"-byl",by,&flg); MyCheckAndOutputInt(flg,by[0],"byl","BC at y lower");
PetscOptionsGetInt(PETSC_NULL,"-byu",by+1,&flg); MyCheckAndOutputInt(flg,by[1],"byu","BC at y upper");
PetscOptionsGetInt(PETSC_NULL,"-bzl",bz,&flg); MyCheckAndOutputInt(flg,bz[0],"bzl","BC at z lower");
PetscOptionsGetInt(PETSC_NULL,"-bzu",bz+1,&flg); MyCheckAndOutputInt(flg,bz[1],"bzu","BC at z upper");
double epssub, RRT, sigmax, sigmay, sigmaz ;
PetscOptionsGetReal(PETSC_NULL,"-hx",&hx,&flg); MyCheckAndOutputDouble(flg,hx,"hx","hx");
hy = hx;
hz = hx;
hxyz = (Nz==1)*hx*hy + (Nz>1)*hx*hy*hz;
double omega, omegaone, omegatwo, wratio;
PetscOptionsGetReal(PETSC_NULL,"-omega",&omega,&flg); MyCheckAndOutputDouble(flg,omega,"omega","omega");
PetscOptionsGetReal(PETSC_NULL,"-wratio",&wratio,&flg); MyCheckAndOutputDouble(flg,wratio,"wratio","wratio");
omegaone=omega;
omegatwo=wratio*omega;
PetscPrintf(PETSC_COMM_WORLD,"---omegaone is %.16e and omegatwo is %.16e ---\n",omegaone, omegatwo);
PetscOptionsGetReal(PETSC_NULL,"-Qabs",&Qabs,&flg);
if (flg && Qabs>1e+15)
Qabs=1.0/0.0;
MyCheckAndOutputDouble(flg,Qabs,"Qabs","Qabs");
PetscOptionsGetReal(PETSC_NULL,"-epsair",&epsair,&flg); MyCheckAndOutputDouble(flg,epsair,"epsair","epsair");
PetscOptionsGetReal(PETSC_NULL,"-epssub",&epssub,&flg); MyCheckAndOutputDouble(flg,epssub,"epssub","epssub");
PetscOptionsGetReal(PETSC_NULL,"-RRT",&RRT,&flg); MyCheckAndOutputDouble(flg,RRT,"RRT","RRT given");
sigmax = pmlsigma(RRT,Npmlx*hx);
sigmay = pmlsigma(RRT,Npmly*hy);
sigmaz = pmlsigma(RRT,Npmlz*hz);
PetscPrintf(PETSC_COMM_WORLD,"----sigmax is %.12e \n",sigmax);
PetscPrintf(PETSC_COMM_WORLD,"----sigmay is %.12e \n",sigmay);
PetscPrintf(PETSC_COMM_WORLD,"----sigmaz is %.12e \n",sigmaz);
char initialdata[PETSC_MAX_PATH_LEN]; //filenameComm[PETSC_MAX_PATH_LEN];
PetscOptionsGetString(PETSC_NULL,"-initialdata",initialdata,PETSC_MAX_PATH_LEN,&flg); MyCheckAndOutputChar(flg,initialdata,"initialdata","Inputdata file");
PetscOptionsGetString(PETSC_NULL,"-filenameComm",filenameComm,PETSC_MAX_PATH_LEN,&flg); MyCheckAndOutputChar(flg,filenameComm,"filenameComm","Output filenameComm");
// add cx, cy, cz to indicate where the diapole current is;
int cx, cy, cz;
PetscOptionsGetInt(PETSC_NULL,"-cx",&cx,&flg);
if (!flg)
{cx=(LowerPML)*floor(Nx/2); PetscPrintf(PETSC_COMM_WORLD,"cx is %d by default \n",cx);}
else
{PetscPrintf(PETSC_COMM_WORLD,"the current poisiont cx is %d \n",cx);}
PetscOptionsGetInt(PETSC_NULL,"-cy",&cy,&flg);
if (!flg)
{cy=(LowerPML)*floor(Ny/2); PetscPrintf(PETSC_COMM_WORLD,"cy is %d by default \n",cy);}
else
{PetscPrintf(PETSC_COMM_WORLD,"the current poisiont cy is %d \n",cy);}
PetscOptionsGetInt(PETSC_NULL,"-cz",&cz,&flg);
if (!flg)
{cz=(LowerPML)*floor(Nz/2); PetscPrintf(PETSC_COMM_WORLD,"cz is %d by default \n",cz);}
else
{PetscPrintf(PETSC_COMM_WORLD,"the current poisiont cz is %d \n",cz);}
posj = (cx*Ny+ cy)*Nz + cz;
PetscPrintf(PETSC_COMM_WORLD,"the posj is %d \n. ", posj);
int fixpteps;
PetscOptionsGetInt(PETSC_NULL,"-fixpteps",&fixpteps,&flg); MyCheckAndOutputInt(flg,fixpteps,"fixpteps","fixpteps");
// Get minapproach;
PetscOptionsGetInt(PETSC_NULL,"-minapproach",&minapproach,&flg); MyCheckAndOutputInt(flg,minapproach,"minapproach","minapproach");
// Get withepsinldos;
PetscOptionsGetInt(PETSC_NULL,"-withepsinldos",&withepsinldos,&flg); MyCheckAndOutputInt(flg,withepsinldos,"withepsinldos","withepsinldos");
// Get outputbase;
PetscOptionsGetInt(PETSC_NULL,"-outputbase",&outputbase,&flg); MyCheckAndOutputInt(flg,outputbase,"outputbase","outputbase");
// Get cavityverbose;
PetscOptionsGetInt(PETSC_NULL,"-cavityverbose",&cavityverbose,&flg);
if(!flg) cavityverbose=0;
PetscPrintf(PETSC_COMM_WORLD,"the cavity verbose is set as %d \n", cavityverbose);
// Get refinedldos;
PetscOptionsGetInt(PETSC_NULL,"-refinedldos",&refinedldos,&flg);
if(!flg) refinedldos=0;
PetscPrintf(PETSC_COMM_WORLD,"the refinedldos is set as %d \n", refinedldos);
// Get cmpwrhs;
int cmpwrhs;
PetscOptionsGetInt(PETSC_NULL,"-cmpwrhs",&cmpwrhs,&flg);
if(!flg) cmpwrhs=0;
PetscPrintf(PETSC_COMM_WORLD,"the cmpwrhs is set as %d \n", cmpwrhs);
// Get lrzsqr;
PetscOptionsGetInt(PETSC_NULL,"-lrzsqr",&lrzsqr,&flg);
if(!flg) lrzsqr=0;
PetscPrintf(PETSC_COMM_WORLD,"the lrzsqr is set as %d \n", lrzsqr);
// Get newQdef;
PetscOptionsGetInt(PETSC_NULL,"-newQdef",&newQdef,&flg);
if(!flg) newQdef=0;
PetscPrintf(PETSC_COMM_WORLD,"the newQdef is set as %d \n", newQdef);
/*--------------------------------------------------------*/
/*--------------------------------------------------------*/
/*---------- Set the current source---------*/
//Mat D; //ImaginaryIMatrix;
ImagIMat(PETSC_COMM_WORLD, &D,6*Nxyz);
Vec J;
ierr = VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, 6*Nxyz, &J);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) J, "Source");CHKERRQ(ierr);
VecSet(J,0.0); //initialization;
if (Jdirection == 1)
SourceSingleSetX(PETSC_COMM_WORLD, J, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else if (Jdirection ==2)
SourceSingleSetY(PETSC_COMM_WORLD, J, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else if (Jdirection == 3)
SourceSingleSetZ(PETSC_COMM_WORLD, J, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else
PetscPrintf(PETSC_COMM_WORLD," Please specify correct direction of current: x (1) , y (2) or z (3)\n ");
Vec Jtwo;
ierr = VecDuplicate(J, &Jtwo);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) Jtwo, "Sourcetwo");CHKERRQ(ierr);
VecSet(Jtwo,0.0); //initialization;
if (Jdirectiontwo == 1)
SourceSingleSetX(PETSC_COMM_WORLD, Jtwo, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else if (Jdirectiontwo ==2)
SourceSingleSetY(PETSC_COMM_WORLD, Jtwo, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else if (Jdirectiontwo == 3)
SourceSingleSetZ(PETSC_COMM_WORLD, Jtwo, Nx, Ny, Nz, cx, cy, cz,1.0/hxyz);
else
PetscPrintf(PETSC_COMM_WORLD," Please specify correct direction of current two: x (1) , y (2) or z (3)\n ");
//Vec b; // b= i*omega*J;
Vec bone, btwo;
ierr = VecDuplicate(J,&b);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) b, "rhsone");CHKERRQ(ierr);
ierr = VecDuplicate(J,&bone);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) bone, "rhsone");CHKERRQ(ierr);
ierr = VecDuplicate(Jtwo,&btwo);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) btwo, "rhstwo");CHKERRQ(ierr);
if (cmpwrhs==0)
{
ierr = MatMult(D,J,b);CHKERRQ(ierr);
ierr = MatMult(D,Jtwo,btwo);CHKERRQ(ierr);
VecCopy(b,bone);
VecScale(bone,omegaone);
VecScale(btwo,omegatwo);
VecScale(b,omega);
}
else
{
double complex cmpiomega;
cmpiomega = cpow(1+I/Qabs,newQdef+1);
double sqrtiomegaR = -omega*cimag(csqrt(cmpiomega));
double sqrtiomegaI = omega*creal(csqrt(cmpiomega));
PetscPrintf(PETSC_COMM_WORLD,"the real part of sqrt cmpomega is %g and imag sqrt is % g ", sqrtiomegaR, sqrtiomegaI);
Vec tmpi;
ierr = VecDuplicate(J,&tmpi);
VecSet(b,0.0);
VecSet(tmpi,0.0);
CmpVecScale(J,b,sqrtiomegaR,sqrtiomegaI,D,tmpi);
VecDestroy(&tmpi);
}
/*-------Get the weight vector ------------------*/
//Vec weight;
ierr = VecDuplicate(J,&weight); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) weight, "weight");CHKERRQ(ierr);
if(LowerPML==0)
GetWeightVec(weight, Nx, Ny,Nz); // new code handles both 3D and 2D;
else
VecSet(weight,1.0);
Vec weightedJ;
ierr = VecDuplicate(J,&weightedJ); CHKERRQ(ierr);
ierr = VecPointwiseMult(weightedJ,J,weight);
ierr = PetscObjectSetName((PetscObject) weightedJ, "weightedJ");CHKERRQ(ierr);
Vec weightedJtwo;
ierr = VecDuplicate(Jtwo,&weightedJtwo); CHKERRQ(ierr);
ierr = VecPointwiseMult(weightedJtwo,Jtwo,weight);
ierr = PetscObjectSetName((PetscObject) weightedJtwo, "weightedJtwo");CHKERRQ(ierr);
//Vec vR;
ierr = VecDuplicate(J,&vR); CHKERRQ(ierr);
GetRealPartVec(vR, 6*Nxyz);
// VecFReal;
if (lrzsqr)
{ ierr = VecDuplicate(J,&epsFReal); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epsFReal, "epsFReal");CHKERRQ(ierr);
if (newQdef==0)
{
sqrtomegaI = omega*cimag(csqrt(1+I/Qabs));
PetscPrintf(PETSC_COMM_WORLD,"the real part of sqrt cmpomega is %g and imag sqrt is % g ", omega*creal(csqrt(1+I/Qabs)), sqrtomegaI);
betar = 2*sqrtomegaI;
betai = betar/Qabs;
}
else
{
double gamma;
gamma = omega/Qabs;
betar = 2*gamma*(1-1.0/pow(Qabs,2));
betai = 2*gamma*(2.0/Qabs);
}
ierr = VecDuplicate(J,&nb); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) nb, "nb"); CHKERRQ(ierr);
ierr = VecDuplicate(J,&y); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, "y"); CHKERRQ(ierr);
ierr = VecDuplicate(J,&xsqr); CHKERRQ(ierr); // xsqr = x*x;
ierr = PetscObjectSetName((PetscObject) xsqr, "xsqr"); CHKERRQ(ierr);
CongMat(PETSC_COMM_WORLD, &C, 6*Nxyz);
}
/*----------- Define PML muinv vectors */
Vec muinvpml;
MuinvPMLFull(PETSC_COMM_SELF, &muinvpml,Nx,Ny,Nz,Npmlx,Npmly,Npmlz,sigmax,sigmay,sigmaz,omega, LowerPML);
//double *muinv;
muinv = (double *) malloc(sizeof(double)*6*Nxyz);
int add=0;
AddMuAbsorption(muinv,muinvpml,Qabs,add);
ierr = VecDestroy(&muinvpml); CHKERRQ(ierr);
/*---------- Define PML eps vectors: epspml---------- */
Vec epspml; //epspmlQ, epscoef;
ierr = VecDuplicate(J,&epspml);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epspml,"EpsPMLFull"); CHKERRQ(ierr);
EpsPMLFull(PETSC_COMM_WORLD, epspml,Nx,Ny,Nz,Npmlx,Npmly,Npmlz,sigmax,sigmay,sigmaz,omega, LowerPML);
ierr = VecDuplicate(J,&epspmlQ);CHKERRQ(ierr);
Vec epscoefone, epscoeftwo;
ierr = VecDuplicate(J,&epscoefone);CHKERRQ(ierr);
ierr = VecDuplicate(J,&epscoeftwo);CHKERRQ(ierr);
// compute epspmlQ,epscoef;
EpsCombine(D, weight, epspml, epspmlQ, epscoefone, Qabs, omegaone);
EpsCombine(D, weight, epspml, epspmlQ, epscoeftwo, Qabs, omegatwo);
/*--------- Setup the interp matrix ----------------------- */
/* for a samll eps block, interp it into yee-lattice. The interp matrix A and PML epspml only need to generated once;*/
//Mat A;
//new routine for myinterp;
myinterp(PETSC_COMM_WORLD, &A, Nx,Ny,Nz, LowerPML*floor((Nx-Mx)/2),LowerPML*floor((Ny-My)/2),LowerPML*floor((Nz-Mz)/2), Mx,My,Mz,Mzslab, anisotropic); // LoweerPML*Npmlx,..,.., specify where the interp starts;
//Vec epsSReal, epsgrad, vgrad; // create compatiable vectors with A.
ierr = MatGetVecs(A,&epsSReal, &epsgrad); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epsgrad, "epsgrad");CHKERRQ(ierr);
ierr = VecDuplicate(epsSReal, &vgrad); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epsSReal, "epsSReal");CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) vgrad, "vgrad");CHKERRQ(ierr);
/*---------Setup the epsmedium vector----------------*/
//Vec epsmedium;
ierr = VecDuplicate(J,&epsmedium); CHKERRQ(ierr);
GetMediumVec(epsmedium,Nz,Mz,epsair,epssub);
/*--------- Setup the finitie difference matrix-------------*/
//Mat M;
MoperatorGeneral(PETSC_COMM_WORLD, &M, Nx,Ny,Nz,hx,hy,hz, bx, by, bz,muinv,BCPeriod);
free(muinv);
/*--------Setup the KSP variables ---------------*/
KSP kspone;
PC pcone;
ierr = KSPCreate(PETSC_COMM_WORLD,&kspone);CHKERRQ(ierr);
//ierr = KSPSetType(ksp, KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetType(kspone, KSPGMRES);CHKERRQ(ierr);
ierr = KSPGetPC(kspone,&pcone);CHKERRQ(ierr);
ierr = PCSetType(pcone,PCLU);CHKERRQ(ierr);
ierr = PCFactorSetMatSolverPackage(pcone,MATSOLVERPASTIX);CHKERRQ(ierr);
ierr = PCSetFromOptions(pcone);
int maxkspit = 20;
ierr = KSPSetTolerances(kspone,1e-14,PETSC_DEFAULT,PETSC_DEFAULT,maxkspit);CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspone);CHKERRQ(ierr);
KSP ksptwo;
PC pctwo;
ierr = KSPCreate(PETSC_COMM_WORLD,&ksptwo);CHKERRQ(ierr);
//ierr = KSPSetType(ksp, KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetType(ksptwo, KSPGMRES);CHKERRQ(ierr);
ierr = KSPGetPC(ksptwo,&pctwo);CHKERRQ(ierr);
ierr = PCSetType(pctwo,PCLU);CHKERRQ(ierr);
ierr = PCFactorSetMatSolverPackage(pctwo,MATSOLVERPASTIX);CHKERRQ(ierr);
ierr = PCSetFromOptions(pctwo);
ierr = KSPSetTolerances(ksptwo,1e-14,PETSC_DEFAULT,PETSC_DEFAULT,maxkspit);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksptwo);CHKERRQ(ierr);
/*--------- Create the space for solution vector -------------*/
//Vec x;
ierr = VecDuplicate(J,&x);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Solution");CHKERRQ(ierr);
/*----------- Create the space for final eps -------------*/
//Vec epsC, epsCi, epsP;
ierr = VecDuplicate(J,&epsC);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) epsC, "EpsC");CHKERRQ(ierr);
ierr = VecDuplicate(J,&epsCi);CHKERRQ(ierr);
ierr = VecDuplicate(J,&epsP);CHKERRQ(ierr);
ierr = VecSet(epsP,0.0); CHKERRQ(ierr);
ierr = VecAssemblyBegin(epsP); CHKERRQ(ierr);
ierr = VecAssemblyEnd(epsP); CHKERRQ(ierr);
/*------------ Create space used in the solver ------------*/
//Vec vgradlocal,tmp, tmpa,tmpb;
ierr = VecCreateSeq(PETSC_COMM_SELF, DegFree, &vgradlocal); CHKERRQ(ierr);
ierr = VecDuplicate(J,&tmp); CHKERRQ(ierr);
ierr = VecDuplicate(J,&tmpa); CHKERRQ(ierr);
ierr = VecDuplicate(J,&tmpb); CHKERRQ(ierr);
// Vec pickposvec; this vector is zero except that first entry is one;
if (withepsinldos)
{ ierr = VecDuplicate(J,&pickposvec); CHKERRQ(ierr);
ierr = VecSet(pickposvec,0.0); CHKERRQ(ierr);
ierr = VecSetValue(pickposvec,posj+Jdirection*Nxyz,1.0,INSERT_VALUES);
VecAssemblyBegin(pickposvec);
VecAssemblyEnd(pickposvec);
}
/*------------ Create scatter used in the solver -----------*/
//VecScatter scatter;
//IS from, to;
ierr =ISCreateStride(PETSC_COMM_SELF,DegFree,0,1,&from); CHKERRQ(ierr);
ierr =ISCreateStride(PETSC_COMM_SELF,DegFree,0,1,&to); CHKERRQ(ierr);
/*-------------Read the input file -------------------------*/
double *epsoptAll;
epsoptAll = (double *) malloc(DegFreeAll*sizeof(double));
FILE *ptf;
ptf = fopen(initialdata,"r");
PetscPrintf(PETSC_COMM_WORLD,"reading from input files \n");
int i;
// set the dielectric at the center is fixed, and alwyas high
//epsopt[0]=myub; is defined below near lb and ub;
for (i=0;i<DegFree;i++)
{ //PetscPrintf(PETSC_COMM_WORLD,"current eps reading is %lf \n",epsopt[i]);
fscanf(ptf,"%lf",&epsoptAll[i]);
}
epsoptAll[DegFreeAll-1]=0; //initialize auxiliary variable;
fclose(ptf);
/*----declare these data types, althought they may not be used for job 2 -----------------*/
double mylb,myub, *lb=NULL, *ub=NULL;
int maxeval, maxtime, mynloptalg;
double maxf;
nlopt_opt opt;
nlopt_result result;
/*--------------------------------------------------------------*/
/*----Now based on Command Line, Do the corresponding job----*/
/*----------------------------------------------------------------*/
//int Job; set Job to be gloabl variables;
PetscOptionsGetInt(PETSC_NULL,"-Job",&Job,&flg); MyCheckAndOutputInt(flg,Job,"Job","The Job indicator you set");
int numofvar=(Job==1)*DegFreeAll + (Job==3);
/*-------- convert the epsopt array to epsSReal (if job!=optmization) --------*/
if (Job==2 || Job ==3)
{
// copy epsilon from file to epsSReal; (different from FindOpt.c, because epsilon is not degree-of-freedoms in computeQ.
// i) create a array to read file (done above in epsopt); ii) convert the array to epsSReal;
int ns, ne;
ierr = VecGetOwnershipRange(epsSReal,&ns,&ne);
for(i=ns;i<ne;i++)
{ ierr=VecSetValue(epsSReal,i,epsoptAll[i],INSERT_VALUES);
CHKERRQ(ierr); }
if(withepsinldos)
{ epsatinterest = epsoptAll[cx*Ny*Nz + cy*Nz + cz] + epsair;
PetscPrintf(PETSC_COMM_WORLD, " the relative permitivity at the point of current is %.16e \n ",epsatinterest);}
ierr = VecAssemblyBegin(epsSReal); CHKERRQ(ierr);
ierr = VecAssemblyEnd(epsSReal); CHKERRQ(ierr);
}
if (Job==1 || Job==3) // optimization bounds setup;
{
PetscOptionsGetInt(PETSC_NULL,"-maxeval",&maxeval,&flg); MyCheckAndOutputInt(flg,maxeval,"maxeval","max number of evaluation");
PetscOptionsGetInt(PETSC_NULL,"-maxtime",&maxtime,&flg); MyCheckAndOutputInt(flg,maxtime,"maxtime","max time of evaluation");
PetscOptionsGetInt(PETSC_NULL,"-mynloptalg",&mynloptalg,&flg); MyCheckAndOutputInt(flg,mynloptalg,"mynloptalg","The algorithm used ");
PetscOptionsGetReal(PETSC_NULL,"-mylb",&mylb,&flg); MyCheckAndOutputDouble(flg,mylb,"mylb","optimization lb");
PetscOptionsGetReal(PETSC_NULL,"-myub",&myub,&flg); MyCheckAndOutputDouble(flg,myub,"myub","optimization ub");
lb = (double *) malloc(numofvar*sizeof(double));
ub = (double *) malloc(numofvar*sizeof(double));
// the dielectric constant at center is fixed!
for(i=0;i<numofvar;i++)
{
lb[i] = mylb;
ub[i] = myub;
} //initial guess, lower bounds, upper bounds;
// set lower and upper bounds for auxiliary variable;
lb[numofvar-1]=0;
ub[numofvar-1]=1.0/0.0;
//fix the dielectric at the center to be high for topology optimization;
if (Job==1 && fixpteps==1)
{
epsoptAll[0]=myub;
lb[0]=myub;
ub[0]=myub;
}
opt = nlopt_create(mynloptalg, numofvar);
myfundatatypeshg data[2] = {{omegaone, bone, weightedJ, epscoefone,kspone},{omegatwo, btwo, weightedJtwo, epscoeftwo,ksptwo}};
nlopt_add_inequality_constraint(opt,ldosconstraint, &data[0], 1e-8);
nlopt_add_inequality_constraint(opt,ldosconstraint, &data[1], 1e-8);
nlopt_set_lower_bounds(opt,lb);
nlopt_set_upper_bounds(opt,ub);
nlopt_set_maxeval(opt,maxeval);
nlopt_set_maxtime(opt,maxtime);
/*add functionality to choose local optimizer; */
int mynloptlocalalg;
nlopt_opt local_opt;
PetscOptionsGetInt(PETSC_NULL,"-mynloptlocalalg",&mynloptlocalalg,&flg); MyCheckAndOutputInt(flg,mynloptlocalalg,"mynloptlocalalg","The local optimization algorithm used ");
if (mynloptlocalalg)
{
local_opt=nlopt_create(mynloptlocalalg,numofvar);
nlopt_set_ftol_rel(local_opt, 1e-14);
nlopt_set_maxeval(local_opt,100000);
nlopt_set_local_optimizer(opt,local_opt);
}
}
switch (Job)
{
case 1:
{
if (minapproach)
nlopt_set_min_objective(opt,maxminobjfun,NULL);// NULL: no data to be passed because of global variables;
else
nlopt_set_max_objective(opt,maxminobjfun,NULL);
result = nlopt_optimize(opt,epsoptAll,&maxf);
}
break;
case 2 : //AnalyzeStructure
{
int Linear, Eig, maxeigit;
PetscOptionsGetInt(PETSC_NULL,"-Linear",&Linear,&flg); MyCheckAndOutputInt(flg,Linear,"Linear","Linear solver indicator");
PetscOptionsGetInt(PETSC_NULL,"-Eig",&Eig,&flg); MyCheckAndOutputInt(flg,Eig,"Eig","Eig solver indicator");
PetscOptionsGetInt(PETSC_NULL,"-maxeigit",&maxeigit,&flg); MyCheckAndOutputInt(flg,maxeigit,"maxeigit","maximum number of Eig solver iterations is");
/*----------------------------------*/
//EigenSolver(Linear, Eig, maxeigit);
/*----------------------------------*/
OutputVec(PETSC_COMM_WORLD, weight,filenameComm, "weight.m");
}
break;
default:
PetscPrintf(PETSC_COMM_WORLD,"--------Interesting! You're doing nothing!--------\n ");
}
if(Job==1 || Job==3)
{
/* print the optimization parameters */
#if 0
double xrel, frel, fabs;
// double *xabs;
frel=nlopt_get_ftol_rel(opt);
fabs=nlopt_get_ftol_abs(opt);
xrel=nlopt_get_xtol_rel(opt);
PetscPrintf(PETSC_COMM_WORLD,"nlopt frel is %g \n",frel);
PetscPrintf(PETSC_COMM_WORLD,"nlopt fabs is %g \n",fabs);
PetscPrintf(PETSC_COMM_WORLD,"nlopt xrel is %g \n",xrel);
//nlopt_result nlopt_get_xtol_abs(const nlopt_opt opt, double *tol);
#endif
/*--------------*/
if (result < 0) {
PetscPrintf(PETSC_COMM_WORLD,"nlopt failed! \n", result);
}
else {
PetscPrintf(PETSC_COMM_WORLD,"found extremum %0.16e\n", minapproach?1.0/maxf:maxf);
}
PetscPrintf(PETSC_COMM_WORLD,"nlopt returned value is %d \n", result);
if(Job==1)
{ //OutputVec(PETSC_COMM_WORLD, epsopt,filenameComm, "epsopt.m");
//OutputVec(PETSC_COMM_WORLD, epsgrad,filenameComm, "epsgrad.m");
//OutputVec(PETSC_COMM_WORLD, vgrad,filenameComm, "vgrad.m");
//OutputVec(PETSC_COMM_WORLD, x,filenameComm, "x.m");
int rankA;
MPI_Comm_rank(PETSC_COMM_WORLD, &rankA);
if(rankA==0)
{
ptf = fopen(strcat(filenameComm,"epsopt.txt"),"w");
for (i=0;i<DegFree;i++)
fprintf(ptf,"%0.16e \n",epsoptAll[i]);
fclose(ptf);
PetscPrintf(PETSC_COMM_WORLD,"the t parameter is %.8e \n",epsoptAll[DegFreeAll-1]);
}
}
nlopt_destroy(opt);
}
ierr = PetscPrintf(PETSC_COMM_WORLD,"--------Done!--------\n ");CHKERRQ(ierr);
/*------------------------------------*/
/* ----------------------Destroy Vecs and Mats----------------------------*/
free(epsoptAll);
free(lb);
free(ub);
ierr = VecDestroy(&J); CHKERRQ(ierr);
ierr = VecDestroy(&b); CHKERRQ(ierr);
ierr = VecDestroy(&weight); CHKERRQ(ierr);
ierr = VecDestroy(&weightedJ); CHKERRQ(ierr);
ierr = VecDestroy(&vR); CHKERRQ(ierr);
ierr = VecDestroy(&epspml); CHKERRQ(ierr);
ierr = VecDestroy(&epspmlQ); CHKERRQ(ierr);
ierr = VecDestroy(&epsSReal); CHKERRQ(ierr);
ierr = VecDestroy(&epsgrad); CHKERRQ(ierr);
ierr = VecDestroy(&vgrad); CHKERRQ(ierr);
ierr = VecDestroy(&epsmedium); CHKERRQ(ierr);
ierr = VecDestroy(&epsC); CHKERRQ(ierr);
ierr = VecDestroy(&epsCi); CHKERRQ(ierr);
ierr = VecDestroy(&epsP); CHKERRQ(ierr);
ierr = VecDestroy(&x); CHKERRQ(ierr);
ierr = VecDestroy(&vgradlocal);CHKERRQ(ierr);
ierr = VecDestroy(&tmp); CHKERRQ(ierr);
ierr = VecDestroy(&tmpa); CHKERRQ(ierr);
ierr = VecDestroy(&tmpb); CHKERRQ(ierr);
ierr = MatDestroy(&A); CHKERRQ(ierr);
ierr = MatDestroy(&D); CHKERRQ(ierr);
ierr = MatDestroy(&M); CHKERRQ(ierr);
ierr = VecDestroy(&epscoefone); CHKERRQ(ierr);
ierr = VecDestroy(&epscoeftwo); CHKERRQ(ierr);
ierr = KSPDestroy(&kspone);CHKERRQ(ierr);
ierr = KSPDestroy(&ksptwo);CHKERRQ(ierr);
ISDestroy(&from);
ISDestroy(&to);
if (withepsinldos)
{ierr=VecDestroy(&pickposvec); CHKERRQ(ierr);}
if (lrzsqr)
{
ierr=VecDestroy(&epsFReal); CHKERRQ(ierr);
ierr=VecDestroy(&xsqr); CHKERRQ(ierr);
ierr=VecDestroy(&y); CHKERRQ(ierr);
ierr=VecDestroy(&nb); CHKERRQ(ierr);
ierr=MatDestroy(&C); CHKERRQ(ierr);
}
ierr = VecDestroy(&bone); CHKERRQ(ierr);
ierr = VecDestroy(&btwo); CHKERRQ(ierr);
ierr = VecDestroy(&Jtwo); CHKERRQ(ierr);
/*------------ finalize the program -------------*/
{
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
//if (rank == 0) fgetc(stdin);
MPI_Barrier(PETSC_COMM_WORLD);
}
ierr = PetscFinalize(); CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt i,n,*ix,*iy,*tomap,start;
Vec x,y;
PetscMPIInt nproc,rank;
IS isx,isy;
const PetscInt *ranges;
VecScatter vscat;
PetscFunctionBegin;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&nproc);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (nproc != 2) SETERRQ(PETSC_COMM_SELF,1,"This test must run with exactly two MPI ranks\n");
/* ====================================================================
(1) test VecScatterRemap on a parallel to parallel (PtoP) vecscatter
====================================================================
*/
n = 64; /* long enough to trigger memcpy optimizations both in local scatter and remote scatter */
/* create two MPI vectors x, y of length n=64, N=128 */
ierr = VecCreateMPI(PETSC_COMM_WORLD,n,PETSC_DECIDE,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
/* Initialize x as {0~127} */
ierr = VecGetOwnershipRanges(x,&ranges);CHKERRQ(ierr);
for (i=ranges[rank]; i<ranges[rank+1]; i++) { ierr = VecSetValue(x,i,(PetscScalar)i,INSERT_VALUES);CHKERRQ(ierr); }
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
/* create two general index sets isx = {0~127} and isy = {32~63,64~95,96~127,0~31}. isx is sequential, but we use
it as general and let PETSc detect the pattern and optimize it. indices in isy are set to make the vecscatter
have both local scatter and remote scatter (i.e., MPI communication)
*/
ierr = PetscMalloc2(n,&ix,n,&iy);CHKERRQ(ierr);
start = ranges[rank];
for (i=ranges[rank]; i<ranges[rank+1]; i++) ix[i-start] = i;
ierr = ISCreateGeneral(PETSC_COMM_WORLD,n,ix,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
if (!rank) { for (i=0; i<n; i++) iy[i] = i+32; }
else for (i=0; i<n/2; i++) { iy[i] = i+96; iy[i+n/2] = i; }
ierr = ISCreateGeneral(PETSC_COMM_WORLD,n,iy,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
/* create a vecscatter that shifts x to the tail by quater periodically and puts the results in y */
ierr = VecScatterCreateWithData(x,isx,y,isy,&vscat);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* view y to check the result. y should be {Q3,Q0,Q1,Q2} of x, that is {96~127,0~31,32~63,64~95} */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Before VecScatterRemap on PtoP, MPI vector y is:\n");CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* now call the weird subroutine VecScatterRemap to slightly change the vecscatter. It changes where we read vector
x entries to send out, but does not change the communication pattern (i.e., send/recv pairs and msg lengths).
We create tomap as {32~63,0~31}. Originaly, we read from indices {0~64} of the local x to send out. The remap
does indices[i] = tomap[indices[i]]. Therefore, after the remap, we read from indices {32~63,0~31} of the local x.
isy is unchanged. So, we will shift x to {Q2,Q1,Q0,Q3}, that is {64~95,32~63,0~31,96~127}
*/
ierr = PetscMalloc1(n,&tomap);CHKERRQ(ierr);
for (i=0; i<n/2; i++) { tomap[i] = i+n/2; tomap[i+n/2] = i; };
ierr = VecScatterRemap(vscat,tomap,NULL);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* view y to check the result. y should be {64~95,32~63,0~31,96~127} */
ierr = PetscPrintf(PETSC_COMM_WORLD,"After VecScatterRemap on PtoP, MPI vector y is:\n");CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* destroy everything before we recreate them in different types */
ierr = PetscFree2(ix,iy);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&isx);CHKERRQ(ierr);
ierr = ISDestroy(&isy);CHKERRQ(ierr);
ierr = PetscFree(tomap);CHKERRQ(ierr);
ierr = VecScatterDestroy(&vscat);CHKERRQ(ierr);
/* ==========================================================================================
(2) test VecScatterRemap on a sequential general to sequential general (SGToSG) vecscatter
==========================================================================================
*/
n = 64; /* long enough to trigger memcpy optimizations in local scatter */
/* create two seq vectors x, y of length n */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
/* Initialize x as {0~63} */
for (i=0; i<n; i++) { ierr = VecSetValue(x,i,(PetscScalar)i,INSERT_VALUES);CHKERRQ(ierr); }
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
/* create two general index sets isx = isy = {0~63}, which are sequential, but we use them as
general and let PETSc detect the pattern and optimize it */
ierr = PetscMalloc2(n,&ix,n,&iy);CHKERRQ(ierr);
for (i=0; i<n; i++) ix[i] = i;
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,ix,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISDuplicate(isx,&isy);CHKERRQ(ierr);
/* create a vecscatter that just copies x to y */
ierr = VecScatterCreateWithData(x,isx,y,isy,&vscat);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* view y to check the result. y should be {0~63} */
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nBefore VecScatterRemap on SGToSG, SEQ vector y is:\n");CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* now call the weird subroutine VecScatterRemap to slightly change the vecscatter.
Create tomap as {32~63,0~31}. Originaly, we read from indices {0~64} of seq x to write to y. The remap
does indices[i] = tomap[indices[i]]. Therefore, after the remap, we read from indices{32~63,0~31} of seq x.
*/
ierr = PetscMalloc1(n,&tomap);CHKERRQ(ierr);
for (i=0; i<n/2; i++) { tomap[i] = i+n/2; tomap[i+n/2] = i; };
ierr = VecScatterRemap(vscat,tomap,NULL);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* view y to check the result. y should be {32~63,0~31} */
ierr = PetscPrintf(PETSC_COMM_WORLD,"After VecScatterRemap on SGToSG, SEQ vector y is:\n");CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* destroy everything before we recreate them in different types */
ierr = PetscFree2(ix,iy);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&isx);CHKERRQ(ierr);
ierr = ISDestroy(&isy);CHKERRQ(ierr);
ierr = PetscFree(tomap);CHKERRQ(ierr);
ierr = VecScatterDestroy(&vscat);CHKERRQ(ierr);
/* ===================================================================================================
(3) test VecScatterRemap on a sequential general to sequential stride 1 (SGToSS_Stride1) vecscatter
===================================================================================================
*/
n = 64; /* long enough to trigger memcpy optimizations in local scatter */
/* create two seq vectors x of length n, and y of length n/2 */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&x);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n/2,&y);CHKERRQ(ierr);
/* Initialize x as {0~63} */
for (i=0; i<n; i++) { ierr = VecSetValue(x,i,(PetscScalar)i,INSERT_VALUES);CHKERRQ(ierr); }
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
/* create a general index set isx = {0:63:2}, which actually is a stride IS with first=0, n=32, step=2,
but we use it as general and let PETSc detect the pattern and optimize it. */
ierr = PetscMalloc2(n/2,&ix,n/2,&iy);CHKERRQ(ierr);
for (i=0; i<n/2; i++) ix[i] = i*2;
ierr = ISCreateGeneral(PETSC_COMM_SELF,n/2,ix,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
/* create a stride1 index set isy = {0~31}. We intentionally set the step to 1 to trigger optimizations */
ierr = ISCreateStride(PETSC_COMM_SELF,32,0,1,&isy);CHKERRQ(ierr);
/* create a vecscatter that just copies even entries of x to y */
ierr = VecScatterCreateWithData(x,isx,y,isy,&vscat);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* view y to check the result. y should be {0:63:2} */
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nBefore VecScatterRemap on SGToSS_Stride1, SEQ vector y is:\n");CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* now call the weird subroutine VecScatterRemap to slightly change the vecscatter.
Create tomap as {32~63,0~31}. Originaly, we read from indices{0:63:2} of seq x to write to y. The remap
does indices[i] = tomap[indices[i]]. Therefore, after the remap, we read from indices{32:63:2,0:31:2} of seq x.
*/
ierr = PetscMalloc1(n,&tomap);CHKERRQ(ierr);
for (i=0; i<n/2; i++) { tomap[i] = i+n/2; tomap[i+n/2] = i; };
ierr = VecScatterRemap(vscat,tomap,NULL);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* view y to check the result. y should be {32:63:2,0:31:2} */
ierr = PetscPrintf(PETSC_COMM_WORLD,"After VecScatterRemap on SGToSS_Stride1, SEQ vector y is:\n");CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* destroy everything before PetscFinalize */
ierr = PetscFree2(ix,iy);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&isx);CHKERRQ(ierr);
ierr = ISDestroy(&isy);CHKERRQ(ierr);
ierr = PetscFree(tomap);CHKERRQ(ierr);
ierr = VecScatterDestroy(&vscat);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
TS ts; /* time integrator */
TSAdapt adapt;
Vec X; /* solution vector */
Mat J; /* Jacobian matrix */
PetscInt steps,maxsteps,ncells,xs,xm,i;
PetscErrorCode ierr;
PetscReal ftime,dt;
char chemfile[PETSC_MAX_PATH_LEN] = "chem.inp",thermofile[PETSC_MAX_PATH_LEN] = "therm.dat";
struct _User user;
TSConvergedReason reason;
PetscBool showsolutions = PETSC_FALSE;
char **snames,*names;
Vec lambda; /* used with TSAdjoint for sensitivities */
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"Chemistry solver options","");CHKERRQ(ierr);
ierr = PetscOptionsString("-chem","CHEMKIN input file","",chemfile,chemfile,sizeof(chemfile),NULL);CHKERRQ(ierr);
ierr = PetscOptionsString("-thermo","NASA thermo input file","",thermofile,thermofile,sizeof(thermofile),NULL);CHKERRQ(ierr);
user.pressure = 1.01325e5; /* Pascal */
ierr = PetscOptionsReal("-pressure","Pressure of reaction [Pa]","",user.pressure,&user.pressure,NULL);CHKERRQ(ierr);
user.Tini = 1550;
ierr = PetscOptionsReal("-Tini","Initial temperature [K]","",user.Tini,&user.Tini,NULL);CHKERRQ(ierr);
user.diffus = 100;
ierr = PetscOptionsReal("-diffus","Diffusion constant","",user.diffus,&user.diffus,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-draw_solution","Plot the solution for each cell","",showsolutions,&showsolutions,NULL);CHKERRQ(ierr);
user.diffusion = PETSC_TRUE;
ierr = PetscOptionsBool("-diffusion","Have diffusion","",user.diffusion,&user.diffusion,NULL);CHKERRQ(ierr);
user.reactions = PETSC_TRUE;
ierr = PetscOptionsBool("-reactions","Have reactions","",user.reactions,&user.reactions,NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = TC_initChem(chemfile, thermofile, 0, 1.0);TCCHKERRQ(ierr);
user.Nspec = TC_getNspec();
user.Nreac = TC_getNreac();
ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_PERIODIC,-1,user.Nspec+1,1,NULL,&user.dm);CHKERRQ(ierr);
ierr = DMDAGetInfo(user.dm,NULL,&ncells,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);CHKERRQ(ierr);
user.dx = 1.0/ncells; /* Set the coordinates of the cell centers; note final ghost cell is at x coordinate 1.0 */
ierr = DMDASetUniformCoordinates(user.dm,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
/* set the names of each field in the DMDA based on the species name */
ierr = PetscMalloc1((user.Nspec+1)*LENGTHOFSPECNAME,&names);CHKERRQ(ierr);
ierr = PetscStrcpy(names,"Temp");CHKERRQ(ierr);
TC_getSnames(user.Nspec,names+LENGTHOFSPECNAME);CHKERRQ(ierr);
ierr = PetscMalloc1((user.Nspec+2),&snames);CHKERRQ(ierr);
for (i=0; i<user.Nspec+1; i++) snames[i] = names+i*LENGTHOFSPECNAME;
snames[user.Nspec+1] = NULL;
ierr = DMDASetFieldNames(user.dm,(const char * const *)snames);CHKERRQ(ierr);
ierr = PetscFree(snames);CHKERRQ(ierr);
ierr = PetscFree(names);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.dm,&J);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(user.dm,&X);CHKERRQ(ierr);
ierr = PetscMalloc3(user.Nspec+1,&user.tchemwork,PetscSqr(user.Nspec+1),&user.Jdense,user.Nspec+1,&user.rows);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create timestepping solver context
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr);
ierr = TSSetDM(ts,user.dm);CHKERRQ(ierr);
ierr = TSSetType(ts,TSARKIMEX);CHKERRQ(ierr);
ierr = TSARKIMEXSetFullyImplicit(ts,PETSC_TRUE);CHKERRQ(ierr);
ierr = TSARKIMEXSetType(ts,TSARKIMEX4);CHKERRQ(ierr);
ierr = TSSetRHSFunction(ts,NULL,FormRHSFunction,&user);CHKERRQ(ierr);
ierr = TSSetRHSJacobian(ts,J,J,FormRHSJacobian,&user);CHKERRQ(ierr);
ftime = 1.0;
maxsteps = 10000;
ierr = TSSetDuration(ts,maxsteps,ftime);CHKERRQ(ierr);
ierr = TSSetExactFinalTime(ts,TS_EXACTFINALTIME_STEPOVER);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set initial conditions
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = FormInitialSolution(ts,X,&user);CHKERRQ(ierr);
ierr = TSSetSolution(ts,X);CHKERRQ(ierr);
dt = 1e-10; /* Initial time step */
ierr = TSSetInitialTimeStep(ts,0.0,dt);CHKERRQ(ierr);
ierr = TSGetAdapt(ts,&adapt);CHKERRQ(ierr);
ierr = TSAdaptSetStepLimits(adapt,1e-12,1e-4);CHKERRQ(ierr); /* Also available with -ts_adapt_dt_min/-ts_adapt_dt_max */
ierr = TSSetMaxSNESFailures(ts,-1);CHKERRQ(ierr); /* Retry step an unlimited number of times */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Pass information to graphical monitoring routine
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
if (showsolutions) {
ierr = DMDAGetCorners(user.dm,&xs,NULL,NULL,&xm,NULL,NULL);CHKERRQ(ierr);
for (i=xs;i<xs+xm;i++) {
ierr = MonitorCell(ts,&user,i);CHKERRQ(ierr);
}
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set runtime options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set final conditions for sensitivities
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DMCreateGlobalVector(user.dm,&lambda);CHKERRQ(ierr);
ierr = TSSetCostGradients(ts,1,&lambda,NULL);CHKERRQ(ierr);
ierr = VecSetValue(lambda,0,1.0,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(lambda);CHKERRQ(ierr);
ierr = VecAssemblyEnd(lambda);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve ODE
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSSolve(ts,X);CHKERRQ(ierr);
ierr = TSGetSolveTime(ts,&ftime);CHKERRQ(ierr);
ierr = TSGetTimeStepNumber(ts,&steps);CHKERRQ(ierr);
ierr = TSGetConvergedReason(ts,&reason);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"%s at time %g after %D steps\n",TSConvergedReasons[reason],(double)ftime,steps);CHKERRQ(ierr);
{
Vec max;
const char * const *names;
PetscInt i;
const PetscReal *bmax;
ierr = TSMonitorEnvelopeGetBounds(ts,&max,NULL);CHKERRQ(ierr);
if (max) {
ierr = TSMonitorLGGetVariableNames(ts,&names);CHKERRQ(ierr);
if (names) {
ierr = VecGetArrayRead(max,&bmax);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"Species - maximum mass fraction\n");CHKERRQ(ierr);
for (i=1; i<user.Nspec; i++) {
if (bmax[i] > .01) {ierr = PetscPrintf(PETSC_COMM_SELF,"%s %g\n",names[i],bmax[i]);CHKERRQ(ierr);}
}
ierr = VecRestoreArrayRead(max,&bmax);CHKERRQ(ierr);
}
}
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free work space.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
TC_reset();
ierr = DMDestroy(&user.dm);CHKERRQ(ierr);
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&lambda);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
ierr = PetscFree3(user.tchemwork,user.Jdense,user.rows);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat Amat;
PetscErrorCode ierr;
SNES snes;
KSP ksp;
MPI_Comm comm;
PetscMPIInt npe,rank;
PetscLogStage stage[7];
PetscBool test_nonzero_cols=PETSC_FALSE,use_nearnullspace=PETSC_TRUE;
Vec xx,bb;
PetscInt iter,i,N,dim=3,cells[3]={1,1,1},max_conv_its,local_sizes[7],run_type=1;
DM dm,distdm,basedm;
PetscBool flg;
char convType[256];
PetscReal Lx,mdisp[10],err[10];
const char * const options[10] = {"-ex56_dm_refine 0",
"-ex56_dm_refine 1",
"-ex56_dm_refine 2",
"-ex56_dm_refine 3",
"-ex56_dm_refine 4",
"-ex56_dm_refine 5",
"-ex56_dm_refine 6",
"-ex56_dm_refine 7",
"-ex56_dm_refine 8",
"-ex56_dm_refine 9"};
PetscFunctionBeginUser;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
comm = PETSC_COMM_WORLD;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &npe);CHKERRQ(ierr);
/* options */
ierr = PetscOptionsBegin(comm,NULL,"3D bilinear Q1 elasticity options","");CHKERRQ(ierr);
{
i = 3;
ierr = PetscOptionsIntArray("-cells", "Number of (flux tube) processor in each dimension", "ex56.c", cells, &i, NULL);CHKERRQ(ierr);
Lx = 1.; /* or ne for rod */
max_conv_its = 3;
ierr = PetscOptionsInt("-max_conv_its","Number of iterations in convergence study","",max_conv_its,&max_conv_its,NULL);CHKERRQ(ierr);
if (max_conv_its<=0 || max_conv_its>7) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_USER, "Bad number of iterations for convergence test (%D)",max_conv_its);
ierr = PetscOptionsReal("-lx","Length of domain","",Lx,&Lx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-alpha","material coefficient inside circle","",s_soft_alpha,&s_soft_alpha,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-test_nonzero_cols","nonzero test","",test_nonzero_cols,&test_nonzero_cols,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-use_mat_nearnullspace","MatNearNullSpace API test","",use_nearnullspace,&use_nearnullspace,NULL);CHKERRQ(ierr);
ierr = PetscOptionsInt("-run_type","0: twisting load on cantalever, 1: 3rd order accurate convergence test","",run_type,&run_type,NULL);CHKERRQ(ierr);
i = 3;
ierr = PetscOptionsInt("-mat_block_size","","",i,&i,&flg);CHKERRQ(ierr);
if (!flg || i!=3) SETERRQ2(PETSC_COMM_WORLD, PETSC_ERR_USER, "'-mat_block_size 3' must be set (%D) and = 3 (%D)",flg,flg? i : 3);
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscLogStageRegister("Mesh Setup", &stage[6]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("1st Setup", &stage[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("1st Solve", &stage[1]);CHKERRQ(ierr);
/* create DM, Plex calls DMSetup */
ierr = PetscLogStagePush(stage[6]);CHKERRQ(ierr);
ierr = DMPlexCreateHexBoxMesh(comm, dim, cells, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, &dm);CHKERRQ(ierr);
{
DMLabel label;
IS is;
ierr = DMCreateLabel(dm, "boundary");CHKERRQ(ierr);
ierr = DMGetLabel(dm, "boundary", &label);CHKERRQ(ierr);
ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
if (run_type==0) {
ierr = DMGetStratumIS(dm, "boundary", 1, &is);CHKERRQ(ierr);
ierr = DMCreateLabel(dm,"Faces");CHKERRQ(ierr);
if (is) {
PetscInt d, f, Nf;
const PetscInt *faces;
PetscInt csize;
PetscSection cs;
Vec coordinates ;
DM cdm;
ierr = ISGetLocalSize(is, &Nf);CHKERRQ(ierr);
ierr = ISGetIndices(is, &faces);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
ierr = DMGetDefaultSection(cdm, &cs);CHKERRQ(ierr);
/* Check for each boundary face if any component of its centroid is either 0.0 or 1.0 */
for (f = 0; f < Nf; ++f) {
PetscReal faceCoord;
PetscInt b,v;
PetscScalar *coords = NULL;
PetscInt Nv;
ierr = DMPlexVecGetClosure(cdm, cs, coordinates, faces[f], &csize, &coords);CHKERRQ(ierr);
Nv = csize/dim; /* Calculate mean coordinate vector */
for (d = 0; d < dim; ++d) {
faceCoord = 0.0;
for (v = 0; v < Nv; ++v) faceCoord += PetscRealPart(coords[v*dim+d]);
faceCoord /= Nv;
for (b = 0; b < 2; ++b) {
if (PetscAbs(faceCoord - b) < PETSC_SMALL) { /* domain have not been set yet, still [0,1]^3 */
ierr = DMSetLabelValue(dm, "Faces", faces[f], d*2+b+1);CHKERRQ(ierr);
}
}
}
ierr = DMPlexVecRestoreClosure(cdm, cs, coordinates, faces[f], &csize, &coords);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(is, &faces);CHKERRQ(ierr);
}
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = DMGetLabel(dm, "Faces", &label);CHKERRQ(ierr);
ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr);
}
}
{
PetscInt dimEmbed, i;
PetscInt nCoords;
PetscScalar *coords,bounds[] = {0,Lx,-.5,.5,-.5,.5,}; /* x_min,x_max,y_min,y_max */
Vec coordinates;
if (run_type==1) {
for (i = 0; i < 2*dim; i++) bounds[i] = (i%2) ? 1 : 0;
}
ierr = DMGetCoordinatesLocal(dm,&coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateDim(dm,&dimEmbed);CHKERRQ(ierr);
if (dimEmbed != dim) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"dimEmbed != dim %D",dimEmbed);CHKERRQ(ierr);
ierr = VecGetLocalSize(coordinates,&nCoords);CHKERRQ(ierr);
if (nCoords % dimEmbed) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Coordinate vector the wrong size");CHKERRQ(ierr);
ierr = VecGetArray(coordinates,&coords);CHKERRQ(ierr);
for (i = 0; i < nCoords; i += dimEmbed) {
PetscInt j;
PetscScalar *coord = &coords[i];
for (j = 0; j < dimEmbed; j++) {
coord[j] = bounds[2 * j] + coord[j] * (bounds[2 * j + 1] - bounds[2 * j]);
}
}
ierr = VecRestoreArray(coordinates,&coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dm,coordinates);CHKERRQ(ierr);
}
/* convert to p4est, and distribute */
ierr = PetscOptionsBegin(comm, "", "Mesh conversion options", "DMPLEX");CHKERRQ(ierr);
ierr = PetscOptionsFList("-dm_type","Convert DMPlex to another format (should not be Plex!)","ex56.c",DMList,DMPLEX,convType,256,&flg);CHKERRQ(ierr);
ierr = PetscOptionsEnd();
if (flg) {
DM newdm;
ierr = DMConvert(dm,convType,&newdm);CHKERRQ(ierr);
if (newdm) {
const char *prefix;
PetscBool isForest;
ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject)newdm,prefix);CHKERRQ(ierr);
ierr = DMIsForest(newdm,&isForest);CHKERRQ(ierr);
if (isForest) {
} else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Converted to non Forest?");
ierr = DMDestroy(&dm);CHKERRQ(ierr);
dm = newdm;
} else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Convert failed?");
} else {
/* Plex Distribute mesh over processes */
ierr = DMPlexDistribute(dm, 0, NULL, &distdm);CHKERRQ(ierr);
if (distdm) {
const char *prefix;
ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject)distdm,prefix);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
dm = distdm;
}
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
basedm = dm; dm = NULL;
for (iter=0 ; iter<max_conv_its ; iter++) {
ierr = PetscLogStagePush(stage[6]);CHKERRQ(ierr);
/* make new DM */
ierr = DMClone(basedm, &dm);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, "ex56_");CHKERRQ(ierr);
ierr = PetscObjectSetName( (PetscObject)dm,"Mesh");CHKERRQ(ierr);
ierr = PetscOptionsClearValue(NULL,"-ex56_dm_refine");CHKERRQ(ierr);
ierr = PetscOptionsInsertString(NULL,options[iter]);CHKERRQ(ierr);
ierr = DMSetFromOptions(dm);CHKERRQ(ierr); /* refinement done here in Plex, p4est */
/* snes */
ierr = SNESCreate(comm, &snes);CHKERRQ(ierr);
ierr = SNESSetDM(snes, dm);CHKERRQ(ierr);
/* fem */
{
const PetscInt Ncomp = dim;
const PetscInt components[] = {0,1,2};
const PetscInt Nfid = 1, Npid = 1;
const PetscInt fid[] = {1}; /* The fixed faces (x=0) */
const PetscInt pid[] = {2}; /* The faces with loading (x=L_x) */
PetscFE fe;
PetscDS prob;
DM cdm = dm;
ierr = PetscFECreateDefault(dm, dim, dim, PETSC_FALSE, NULL, PETSC_DECIDE, &fe);CHKERRQ(ierr); /* elasticity */
ierr = PetscObjectSetName((PetscObject) fe, "deformation");CHKERRQ(ierr);
/* FEM prob */
ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
ierr = PetscDSSetDiscretization(prob, 0, (PetscObject) fe);CHKERRQ(ierr);
/* setup problem */
if (run_type==1) {
ierr = PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu_3d);CHKERRQ(ierr);
ierr = PetscDSSetResidual(prob, 0, f0_u_x4, f1_u_3d);CHKERRQ(ierr);
} else {
ierr = PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu_3d_alpha);CHKERRQ(ierr);
ierr = PetscDSSetResidual(prob, 0, f0_u, f1_u_3d_alpha);CHKERRQ(ierr);
ierr = PetscDSSetBdResidual(prob, 0, f0_bd_u_3d, f1_bd_u);CHKERRQ(ierr);
}
/* bcs */
if (run_type==1) {
PetscInt id = 1;
ierr = DMAddBoundary(dm, DM_BC_ESSENTIAL, "wall", "boundary", 0, 0, NULL, (void (*)()) zero, 1, &id, NULL);CHKERRQ(ierr);
} else {
ierr = PetscDSAddBoundary(prob, DM_BC_ESSENTIAL, "fixed", "Faces", 0, Ncomp, components, (void (*)()) zero, Nfid, fid, NULL);CHKERRQ(ierr);
ierr = PetscDSAddBoundary(prob, DM_BC_NATURAL, "traction", "Faces", 0, Ncomp, components, NULL, Npid, pid, NULL);CHKERRQ(ierr);
}
while (cdm) {
ierr = DMSetDS(cdm,prob);CHKERRQ(ierr);
ierr = DMGetCoarseDM(cdm, &cdm);CHKERRQ(ierr);
}
ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
}
/* vecs & mat */
ierr = DMCreateGlobalVector(dm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx, &bb);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) bb, "b");CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) xx, "u");CHKERRQ(ierr);
ierr = DMCreateMatrix(dm, &Amat);CHKERRQ(ierr);
ierr = VecGetSize(bb,&N);CHKERRQ(ierr);
local_sizes[iter] = N;
ierr = PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %d global equations, %d vertices\n",rank,PETSC_FUNCTION_NAME,N,N/dim);CHKERRQ(ierr);
if (use_nearnullspace && N/dim > 1) {
/* Set up the near null space (a.k.a. rigid body modes) that will be used by the multigrid preconditioner */
DM subdm;
MatNullSpace nearNullSpace;
PetscInt fields = 0;
PetscObject deformation;
ierr = DMCreateSubDM(dm, 1, &fields, NULL, &subdm);CHKERRQ(ierr);
ierr = DMPlexCreateRigidBody(subdm, &nearNullSpace);CHKERRQ(ierr);
ierr = DMGetField(dm, 0, &deformation);CHKERRQ(ierr);
ierr = PetscObjectCompose(deformation, "nearnullspace", (PetscObject) nearNullSpace);CHKERRQ(ierr);
ierr = DMDestroy(&subdm);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nearNullSpace);CHKERRQ(ierr); /* created by DM and destroyed by Mat */
}
ierr = DMPlexSetSNESLocalFEM(dm,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, Amat, Amat, NULL, NULL);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr);
/* ksp */
ierr = SNESGetKSP(snes, &ksp);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE);CHKERRQ(ierr);
/* test BCs */
ierr = VecZeroEntries(xx);CHKERRQ(ierr);
if (test_nonzero_cols) {
if (rank==0) ierr = VecSetValue(xx,0,1.0,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(xx);CHKERRQ(ierr);
ierr = VecAssemblyEnd(xx);CHKERRQ(ierr);
}
ierr = VecZeroEntries(bb);CHKERRQ(ierr);
ierr = VecGetSize(bb,&i);CHKERRQ(ierr);
local_sizes[iter] = i;
ierr = PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %d equations in vector, %d vertices\n",rank,PETSC_FUNCTION_NAME,i,i/dim);CHKERRQ(ierr);
/* setup solver, dummy solve to really setup */
if (0) {
ierr = KSPSetTolerances(ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,1);CHKERRQ(ierr);
ierr = SNESSolve(snes, bb, xx);CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,50);CHKERRQ(ierr);
ierr = VecZeroEntries(xx);CHKERRQ(ierr);
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
/* solve */
ierr = PetscLogStagePush(stage[1]);CHKERRQ(ierr);
ierr = SNESSolve(snes, bb, xx);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = VecNorm(xx,NORM_INFINITY,&mdisp[iter]);CHKERRQ(ierr);
ierr = DMViewFromOptions(dm, NULL, "-dm_view");CHKERRQ(ierr);
{
PetscViewer viewer = NULL;
PetscViewerFormat fmt;
ierr = PetscOptionsGetViewer(comm,"ex56_","-vec_view",&viewer,&fmt,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscViewerPushFormat(viewer,fmt);CHKERRQ(ierr);
ierr = VecView(xx,viewer);CHKERRQ(ierr);
ierr = VecView(bb,viewer);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
}
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
/* Free work space */
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&bb);CHKERRQ(ierr);
ierr = MatDestroy(&Amat);CHKERRQ(ierr);
}
ierr = DMDestroy(&basedm);CHKERRQ(ierr);
if (run_type==1) {
err[0] = 59.975208 - mdisp[0]; /* error with what I think is the exact solution */
} else {
err[0] = 171.038 - mdisp[0];
}
for (iter=1 ; iter<max_conv_its ; iter++) {
if (run_type==1) {
err[iter] = 59.975208 - mdisp[iter];
} else {
err[iter] = 171.038 - mdisp[iter];
}
PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %D) N=%12D, max displ=%9.7e, disp diff=%9.2e, error=%4.3e, rate=%3.2g\n",
rank,PETSC_FUNCTION_NAME,iter,local_sizes[iter],mdisp[iter],
mdisp[iter]-mdisp[iter-1],err[iter],log(err[iter-1]/err[iter])/log(2.));
}
ierr = PetscFinalize();
return ierr;
}
/*
DMCoarsen_SNESVI - Computes the regular coarsened DM then computes additional information about its inactive set
*/
PetscErrorCode DMCoarsen_SNESVI(DM dm1,MPI_Comm comm,DM *dm2)
{
PetscErrorCode ierr;
PetscContainer isnes;
DM_SNESVI *dmsnesvi1;
Vec finemarked,coarsemarked;
IS inactive;
VecScatter inject;
const PetscInt *index;
PetscInt n,k,cnt = 0,rstart,*coarseindex;
PetscScalar *marked;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)dm1,"VI",(PetscObject *)&isnes);CHKERRQ(ierr);
if (!isnes) SETERRQ(((PetscObject)dm1)->comm,PETSC_ERR_PLIB,"Composed VI data structure is missing");
ierr = PetscContainerGetPointer(isnes,(void**)&dmsnesvi1);CHKERRQ(ierr);
/* get the original coarsen */
ierr = (*dmsnesvi1->coarsen)(dm1,comm,dm2);CHKERRQ(ierr);
/* not sure why this extra reference is needed, but without the dm2 disappears too early */
ierr = PetscObjectReference((PetscObject)*dm2);CHKERRQ(ierr);
/* need to set back global vectors in order to use the original injection */
ierr = DMClearGlobalVectors(dm1);CHKERRQ(ierr);
dm1->ops->createglobalvector = dmsnesvi1->createglobalvector;
ierr = DMCreateGlobalVector(dm1,&finemarked);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(*dm2,&coarsemarked);CHKERRQ(ierr);
/*
fill finemarked with locations of inactive points
*/
ierr = ISGetIndices(dmsnesvi1->inactive,&index);CHKERRQ(ierr);
ierr = ISGetLocalSize(dmsnesvi1->inactive,&n);CHKERRQ(ierr);
ierr = VecSet(finemarked,0.0);CHKERRQ(ierr);
for (k=0;k<n;k++){
ierr = VecSetValue(finemarked,index[k],1.0,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(finemarked);CHKERRQ(ierr);
ierr = VecAssemblyEnd(finemarked);CHKERRQ(ierr);
ierr = DMCreateInjection(*dm2,dm1,&inject);CHKERRQ(ierr);
ierr = VecScatterBegin(inject,finemarked,coarsemarked,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(inject,finemarked,coarsemarked,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&inject);CHKERRQ(ierr);
/*
create index set list of coarse inactive points from coarsemarked
*/
ierr = VecGetLocalSize(coarsemarked,&n);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(coarsemarked,&rstart,PETSC_NULL);CHKERRQ(ierr);
ierr = VecGetArray(coarsemarked,&marked);CHKERRQ(ierr);
for (k=0; k<n; k++) {
if (marked[k] != 0.0) cnt++;
}
ierr = PetscMalloc(cnt*sizeof(PetscInt),&coarseindex);CHKERRQ(ierr);
cnt = 0;
for (k=0; k<n; k++) {
if (marked[k] != 0.0) coarseindex[cnt++] = k + rstart;
}
ierr = VecRestoreArray(coarsemarked,&marked);CHKERRQ(ierr);
ierr = ISCreateGeneral(((PetscObject)coarsemarked)->comm,cnt,coarseindex,PETSC_OWN_POINTER,&inactive);CHKERRQ(ierr);
ierr = DMClearGlobalVectors(dm1);CHKERRQ(ierr);
dm1->ops->createglobalvector = DMCreateGlobalVector_SNESVI;
ierr = DMSetVI(*dm2,inactive);CHKERRQ(ierr);
ierr = VecDestroy(&finemarked);CHKERRQ(ierr);
ierr = VecDestroy(&coarsemarked);CHKERRQ(ierr);
ierr = ISDestroy(&inactive);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
Vec t,v;
Mat Q,M;
BV X,Y;
PetscInt i,j,n=10,kx=6,lx=3,ky=5,ly=2;
PetscScalar *q,*z;
PetscReal nrm;
PetscViewer view;
PetscBool verbose,trans;
SlepcInitialize(&argc,&argv,(char*)0,help);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-kx",&kx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-lx",&lx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-ky",&ky,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-ly",&ly,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,"-verbose",&verbose);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"First BV with %D active columns (%D leading columns) of dimension %D.\n",kx,lx,n);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Second BV with %D active columns (%D leading columns) of dimension %D.\n",ky,ly,n);CHKERRQ(ierr);
/* Create template vector */
ierr = VecCreate(PETSC_COMM_WORLD,&t);CHKERRQ(ierr);
ierr = VecSetSizes(t,PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetFromOptions(t);CHKERRQ(ierr);
/* Create BV object X */
ierr = BVCreate(PETSC_COMM_WORLD,&X);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)X,"X");CHKERRQ(ierr);
ierr = BVSetSizesFromVec(X,t,kx+2);CHKERRQ(ierr); /* two extra columns to test active columns */
ierr = BVSetFromOptions(X);CHKERRQ(ierr);
ierr = BVSetActiveColumns(X,lx,kx);CHKERRQ(ierr);
/* Set up viewer */
ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&view);CHKERRQ(ierr);
if (verbose) {
ierr = PetscViewerPushFormat(view,PETSC_VIEWER_ASCII_MATLAB);CHKERRQ(ierr);
}
/* Fill X entries */
for (j=0;j<kx+2;j++) {
ierr = BVGetColumn(X,j,&v);CHKERRQ(ierr);
ierr = VecZeroEntries(v);CHKERRQ(ierr);
for (i=0;i<4;i++) {
if (i+j<n) {
ierr = VecSetValue(v,i+j,(PetscScalar)(3*i+j-2),INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = VecAssemblyBegin(v);CHKERRQ(ierr);
ierr = VecAssemblyEnd(v);CHKERRQ(ierr);
ierr = BVRestoreColumn(X,j,&v);CHKERRQ(ierr);
}
if (verbose) {
ierr = BVView(X,view);CHKERRQ(ierr);
}
/* Create BV object Y */
ierr = BVCreate(PETSC_COMM_WORLD,&Y);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)Y,"Y");CHKERRQ(ierr);
ierr = BVSetSizesFromVec(Y,t,ky+1);CHKERRQ(ierr);
ierr = BVSetFromOptions(Y);CHKERRQ(ierr);
ierr = BVSetActiveColumns(Y,ly,ky);CHKERRQ(ierr);
/* Fill Y entries */
for (j=0;j<ky+1;j++) {
ierr = BVGetColumn(Y,j,&v);CHKERRQ(ierr);
ierr = VecSet(v,(PetscScalar)(j+1)/4.0);CHKERRQ(ierr);
ierr = BVRestoreColumn(Y,j,&v);CHKERRQ(ierr);
}
if (verbose) {
ierr = BVView(Y,view);CHKERRQ(ierr);
}
/* Create Mat */
ierr = MatCreateSeqDense(PETSC_COMM_SELF,kx,ky,NULL,&Q);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)Q,"Q");CHKERRQ(ierr);
ierr = MatDenseGetArray(Q,&q);CHKERRQ(ierr);
for (i=0;i<kx;i++)
for (j=0;j<ky;j++)
q[i+j*kx] = (i<j)? 2.0: -0.5;
ierr = MatDenseRestoreArray(Q,&q);CHKERRQ(ierr);
if (verbose) {
ierr = MatView(Q,NULL);CHKERRQ(ierr);
}
/* Test BVMult */
ierr = BVMult(Y,2.0,1.0,X,Q);CHKERRQ(ierr);
if (verbose) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"After BVMult - - - - - - - - -\n");CHKERRQ(ierr);
ierr = BVView(Y,view);CHKERRQ(ierr);
}
/* Test BVMultVec */
ierr = BVGetColumn(Y,0,&v);CHKERRQ(ierr);
ierr = PetscMalloc1(kx-lx,&z);CHKERRQ(ierr);
z[0] = 2.0;
for (i=1;i<kx-lx;i++) z[i] = -0.5*z[i-1];
ierr = BVMultVec(X,-1.0,1.0,v,z);CHKERRQ(ierr);
ierr = PetscFree(z);CHKERRQ(ierr);
ierr = BVRestoreColumn(Y,0,&v);CHKERRQ(ierr);
if (verbose) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"After BVMultVec - - - - - - -\n");CHKERRQ(ierr);
ierr = BVView(Y,view);CHKERRQ(ierr);
}
/* Test BVDot */
ierr = MatCreateSeqDense(PETSC_COMM_SELF,ky,kx,NULL,&M);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)M,"M");CHKERRQ(ierr);
ierr = BVDot(X,Y,M);CHKERRQ(ierr);
if (verbose) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"After BVDot - - - - - - - - -\n");CHKERRQ(ierr);
ierr = MatView(M,NULL);CHKERRQ(ierr);
}
/* Test BVDotVec */
ierr = BVGetColumn(Y,0,&v);CHKERRQ(ierr);
ierr = PetscMalloc1(kx-lx,&z);CHKERRQ(ierr);
ierr = BVDotVec(X,v,z);CHKERRQ(ierr);
ierr = BVRestoreColumn(Y,0,&v);CHKERRQ(ierr);
if (verbose) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"After BVDotVec - - - - - - -\n");CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,kx-lx,z,&v);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)v,"z");CHKERRQ(ierr);
ierr = VecView(v,view);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
}
ierr = PetscFree(z);CHKERRQ(ierr);
/* Test BVMultInPlace and BVScale */
ierr = PetscOptionsHasName(NULL,"-trans",&trans);CHKERRQ(ierr);
if (trans) {
Mat Qt;
ierr = MatTranspose(Q,MAT_INITIAL_MATRIX,&Qt);CHKERRQ(ierr);
ierr = BVMultInPlaceTranspose(X,Qt,lx+1,ky);CHKERRQ(ierr);
ierr = MatDestroy(&Qt);CHKERRQ(ierr);
} else {
ierr = BVMultInPlace(X,Q,lx+1,ky);CHKERRQ(ierr);
}
ierr = BVScale(X,2.0);CHKERRQ(ierr);
if (verbose) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"After BVMultInPlace - - - - -\n");CHKERRQ(ierr);
ierr = BVView(X,view);CHKERRQ(ierr);
}
/* Test BVNorm */
ierr = BVNormColumn(X,lx,NORM_2,&nrm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"2-Norm or X[%D] = %g\n",lx,(double)nrm);CHKERRQ(ierr);
ierr = BVNorm(X,NORM_FROBENIUS,&nrm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Frobenius Norm or X = %g\n",(double)nrm);CHKERRQ(ierr);
ierr = BVDestroy(&X);CHKERRQ(ierr);
ierr = BVDestroy(&Y);CHKERRQ(ierr);
ierr = MatDestroy(&Q);CHKERRQ(ierr);
ierr = MatDestroy(&M);CHKERRQ(ierr);
ierr = VecDestroy(&t);CHKERRQ(ierr);
ierr = SlepcFinalize();
return 0;
}
PetscErrorCode TestMatZeroRows(Mat A, Mat Afull, PetscBool squaretest, IS is, PetscScalar diag)
{
Mat B,Bcheck,B2 = NULL,lB;
Vec x = NULL, b = NULL, b2 = NULL;
ISLocalToGlobalMapping l2gr,l2gc;
PetscReal error;
char diagstr[16];
const PetscInt *idxs;
PetscInt rst,ren,i,n,N,d;
PetscMPIInt rank;
PetscBool miss,haszerorows;
PetscErrorCode ierr;
PetscFunctionBeginUser;
if (diag == 0.) {
ierr = PetscStrcpy(diagstr,"zero");CHKERRQ(ierr);
} else {
ierr = PetscStrcpy(diagstr,"nonzero");CHKERRQ(ierr);
}
ierr = ISView(is,NULL);CHKERRQ(ierr);
ierr = MatGetLocalToGlobalMapping(A,&l2gr,&l2gc);CHKERRQ(ierr);
/* tests MatDuplicate and MatCopy */
if (diag == 0.) {
ierr = MatDuplicate(A,MAT_COPY_VALUES,&B);CHKERRQ(ierr);
} else {
ierr = MatDuplicate(A,MAT_DO_NOT_COPY_VALUES,&B);CHKERRQ(ierr);
ierr = MatCopy(A,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
ierr = MatISGetLocalMat(B,&lB);CHKERRQ(ierr);
ierr = MatHasOperation(lB,MATOP_ZERO_ROWS,&haszerorows);CHKERRQ(ierr);
if (squaretest && haszerorows) {
ierr = MatCreateVecs(B,&x,&b);CHKERRQ(ierr);
ierr = MatDuplicate(B,MAT_COPY_VALUES,&B2);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(b,l2gr);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(x,l2gc);CHKERRQ(ierr);
ierr = VecSetRandom(x,NULL);CHKERRQ(ierr);
ierr = VecSetRandom(b,NULL);CHKERRQ(ierr);
/* mimic b[is] = x[is] */
ierr = VecDuplicate(b,&b2);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(b2,l2gr);CHKERRQ(ierr);
ierr = VecCopy(b,b2);CHKERRQ(ierr);
ierr = ISGetLocalSize(is,&n);CHKERRQ(ierr);
ierr = ISGetIndices(is,&idxs);CHKERRQ(ierr);
ierr = VecGetSize(x,&N);CHKERRQ(ierr);
for (i=0;i<n;i++) {
if (0 <= idxs[i] && idxs[i] < N) {
ierr = VecSetValue(b2,idxs[i],diag,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecSetValue(x,idxs[i],1.,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = VecAssemblyBegin(b2);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b2);CHKERRQ(ierr);
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&idxs);CHKERRQ(ierr);
/* test ZeroRows on MATIS */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsIS(B,is,diag,x,b);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRowsColumns (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsColumnsIS(B2,is,diag,NULL,NULL);CHKERRQ(ierr);
} else if (haszerorows) {
/* test ZeroRows on MATIS */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsIS(B,is,diag,NULL,NULL);CHKERRQ(ierr);
b = b2 = x = NULL;
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Skipping MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
b = b2 = x = NULL;
}
if (squaretest && haszerorows) {
ierr = VecAXPY(b2,-1.,b);CHKERRQ(ierr);
ierr = VecNorm(b2,NORM_INFINITY,&error);CHKERRQ(ierr);
if (error > PETSC_SQRT_MACHINE_EPSILON) SETERRQ2(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"ERROR IN ZEROROWS ON B %g (diag %s)",error,diagstr);
}
/* test MatMissingDiagonal */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatMissingDiagonal\n");CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MatMissingDiagonal(B,&miss,&d);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(B,&rst,&ren);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD, "[%d] [%D,%D) Missing %d, row %D (diag %s)\n",rank,rst,ren,(int)miss,d,diagstr);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&b2);CHKERRQ(ierr);
/* check the result of ZeroRows with that from MPIAIJ routines
assuming that MatConvert_IS_XAIJ and MatZeroRows_MPIAIJ work fine */
if (haszerorows) {
ierr = MatDuplicate(Afull,MAT_COPY_VALUES,&Bcheck);CHKERRQ(ierr);
ierr = MatSetOption(Bcheck,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatZeroRowsIS(Bcheck,is,diag,NULL,NULL);CHKERRQ(ierr);
ierr = CheckMat(B,Bcheck,PETSC_FALSE,"Zerorows");CHKERRQ(ierr);
ierr = MatDestroy(&Bcheck);CHKERRQ(ierr);
}
ierr = MatDestroy(&B);CHKERRQ(ierr);
if (B2) { /* test MatZeroRowsColumns */
ierr = MatDuplicate(Afull,MAT_COPY_VALUES,&B);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatZeroRowsColumnsIS(B,is,diag,NULL,NULL);CHKERRQ(ierr);
ierr = CheckMat(B2,B,PETSC_FALSE,"MatZeroRowsColumns");CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatDestroy(&B2);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCNullSpaceAssembleCorrection(PC pc, PetscBool isdir, IS local_dofs)
{
PC_BDDC *pcbddc = (PC_BDDC*)pc->data;
PC_IS *pcis = (PC_IS*)pc->data;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
KSP local_ksp;
PC newpc;
NullSpaceCorrection_ctx shell_ctx;
Mat local_mat,local_pmat,small_mat,inv_small_mat;
Vec work1,work2;
const Vec *nullvecs;
VecScatter scatter_ctx;
IS is_aux;
MatFactorInfo matinfo;
PetscScalar *basis_mat,*Kbasis_mat,*array,*array_mat;
PetscScalar one = 1.0,zero = 0.0, m_one = -1.0;
PetscInt basis_dofs,basis_size,nnsp_size,i,k;
PetscBool nnsp_has_cnst;
PetscErrorCode ierr;
PetscFunctionBegin;
/* Infer the local solver */
ierr = ISGetSize(local_dofs,&basis_dofs);CHKERRQ(ierr);
if (isdir) {
/* Dirichlet solver */
local_ksp = pcbddc->ksp_D;
} else {
/* Neumann solver */
local_ksp = pcbddc->ksp_R;
}
ierr = KSPGetOperators(local_ksp,&local_mat,&local_pmat);CHKERRQ(ierr);
/* Get null space vecs */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nnsp_has_cnst,&nnsp_size,&nullvecs);CHKERRQ(ierr);
basis_size = nnsp_size;
if (nnsp_has_cnst) {
basis_size++;
}
if (basis_dofs) {
/* Create shell ctx */
ierr = PetscNew(&shell_ctx);CHKERRQ(ierr);
/* Create work vectors in shell context */
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_small_1,basis_size,basis_size);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_small_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_small_1,&shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_full_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_full_1,basis_dofs,basis_dofs);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_full_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&shell_ctx->work_full_2);CHKERRQ(ierr);
/* Allocate workspace */
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->basis_mat );CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->Kbasis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Restrict local null space on selected dofs (Dirichlet or Neumann)
and compute matrices N and K*N */
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,local_dofs,work1,(IS)0,&scatter_ctx);CHKERRQ(ierr);
}
for (k=0;k<nnsp_size;k++) {
ierr = VecScatterBegin(matis->rctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->rctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (basis_dofs) {
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
}
if (basis_dofs) {
if (nnsp_has_cnst) {
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecSet(work1,one);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter_ctx);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Assemble another Mat object in shell context */
ierr = MatTransposeMatMult(shell_ctx->basis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&small_mat);CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&matinfo);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,basis_size,0,1,&is_aux);CHKERRQ(ierr);
ierr = MatLUFactor(small_mat,is_aux,is_aux,&matinfo);CHKERRQ(ierr);
ierr = ISDestroy(&is_aux);CHKERRQ(ierr);
ierr = PetscMalloc1(basis_size*basis_size,&array_mat);CHKERRQ(ierr);
for (k=0;k<basis_size;k++) {
ierr = VecSet(shell_ctx->work_small_1,zero);CHKERRQ(ierr);
ierr = VecSetValue(shell_ctx->work_small_1,k,one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecAssemblyEnd(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = MatSolve(small_mat,shell_ctx->work_small_1,shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecGetArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
for (i=0;i<basis_size;i++) {
array_mat[i*basis_size+k]=array[i];
}
ierr = VecRestoreArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
}
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_size,basis_size,array_mat,&inv_small_mat);CHKERRQ(ierr);
ierr = MatMatMult(shell_ctx->basis_mat,inv_small_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&shell_ctx->Lbasis_mat);CHKERRQ(ierr);
ierr = PetscFree(array_mat);CHKERRQ(ierr);
ierr = MatDestroy(&inv_small_mat);CHKERRQ(ierr);
ierr = MatDestroy(&small_mat);CHKERRQ(ierr);
ierr = MatScale(shell_ctx->Kbasis_mat,m_one);CHKERRQ(ierr);
/* Rebuild local PC */
ierr = KSPGetPC(local_ksp,&shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PCCreate(PETSC_COMM_SELF,&newpc);CHKERRQ(ierr);
ierr = PCSetOperators(newpc,local_mat,local_mat);CHKERRQ(ierr);
ierr = PCSetType(newpc,PCSHELL);CHKERRQ(ierr);
ierr = PCShellSetContext(newpc,shell_ctx);CHKERRQ(ierr);
ierr = PCShellSetApply(newpc,PCBDDCApplyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCShellSetDestroy(newpc,PCBDDCDestroyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCSetUp(newpc);CHKERRQ(ierr);
ierr = KSPSetPC(local_ksp,newpc);CHKERRQ(ierr);
ierr = PCDestroy(&newpc);CHKERRQ(ierr);
ierr = KSPSetUp(local_ksp);CHKERRQ(ierr);
}
/* test */
if (pcbddc->dbg_flag && basis_dofs) {
KSP check_ksp;
PC check_pc;
Mat test_mat;
Vec work3;
PetscReal test_err,lambda_min,lambda_max;
PetscBool setsym,issym=PETSC_FALSE;
PetscInt tabs;
ierr = PetscViewerASCIIGetTab(pcbddc->dbg_viewer,&tabs);CHKERRQ(ierr);
ierr = KSPGetPC(local_ksp,&check_pc);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work3);CHKERRQ(ierr);
ierr = VecSetRandom(shell_ctx->work_small_1,NULL);CHKERRQ(ierr);
ierr = MatMult(shell_ctx->basis_mat,shell_ctx->work_small_1,work1);CHKERRQ(ierr);
ierr = VecCopy(work1,work2);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work3);CHKERRQ(ierr);
ierr = PCApply(check_pc,work3,work1);CHKERRQ(ierr);
ierr = VecAXPY(work1,m_one,work2);CHKERRQ(ierr);
ierr = VecNorm(work1,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for nullspace correction for ",PetscGlobalRank);CHKERRQ(ierr);
ierr = PetscViewerASCIIUseTabs(pcbddc->dbg_viewer,PETSC_FALSE);CHKERRQ(ierr);
if (isdir) {
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Dirichlet ");CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Neumann ");CHKERRQ(ierr);
}
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"solver is :%1.14e\n",test_err);CHKERRQ(ierr);
ierr = PetscViewerASCIISetTab(pcbddc->dbg_viewer,tabs);CHKERRQ(ierr);
ierr = PetscViewerASCIIUseTabs(pcbddc->dbg_viewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatTransposeMatMult(shell_ctx->Lbasis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&test_mat);CHKERRQ(ierr);
ierr = MatShift(test_mat,one);CHKERRQ(ierr);
ierr = MatNorm(test_mat,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = MatDestroy(&test_mat);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for nullspace matrices is :%1.14e\n",PetscGlobalRank,test_err);CHKERRQ(ierr);
/* Create ksp object suitable for extreme eigenvalues' estimation */
ierr = KSPCreate(PETSC_COMM_SELF,&check_ksp);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(check_ksp,pc->erroriffailure);CHKERRQ(ierr);
ierr = KSPSetOperators(check_ksp,local_mat,local_mat);CHKERRQ(ierr);
ierr = KSPSetTolerances(check_ksp,1.e-8,1.e-8,PETSC_DEFAULT,basis_dofs);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues(check_ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatIsSymmetricKnown(pc->pmat,&setsym,&issym);CHKERRQ(ierr);
if (issym) {
ierr = KSPSetType(check_ksp,KSPCG);CHKERRQ(ierr);
}
ierr = KSPSetPC(check_ksp,check_pc);CHKERRQ(ierr);
ierr = KSPSetUp(check_ksp);CHKERRQ(ierr);
ierr = VecSetRandom(work1,NULL);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = KSPSolve(check_ksp,work2,work2);CHKERRQ(ierr);
ierr = VecAXPY(work2,m_one,work1);CHKERRQ(ierr);
ierr = VecNorm(work2,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = KSPComputeExtremeSingularValues(check_ksp,&lambda_max,&lambda_min);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(check_ksp,&k);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for adapted KSP %1.14e (it %d, eigs %1.6e %1.6e)\n",PetscGlobalRank,test_err,k,lambda_min,lambda_max);CHKERRQ(ierr);
ierr = KSPDestroy(&check_ksp);CHKERRQ(ierr);
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecDestroy(&work3);CHKERRQ(ierr);
}
/* all processes shoud call this, even the void ones */
if (pcbddc->dbg_flag) {
ierr = PetscViewerFlush(pcbddc->dbg_viewer);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
void
construct_operator(Elliptic1D* p, int levels) {
// Start out with a simple tridiagonal matrix.
// problem solves laplacian u = -1 on -1 to 1 with homogeneous
// dirichlet boundary conditions. Code
// implements a finite difference to approximate this
// equation system.
p->levels = levels;
p->npoints = (2<<levels)-1;
p->num_seg = 2<<levels;
PetscScalar h = 2./p->num_seg;
MatCreateMPIAIJ(PETSC_COMM_WORLD,
PETSC_DECIDE, //number of local rows
PETSC_DECIDE, //number of local cols
p->npoints, //global rows
p->npoints, //global cols
3, // upper bound of diagonal nnz per row
PETSC_NULL, // array of diagonal nnz per row
1, // upper bound of off-processor nnz per row
PETSC_NULL, // array of off-processor nnz per row
&p->A); // matrix
MatSetFromOptions(p->A);
PetscInt start;
PetscInt end;
MatGetOwnershipRange(p->A, &start, &end);
int ii;
for (ii=start; ii<end; ii++) {
PetscInt col_index[3] = {ii-1, ii, ii+1};
PetscInt row_index = ii;
PetscScalar stencil[3] = {-1./(h*h), 2./(h*h), -1./(h*h)};
// handle corner cases at beginning and end of matrix.
if (ii+1 == p->npoints) {
col_index[2] = -1;
} else if (ii == 0) {
col_index[0] = -1;
}
MatSetValues(p->A, 1, &row_index, 3, col_index, stencil, INSERT_VALUES);
}
MatAssemblyBegin(p->A, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(p->A, MAT_FINAL_ASSEMBLY);
//Create the corresponding vectors
VecCreateMPI(PETSC_COMM_WORLD,
PETSC_DECIDE,
p->npoints,
&p->x
);
VecSetFromOptions(p->x);
VecDuplicate(p->x, &p->b);
VecSet(p->b, 1);
VecZeroEntries(p->x);
//Fill in a random initial guess
PetscInt high;
PetscInt low;
VecGetOwnershipRange(p->x, &low, &high);
for (ii=low; ii<high; ii++) {
VecSetValue(p->x, ii, frand(), INSERT_VALUES);
}
VecAssemblyBegin(p->x);
VecAssemblyEnd(p->x);
}
