void Delta::BuildDensityGrid(Particle& pp, Vec& delta)
// RS 2010/07/06: Commented!
{
_mydestroy(delta); // Clean up delta
DensityGrid dg2(N, L);
DensityGrid dg3(N, L);
delta = dg1.Allocate();
bool rmask = (rmasksmooth > 1e-5); // RS: boolean shorthand
// First the particles
// RS 2010/07/06: CIC assign the list of particles to delta.
// If we're using a mask made from smoothed randoms, don't bother trimming,
// because the mask will just be a "dummy mask".
if (!rmask) pp.TrimMask(*maskptr);
pp.SlabDecompose(dg1);
dg1.CIC(delta,pp, false);
PetscPrintf(PETSC_COMM_WORLD, "Particles assigned to the grid.....\n");
// RS 2010/07/06: If rmasksmooth > 0, smooth CIC-assigned particles before
// computing overdensity. Otherwise we'll get unpleasant edge effects.
if (rmask) dg1.GaussSmooth(delta, rmasksmooth);
// Normalization factor
// RS: This equals the *average* number (per allowed cell) of "randoms"
// compared to the number of particles from which we're building delta.
VecPointwiseMult(delta, W);
double fac;
VecSum(delta, &fac);
fac = nrand/fac;
// Now construct delta
int lo, hi;
dg1.slab(lo, hi);
// RS: Set up the wrappers to allow us to index delta by (x,y,z).
dg1.Pull(delta); dg2.Pull(W); dg3.Pull(_W);
for (int ix=lo; ix < hi; ++ix)
for (int iy=0; iy < N; ++iy)
for (int iz=0; iz < N; ++iz)
if (dg2(ix, iy,iz) > 1.e-5) {
// RS: In each cell location, if W > 0, divide by _W/fac.
// Since _W is the average density of regular-grid "randoms" in
// each cell (accounting for edge effects), _W/fac corresponds to
// the average density of pp particles. So W/(_W/fac) = W/_W*fac,
// i.e., the density contrast, which is what we want.
dg1(ix, iy, iz) = (dg1(ix, iy, iz)/dg3(ix, iy, iz))*fac - 1.0;
} else {
// RS: If W = 0 (to within epsilon), set delta to zero.
dg1(ix, iy, iz) = 0.0;
}
// RS: Deallocate the memory we just set up... and we'll be done.
dg1.Push(delta, INSERT_VALUES, false);
dg2.Push(W, INSERT_VALUES, false);
dg3.Push(_W, INSERT_VALUES, false);
VecSum(delta, &fac);
PetscPrintf(PETSC_COMM_WORLD, "Sum (delta).....%e\n", fac);
}
void Delta::ZeroMean(Vec &delta) {
// Compute mean
double sum, npts, norm;
VecPointwiseMult(delta, W, delta);
VecSum(delta, &sum);
VecSum(W, &npts);
norm = sum/npts;
// Subtract mean
VecAXPY(delta, -norm, W);
}
PetscErrorCode PostStep(TS ts)
{
PetscErrorCode ierr;
Vec X;
AppCtx *user;
PetscReal t;
PetscScalar asum;
PetscFunctionBegin;
ierr = TSGetApplicationContext(ts,&user);CHKERRQ(ierr);
ierr = TSGetTime(ts,&t);CHKERRQ(ierr);
ierr = TSGetSolution(ts,&X);CHKERRQ(ierr);
/*
if (t >= .2) {
ierr = TSGetSolution(ts,&X);CHKERRQ(ierr);
ierr = VecView(X,PETSC_VIEWER_BINARY_WORLD);CHKERRQ(ierr);
exit(0);
results in initial conditions after fault in binaryoutput
}*/
if ((t > user->tf) && (t < user->tcl)) user->Pmax = 0.0; /* A short-circuit that drives the electrical power output (Pmax*sin(delta)) to zero */
else user->Pmax = user->Pmax_s;
ierr = VecSum(X,&asum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"sum(p) at t = %f = %f\n",(double)t,(double)(asum));CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Checks rhs of Lp systems is in the null space of Lp, i.e. {rhs} \centerdot {null_space} = 0
Since Lp should contain the null space {1}, we just check the \sum_i rhs_i = 0
*/
PetscErrorCode BSSCRBSSCR_Lp_monitor_check_rhs_consistency( KSP ksp, Vec rhs, PetscInt index )
{
PetscScalar dot;
#if 0
Vec one;
VecDuplicate( rhs, &one );
VecSet( one, 1.0 );
VecDot( rhs, one, &dot );
if( PetscAbsReal(dot) > 1.0e-8 ) {
PetscPrintf(ksp->comm," ($D) Lp z = r: ******* WARNING ******* RHS is not consistent. {b}.{1} = %5.5e \n", index, dot );
}
Stg_VecDestroy(&one );
PetscFunctionReturn(0);
#endif
#if 0
VecSum( rhs, &dot );
if( PetscAbsReal(dot) > 1.0e-8 ) {
PetscPrintf(((PetscObject)ksp)->comm," (%D) Lp z = r: ******* WARNING ******* RHS is not consistent. {b}.{1} = %5.5e \n",
index, dot );
BSSCR_VecRemoveConstNullspace( rhs, PETSC_NULL );
}
#endif
PetscFunctionReturn(0);
}
/*@C
MatNullSpaceRemove - Removes all the components of a null space from a vector.
Collective on MatNullSpace
Input Parameters:
+ sp - the null space context (if this is NULL then no null space is removed)
- vec - the vector from which the null space is to be removed
Level: advanced
.keywords: PC, null space, remove
.seealso: MatNullSpaceCreate(), MatNullSpaceDestroy(), MatNullSpaceSetFunction()
@*/
PetscErrorCode MatNullSpaceRemove(MatNullSpace sp,Vec vec)
{
PetscScalar sum;
PetscInt i,N;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!sp) PetscFunctionReturn(0);
PetscValidHeaderSpecific(sp,MAT_NULLSPACE_CLASSID,1);
PetscValidHeaderSpecific(vec,VEC_CLASSID,2);
if (sp->has_cnst) {
ierr = VecGetSize(vec,&N);CHKERRQ(ierr);
if (N > 0) {
ierr = VecSum(vec,&sum);CHKERRQ(ierr);
sum = sum/((PetscScalar)(-1.0*N));
ierr = VecShift(vec,sum);CHKERRQ(ierr);
}
}
if (sp->n) {
ierr = VecMDot(vec,sp->n,sp->vecs,sp->alpha);CHKERRQ(ierr);
for (i=0; i<sp->n; i++) sp->alpha[i] = -sp->alpha[i];
ierr = VecMAXPY(vec,sp->n,sp->alpha,sp->vecs);CHKERRQ(ierr);
}
if (sp->remove) {
ierr = (*sp->remove)(sp,vec,sp->rmctx);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode printVecSum(Vec v){
PetscErrorCode err;
PetscScalar sum;
err = VecSum(v, &sum); CHKERRQ(err);
err = PetscPrintf(PETSC_COMM_WORLD, "%lf\n", sum);
return err;
}
Delta::Delta(int _N, double _L, int _nover, double thresh, Mask3D& _mask,
Particle& pr, double smooth)
// RS 2010/07/06: Version of Delta constructor that takes as input a set
// of random particles to define where the mask is or is not. This is in
// contrast to the regular grid of "randoms" set up to compute the average
// density in the normal Delta constructor above.
{
// We're not going to use the Mask3D& for anything, but I make the user
// pass one in anyway, because Delta can't manage the memory for it without
// creating the risk either of a memory leak or a seg fault.
N=_N; L=_L; nover=_nover; maskptr=&_mask; rmasksmooth=smooth;
// W is the mask, _W is the average density of randoms.
// dg1 and dg2 are DensityGrid wrappers for these vectors.
DensityGrid dg2(N, L);
dg1.Init(N, L);
_W = dg1.Allocate();
W = dg1.Allocate(); VecSet(W, 0.0);
pr.SlabDecompose(dg1);
dg1.CIC(_W, pr, false); // Assign randoms to _W ("no overdensity")
dg1.GaussSmooth(_W, smooth); // Smooth by smoothing scale
PetscPrintf(PETSC_COMM_WORLD, "%d grid particles accumulated....\n", pr.npart);
// RS: For each cell, if the contents of _W exceed thresh, set W = 1.
// Use dg1 and dg2 as wrappers to index the copies of _W and W.
int lo, hi;
dg1.slab(lo, hi);
dg1.Pull(_W); dg2.Pull(W);
for (int ix=lo; ix < hi; ++ix)
for (int iy=0; iy < N; ++iy)
for (int iz=0; iz < N; ++iz)
if (dg1(ix,iy,iz) > thresh) {
dg2(ix,iy,iz) = 1.0;
} else {
dg1(ix, iy, iz) = 0.0; // Good to set this explicitly to zero as well
}
dg1.Push(_W, INSERT_VALUES, false);
dg2.Push(W, INSERT_VALUES, false);
double tmp;
VecSum(W, &tmp);
PetscPrintf(PETSC_COMM_WORLD,"Nonzero grid points in W : %f\n",tmp);
VecSum(_W, &nrand);
PetscPrintf(PETSC_COMM_WORLD,"Sum of background density vector : %f\n",nrand);
}
PetscErrorCode ini_bou(Vec X,AppCtx* user)
{
PetscErrorCode ierr;
DM cda;
DMDACoor2d **coors;
PetscScalar **p;
Vec gc;
PetscInt i,j;
PetscInt xs,ys,xm,ym,M,N;
PetscScalar xi,yi;
PetscScalar sigmax=user->sigmax,sigmay=user->sigmay;
PetscScalar rho =user->rho;
PetscScalar mux =user->mux,muy=user->muy;
PetscMPIInt rank;
PetscScalar sum;
PetscFunctionBeginUser;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = DMDAGetInfo(user->da,NULL,&M,&N,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
user->dx = (user->xmax - user->xmin)/(M-1); user->dy = (user->ymax - user->ymin)/(N-1);
ierr = DMGetCoordinateDM(user->da,&cda);CHKERRQ(ierr);
ierr = DMGetCoordinates(user->da,&gc);CHKERRQ(ierr);
ierr = DMDAVecGetArray(cda,gc,&coors);CHKERRQ(ierr);
ierr = DMDAVecGetArray(user->da,X,&p);CHKERRQ(ierr);
ierr = DMDAGetCorners(cda,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
/* mux and muy need to be grid points in the x and y-direction otherwise the solution goes unstable
muy is set by choosing the y domain, no. of grid points along y-direction so that muy is a grid point
in the y-direction. We only modify mux here
*/
mux = user->mux = coors[0][M/2+10].x; /* For -pi < x < pi, this should be some angle between 0 and pi/2 */
if (user->nonoiseinitial) {
for (i=xs; i < xs+xm; i++) {
for (j=ys; j < ys+ym; j++) {
xi = coors[j][i].x; yi = coors[j][i].y;
if ((xi == mux) && (yi == muy)) {
p[j][i] = 1.0;
}
}
}
} else {
/* Change PM_min accordingly */
user->PM_min = user->Pmax*sin(mux);
for (i=xs; i < xs+xm; i++) {
for (j=ys; j < ys+ym; j++) {
xi = coors[j][i].x; yi = coors[j][i].y;
p[j][i] = (0.5/(PETSC_PI*sigmax*sigmay*PetscSqrtScalar(1.0-rho*rho)))*PetscExpScalar(-0.5/(1-rho*rho)*(PetscPowScalar((xi-mux)/sigmax,2) + PetscPowScalar((yi-muy)/sigmay,2) - 2*rho*(xi-mux)*(yi-muy)/(sigmax*sigmay)));
}
}
}
ierr = DMDAVecRestoreArray(cda,gc,&coors);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(user->da,X,&p);CHKERRQ(ierr);
ierr = VecSum(X,&sum);CHKERRQ(ierr);
ierr = VecScale(X,1.0/sum);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void _remove_constant_nullsp( Vec v )
{
PetscInt N;
PetscScalar sum;
VecGetSize( v, &N );
if( N > 0 ) {
VecSum( v, &sum );
sum = sum/( -1.0*N );
VecShift( v, sum );
}
}
T PetscVector<T>::sum () const
{
this->_restore_array();
libmesh_assert(this->closed());
PetscErrorCode ierr=0;
PetscScalar value=0.;
ierr = VecSum (_vec, &value);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
return static_cast<T>(value);
}
PetscScalar SingleLongPipe::computeCrossSectionSum() {
PetscScalar sum_r_2 = 0.0;
PetscScalar* localRadii = radii.getLocalArray();
for (int i = 0; i < local_n; ++i) {
PetscScalar radius = localRadii[i];
sum_r_2 += radius*radius;
}
PetscScalar* localOneOverR4 = one_over_r4.getLocalArray();
*localOneOverR4 = sum_r_2;
one_over_r4.restoreVec(localOneOverR4);
PetscScalar globalR2;
VecSum(one_over_r4.getVec(), &globalR2);
return globalR2;
}
PetscScalar SingleLongPipe::computeTransmissivity() {
PetscScalar sum_r_4 = 0.0;
PetscScalar* localRadii = radii.getLocalArray();
for (int i = 0; i < local_n; ++i) {
PetscScalar radius = localRadii[i];
sum_r_4 += 1.0/(radius*radius*radius*radius);
}
PetscScalar* localOneOverR4 = one_over_r4.getLocalArray();
*localOneOverR4 = sum_r_4;
one_over_r4.restoreVec(localOneOverR4);
PetscScalar globalOneOverR4;
VecSum(one_over_r4.getVec(), &globalOneOverR4);
PetscScalar transmissivity = 1.0/globalOneOverR4;
return transmissivity;
}
PetscErrorCode PostStep(TS ts)
{
PetscErrorCode ierr;
Vec X;
AppCtx *user;
PetscScalar sum;
PetscReal t;
PetscFunctionBegin;
ierr = TSGetApplicationContext(ts,&user);CHKERRQ(ierr);
ierr = TSGetTime(ts,&t);CHKERRQ(ierr);
ierr = TSGetSolution(ts,&X);CHKERRQ(ierr);
ierr = VecSum(X,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"sum(p)*dw*dtheta at t = %3.2f = %3.6f\n",t,sum*user->dx*user->dy);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
// Test Mat BNHat up to N=10 when Op is a diagonal Mat
TEST(operatorsCreateBnHeadTest, testBNHatDiagonalOp)
{
Mat Op; // operator (e.g., Laplacian)
PetscReal dt = 2.3, // time-step size
c = 0.5, // time-scheme coefficient of the implicit diffusive term
val = 2.0 / dt; // value set on the diagonal of the operator
PetscInt nx = 10, // number of points in the x-direction
ny = 12; // number of points in the y-direction
PetscReal ans = nx * ny * dt; // expected sum of all elements of B1Hat
// Create and assemble operator
MatCreate(PETSC_COMM_WORLD, &Op);
MatSetSizes(Op, PETSC_DECIDE, PETSC_DECIDE, nx * ny, nx * ny);
MatSetType(Op, MATAIJ);
MatSetUp(Op);
for (PetscInt i = 0; i < nx * ny; i++)
MatSetValues(Op, 1, &i, 1, &i, &val, INSERT_VALUES);
MatAssemblyBegin(Op, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(Op, MAT_FINAL_ASSEMBLY);
for (PetscInt N = 1; N <= 10; N++)
{
Mat BNHat; // Nth-order approximation of the inverse of (I/dt - c*Op)
// Call function to test
petibm::operators::createBnHead(Op, dt, c, N, BNHat);
// Check size of Mat BNHat
{
PetscInt nrows, ncols;
MatGetSize(BNHat, &nrows, &ncols);
ASSERT_EQ(nx * ny, nrows);
ASSERT_EQ(nx * ny, ncols);
}
// Check sum of elements of BNHat is the expected value
if (N > 1) ans += dt * nx * ny * std::pow(c * dt * val, N - 1);
{
PetscReal sum;
Vec v;
MatCreateVecs(Op, &v, nullptr);
MatGetRowSum(BNHat, v);
VecSum(v, &sum);
ASSERT_TRUE(std::abs(sum - ans) <= 1.0E-11);
VecDestroy(&v);
}
MatDestroy(&BNHat);
}
MatDestroy(&Op);
}
PetscErrorCode PCISApplyInvSchur(PC pc, Vec b, Vec x, Vec vec1_N, Vec vec2_N)
{
PetscErrorCode ierr;
PC_IS *pcis = (PC_IS*)(pc->data);
PetscFunctionBegin;
/*
Neumann solvers.
Applying the inverse of the local Schur complement, i.e, solving a Neumann
Problem with zero at the interior nodes of the RHS and extracting the interface
part of the solution. inverse Schur complement is applied to b and the result
is stored in x.
*/
/* Setting the RHS vec1_N */
ierr = VecSet(vec1_N,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,b,vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,b,vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* Checking for consistency of the RHS */
{
PetscBool flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-pc_is_check_consistency",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscScalar average;
PetscViewer viewer;
ierr = PetscViewerASCIIGetStdout(PetscObjectComm((PetscObject)pc),&viewer);CHKERRQ(ierr);
ierr = VecSum(vec1_N,&average);CHKERRQ(ierr);
average = average / ((PetscReal)pcis->n);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);CHKERRQ(ierr);
if (pcis->pure_neumann) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d is floating. Average = % 1.14e\n",PetscGlobalRank,PetscAbsScalar(average));CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d is fixed. Average = % 1.14e\n",PetscGlobalRank,PetscAbsScalar(average));CHKERRQ(ierr);
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);CHKERRQ(ierr);
}
}
/* Solving the system for vec2_N */
ierr = KSPSolve(pcis->ksp_N,vec1_N,vec2_N);CHKERRQ(ierr);
/* Extracting the local interface vector out of the solution */
ierr = VecScatterBegin(pcis->N_to_B,vec2_N,x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,vec2_N,x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Computes the wind speed using Weibull distribution */
PetscErrorCode WindSpeeds(AppCtx *user)
{
PetscErrorCode ierr;
PetscScalar *x,*t,avg_dev,sum;
PetscInt i;
PetscFunctionBegin;
user->cw = 5;
user->kw = 2; /* Rayleigh distribution */
user->nsamples = 2000;
user->Tw = 0.2;
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,PETSC_NULL,"Wind Speed Options","");CHKERRQ(ierr);
{
ierr = PetscOptionsReal("-cw","","",user->cw,&user->cw,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-kw","","",user->kw,&user->kw,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-nsamples","","",user->nsamples,&user->nsamples,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-Tw","","",user->Tw,&user->Tw,PETSC_NULL);CHKERRQ(ierr);
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&user->wind_data);CHKERRQ(ierr);
ierr = VecSetSizes(user->wind_data,PETSC_DECIDE,user->nsamples);CHKERRQ(ierr);
ierr = VecSetFromOptions(user->wind_data);CHKERRQ(ierr);
ierr = VecDuplicate(user->wind_data,&user->t_wind);CHKERRQ(ierr);
ierr = VecGetArray(user->t_wind,&t);CHKERRQ(ierr);
for(i=0;i < user->nsamples;i++) t[i] = (i+1)*tmax/user->nsamples;
ierr = VecRestoreArray(user->t_wind,&t);CHKERRQ(ierr);
/* Wind speed deviation = (-log(rand)/cw)^(1/kw) */
ierr = VecSetRandom(user->wind_data,PETSC_NULL);CHKERRQ(ierr);
ierr = VecLog(user->wind_data);CHKERRQ(ierr);
ierr = VecScale(user->wind_data,-1/user->cw);CHKERRQ(ierr);
ierr = VecGetArray(user->wind_data,&x);CHKERRQ(ierr);
for(i=0;i < user->nsamples;i++) {
x[i] = PetscPowScalar(x[i],(1/user->kw));
}
ierr = VecRestoreArray(user->wind_data,&x);CHKERRQ(ierr);
ierr = VecSum(user->wind_data,&sum);CHKERRQ(ierr);
avg_dev = sum/user->nsamples;
/* Wind speed (t) = (1 + wind speed deviation(t) - avg_dev)*average wind speed */
ierr = VecShift(user->wind_data,(1-avg_dev));CHKERRQ(ierr);
ierr = VecScale(user->wind_data,vwa);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void ProfilerTest::test_petsc_memory() {
int ierr, mpi_rank;
ierr = MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
EXPECT_EQ( ierr, 0 );
Profiler::initialize(); {
PetscLogDouble mem;
START_TIMER("A");
PetscInt size = 100*1000;
PetscScalar value = 0.1;
Vec tmp_vector;
VecCreateSeq(PETSC_COMM_SELF, size, &tmp_vector);
VecSet(tmp_vector, value);
// VecSetRandom(tmp_vector, NULL);
END_TIMER("A");
START_TIMER("A");
// allocated memory MUST be greater or equal to size * size of double
EXPECT_GE(AN.petsc_memory_difference, size*sizeof(double));
END_TIMER("A");
START_TIMER("B");
PetscScalar sum;
VecSum(tmp_vector, &sum);
END_TIMER("B");
START_TIMER("C");
VecDestroy(&tmp_vector);
END_TIMER("C");
START_TIMER("C");
// since we are destroying vector, we expect to see negative memory difference
EXPECT_LE(AN.petsc_memory_difference, 0);
END_TIMER("C");
}
PI->output(MPI_COMM_WORLD, cout);
Profiler::uninitialize();
}
/*
Stokes output:
---------------------------------
Operator summary:
K
G
f,
h
u
p
---------------------------------
Solution summary:
max_u
min_u
average_u
|r_1|
|r_2|
---------------------------------
Solver summary:
name
---------------------------------
Petsc build summary:
*/
PetscErrorCode BSSCR_stokes_output( PetscViewer v, Mat stokes_A, Vec stokes_b, Vec stokes_x, KSP ksp, PetscInt monitor_index )
{
Mat K,G,D,C;
Vec f,h, u,p;
K = G = D = C = PETSC_NULL;
f = h = PETSC_NULL;
u = p = PETSC_NULL;
MatNestGetSubMat( stokes_A, 0,0, &K );
MatNestGetSubMat( stokes_A, 0,1, &G );
MatNestGetSubMat( stokes_A, 1,0, &D );
MatNestGetSubMat( stokes_A, 1,1, &C );
VecNestGetSubVec( stokes_x, 0, &u );
VecNestGetSubVec( stokes_x, 1, &p );
VecNestGetSubVec( stokes_b, 0, &f );
VecNestGetSubVec( stokes_b, 1, &h );
PetscViewerASCIIPrintf( v, "Stokes Output:\n");
PetscViewerASCIIPushTab( v );
/*--------------------------------------------------------------------------------------------*/
PetscViewerASCIIPrintf( v, "--------------------------------------------------\n");
PetscViewerASCIIPrintf( v, "Operator summary:\n");
PetscViewerASCIIPushTab( v );
if (K) {
BSSCR_MatInfoLog(v,K, "stokes_A11");
PetscViewerASCIIPrintf( v, "\n");
}
if (G) {
BSSCR_MatInfoLog(v,G, "stokes_A12");
PetscViewerASCIIPrintf( v, "\n");
}
if (D) {
BSSCR_MatInfoLog(v,D, "stokes_A21");
PetscViewerASCIIPrintf( v, "\n");
}
if (C) {
BSSCR_MatInfoLog(v,C, "stokes_A22");
PetscViewerASCIIPrintf( v, "\n");
}
if (f) {
BSSCR_VecInfoLog(v,f,"stokes_b1");
PetscViewerASCIIPrintf( v, "\n");
}
if (h) {
BSSCR_VecInfoLog(v,h,"stokes_b2");
PetscViewerASCIIPrintf( v, "\n");
}
PetscViewerASCIIPopTab( v );
/*--------------------------------------------------------------------------------------------*/
PetscViewerASCIIPrintf( v, "--------------------------------------------------\n");
PetscViewerASCIIPrintf( v, "Solution summary:\n");
PetscViewerASCIIPushTab( v );
if (u) {
BSSCR_VecInfoLog(v,u,"x1");
PetscViewerASCIIPrintf( v, "\n");
}
if (p) {
BSSCR_VecInfoLog(v,p,"x2");
PetscViewerASCIIPrintf( v, "\n");
}
{
PetscScalar s,sum;
PetscReal r,max,min;
PetscInt N, loc;
double r1,r2;
Vec K_d;
PetscInt loc_max, loc_min;
VecGetSize( u, &N );
VecMax( u, &loc, &r );
PetscViewerASCIIPrintf( v, "u_max: %1.12e [%d] \n", r, loc );
VecMin( u, &loc, &r );
PetscViewerASCIIPrintf( v, "u_min: %1.12e [%d] \n", r, loc );
VecDot( u,u, &s );
PetscViewerASCIIPrintf( v, "u_rms: %1.12e \n", sqrt( PetscRealPart(s) )/N );
VecDuplicate( u, &K_d );
MatGetDiagonal( K, K_d );
VecMax( K_d, &loc_max, &max );
VecMin( K_d, &loc_min, &min );
PetscViewerASCIIPrintf( v,"Algebraic contrast: max(K_d)=%.3e [%d] , min(K_d)=%.3e [%d] , max(K_d)/min(K_d) = %.8e \n", max,loc_max, min,loc_min, max/min );
MatGetRowMax( K, K_d, PETSC_NULL );
VecMax( K_d, &loc_max, &max );
MatGetRowMin( K, K_d, PETSC_NULL );
VecAbs( K_d );
VecMin( K_d, &loc_min, &min );
PetscViewerASCIIPrintf( v,"Algebraic contrast: max(K)=%.3e [%d] , |min(K)|=%.3e [%d] , max(K)/|min(K)| = %.8e \n", max,loc_max, min,loc_min, max/min );
Stg_VecDestroy(&K_d );
PetscViewerASCIIPrintf( v, "\n");
VecGetSize( p, &N );
VecMax( p, &loc, &r );
PetscViewerASCIIPrintf( v, "p_max: %1.12e [%d] \n", r, loc );
VecMin( p, &loc, &r );
PetscViewerASCIIPrintf( v, "p_min: %1.12e [%d] \n", r, loc );
VecDot( p,p, &s );
PetscViewerASCIIPrintf( v, "p_rms: %1.12e \n", sqrt( PetscRealPart(s) )/N );
VecSum( p, &sum );
PetscViewerASCIIPrintf( v, "sum(p): %1.12e \n", sum );
PetscViewerASCIIPrintf( v, "\n");
r1 = BSSCR_StokesMomentumResidual( K,G,f, u,p );
PetscViewerASCIIPrintf( v, "|r1| = %1.12e <momentum> \n", r1 );
r2 = BSSCR_StokesContinuityResidual( G,C,h, u,p );
PetscViewerASCIIPrintf( v, "|r2| = %1.12e <continuity> \n", r2 );
PetscViewerASCIIPrintf( v, "\n");
}
PetscViewerASCIIPopTab( v );
/*--------------------------------------------------------------------------------------------*/
if (ksp) {
PetscViewerASCIIPrintf( v, "--------------------------------------------------\n");
PetscViewerASCIIPrintf( v, "Solver summary:\n");
PetscViewerASCIIPushTab( v );
BSSCR_KSPLogSolve( v, monitor_index, ksp );
BSSCR_BSSCR_KSPLogSolveSummary( v, monitor_index, ksp );
PetscViewerASCIIPrintf( v, "\n");
PetscViewerASCIIPopTab( v );
}
/*--------------------------------------------------------------------------------------------*/
PetscViewerASCIIPrintf( v, "--------------------------------------------------\n");
PetscViewerASCIIPrintf( v, "Petsc build summary:\n");
PetscViewerASCIIPushTab( v );
BSSCR_GeneratePetscHeader_for_viewer( v );
PetscViewerASCIIPrintf( v, "\n");
PetscViewerASCIIPopTab( v );
/*--------------------------------------------------------------------------------------------*/
PetscViewerASCIIPopTab(v);
PetscFunctionReturn(0);
}
void BuildDensityGrid(int N, double L, Mask3D& mask, Particle& pp, int nover, double thresh, Vec& delta, Vec& W, double smooth) {
/* Parameters :
* INPUT ---
* N : grid size
* L : box size
* mask : A 3D positive mask (assumed to be 1 or 0)
* nover : Oversampling rate for defining the mean density (we use the GridInitialize routine)
* This determines the displacement.
* For a periodic box, nover means there will be nover**3 particles per grid.
* thresh: Fractional threshold below which to remove grid point from survey
* smooth: This is an optional parameter -- if set to > 0, it will smooth the density and mask
* fields before computing the overdensity.
* OUTPUT ---
* delta : What do think this is???
* W : Window function, 1 in survey, 0 elsewhere.
* Note that the output vectors are not cleared, so you had better make sure you don't leak memory.
*/
// Allocate two density grids
DensityGrid dg1(N,L), dg2(N,L);
delta = dg1.Allocate();
W = dg2.Allocate();
// First the particles
pp.TrimMask(mask);
pp.SlabDecompose(dg1);
dg2.CIC(delta,pp, false);
PetscPrintf(PETSC_COMM_WORLD, "Particles assigned to the grid.....\n");
// Now do the randoms
double nrand=0.0, nexp, nthresh;
nexp = pow((double) nover, 3);
nthresh = nexp*thresh;
{
double tmp = 1./nover, dx, dy, dz;
VecSet(W, 0.0); // Manually set to zero
Vec W1; VecDuplicate(W, &W1);
for (int ix = 0; ix < nover; ++ix) {
PetscPrintf(PETSC_COMM_WORLD,"Starting random loop %i of %i....\n",ix,nover-1);
dx = ix*tmp + tmp/2.0;
for (int iy = 0; iy < nover; ++iy) {
dy = iy*tmp + tmp/2.0;
for (int iz = 0; iz < nover; ++iz) {
dz = iz*tmp + tmp/2.0;
Particle rr; // Definition here is important, since it ensures destruction, before recreating
rr.GridInitialize(N, dx, dy, dz, L);
rr.TrimMask(mask); nrand += rr.npart;
// rr.SlabDecompose(dg2); We don't need to slab decompose because nproc divides N by construction
dg2.CIC(W1, rr, false); // No overdensity
VecAXPY(W, 1.0, W1);
}
}
}
}
PetscPrintf(PETSC_COMM_WORLD, "%f grid particles accumulated....\n", nrand);
PetscPrintf(PETSC_COMM_WORLD, "Expected number of particles=%f, threshold=%f\n", nexp, nthresh);
// Smooth if desired
if (smooth > 1.e-5) {
dg1.GaussSmooth(delta, smooth);
dg2.GaussSmooth(W, smooth);
}
// Now build up delta
int lo, hi;
nrand /= pp.npart;
dg1.slab(lo, hi);
dg1.Pull(delta); dg2.Pull(W);
for (int ix=lo; ix < hi; ++ix)
for (int iy=0; iy < N; ++iy)
for (int iz=0; iz < N; ++iz) {
if (dg2(ix, iy,iz) > nthresh) {
dg1(ix, iy,iz) = (dg1(ix,iy,iz)/dg2(ix, iy, iz)) * nrand - 1.0;
dg2(ix, iy,iz) = 1.0;
} else {
dg1(ix, iy, iz) = 0.0;
dg2(ix, iy, iz) = 0.0;
}
}
dg1.Push(delta, INSERT_VALUES, false);
dg2.Push(W, INSERT_VALUES, false);
VecSum(W, &nrand);
PetscPrintf(PETSC_COMM_WORLD,"Nonzero grid points in W : %f\n",nrand);
}
void _Stokes_SLE_UzawaSolver_Solve( void* solver, void* stokesSLE ) {
Stokes_SLE_UzawaSolver* self = (Stokes_SLE_UzawaSolver*)solver;
Stokes_SLE* sle = (Stokes_SLE*)stokesSLE;
/* Create shortcuts to stuff needed on sle */
Mat K_Mat = sle->kStiffMat->matrix;
Mat G_Mat = sle->gStiffMat->matrix;
Mat D_Mat = NULL;
Mat M_Mat = NULL;
Vec uVec = sle->uSolnVec->vector;
Vec qVec = sle->pSolnVec->vector;
Vec fVec = sle->fForceVec->vector;
Vec hVec = sle->hForceVec->vector;
/* Create shortcuts to solver related stuff */
Vec qTempVec = self->pTempVec;
Vec rVec = self->rVec;
Vec sVec = self->sVec;
Vec fTempVec = self->fTempVec;
Vec vStarVec = self->vStarVec;
KSP velSolver = self->velSolver; /* Inner velocity solver */
KSP pcSolver = self->pcSolver; /* Preconditioner */
Iteration_Index maxIterations = self->maxUzawaIterations;
Iteration_Index minIterations = self->minUzawaIterations;
Iteration_Index iteration_I = 0;
Iteration_Index outputInterval = 1;
double zdotr_current = 0.0;
double zdotr_previous = 1.0;
double sdotGTrans_v;
double alpha, beta;
double absResidual;
double relResidual;
double* chosenResidual; /* We can opt to use either the absolute or relative residual in termination condition */
double uzawaRhsScale;
double divU;
double weightedResidual;
double weightedVelocityScale;
double momentumEquationResidual;
Iteration_Index innerLoopIterations;
Stream* errorStream = Journal_Register( Error_Type, (Name)Stokes_SLE_UzawaSolver_Type );
PetscInt fVecSize, qTempVecSize, uVecSize, qVecSize;
PetscScalar fVecNorm, qTempVecNorm, uVecNorm, rVecNorm, fTempVecNorm, uVecNormInf, qVecNorm, qVecNormInf;
double qGlobalProblemScale;
double qReciprocalGlobalProblemScale;
int init_info_stream_rank;
PetscScalar p_sum;
/* Bool nullsp_present; */
Bool uzawa_summary;
double time,t0,rnorm0;
PetscTruth flg;
double ksptime;
VecGetSize( qTempVec, &qTempVecSize );
qGlobalProblemScale = sqrt( (double) qTempVecSize );
qReciprocalGlobalProblemScale = 1.0 / qGlobalProblemScale;
init_info_stream_rank = Stream_GetPrintingRank( self->info );
Stream_SetPrintingRank( self->info, 0 );
/* DEFINITIONS:
See accompanying documentation
u - the displacement / velocity solution (to which constraints are applied)
q - the pressure-like variable which constrains the divergence displacement / velocity (= pressure for incompressible)
F - standard FE force vector
Fhat - Uzawa RHS = K^{-1} G F - h
K - standard FE stiffness matrix
Khat - Uzawa transformed stiffness matrix = G^T K^{-1} G
G matrix - discrete gradient operator
D matrix - discrete divergence operator = G^T for this particular algorithm
C matrix - Mass matrix (M) for compressibility
LM & DAM
*/
/* CHOICE OF RESIDUAL:
we may opt to converge on the absolute value (self->useAbsoluteTolerance == True ... default)
or the relative value of the residual (self->useAbsoluteTolerance == False)
(another possibility would be always to improve the residual by a given tolerance)
The Moresi & Solomatov (Phys Fluids, 1995) approach is to use the relative tolerance
*/
VecNorm( fVec, NORM_2, &fVecNorm );
VecGetSize( fVec, &fVecSize );
if ( fVecNorm / sqrt( (double)fVecSize ) <= 1e-99 ) {
Journal_Printf( errorStream,
"Error in func %s: The momentum force vector \"%s\" is zero. "
"The force vector should be non-zero either because of your chosen boundary "
"conditions, or because of the element force vector assembly. You have %d "
"element force vectors attached.\n",
__func__, sle->fForceVec->name, sle->fForceVec->assembleForceVector->hooks->count );
if ( sle->fForceVec->assembleForceVector->hooks->count > 0 ) {
Journal_Printf( errorStream, "You used the following force vector assembly terms:\n" );
EntryPoint_PrintConcise( sle->fForceVec->assembleForceVector, errorStream );
/* TODO : need to print the elementForceVector assembly, not the global guy!! */
}
Journal_Printf( errorStream,
"Please check values for building the force vector.\n" );
Journal_Firewall( 0, errorStream, "Exiting.\n" );
}
Journal_DPrintf( self->debug, "In %s:\n", __func__ );
Journal_RPrintfL( self->debug, 2, "Conjugate Gradient Uzawa solver with:\n");
Stream_IndentBranch( StgFEM_Debug );
Journal_RPrintfL( self->debug, 2, "Compressibility %s\n", (sle->cStiffMat)? "on" : "off");
Journal_RPrintfL( self->debug, 2, "Preconditioning %s\n", (pcSolver)? "on" : "off" );
if ( sle->cStiffMat ) {
Journal_DPrintfL( self->debug, 2, "(compressibility active)\n" );
M_Mat = sle->cStiffMat->matrix;
}
else {
Journal_DPrintfL( self->debug, 2, "(compressibility inactive)\n" );
}
if ( sle->dStiffMat ) {
Journal_DPrintfL( self->debug, 2, "(asymmetric geometry: handling D Matrix [incorrectly - will be ignored])\n" );
D_Mat = sle->dStiffMat->matrix;
}
else {
Journal_DPrintfL( self->debug, 2, "(No D -> symmetric geometry: D = Gt)\n" );
}
#if DEBUG
if ( Stream_IsPrintableLevel( self->debug, 3 ) ) {
Journal_DPrintf( self->debug, "Matrices and Vectors to solve are:\n" );
Journal_DPrintf( self->debug, "K Matrix:\n" );
/* No nice way of viewing Matrices, so commented out as incompatible with
* new 3D decomp at present --Kathleen Humble 30-04-07
* Matrix_View( sle->kStiffMat->matrix, self->debug ); */
Journal_DPrintf( self->debug, "G Matrix:\n" );
if ( D_Mat ) {
Journal_DPrintf( self->debug, "D Matrix:\n" );
}
if ( M_Mat ) {
Journal_DPrintf( self->debug, "M Matrix:\n" );
}
Journal_DPrintf( self->debug, "Z (preconditioner) Matrix:\n" );
Journal_DPrintf( self->debug, "f Vector:\n" );
_SLE_VectorView( fVec, self->debug );
Journal_DPrintf( self->debug, "h Vector:\n" );
_SLE_VectorView( hVec, self->debug );
}
#endif
/* STEP 1: Estimate the magnitude of the RHS for the transformed problem
we compute (usually to lower accuracy than elsewhere) the RHS (Fhat - h)
and store the result in qTempVec.
LM & DAM
*/
Journal_DPrintfL( self->debug, 2, "Building Fhat - h.\n" );
PetscOptionsHasName(PETSC_NULL,"-uzawa_printksptimes",&flg);
KSPSetTolerances( velSolver, self->tolerance, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT );
if (flg) {
ksptime = MPI_Wtime();
}
KSPSolve( velSolver, fVec, vStarVec );
if (flg) {
ksptime = MPI_Wtime() - ksptime;
PetscPrintf( PETSC_COMM_WORLD, "KSP on velSolver took %lf seconds in Building Fhat step\n", ksptime);
}
KSPGetIterationNumber( velSolver, &innerLoopIterations );
Journal_DPrintfL( self->debug, 2, "Fhat inner solution: Number of iterations: %d\n", innerLoopIterations );
if ( D_Mat ) {
MatMult( D_Mat, vStarVec, qTempVec );
}
else {
MatMultTranspose( G_Mat, vStarVec, qTempVec );
}
VecAXPY( qTempVec, -1.0, hVec );
/* WARNING:
If D != G^T then the resulting \hat{K} is not likely to be symmetric, positive definite as
required by this implementation of the Uzawa iteration. This next piece of code
is VERY unlikely to work properly so it's in the sin bin for the time being - LM.
if ( D_Mat ) {
MatrixMultiply( D_Mat, vStarVec, qTempVec );
}
else {
MatrixTransposeMultiply( G_Mat, vStarVec, qTempVec );
}
LM & DAM
*/
/* STEP 2: The problem scaling - optionally normalize the uzawa residual by the magnitude of the RHS (use a relative tolerance)
For the inner velocity solver, Citcom uses a relative tolerance equal to that used for the Uzawa iteration as a whole
LM & DAM
*/
if (self->useAbsoluteTolerance) {
chosenResidual = &absResidual;
Journal_PrintfL( self->info, 2, "Absolute residual < %g for Uzawa stopping condition\n", self->tolerance);
/* We should calculate the effective relative tolerance and insert that here !! */
KSPSetTolerances( velSolver, 0.1 * self->tolerance, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT );
}
else { /* The CITCOM compatible choice */
chosenResidual = &relResidual;
Journal_PrintfL( self->info, 2, "Relative residual < %g for Uzawa stopping condition\n", self->tolerance);
KSPSetTolerances( velSolver, 0.1 * self->tolerance, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT );
}
Journal_DPrintfL( self->debug, 2, "Determining scaling factor for residual:\n" );
VecNorm( qTempVec, NORM_2, &qTempVecNorm );
uzawaRhsScale = ((double)qTempVecNorm) * qReciprocalGlobalProblemScale;
Journal_DPrintfL( self->debug, 2, "uzawaRhsScale = %f\n", uzawaRhsScale );
Journal_Firewall( isGoodNumber( uzawaRhsScale ), errorStream,
"Error in func '%s' for %s '%s' - uzawaRhsScale has illegal value '%g'.\n", __func__, self->type, self->name, uzawaRhsScale );
/* STEP 3: Calculate initial residual for transformed equation (\hat{F} - h - \hat{K} q_0)
Compute the solution to K u_0 = F - G q_0 (u_0 unknown)
Then G^T u* = \hat{F} - \hat{K} q_0
u_0 is also the initial velocity solution to which the constraint is applied by the subsequent iteration
LM & DAM
*/
Journal_DPrintfL( self->debug, 2, "Solving for transformed Uzawa RHS.\n" );
VecCopy( fVec, fTempVec );
VecScale( fTempVec, -1.0 );
MatMultAdd( G_Mat, qVec, fTempVec, fTempVec );
VecScale( fTempVec, -1.0 );
KSPSolve( velSolver, fTempVec, uVec );
/* Handling for NON-SYMMETRIC: relegated to sin bin (see comment above) LM & DAM */
if ( D_Mat ) {
MatMult( D_Mat, uVec, rVec );
}
else {
MatMultTranspose( G_Mat, uVec, rVec );
}
VecNorm( rVec, NORM_2, &rnorm0 );
VecNorm( uVec, NORM_2, &uVecNorm );
divU = rnorm0 / uVecNorm;
Journal_PrintfL( self->info, 2, "Initial l2Norm( Div u ) / l2Norm( u ) = %f \n", divU);
Journal_Firewall( isGoodNumber( divU ), errorStream,
"Error in func '%s' for %s '%s' - l2Norm( Div u ) has illegal value '%g'.\n",
__func__, self->type, self->name, divU );
Journal_DPrintfL( self->debug, 2, "Adding compressibility and prescribed divergence terms.\n" );
if ( M_Mat ) {
MatMultAdd( M_Mat, qVec, rVec, rVec );
}
VecAXPY( rVec, -1.0, hVec );
/* Check for existence of constant null space */
#if 0
nullsp_present = _check_if_constant_nullsp_present( self, K_Mat,G_Mat,M_Mat, fTempVec,vStarVec,qTempVec,sVec, velSolver );
#endif
/* STEP 4: Preconditioned conjugate gradient iteration loop */
Journal_DPrintfL( self->debug, 1, "Beginning main Uzawa conjugate gradient loop:\n" );
iteration_I = 0;
/* outer_it, residual, time */
uzawa_summary = self->monitor;
time = 0.0;
t0 = MPI_Wtime();
// Journal_PrintfL( self->info, 1, " |r0| = %.8e \n", rnorm0 );
do{
/* reset initial time and end time for inner its back to 0 - probs don't need to do this but just in case */
self->outeritsinitialtime = 0;
self->outeritsendtime = 0;
//BEGINNING OF OUTER ITERATIONS!!!!!
/*get wall time for start of outer loop*/
self->outeritsinitialtime = MPI_Wtime();
Journal_DPrintfL( self->debug, 2, "Beginning solve '%u'.\n", iteration_I );
Stream_IndentBranch( StgFEM_Debug );
/* STEP 4.1: Preconditioner
Solve:
Q_\hat{K} z_1 = r_1
Q_\hat{K} is an approximation to \hat{K} which is simple / trivial / quick to invert
LM & DAM
*/
if ( pcSolver ) {
PetscOptionsHasName(PETSC_NULL,"-uzawa_printksptimes",&flg);
if (flg) {
ksptime = MPI_Wtime();
}
KSPSolve( pcSolver, rVec, qTempVec );
if (flg) {
ksptime = MPI_Wtime() - ksptime;
PetscPrintf( PETSC_COMM_WORLD, "KSP on pcSolver took %lf seconds\n", ksptime);
}
}
else {
VecCopy( rVec, qTempVec );
}
/* Remove the constant null space, but only if NOT compressible */
#if 0
if( nullsp_present == True ) {
_remove_constant_nullsp( qTempVec );
}
#endif
/* STEP 4.2: Calculate s_I, the pressure search direction
z_{I-1} . r_{I-1}
\beta = (z_{I-1} . r_{I-1}) / (z_{I-2} . r_{I-2})
\beta = 0 for the first iteration
s_I = z_(I-1) + \beta * s_(I-1)
LM & DAM
*/
VecDot( qTempVec, rVec, &zdotr_current );
VecNorm( qTempVec, NORM_2, &qTempVecNorm );
VecNorm( rVec, NORM_2, &rVecNorm );
Journal_DPrintfL( self->debug, 2, "l2Norm (qTempVec) %g; (rVec) %g \n",
qTempVecNorm * qReciprocalGlobalProblemScale,
rVecNorm * qReciprocalGlobalProblemScale );
if ( iteration_I == 0 ) {
VecCopy( qTempVec, sVec );
}
else {
beta = zdotr_current/zdotr_previous;
VecAYPX( sVec, beta, qTempVec );
}
/* STEP 4.3: Velocity search direction corresponding to s_I is found by solving
K u* = G s_I
LM & DAM
*/
MatMult( G_Mat, sVec, fTempVec );
Journal_DPrintfL( self->debug, 2, "Uzawa inner iteration step\n");
//START OF INNER ITERATIONS!!!!
PetscOptionsHasName(PETSC_NULL,"-uzawa_printksptimes",&flg);
/*get initial wall time for inner loop*/
self->inneritsinitialtime = MPI_Wtime();
if (flg) {
ksptime = MPI_Wtime();
}
KSPSolve( velSolver, fTempVec, vStarVec );
if (flg) {
ksptime = MPI_Wtime() - ksptime;
PetscPrintf( PETSC_COMM_WORLD, "KSP on velSolver took %lf seconds in Uzawa inner iteration step\n", ksptime);
}
/*get end wall time for inner loop*/
self->inneritsendtime = MPI_Wtime();
/* add time to total time inner its: */
self->totalinneritstime = self->totalinneritstime + (-self->inneritsinitialtime + self->inneritsendtime);
/* reset initial time and end time for inner its back to 0 - probs don't need to do this but just in case */
self->inneritsinitialtime = 0;
self->inneritsendtime = 0;
KSPGetIterationNumber( velSolver, &innerLoopIterations );
/* add the inner loop iterations to the total inner iterations */
self->totalnuminnerits = self->totalnuminnerits + innerLoopIterations;
Journal_DPrintfL( self->debug, 2, "Completed Uzawa inner iteration in '%u' iterations \n", innerLoopIterations );
/* STEP 4.4: Calculate the step size ( \alpha = z_{I-1} . r_{I-1} / (s_I . \hat{K} s_I) )
\hat{K} s_I = G^T u* - M s_I (u* from step 4.3)
LM & DAM
*/
if ( D_Mat ) {
MatMult( D_Mat, vStarVec, qTempVec );
}
else {
MatMultTranspose( G_Mat, vStarVec, qTempVec );
}
/* Handling for NON-SYMMETRIC: relegated to sin bin (see comment above)
if ( D_Mat ) {
MatrixMultiply( D_Mat, vStarVec, qTempVec );
}
else {
MatrixTransposeMultiply( G_Mat, vStarVec, qTempVec );
}
LM & DAM
*/
if ( M_Mat ) {
Journal_DPrintfL( self->debug, 2, "Correcting for Compressibility\n" );
VecScale( qTempVec, -1.0 );
MatMultAdd( M_Mat, sVec, qTempVec, qTempVec );
VecScale( qTempVec, -1.0 );
}
VecDot( sVec, qTempVec, &sdotGTrans_v );
alpha = zdotr_current/sdotGTrans_v;
/* STEP 4.5: Update pressure, velocity and value of residual
by \alpha times corresponding search direction
LM & DAM
*/
Journal_DPrintfL( self->debug, 2, "zdotr_current = %g \n", zdotr_current);
Journal_DPrintfL( self->debug, 2, "sdotGTrans_v = %g \n", sdotGTrans_v);
Journal_DPrintfL( self->debug, 2, "alpha = %g \n", alpha);
Journal_Firewall(
isGoodNumber( zdotr_current ) && isGoodNumber( sdotGTrans_v ) && isGoodNumber( alpha ),
errorStream,
"Error in func '%s' for %s '%s' - zdotr_current, sdotGTrans_v or alpha has an illegal value: '%g','%g' or '%g'\n",
__func__, self->type, self->name, zdotr_current, sdotGTrans_v, alpha );
VecAXPY( qVec, alpha, sVec );
VecAXPY( uVec, -alpha, vStarVec );
VecAXPY( rVec, -alpha, qTempVec );
/* STEP 4.6: store the value of z_{I-1} . r_{I-1} for the next iteration
LM & DAM
*/
zdotr_previous = zdotr_current;
VecNorm( rVec, NORM_2, &rVecNorm );
absResidual = rVecNorm * qReciprocalGlobalProblemScale;
relResidual = absResidual / uzawaRhsScale;
Stream_UnIndentBranch( StgFEM_Debug );
if( iteration_I % outputInterval == 0 ) {
Journal_PrintfL( self->info, 2, "\tLoop = %u, absResidual = %.8e, relResidual = %.8e\n",
iteration_I, absResidual, relResidual );
}
Journal_Firewall( isGoodNumber( absResidual ), errorStream,
"Error in func '%s' for %s '%s' - absResidual has an illegal value: '%g'\n",
__func__, self->type, self->name, absResidual );
Journal_Firewall( iteration_I < maxIterations,
errorStream, "In func %s: Reached maximum number of iterations %u without converging; absResidual = %.5g, relResidual = %.5g \n",
__func__, iteration_I, absResidual, relResidual );
/* TODO: test for small change in 10 iterations and if so restart? */
time = MPI_Wtime()-t0;
if (uzawa_summary) {
Journal_PrintfL( self->info, 1, " %1.4d uzawa residual norm %12.13e, cpu time %5.5e\n", iteration_I+1,*chosenResidual,time );
}
iteration_I++;
//END OF OUTER ITERATION LOOP!!!
/*get wall time for end of outer loop*/
self->outeritsendtime = MPI_Wtime();
/* add time to total time inner its: */
self->totalouteritstime = self->totalouteritstime + (-self->outeritsinitialtime + self->outeritsendtime);
/* reset initial time and end time for inner its back to 0 - probs don't need to do this but just in case */
self->outeritsinitialtime = 0;
self->outeritsendtime = 0;
/* add the outer loop iterations to the total outer iterations */
self->totalnumouterits++;
} while ( (*chosenResidual > self->tolerance) || (iteration_I<minIterations) );
// } while ( *chosenResidual > self->tolerance );
Journal_DPrintfL( self->debug, 1, "Pressure solution converged. Exiting uzawa \n ");
/* STEP 5: Check all the relevant residuals and report back */
if (Stream_IsEnable( self->info ) ) {
/* This information should be in an info stream */
Journal_PrintfL( self->info, 1, "Summary:\n");
Journal_PrintfL( self->info, 1, " Uzawa its. = %04d , Uzawa residual = %12.13e\n", iteration_I, relResidual );
MatMultTranspose( G_Mat, uVec, rVec );
VecNorm( rVec, NORM_2, &rVecNorm );
VecNorm( uVec, NORM_2, &uVecNorm );
divU = rVecNorm / uVecNorm;
Journal_PrintfL( self->info, 1, " |G^T u|/|u| = %.8e\n", divU);
/* Residual for the momentum equation
Compute r = || F - Ku - Gp || / || F ||
*/
MatMult( G_Mat, qVec, vStarVec );
MatMultAdd( K_Mat, uVec, vStarVec, fTempVec );
VecAYPX( fTempVec, -1.0, fVec );
VecNorm( fTempVec, NORM_2, &fTempVecNorm );
VecNorm( fVec, NORM_2, &fVecNorm );
momentumEquationResidual = fTempVecNorm / fVecNorm;
Journal_PrintfL( self->info, 1, " |f - K u - G p|/|f| = %.8e\n", momentumEquationResidual );
Journal_Firewall( isGoodNumber( momentumEquationResidual ), errorStream,
"Bad residual for the momentum equation (|| F - Ku - Gp || / || F || = %g):\n"
"\tCheck to see if forcing term is zero or nan - \n\t|| F - Ku - Gp || = %g \n\t|| F || = %g.\n",
momentumEquationResidual,
fTempVecNorm, fVecNorm );
/* "Preconditioned" residual for the momentum equation
r_{w} = || Q_{K}(r) || / || Q_{K}(F)
fTempVec contains the residual but is overwritten once used
vStarVec is used to hold the diagonal preconditioner Q_{K}
*/
MatGetDiagonal( K_Mat, vStarVec );
VecReciprocal( vStarVec );
VecPointwiseMult( vStarVec, fTempVec, fTempVec );
VecNorm( fTempVec, NORM_2, &weightedResidual );
VecPointwiseMult( vStarVec, fVec, fTempVec );
VecNorm( fTempVec, NORM_2, &weightedVelocityScale );
Journal_PrintfL( self->info, 1, " |f - K u - G p|_w/|f|_w = %.8e\n", weightedResidual / weightedVelocityScale );
/* Report back on the solution - velocity and pressure
Note - correction for dof in Vrms ??
*/
VecNorm( uVec, NORM_INFINITY, &uVecNormInf );
VecNorm( uVec, NORM_2, &uVecNorm );
VecGetSize( uVec, &uVecSize );
VecNorm( qVec, NORM_INFINITY, &qVecNormInf );
VecNorm( qVec, NORM_2, &qVecNorm );
VecGetSize( qVec, &qVecSize );
Journal_PrintfL( self->info, 1, " |u|_{\\infty} = %.8e , u_rms = %.8e\n",
uVecNormInf, uVecNorm / sqrt( (double)uVecSize ) );
Journal_PrintfL( self->info, 1, " |p|_{\\infty} = %.8e , p_rms = %.8e\n",
qVecNormInf, qVecNorm / sqrt( (double)qVecSize ) );
{ PetscInt lmin,lmax;
PetscReal min,max;
VecMax( uVec, &lmax, &max );
VecMin( uVec, &lmin, &min );
Journal_PrintfL( self->info, 1, " min/max(u) = %.8e [%d] / %.8e [%d]\n",min,lmin,max,lmax);
VecMax( qVec, &lmax, &max );
VecMin( qVec, &lmin, &min );
Journal_PrintfL( self->info, 1, " min/max(p) = %.8e [%d] / %.8e [%d]\n",min,lmin,max,lmax);
}
VecSum( qVec, &p_sum );
Journal_PrintfL( self->info, 1, " \\sum_i p_i = %.8e \n", p_sum );
} /* journal stream enabled */
#if DEBUG
if ( Stream_IsPrintableLevel( self->debug, 3 ) ) {
Journal_DPrintf( self->debug, "Velocity solution:\n" );
_SLE_VectorView( uVec, self->debug );
Journal_DPrintf( self->debug, "Pressure solution:\n" );
_SLE_VectorView( qVec, self->debug );
}
#endif
Stream_UnIndentBranch( StgFEM_Debug );
Stream_SetPrintingRank( self->info, init_info_stream_rank );
/* Now gather up data for printing out to FrequentOutput file: */
/*!!! if non-linear need to divide by number of nonlinear iterations and we do this in SystemLinearEquations */
if((sle->isNonLinear != True)){
self->avgnuminnerits = self->totalnuminnerits/self->totalnumouterits;
self->avgnumouterits = self->totalnumouterits;
self->avgtimeouterits = (self->totalouteritstime - self->totalinneritstime)/self->totalnumouterits;
self->avgtimeinnerits = self->totalinneritstime/self->totalnuminnerits;
}
}
Delta::Delta(int _N, double _L, int _nover, double thresh, Mask3D& _mask)
// RS 2010/07/06: Comments!
{
// Cache values
N=_N; L=_L; nover=_nover; maskptr=&_mask; rmasksmooth = 0.0;
// Set up the density
// RS: dg2 is some kind of local variable; dg1 is a class member.
// W and _W are contents of a grid of the same size and dimensions
// as dg1 and dg2.
DensityGrid dg2(N, L);
dg1.Init(N, L);
_W = dg1.Allocate();
W = dg1.Allocate(); VecSet(W, 0.0);
// Now do the randoms
// RS: "Doing" the randoms in this context means "oversampling" each grid
// cell into subcells, where the main cell is nover times as large as each
// subcell and hence there are nover^3 subcells to a cell.
nrand=0.0;
double nexp, nthresh;
nexp = pow((double) nover, 3);
// RS: thresh is the *fraction* of the main cell's volume to be included
// inside the mask for it to be counted as not masked, so nthresh is the
// corresponding number of subcells.
nthresh = nexp*thresh;
{
double tmp = 1./nover, dx, dy, dz;
VecSet(_W, 0.0); // Manually set to zero
Vec W1; VecDuplicate(_W, &W1);
// RS: Loop over all subcell *offsets*. The loop over actual grid cells
// is implicit in the calls to methods like GridInitialize below.
for (int ix = 0; ix < nover; ++ix) {
PetscPrintf(PETSC_COMM_WORLD,"Starting random loop %i of %i....\n",ix,nover-1);
dx = ix*tmp + tmp/2.0;
for (int iy = 0; iy < nover; ++iy) {
dy = iy*tmp + tmp/2.0;
for (int iz = 0; iz < nover; ++iz) {
dz = iz*tmp + tmp/2.0;
Particle rr; // Definition here is important, since it ensures destruction, before recreating
// RS: Generate a bunch of particles uniformly spaced, and offset
// from the low-(x,y,z) corner of each grid cell by (dx,dy,dz).
rr.GridInitialize(N, dx, dy, dz, L);
// RS: Remove from the list any particles which were not allowed
// by the mask. Add to the total.
rr.TrimMask(*maskptr); nrand += rr.npart;
// rr.SlabDecompose(dg2); We don't need to slab decompose because nproc divides N by construction
// RS: CIC assign remaining particles to temporary grid vector W1.
dg1.CIC(W1, rr, false); // No overdensity
// RS: Now add to _W the CIC contents of W1.
VecAXPY(_W, 1.0, W1);
}
}
}
}
// RS: At this point, nrand equals the total number of oversampled subcells
// found to be allowed by the masked region. _W contains the number of
// particles allowed by an oversampled mask at each grid location.
PetscPrintf(PETSC_COMM_WORLD, "%f grid particles accumulated....\n", nrand);
// Now set the mask
// RS: For each cell, if the contents of _W exceed nthresh, set W = 1.
// Use dg1 and dg2 as wrappers to index the copies of _W and W.
int lo, hi;
dg1.slab(lo, hi);
dg1.Pull(_W); dg2.Pull(W);
for (int ix=lo; ix < hi; ++ix)
for (int iy=0; iy < N; ++iy)
for (int iz=0; iz < N; ++iz)
if (dg1(ix, iy,iz) > nthresh) {
dg2(ix, iy,iz) = 1.0;
} else {
dg1(ix, iy, iz) = 0.0; // Good to explicitly set this to zero as well
}
dg1.Push(_W, INSERT_VALUES, false);
dg2.Push(W, INSERT_VALUES, false);
double tmp;
VecSum(W, &tmp);
PetscPrintf(PETSC_COMM_WORLD,"Nonzero grid points in W : %f\n",tmp);
VecSum(_W, &nrand);
PetscPrintf(PETSC_COMM_WORLD,"Sum of background density vector : %f\n",nrand);
}
int main(int argc,char **args)
{
PetscErrorCode ierr;
DomainData dd;
PetscReal norm,maxeig,mineig;
PetscScalar scalar_value;
PetscInt ndofs,its;
Mat A =0,F=0;
KSP KSPwithBDDC =0,KSPwithFETIDP=0;
Vec fetidp_solution_all=0,bddc_solution=0,bddc_rhs=0;
Vec exact_solution =0,fetidp_solution=0,fetidp_rhs=0;
/* Init PETSc */
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
/* Initialize DomainData */
ierr = InitializeDomainData(&dd);CHKERRQ(ierr);
/* Decompose domain */
ierr = DomainDecomposition(&dd);CHKERRQ(ierr);
#if DEBUG
printf("Subdomain data\n");
printf("IPS : %d %d %d\n",dd.ipx,dd.ipy,dd.ipz);
printf("NEG : %d %d %d\n",dd.nex,dd.ney,dd.nez);
printf("NEL : %d %d %d\n",dd.nex_l,dd.ney_l,dd.nez_l);
printf("LDO : %d %d %d\n",dd.xm_l,dd.ym_l,dd.zm_l);
printf("SIZES : %d %d %d\n",dd.xm,dd.ym,dd.zm);
printf("STARTS: %d %d %d\n",dd.startx,dd.starty,dd.startz);
#endif
/* assemble global matrix */
ierr = ComputeMatrix(dd,&A);CHKERRQ(ierr);
/* get work vectors */
ierr = MatCreateVecs(A,&bddc_solution,NULL);CHKERRQ(ierr);
ierr = VecDuplicate(bddc_solution,&bddc_rhs);CHKERRQ(ierr);
ierr = VecDuplicate(bddc_solution,&fetidp_solution_all);CHKERRQ(ierr);
ierr = VecDuplicate(bddc_solution,&exact_solution);CHKERRQ(ierr);
/* create and customize KSP/PC for BDDC */
ierr = ComputeKSPBDDC(dd,A,&KSPwithBDDC);CHKERRQ(ierr);
/* create KSP/PC for FETIDP */
ierr = ComputeKSPFETIDP(dd,KSPwithBDDC,&KSPwithFETIDP);CHKERRQ(ierr);
/* create random exact solution */
ierr = VecSetRandom(exact_solution,NULL);CHKERRQ(ierr);
ierr = VecShift(exact_solution,-0.5);CHKERRQ(ierr);
ierr = VecScale(exact_solution,100.0);CHKERRQ(ierr);
ierr = VecGetSize(exact_solution,&ndofs);CHKERRQ(ierr);
if (dd.pure_neumann) {
ierr = VecSum(exact_solution,&scalar_value);CHKERRQ(ierr);
scalar_value = -scalar_value/(PetscScalar)ndofs;
ierr = VecShift(exact_solution,scalar_value);CHKERRQ(ierr);
}
/* assemble BDDC rhs */
ierr = MatMult(A,exact_solution,bddc_rhs);CHKERRQ(ierr);
/* test ksp with BDDC */
ierr = KSPSolve(KSPwithBDDC,bddc_rhs,bddc_solution);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(KSPwithBDDC,&its);CHKERRQ(ierr);
ierr = KSPComputeExtremeSingularValues(KSPwithBDDC,&maxeig,&mineig);CHKERRQ(ierr);
if (dd.pure_neumann) {
ierr = VecSum(bddc_solution,&scalar_value);CHKERRQ(ierr);
scalar_value = -scalar_value/(PetscScalar)ndofs;
ierr = VecShift(bddc_solution,scalar_value);CHKERRQ(ierr);
}
/* check exact_solution and BDDC solultion */
ierr = VecAXPY(bddc_solution,-1.0,exact_solution);CHKERRQ(ierr);
ierr = VecNorm(bddc_solution,NORM_INFINITY,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"---------------------BDDC stats-------------------------------\n");CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"Number of degrees of freedom : %8D \n",ndofs);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"Number of iterations : %8D \n",its);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"Eigenvalues preconditioned operator : %1.2e %1.2e\n",(double)mineig,(double)maxeig);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"Error betweeen exact and computed solution : %1.2e\n",(double)norm);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"--------------------------------------------------------------\n");CHKERRQ(ierr);
/* assemble fetidp rhs on the space of Lagrange multipliers */
ierr = KSPGetOperators(KSPwithFETIDP,&F,NULL);CHKERRQ(ierr);
ierr = MatCreateVecs(F,&fetidp_solution,&fetidp_rhs);CHKERRQ(ierr);
ierr = PCBDDCMatFETIDPGetRHS(F,bddc_rhs,fetidp_rhs);CHKERRQ(ierr);
ierr = VecSet(fetidp_solution,0.0);CHKERRQ(ierr);
/* test ksp with FETIDP */
ierr = KSPSolve(KSPwithFETIDP,fetidp_rhs,fetidp_solution);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(KSPwithFETIDP,&its);CHKERRQ(ierr);
ierr = KSPComputeExtremeSingularValues(KSPwithFETIDP,&maxeig,&mineig);CHKERRQ(ierr);
/* assemble fetidp solution on physical domain */
ierr = PCBDDCMatFETIDPGetSolution(F,fetidp_solution,fetidp_solution_all);CHKERRQ(ierr);
/* check FETIDP sol */
if (dd.pure_neumann) {
ierr = VecSum(fetidp_solution_all,&scalar_value);CHKERRQ(ierr);
scalar_value = -scalar_value/(PetscScalar)ndofs;
ierr = VecShift(fetidp_solution_all,scalar_value);CHKERRQ(ierr);
}
ierr = VecAXPY(fetidp_solution_all,-1.0,exact_solution);CHKERRQ(ierr);
ierr = VecNorm(fetidp_solution_all,NORM_INFINITY,&norm);CHKERRQ(ierr);
ierr = VecGetSize(fetidp_solution,&ndofs);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"------------------FETI-DP stats-------------------------------\n");CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"Number of degrees of freedom : %8D \n",ndofs);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"Number of iterations : %8D \n",its);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"Eigenvalues preconditioned operator : %1.2e %1.2e\n",(double)mineig,(double)maxeig);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"Error betweeen exact and computed solution : %1.2e\n",(double)norm);CHKERRQ(ierr);
ierr = PetscPrintf(dd.gcomm,"--------------------------------------------------------------\n");CHKERRQ(ierr);
/* Free workspace */
ierr = VecDestroy(&exact_solution);CHKERRQ(ierr);
ierr = VecDestroy(&bddc_solution);CHKERRQ(ierr);
ierr = VecDestroy(&fetidp_solution);CHKERRQ(ierr);
ierr = VecDestroy(&fetidp_solution_all);CHKERRQ(ierr);
ierr = VecDestroy(&bddc_rhs);CHKERRQ(ierr);
ierr = VecDestroy(&fetidp_rhs);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = KSPDestroy(&KSPwithBDDC);CHKERRQ(ierr);
ierr = KSPDestroy(&KSPwithFETIDP);CHKERRQ(ierr);
/* Quit PETSc */
ierr = PetscFinalize();
return ierr;
}
/*@C
MatNullSpaceCreate - Creates a data structure used to project vectors
out of null spaces.
Collective on MPI_Comm
Input Parameters:
+ comm - the MPI communicator associated with the object
. has_cnst - PETSC_TRUE if the null space contains the constant vector; otherwise PETSC_FALSE
. n - number of vectors (excluding constant vector) in null space
- vecs - the vectors that span the null space (excluding the constant vector);
these vectors must be orthonormal. These vectors are NOT copied, so do not change them
after this call. You should free the array that you pass in and destroy the vectors (this will reduce the reference count
for them by one).
Output Parameter:
. SP - the null space context
Level: advanced
Notes:
See MatNullSpaceSetFunction() as an alternative way of providing the null space information instead of setting vecs.
If has_cnst is PETSC_TRUE you do not need to pass a constant vector in as a fourth argument to this routine, nor do you
need to pass in a function that eliminates the constant function into MatNullSpaceSetFunction().
Users manual sections:
. sec_singular
.keywords: PC, null space, create
.seealso: MatNullSpaceDestroy(), MatNullSpaceRemove(), MatSetNullSpace(), MatNullSpace, MatNullSpaceSetFunction()
@*/
PetscErrorCode MatNullSpaceCreate(MPI_Comm comm,PetscBool has_cnst,PetscInt n,const Vec vecs[],MatNullSpace *SP)
{
MatNullSpace sp;
PetscErrorCode ierr;
PetscInt i;
PetscFunctionBegin;
if (n < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Number of vectors (given %D) cannot be negative",n);
if (n) PetscValidPointer(vecs,4);
for (i=0; i<n; i++) PetscValidHeaderSpecific(vecs[i],VEC_CLASSID,4);
PetscValidPointer(SP,5);
if (n) {
for (i=0; i<n; i++) {
/* prevent the user from changes values in the vector */
ierr = VecLockPush(vecs[i]);CHKERRQ(ierr);
}
}
#if defined(PETSC_USE_DEBUG)
if (n) {
PetscScalar *dots;
for (i=0; i<n; i++) {
PetscReal norm;
ierr = VecNorm(vecs[i],NORM_2,&norm);CHKERRQ(ierr);
if (PetscAbsReal(norm - 1) > PETSC_SQRT_MACHINE_EPSILON) SETERRQ2(PetscObjectComm((PetscObject)vecs[i]),PETSC_ERR_ARG_WRONG,"Vector %D must have 2-norm of 1.0, it is %g",i,(double)norm);
}
if (has_cnst) {
for (i=0; i<n; i++) {
PetscScalar sum;
ierr = VecSum(vecs[i],&sum);CHKERRQ(ierr);
if (PetscAbsScalar(sum) > PETSC_SQRT_MACHINE_EPSILON) SETERRQ2(PetscObjectComm((PetscObject)vecs[i]),PETSC_ERR_ARG_WRONG,"Vector %D must be orthogonal to constant vector, inner product is %g",i,(double)PetscAbsScalar(sum));
}
}
ierr = PetscMalloc1(n-1,&dots);CHKERRQ(ierr);
for (i=0; i<n-1; i++) {
PetscInt j;
ierr = VecMDot(vecs[i],n-i-1,vecs+i+1,dots);CHKERRQ(ierr);
for (j=0;j<n-i-1;j++) {
if (PetscAbsScalar(dots[j]) > PETSC_SQRT_MACHINE_EPSILON) SETERRQ3(PetscObjectComm((PetscObject)vecs[i]),PETSC_ERR_ARG_WRONG,"Vector %D must be orthogonal to vector %D, inner product is %g",i,i+j+1,(double)PetscAbsScalar(dots[j]));
}
}
PetscFree(dots);CHKERRQ(ierr);
}
#endif
*SP = NULL;
ierr = MatInitializePackage();CHKERRQ(ierr);
ierr = PetscHeaderCreate(sp,MAT_NULLSPACE_CLASSID,"MatNullSpace","Null space","Mat",comm,MatNullSpaceDestroy,MatNullSpaceView);CHKERRQ(ierr);
sp->has_cnst = has_cnst;
sp->n = n;
sp->vecs = 0;
sp->alpha = 0;
sp->remove = 0;
sp->rmctx = 0;
if (n) {
ierr = PetscMalloc1(n,&sp->vecs);CHKERRQ(ierr);
ierr = PetscMalloc1(n,&sp->alpha);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)sp,n*(sizeof(Vec)+sizeof(PetscScalar)));CHKERRQ(ierr);
for (i=0; i<n; i++) {
ierr = PetscObjectReference((PetscObject)vecs[i]);CHKERRQ(ierr);
sp->vecs[i] = vecs[i];
}
}
*SP = sp;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInt rstart,rend,i,k,N,numPoints=1000000;
PetscScalar dummy,result=0,h=1.0/numPoints,*xarray;
Vec x,xend;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/*
Create a parallel vector.
Here we set up our x vector which will be given values below.
The xend vector is a dummy vector to find the value of the
elements at the endpoints for use in the trapezoid rule.
*/
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,numPoints);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecGetSize(x,&N);CHKERRQ(ierr);
ierr = VecSet(x,result);CHKERRQ(ierr);
ierr = VecDuplicate(x,&xend);CHKERRQ(ierr);
result = 0.5;
if (!rank) {
i = 0;
ierr = VecSetValues(xend,1,&i,&result,INSERT_VALUES);CHKERRQ(ierr);
}
if (rank == size-1) {
i = N-1;
ierr = VecSetValues(xend,1,&i,&result,INSERT_VALUES);CHKERRQ(ierr);
}
/*
Assemble vector, using the 2-step process:
VecAssemblyBegin(), VecAssemblyEnd()
Computations can be done while messages are in transition
by placing code between these two statements.
*/
ierr = VecAssemblyBegin(xend);CHKERRQ(ierr);
ierr = VecAssemblyEnd(xend);CHKERRQ(ierr);
/*
Set the x vector elements.
i*h will return 0 for i=0 and 1 for i=N-1.
The function evaluated (2x/(1+x^2)) is defined above.
Each evaluation is put into the local array of the vector without message passing.
*/
ierr = VecGetOwnershipRange(x,&rstart,&rend);CHKERRQ(ierr);
ierr = VecGetArray(x,&xarray);CHKERRQ(ierr);
k = 0;
for (i=rstart; i<rend; i++) {
xarray[k] = i*h;
xarray[k] = func(xarray[k]);
k++;
}
ierr = VecRestoreArray(x,&xarray);CHKERRQ(ierr);
/*
Evaluates the integral. First the sum of all the points is taken.
That result is multiplied by the step size for the trapezoid rule.
Then half the value at each endpoint is subtracted,
this is part of the composite trapezoid rule.
*/
ierr = VecSum(x,&result);CHKERRQ(ierr);
result = result*h;
ierr = VecDot(x,xend,&dummy);CHKERRQ(ierr);
result = result-h*dummy;
/*
Return the value of the integral.
*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"ln(2) is %g\n",(double)result);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&xend);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode port_lsd_bfbt(void)
{
Mat A;
Vec x,b;
KSP ksp_A;
PC pc_A;
IS isu,isp;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = LoadTestMatrices(&A,&x,&b,&isu,&isp);CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp_A);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(ksp_A,"fc_");CHKERRQ(ierr);
ierr = KSPSetOperators(ksp_A,A,A);CHKERRQ(ierr);
ierr = KSPGetPC(ksp_A,&pc_A);CHKERRQ(ierr);
ierr = PCSetType(pc_A,PCFIELDSPLIT);CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc_A,2);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc_A,"velocity",isu);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc_A,"pressure",isp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp_A);CHKERRQ(ierr);
ierr = KSPSolve(ksp_A,b,x);CHKERRQ(ierr);
/* Pull u,p out of x */
{
PetscInt loc;
PetscReal max,norm;
PetscScalar sum;
Vec uvec,pvec;
VecScatter uscat,pscat;
Mat A11,A22;
/* grab matrices and create the compatable u,p vectors */
ierr = MatGetSubMatrix(A,isu,isu,MAT_INITIAL_MATRIX,&A11);CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isp,isp,MAT_INITIAL_MATRIX,&A22);CHKERRQ(ierr);
ierr = MatCreateVecs(A11,&uvec,NULL);CHKERRQ(ierr);
ierr = MatCreateVecs(A22,&pvec,NULL);CHKERRQ(ierr);
/* perform the scatter from x -> (u,p) */
ierr = VecScatterCreate(x,isu,uvec,NULL,&uscat);CHKERRQ(ierr);
ierr = VecScatterBegin(uscat,x,uvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(uscat,x,uvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterCreate(x,isp,pvec,NULL,&pscat);CHKERRQ(ierr);
ierr = VecScatterBegin(pscat,x,pvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(pscat,x,pvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- vector vector values --\n");
ierr = VecMin(uvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(u) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(uvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(u) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(uvec,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(u) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(uvec,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(u) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- pressure vector values --\n");
ierr = VecMin(pvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(pvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(pvec,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(p) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(pvec,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(p) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- Full vector values --\n");
ierr = VecMin(x,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(u,p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(x,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(u,p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(u,p) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(x,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(u,p) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
ierr = VecScatterDestroy(&uscat);CHKERRQ(ierr);
ierr = VecScatterDestroy(&pscat);CHKERRQ(ierr);
ierr = VecDestroy(&uvec);CHKERRQ(ierr);
ierr = VecDestroy(&pvec);CHKERRQ(ierr);
ierr = MatDestroy(&A11);CHKERRQ(ierr);
ierr = MatDestroy(&A22);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp_A);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = ISDestroy(&isu);CHKERRQ(ierr);
ierr = ISDestroy(&isp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Vec x,b,u; /* approx solution, RHS, exact solution */
Mat A; /* linear system matrix */
KSP ksp; /* KSP context */
PetscErrorCode ierr;
PetscInt i,n = 10,col[3],its,i1,i2;
PetscScalar none = -1.0,value[3],avalue;
PetscReal norm;
PC pc;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
/* Create vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);CHKERRQ(ierr);
/* create a solution that is orthogonal to the constants */
ierr = VecGetOwnershipRange(u,&i1,&i2);CHKERRQ(ierr);
for (i=i1; i<i2; i++) {
avalue = i;
VecSetValues(u,1,&i,&avalue,INSERT_VALUES);
}
ierr = VecAssemblyBegin(u);CHKERRQ(ierr);
ierr = VecAssemblyEnd(u);CHKERRQ(ierr);
ierr = VecSum(u,&avalue);CHKERRQ(ierr);
avalue = -avalue/(PetscReal)n;
ierr = VecShift(u,avalue);CHKERRQ(ierr);
/* Create and assemble matrix */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0] = n - 2; col[1] = n - 1; value[1] = 1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0; col[0] = 0; col[1] = 1; value[0] = 1.0; value[1] = -1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatMult(A,u,b);CHKERRQ(ierr);
/* Create KSP context; set operators and options; solve linear system */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
/* Insure that preconditioner has same null space as matrix */
/* currently does not do anything */
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* ierr = KSPView(ksp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Check error */
ierr = VecAXPY(x,none,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %g, Iterations %D\n",(double)norm,its);CHKERRQ(ierr);
/* Free work space */
ierr = VecDestroy(&x);CHKERRQ(ierr);ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode BSSCR_DRIVER_flex( KSP ksp, Mat stokes_A, Vec stokes_x, Vec stokes_b, Mat approxS, KSP ksp_K,
MatStokesBlockScaling BA, PetscTruth sym, KSP_BSSCR * bsscrp_self )
{
char name[PETSC_MAX_PATH_LEN];
char ubefore[100];
char uafter[100];
char pbefore[100];
char pafter[100];
PetscTruth flg, flg2, truth, useAcceleratingSmoothingMG, useFancySmoothingMG;
PetscTruth usePreviousGuess, useNormInfStoppingConditions, useNormInfMonitor, found, extractMats;
Mat K,G,D,C;
Vec u,p,f,h;
Mat S;
Vec h_hat,t,t2,q,v;
KSP ksp_inner;
KSP ksp_S;
KSP ksp_cleaner;
KSPType ksp_inner_type;
PetscTruth has_cnst_nullspace;
PC pc_S, pc_MG, pcInner;
PetscInt monitor_index,max_it,min_it;
Vec nsp_vec = PETSC_NULL;
PetscReal scr_rtol;
PetscReal inner_rtol;
PetscReal vSolver_rtol;
PetscScalar uNormInf, pNormInf;
PetscScalar uNorm, pNorm, rNorm, fNorm;
PetscInt uSize, pSize;
PetscInt lmin,lmax;
PetscInt iterations;
PetscReal min,max;
PetscReal p_sum;
MGContext mgCtx;
PC shellPC;
double t0, t1;
double mgSetupTime, scrSolveTime, a11SingleSolveTime, solutionAnalysisTime;
Index nx,ny,nz;
PetscInt j,start,end;
static int been_here = 0; /* Ha Ha Ha !! */
/* get sub matrix / vector objects */
MatNestGetSubMat( stokes_A, 0,0, &K );
MatNestGetSubMat( stokes_A, 0,1, &G );
MatNestGetSubMat( stokes_A, 1,0, &D );
MatNestGetSubMat( stokes_A, 1,1, &C );
VecNestGetSubVec( stokes_x, 0, &u );
VecNestGetSubVec( stokes_x, 1, &p );
VecNestGetSubVec( stokes_b, 0, &f );
VecNestGetSubVec( stokes_b, 1, &h );
/* PetscPrintf( PETSC_COMM_WORLD, "\t Adress of stokes_x is %p\n", stokes_x); */
/* VecNorm( u, NORM_2, &uNorm ); */
/* PetscPrintf( PETSC_COMM_WORLD, "\t u Norm is %.6e in %s: address is %p\n",uNorm,__func__,u); */
/* VecNorm( p, NORM_2, &pNorm ); */
/* PetscPrintf( PETSC_COMM_WORLD, "\t p Norm is %.6e in %s: addres is %p\n",pNorm,__func__,p); */
/* Create Schur complement matrix */
//MatCreateSchurFromBlock( stokes_A, 0.0, "MatSchur_A11", &S );
MatCreateSchurComplement(K,K,G,D,C, &S);
/* configure inner solver */
if (ksp_K!=PETSC_NULL) {
MatSchurComplementSetKSP( S, ksp_K );
MatSchurComplementGetKSP( S, &ksp_inner );
}
else {
abort();
MatSchurComplementGetKSP( S, &ksp_inner );
KSPSetType( ksp_inner, "cg" );
}
KSPGetPC( ksp_inner, &pcInner );
/* If we're using multigrid, replace the preconditioner here
so we get the same options prefix. */
if(bsscrp_self->mg) {
mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx );
}
/* SETFROMOPTIONS MIGHT F**K MG UP */
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest( ksp_inner );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfMonitor, PETSC_NULL, PETSC_NULL );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_to_norm_2_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/* create right hand side */
/* h_hat = G'*inv(K)*f - h */
MatGetVecs(K,PETSC_NULL,&t);
MatGetVecs( S, PETSC_NULL, &h_hat );
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
KSPSolve(ksp_inner,f,t);/* t=f/K */
//bsscr_writeVec( t, "ts", "Writing t vector");
MatMult(D,t,h_hat);/* G'*t */
VecAXPY(h_hat, -1.0, h);/* h_hat = h_hat - h */
Stg_VecDestroy(&t);
//bsscr_writeVec( h_hat, "h_hat", "Writing h_hat Vector in Solver");
//MatSchurApplyReductionToVecFromBlock( S, stokes_b, h_hat );
/* create solver for S p = h_hat */
KSPCreate( PETSC_COMM_WORLD, &ksp_S );
KSPSetOptionsPrefix( ksp_S, "scr_");
Stg_KSPSetOperators( ksp_S, S,S, SAME_NONZERO_PATTERN );
KSPSetType( ksp_S, "cg" );
/* Build preconditioner for S */
KSPGetPC( ksp_S, &pc_S );
BSSCR_BSSCR_StokesCreatePCSchur2( K,G,D,C,approxS,pc_S,sym, bsscrp_self );
KSPSetFromOptions(ksp_S);
/* Set specific monitor test */
KSPGetTolerances( ksp_S, PETSC_NULL, PETSC_NULL, PETSC_NULL, &max_it );
//BSSCR_KSPLogSetMonitor( ksp_S, max_it, &monitor_index );
/* Pressure / Velocity Solve */
scrSolveTime = MPI_Wtime();
PetscPrintf( PETSC_COMM_WORLD, "\t* Pressure / Velocity Solve \n");
usePreviousGuess = PETSC_FALSE;
if(been_here)
PetscOptionsGetTruth( PETSC_NULL, "-scr_use_previous_guess", &usePreviousGuess, &found );
if(usePreviousGuess) { /* Note this should actually look at checkpoint information */
KSPSetInitialGuessNonzero( ksp_S, PETSC_TRUE );
}
else {
KSPSetInitialGuessNonzero( ksp_S, PETSC_FALSE );
}
//KSPSetRelativeRhsConvergenceTest( ksp_S );
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-scr_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest(ksp_S);
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_S, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
PetscPrintf( PETSC_COMM_WORLD, "\t* KSPSolve( ksp_S, h_hat, p )\n");
/* if h_hat needs to be fixed up ..take out any nullspace vectors here */
/* we want to check that there is no "noise" in the null-space in the h vector */
/* this causes problems when we are trying to solve a Jacobian system when the Residual is almost converged */
if(bsscrp_self->check_pressureNS){
bsscrp_self->buildPNS(ksp);/* build and set nullspace vectors on bsscr - which is on ksp (function pointer is set in KSPSetUp_BSSCR */
}
PetscScalar norm, a, a1, a2, hnorm, pnorm, gnorm;
MatNorm(G,NORM_INFINITY,&gnorm);
VecNorm(h_hat, NORM_2, &hnorm);
hnorm=hnorm/gnorm;
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,1.0/hnorm);
}
/* test to see if v or t are in nullspace of G and orthogonalize wrt h_hat if needed */
KSPRemovePressureNullspace_BSSCR(ksp, h_hat);
/***************************************/
/* set convergence test to use min_it */
found = PETSC_FALSE;
min_it = 0;
PetscOptionsGetInt( PETSC_NULL,"-scr_ksp_set_min_it_converge", &min_it, &found);
if(found && min_it > 0){
BSSCR_KSPSetConvergenceMinIts(ksp_S, min_it, bsscrp_self);
}
KSPSolve( ksp_S, h_hat, p );
sprintf(pafter,"psafter_%d",been_here);
bsscr_writeVec( p, pafter, "Writing p Vector in Solver");
/***************************************/
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,hnorm);
VecScale(p,hnorm);
}
KSPRemovePressureNullspace_BSSCR(ksp, p);
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve( ksp_S, h_hat, p ) Solve Finished in time: %lf seconds\n\n", scrSolveTime);
/* Resolve with this pressure to obtain solution for u */
/* obtain solution for u */
VecDuplicate( u, &t );
MatMult( G, p, t);
VecAYPX( t, -1.0, f ); /* t <- -t + f */
MatSchurComplementGetKSP( S, &ksp_inner );
a11SingleSolveTime = MPI_Wtime(); /* ---------------------------------- Final V Solve */
if(usePreviousGuess)
KSPSetInitialGuessNonzero( ksp_inner, PETSC_TRUE );
KSPSetOptionsPrefix( ksp_inner, "backsolveA11_" );
KSPSetFromOptions( ksp_inner );
KSPSolve( ksp_inner, t, u ); /* Solve, then restore default tolerance and initial guess */
a11SingleSolveTime = MPI_Wtime() - a11SingleSolveTime; /* ------------------ Final V Solve */
PetscPrintf( PETSC_COMM_WORLD, "\n\nSCR Solver Summary:\n\n");
if(bsscrp_self->mg)
PetscPrintf( PETSC_COMM_WORLD, " Multigrid setup: = %.4g secs \n", mgSetupTime);
KSPGetIterationNumber( ksp_S, &iterations);
PetscPrintf( PETSC_COMM_WORLD, " Pressure Solve: = %.4g secs / %d its\n", scrSolveTime, iterations);
KSPGetIterationNumber( ksp_inner, &iterations);
PetscPrintf( PETSC_COMM_WORLD, " Final V Solve: = %.4g secs / %d its\n\n", a11SingleSolveTime, iterations);
/* Analysis of solution:
This can be somewhat time consuming as it requires allocation / de-allocation,
computing vector norms etc. So we make it optional..
This should be put into a proper KSP monitor now?
*/
flg = PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_solution_summary", &flg, &found );
if(flg) {
solutionAnalysisTime = MPI_Wtime();
VecGetSize( u, &uSize );
VecGetSize( p, &pSize );
VecDuplicate( u, &t2 );
MatMult( K, u, t2);
VecAYPX( t2, -1.0, t ); /* t2 <- -t2 + t ... should be the formal residual vector */
VecNorm( t2, NORM_2, &rNorm );
VecNorm( f, NORM_2, &fNorm );
PetscPrintf( PETSC_COMM_WORLD, " |f - K u - G p|/|f| = %.6e\n", rNorm/fNorm );
VecDuplicate( p, &q );
MatMult( D, u, q );
VecNorm( u, NORM_2, &uNorm );
VecNorm( q, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_2/|u|_2 = %.6e\n", sqrt( (double) uSize / (double) pSize ) * rNorm / uNorm);
VecNorm( q, NORM_INFINITY, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_infty/|u|_2 = %.6e\n", sqrt( (double) uSize ) * rNorm / uNorm);
VecNorm( u, NORM_INFINITY, &uNormInf );
VecNorm( u, NORM_2, &uNorm );
VecGetSize( u, &uSize );
VecNorm( p, NORM_INFINITY, &pNormInf );
VecNorm( p, NORM_2, &pNorm );
PetscPrintf( PETSC_COMM_WORLD, " |u|_{\\infty} = %.6e , u_rms = %.6e\n",
uNormInf, uNorm / sqrt( (double) uSize ) );
PetscPrintf( PETSC_COMM_WORLD, " |p|_{\\infty} = %.6e , p_rms = %.6e\n",
pNormInf, pNorm / sqrt( (double) pSize ) );
VecMax( u, &lmax, &max );
VecMin( u, &lmin, &min );
PetscPrintf( PETSC_COMM_WORLD, " min/max(u) = %.6e [%d] / %.6e [%d]\n",min,lmin,max,lmax);
VecMax( p, &lmax, &max );
VecMin( p, &lmin, &min );
PetscPrintf( PETSC_COMM_WORLD, " min/max(p) = %.6e [%d] / %.6e [%d]\n",min,lmin,max,lmax);
VecSum( p, &p_sum );
PetscPrintf( PETSC_COMM_WORLD, " \\sum_i p_i = %.6e \n", p_sum );
solutionAnalysisTime = MPI_Wtime() - solutionAnalysisTime;
PetscPrintf( PETSC_COMM_WORLD, "\n Time for this analysis = %.4g secs\n\n",solutionAnalysisTime);
Stg_VecDestroy(&t2 );
Stg_VecDestroy(&q );
}
if(bsscrp_self->mg) {
//MG_inner_solver_pcmg_shutdown( pcInner );
}
Stg_VecDestroy(&t );
// KSPLogDestroyMonitor( ksp_S );
Stg_KSPDestroy(&ksp_S );
//Stg_KSPDestroy(&ksp_inner );
Stg_VecDestroy(&h_hat );
Stg_MatDestroy(&S );
/* Destroy nullspace vector if it exists. */
if(nsp_vec)
Stg_VecDestroy(&nsp_vec);
//been_here = 1;
been_here++;
PetscFunctionReturn(0);
}
void bsscr_summary(KSP_BSSCR * bsscrp_self, KSP ksp_S, KSP ksp_inner,
Mat K,Mat K2,Mat D,Mat G,Mat C,Vec u,Vec p,Vec f,Vec h,Vec t,
double penaltyNumber,PetscTruth KisJustK,double mgSetupTime,double scrSolveTime,double a11SingleSolveTime){
PetscTruth flg, found;
PetscInt uSize, pSize, lmax, lmin, iterations;
PetscReal rNorm, fNorm, uNorm, uNormInf, pNorm, pNormInf, p_sum, min, max;
Vec q, qq, t2, t3;
double solutionAnalysisTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\nSCR Solver Summary:\n\n");
if(bsscrp_self->mg)
PetscPrintf( PETSC_COMM_WORLD, " Multigrid setup: = %.4g secs \n", mgSetupTime);
KSPGetIterationNumber( ksp_S, &iterations);
bsscrp_self->solver->stats.pressure_its = iterations;
PetscPrintf( PETSC_COMM_WORLD, " Pressure Solve: = %.4g secs / %d its\n", scrSolveTime, iterations);
KSPGetIterationNumber( ksp_inner, &iterations);
bsscrp_self->solver->stats.velocity_backsolve_its = iterations;
PetscPrintf( PETSC_COMM_WORLD, " Final V Solve: = %.4g secs / %d its\n\n", a11SingleSolveTime, iterations);
/***************************************************************************************************************/
flg = PETSC_FALSE; /* Off by default */
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_solution_summary", &flg, &found );
if(flg) {
PetscScalar KuNorm;
solutionAnalysisTime = MPI_Wtime();
VecGetSize( u, &uSize );
VecGetSize( p, &pSize );
VecDuplicate( u, &t2 );
VecDuplicate( u, &t3 );
MatMult( K, u, t3); VecNorm( t3, NORM_2, &KuNorm );
double angle, kdot;
if(penaltyNumber > 1e-10){/* should change this to ifK2built maybe */
MatMult( K2, u, t2); VecNorm( t2, NORM_2, &rNorm );
VecDot(t2,t3,&kdot);
angle = (kdot/(rNorm*KuNorm));
PetscPrintf( PETSC_COMM_WORLD, " <K u, K2 u>/(|K u| |K2 u|) = %.6e\n", angle);
}
VecNorm( t, NORM_2, &rNorm ); /* t = f- G p should be the formal residual vector, calculated on line 267 in auglag-driver-DGTGD.c */
VecDot(t3,t,&kdot);
angle = (kdot/(rNorm*KuNorm));
PetscPrintf( PETSC_COMM_WORLD, " <K u, (f-G p)>/(|K u| |f- G p|) = %.6e\n\n", angle);
MatMult( K, u, t2); VecNorm(t2, NORM_2, &KuNorm);
VecAYPX( t2, -1.0, t ); /* t2 <- -t2 + t : t = f- G p should be the formal residual vector, calculated on line 267 in auglag-driver-DGTGD.c*/
VecNorm( t2, NORM_2, &rNorm );
VecNorm( f, NORM_2, &fNorm );
if(KisJustK){
PetscPrintf( PETSC_COMM_WORLD,"Velocity back-solve with original K matrix\n");
PetscPrintf( PETSC_COMM_WORLD,"Solved K u = G p -f\n");
PetscPrintf( PETSC_COMM_WORLD,"Residual with original K matrix\n");
PetscPrintf( PETSC_COMM_WORLD, " |f - K u - G p| = %.12e\n", rNorm);
PetscPrintf( PETSC_COMM_WORLD, " |f - K u - G p|/|f| = %.12e\n", rNorm/fNorm);
if(penaltyNumber > 1e-10){/* means the restore_K flag was used */
//if(K2 && f2){
MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);/* Computes K = penaltyNumber*K2 + K */
//VecAXPY(f,penaltyNumber,f2); /* f = penaltyNumber*f2 + f */
KisJustK=PETSC_FALSE;
MatMult( K, u, t2);
MatMult( G, p, t);
VecAYPX( t, -1.0, f ); /* t <- -t + f */
VecAYPX( t2, -1.0, t ); /* t2 <- -t2 + t */
VecNorm( t2, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD,"Residual with K+K2 matrix and f rhs vector\n");
PetscPrintf( PETSC_COMM_WORLD, " |(f) - (K + K2) u - G p| = %.12e\n", rNorm);
//}
}
}
else{
PetscPrintf( PETSC_COMM_WORLD,"Velocity back-solve with K+K2 matrix\n");
PetscPrintf( PETSC_COMM_WORLD,"Solved (K + K2) u = G p - (f)\n");
PetscPrintf( PETSC_COMM_WORLD,"Residual with K+K2 matrix and f rhs vector\n");
PetscPrintf( PETSC_COMM_WORLD, " |(f) - (K + K2) u - G p| = %.12e\n", rNorm);
PetscReal KK2Norm,KK2Normf;
MatNorm(K,NORM_1,&KK2Norm);
MatNorm(K,NORM_FROBENIUS,&KK2Normf);
penaltyNumber = -penaltyNumber;
MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);/* Computes K = penaltyNumber*K2 + K */
//VecAXPY(f,penaltyNumber,f2); /* f = penaltyNumber*f2 + f */
KisJustK=PETSC_FALSE;
MatMult( K, u, t2); /* t2 = K*u */
MatMult( G, p, t); /* t = G*p */
VecAYPX( t, -1.0, f ); /* t <- f - t ; t = f - G*p */
VecAYPX( t2, -1.0, t ); /* t2 <- t - t2; t2 = f - G*p - K*u */
VecNorm( t2, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD,"Residual with original K matrix\n");
PetscPrintf( PETSC_COMM_WORLD, " |f - K u - G p| = %.12e\n", rNorm);
PetscPrintf( PETSC_COMM_WORLD, " |f - K u - G p|/|f| = %.12e\n", rNorm/fNorm);
PetscReal KNorm, K2Norm;
MatNorm(K,NORM_1,&KNorm); MatNorm(K2,NORM_1,&K2Norm);
PetscPrintf( PETSC_COMM_WORLD,"K and K2 norm_1 %.12e %.12e ratio %.12e\n",KNorm,K2Norm,K2Norm/KNorm);
MatNorm(K,NORM_INFINITY,&KNorm); MatNorm(K2,NORM_INFINITY,&K2Norm);
PetscPrintf( PETSC_COMM_WORLD,"K and K2 norm_inf %.12e %.12e ratio %.12e\n",KNorm,K2Norm,K2Norm/KNorm);
MatNorm(K,NORM_FROBENIUS,&KNorm); MatNorm(K2,NORM_FROBENIUS,&K2Norm);
PetscPrintf( PETSC_COMM_WORLD,"K and K2 norm_frob %.12e %.12e ratio %.12e\n",KNorm,K2Norm,K2Norm/KNorm);
PetscPrintf( PETSC_COMM_WORLD,"K+r*K2 norm_1 %.12e\n",KK2Norm);
PetscPrintf( PETSC_COMM_WORLD,"K+r*K2 norm_frob %.12e\n",KK2Normf);
penaltyNumber = -penaltyNumber;
MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);/* Computes K = penaltyNumber*K2 + K */
}
PetscPrintf( PETSC_COMM_WORLD,"\n");
PetscPrintf( PETSC_COMM_WORLD, " |K u| = %.12e\n", KuNorm);
if(penaltyNumber > 1e-10){
MatMult( K2, u, t2); VecNorm( t2, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |K2 u| = %.12e\n", rNorm);
PetscPrintf( PETSC_COMM_WORLD,"\n");
}
VecDuplicate( p, &q );
MatMult( D, u, q ); /* q = G'*u = D*u */
VecNorm( u, NORM_2, &uNorm );
VecNorm( q, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_2 = %.6e\n", rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_2/|u|_2 = %.6e\n", sqrt( (double) uSize / (double) pSize ) * rNorm / uNorm);
VecDuplicate( p, &qq );
MatMultTranspose( G, u, qq );
VecNorm( qq, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|/|u| = %.8e\n", rNorm/uNorm ); /* to compare directly with Uzawa */
VecNorm( q, NORM_INFINITY, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_infty/|u|_2 = %.6e\n", sqrt( (double) uSize ) * rNorm / uNorm);
/* create G'*u+C*p-h to check on this constraint */
/* already have q = D*u */
VecZeroEntries(qq);
if(C){
MatMult( C, p, qq );
}
VecAYPX( q, 1.0, qq ); /* q = q+qq; G'*u + C*p*/
VecAXPY( q, -1.0, h ); /* q = q-h; G'*u + C*p - h */
VecNorm( q, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u + C p - h| = %.8e :constraint\n", rNorm );
VecNorm( u, NORM_INFINITY, &uNormInf );
VecNorm( u, NORM_2, &uNorm );
VecGetSize( u, &uSize );
VecNorm( p, NORM_INFINITY, &pNormInf );
VecNorm( p, NORM_2, &pNorm );
PetscPrintf( PETSC_COMM_WORLD, " |u|_{\\infty} = %.6e , u_rms = %.6e\n",
uNormInf, uNorm / sqrt( (double) uSize ) );
PetscPrintf( PETSC_COMM_WORLD, " |p|_{\\infty} = %.6e , p_rms = %.6e\n",
pNormInf, pNorm / sqrt( (double) pSize ) );
VecMax( u, &lmax, &max );
VecMin( u, &lmin, &min );
PetscPrintf( PETSC_COMM_WORLD, " min/max(u) = %.6e [%d] / %.6e [%d]\n",min,lmin,max,lmax);
bsscrp_self->solver->stats.vmin = min;
bsscrp_self->solver->stats.vmax = max;
VecMax( p, &lmax, &max );
VecMin( p, &lmin, &min );
PetscPrintf( PETSC_COMM_WORLD, " min/max(p) = %.6e [%d] / %.6e [%d]\n",min,lmin,max,lmax);
bsscrp_self->solver->stats.pmin = min;
bsscrp_self->solver->stats.pmax = max;
VecSum( p, &p_sum );
PetscPrintf( PETSC_COMM_WORLD, " \\sum_i p_i = %.6e \n", p_sum );
bsscrp_self->solver->stats.p_sum=p_sum;
solutionAnalysisTime = MPI_Wtime() - solutionAnalysisTime;
PetscPrintf( PETSC_COMM_WORLD, "\n Time for this analysis = %.4g secs\n\n",solutionAnalysisTime);
Stg_VecDestroy(&t2 );
Stg_VecDestroy(&t3 );
Stg_VecDestroy(&q );
Stg_VecDestroy(&qq );
}
}
PetscErrorCode PCBDDCSetupFETIDPMatContext(FETIDPMat_ctx fetidpmat_ctx )
{
PetscErrorCode ierr;
PC_IS *pcis=(PC_IS*)fetidpmat_ctx->pc->data;
PC_BDDC *pcbddc=(PC_BDDC*)fetidpmat_ctx->pc->data;
PCBDDCGraph mat_graph=pcbddc->mat_graph;
Mat_IS *matis = (Mat_IS*)fetidpmat_ctx->pc->pmat->data;
MPI_Comm comm;
Mat ScalingMat;
Vec lambda_global;
IS IS_l2g_lambda;
PetscBool skip_node,fully_redundant;
PetscInt i,j,k,s,n_boundary_dofs,n_global_lambda,n_vertices,partial_sum;
PetscInt n_local_lambda,n_lambda_for_dof,dual_size,n_neg_values,n_pos_values;
PetscMPIInt rank,size,buf_size,neigh;
PetscScalar scalar_value;
PetscInt *vertex_indices;
PetscInt *dual_dofs_boundary_indices,*aux_local_numbering_1,*aux_global_numbering;
PetscInt *aux_sums,*cols_B_delta,*l2g_indices;
PetscScalar *array,*scaling_factors,*vals_B_delta;
PetscInt *aux_local_numbering_2;
/* For communication of scaling factors */
PetscInt *ptrs_buffer,neigh_position;
PetscScalar **all_factors,*send_buffer,*recv_buffer;
MPI_Request *send_reqs,*recv_reqs;
/* tests */
Vec test_vec;
PetscBool test_fetidp;
PetscViewer viewer;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)(fetidpmat_ctx->pc),&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
/* Default type of lagrange multipliers is non-redundant */
fully_redundant = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-fetidp_fullyredundant",&fully_redundant,NULL);CHKERRQ(ierr);
/* Evaluate local and global number of lagrange multipliers */
ierr = VecSet(pcis->vec1_N,0.0);CHKERRQ(ierr);
n_local_lambda = 0;
partial_sum = 0;
n_boundary_dofs = 0;
s = 0;
/* Get Vertices used to define the BDDC */
ierr = PCBDDCGetPrimalVerticesLocalIdx(fetidpmat_ctx->pc,&n_vertices,&vertex_indices);CHKERRQ(ierr);
dual_size = pcis->n_B-n_vertices;
ierr = PetscSortInt(n_vertices,vertex_indices);CHKERRQ(ierr);
ierr = PetscMalloc1(dual_size,&dual_dofs_boundary_indices);CHKERRQ(ierr);
ierr = PetscMalloc1(dual_size,&aux_local_numbering_1);CHKERRQ(ierr);
ierr = PetscMalloc1(dual_size,&aux_local_numbering_2);CHKERRQ(ierr);
ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr);
for (i=0;i<pcis->n;i++){
j = mat_graph->count[i]; /* RECALL: mat_graph->count[i] does not count myself */
if ( j > 0 ) {
n_boundary_dofs++;
}
skip_node = PETSC_FALSE;
if ( s < n_vertices && vertex_indices[s]==i) { /* it works for a sorted set of vertices */
skip_node = PETSC_TRUE;
s++;
}
if (j < 1) {
skip_node = PETSC_TRUE;
}
if ( !skip_node ) {
if (fully_redundant) {
/* fully redundant set of lagrange multipliers */
n_lambda_for_dof = (j*(j+1))/2;
} else {
n_lambda_for_dof = j;
}
n_local_lambda += j;
/* needed to evaluate global number of lagrange multipliers */
array[i]=(1.0*n_lambda_for_dof)/(j+1.0); /* already scaled for the next global sum */
/* store some data needed */
dual_dofs_boundary_indices[partial_sum] = n_boundary_dofs-1;
aux_local_numbering_1[partial_sum] = i;
aux_local_numbering_2[partial_sum] = n_lambda_for_dof;
partial_sum++;
}
}
ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr);
ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(matis->ctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (matis->ctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecSum(pcis->vec1_global,&scalar_value);CHKERRQ(ierr);
fetidpmat_ctx->n_lambda = (PetscInt)PetscRealPart(scalar_value);
/* compute global ordering of lagrange multipliers and associate l2g map */
ierr = PCBDDCSubsetNumbering(comm,matis->mapping,partial_sum,aux_local_numbering_1,aux_local_numbering_2,&i,&aux_global_numbering);CHKERRQ(ierr);
if (i != fetidpmat_ctx->n_lambda) {
SETERRQ3(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"Error in %s: global number of multipliers mismatch! (%d!=%d)\n",__FUNCT__,fetidpmat_ctx->n_lambda,i);
}
ierr = PetscFree(aux_local_numbering_2);CHKERRQ(ierr);
/* init data for scaling factors exchange */
partial_sum = 0;
j = 0;
ierr = PetscMalloc1(pcis->n_neigh,&ptrs_buffer);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n_neigh-1,&send_reqs);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n_neigh-1,&recv_reqs);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n,&all_factors);CHKERRQ(ierr);
ptrs_buffer[0]=0;
for (i=1;i<pcis->n_neigh;i++) {
partial_sum += pcis->n_shared[i];
ptrs_buffer[i] = ptrs_buffer[i-1]+pcis->n_shared[i];
}
ierr = PetscMalloc1(partial_sum,&send_buffer);CHKERRQ(ierr);
ierr = PetscMalloc1(partial_sum,&recv_buffer);CHKERRQ(ierr);
ierr = PetscMalloc1(partial_sum,&all_factors[0]);CHKERRQ(ierr);
for (i=0;i<pcis->n-1;i++) {
j = mat_graph->count[i];
all_factors[i+1]=all_factors[i]+j;
}
/* scatter B scaling to N vec */
ierr = VecScatterBegin(pcis->N_to_B,pcis->D,pcis->vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->N_to_B,pcis->D,pcis->vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* communications */
ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr);
for (i=1;i<pcis->n_neigh;i++) {
for (j=0;j<pcis->n_shared[i];j++) {
send_buffer[ptrs_buffer[i-1]+j]=array[pcis->shared[i][j]];
}
ierr = PetscMPIIntCast(ptrs_buffer[i]-ptrs_buffer[i-1],&buf_size);CHKERRQ(ierr);
ierr = PetscMPIIntCast(pcis->neigh[i],&neigh);CHKERRQ(ierr);
ierr = MPI_Isend(&send_buffer[ptrs_buffer[i-1]],buf_size,MPIU_SCALAR,neigh,0,comm,&send_reqs[i-1]);CHKERRQ(ierr);
ierr = MPI_Irecv(&recv_buffer[ptrs_buffer[i-1]],buf_size,MPIU_SCALAR,neigh,0,comm,&recv_reqs[i-1]);CHKERRQ(ierr);
}
ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr);
ierr = MPI_Waitall((pcis->n_neigh-1),recv_reqs,MPI_STATUSES_IGNORE);CHKERRQ(ierr);
/* put values in correct places */
for (i=1;i<pcis->n_neigh;i++) {
for (j=0;j<pcis->n_shared[i];j++) {
k = pcis->shared[i][j];
neigh_position = 0;
while(mat_graph->neighbours_set[k][neigh_position] != pcis->neigh[i]) {neigh_position++;}
all_factors[k][neigh_position]=recv_buffer[ptrs_buffer[i-1]+j];
}
}
ierr = MPI_Waitall((pcis->n_neigh-1),send_reqs,MPI_STATUSES_IGNORE);CHKERRQ(ierr);
ierr = PetscFree(send_reqs);CHKERRQ(ierr);
ierr = PetscFree(recv_reqs);CHKERRQ(ierr);
ierr = PetscFree(send_buffer);CHKERRQ(ierr);
ierr = PetscFree(recv_buffer);CHKERRQ(ierr);
ierr = PetscFree(ptrs_buffer);CHKERRQ(ierr);
/* Compute B and B_delta (local actions) */
ierr = PetscMalloc1(pcis->n_neigh,&aux_sums);CHKERRQ(ierr);
ierr = PetscMalloc1(n_local_lambda,&l2g_indices);CHKERRQ(ierr);
ierr = PetscMalloc1(n_local_lambda,&vals_B_delta);CHKERRQ(ierr);
ierr = PetscMalloc1(n_local_lambda,&cols_B_delta);CHKERRQ(ierr);
ierr = PetscMalloc1(n_local_lambda,&scaling_factors);CHKERRQ(ierr);
n_global_lambda=0;
partial_sum=0;
for (i=0;i<dual_size;i++) {
n_global_lambda = aux_global_numbering[i];
j = mat_graph->count[aux_local_numbering_1[i]];
aux_sums[0]=0;
for (s=1;s<j;s++) {
aux_sums[s]=aux_sums[s-1]+j-s+1;
}
array = all_factors[aux_local_numbering_1[i]];
n_neg_values = 0;
while(n_neg_values < j && mat_graph->neighbours_set[aux_local_numbering_1[i]][n_neg_values] < rank) {n_neg_values++;}
n_pos_values = j - n_neg_values;
if (fully_redundant) {
for (s=0;s<n_neg_values;s++) {
l2g_indices [partial_sum+s]=aux_sums[s]+n_neg_values-s-1+n_global_lambda;
cols_B_delta [partial_sum+s]=dual_dofs_boundary_indices[i];
vals_B_delta [partial_sum+s]=-1.0;
scaling_factors[partial_sum+s]=array[s];
}
for (s=0;s<n_pos_values;s++) {
l2g_indices [partial_sum+s+n_neg_values]=aux_sums[n_neg_values]+s+n_global_lambda;
cols_B_delta [partial_sum+s+n_neg_values]=dual_dofs_boundary_indices[i];
vals_B_delta [partial_sum+s+n_neg_values]=1.0;
scaling_factors[partial_sum+s+n_neg_values]=array[s+n_neg_values];
}
partial_sum += j;
} else {
/* l2g_indices and default cols and vals of B_delta */
for (s=0;s<j;s++) {
l2g_indices [partial_sum+s]=n_global_lambda+s;
cols_B_delta [partial_sum+s]=dual_dofs_boundary_indices[i];
vals_B_delta [partial_sum+s]=0.0;
}
/* B_delta */
if ( n_neg_values > 0 ) { /* there's a rank next to me to the left */
vals_B_delta [partial_sum+n_neg_values-1]=-1.0;
}
if ( n_neg_values < j ) { /* there's a rank next to me to the right */
vals_B_delta [partial_sum+n_neg_values]=1.0;
}
/* scaling as in Klawonn-Widlund 1999*/
for (s=0;s<n_neg_values;s++) {
scalar_value = 0.0;
for (k=0;k<s+1;k++) {
scalar_value += array[k];
}
scaling_factors[partial_sum+s] = -scalar_value;
}
for (s=0;s<n_pos_values;s++) {
scalar_value = 0.0;
for (k=s+n_neg_values;k<j;k++) {
scalar_value += array[k];
}
scaling_factors[partial_sum+s+n_neg_values] = scalar_value;
}
partial_sum += j;
}
}
ierr = PetscFree(aux_global_numbering);CHKERRQ(ierr);
ierr = PetscFree(aux_sums);CHKERRQ(ierr);
ierr = PetscFree(aux_local_numbering_1);CHKERRQ(ierr);
ierr = PetscFree(dual_dofs_boundary_indices);CHKERRQ(ierr);
ierr = PetscFree(all_factors[0]);CHKERRQ(ierr);
ierr = PetscFree(all_factors);CHKERRQ(ierr);
/* Local to global mapping of fetidpmat */
ierr = VecCreate(PETSC_COMM_SELF,&fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSetSizes(fetidpmat_ctx->lambda_local,n_local_lambda,n_local_lambda);CHKERRQ(ierr);
ierr = VecSetType(fetidpmat_ctx->lambda_local,VECSEQ);CHKERRQ(ierr);
ierr = VecCreate(comm,&lambda_global);CHKERRQ(ierr);
ierr = VecSetSizes(lambda_global,PETSC_DECIDE,fetidpmat_ctx->n_lambda);CHKERRQ(ierr);
ierr = VecSetType(lambda_global,VECMPI);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,n_local_lambda,l2g_indices,PETSC_OWN_POINTER,&IS_l2g_lambda);CHKERRQ(ierr);
ierr = VecScatterCreate(fetidpmat_ctx->lambda_local,(IS)0,lambda_global,IS_l2g_lambda,&fetidpmat_ctx->l2g_lambda);CHKERRQ(ierr);
ierr = ISDestroy(&IS_l2g_lambda);CHKERRQ(ierr);
/* Create local part of B_delta */
ierr = MatCreate(PETSC_COMM_SELF,&fetidpmat_ctx->B_delta);
ierr = MatSetSizes(fetidpmat_ctx->B_delta,n_local_lambda,pcis->n_B,n_local_lambda,pcis->n_B);CHKERRQ(ierr);
ierr = MatSetType(fetidpmat_ctx->B_delta,MATSEQAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(fetidpmat_ctx->B_delta,1,NULL);CHKERRQ(ierr);
ierr = MatSetOption(fetidpmat_ctx->B_delta,MAT_IGNORE_ZERO_ENTRIES,PETSC_TRUE);CHKERRQ(ierr);
for (i=0;i<n_local_lambda;i++) {
ierr = MatSetValue(fetidpmat_ctx->B_delta,i,cols_B_delta[i],vals_B_delta[i],INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree(vals_B_delta);CHKERRQ(ierr);
ierr = MatAssemblyBegin(fetidpmat_ctx->B_delta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (fetidpmat_ctx->B_delta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (fully_redundant) {
ierr = MatCreate(PETSC_COMM_SELF,&ScalingMat);
ierr = MatSetSizes(ScalingMat,n_local_lambda,n_local_lambda,n_local_lambda,n_local_lambda);CHKERRQ(ierr);
ierr = MatSetType(ScalingMat,MATSEQAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(ScalingMat,1,NULL);CHKERRQ(ierr);
for (i=0;i<n_local_lambda;i++) {
ierr = MatSetValue(ScalingMat,i,i,scaling_factors[i],INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(ScalingMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (ScalingMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatMatMult(ScalingMat,fetidpmat_ctx->B_delta,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&fetidpmat_ctx->B_Ddelta);CHKERRQ(ierr);
ierr = MatDestroy(&ScalingMat);CHKERRQ(ierr);
} else {
ierr = MatCreate(PETSC_COMM_SELF,&fetidpmat_ctx->B_Ddelta);
ierr = MatSetSizes(fetidpmat_ctx->B_Ddelta,n_local_lambda,pcis->n_B,n_local_lambda,pcis->n_B);CHKERRQ(ierr);
ierr = MatSetType(fetidpmat_ctx->B_Ddelta,MATSEQAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(fetidpmat_ctx->B_Ddelta,1,NULL);CHKERRQ(ierr);
for (i=0;i<n_local_lambda;i++) {
ierr = MatSetValue(fetidpmat_ctx->B_Ddelta,i,cols_B_delta[i],scaling_factors[i],INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(fetidpmat_ctx->B_Ddelta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (fetidpmat_ctx->B_Ddelta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
ierr = PetscFree(scaling_factors);CHKERRQ(ierr);
ierr = PetscFree(cols_B_delta);CHKERRQ(ierr);
/* Create some vectors needed by fetidp */
ierr = VecDuplicate(pcis->vec1_B,&fetidpmat_ctx->temp_solution_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&fetidpmat_ctx->temp_solution_D);CHKERRQ(ierr);
test_fetidp = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-fetidp_check",&test_fetidp,NULL);CHKERRQ(ierr);
if (test_fetidp && !pcbddc->use_deluxe_scaling) {
PetscReal real_value;
ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"----------FETI_DP TESTS--------------\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"All tests should return zero!\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"FETIDP MAT context in the ");CHKERRQ(ierr);
if (fully_redundant) {
ierr = PetscViewerASCIIPrintf(viewer,"fully redundant case for lagrange multipliers.\n");CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"Non-fully redundant case for lagrange multiplier.\n");CHKERRQ(ierr);
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
/******************************************************************/
/* TEST A/B: Test numbering of global lambda dofs */
/******************************************************************/
ierr = VecDuplicate(fetidpmat_ctx->lambda_local,&test_vec);CHKERRQ(ierr);
ierr = VecSet(lambda_global,1.0);CHKERRQ(ierr);
ierr = VecSet(test_vec,1.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
scalar_value = -1.0;
ierr = VecAXPY(test_vec,scalar_value,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecNorm(test_vec,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = VecDestroy(&test_vec);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"A[%04d]: CHECK glob to loc: % 1.14e\n",rank,real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (fully_redundant) {
ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr);
ierr = VecSet(fetidpmat_ctx->lambda_local,0.5);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecSum(lambda_global,&scalar_value);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"B[%04d]: CHECK loc to glob: % 1.14e\n",rank,PetscRealPart(scalar_value)-fetidpmat_ctx->n_lambda);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
}
/******************************************************************/
/* TEST C: It should holds B_delta*w=0, w\in\widehat{W} */
/* This is the meaning of the B matrix */
/******************************************************************/
ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr);
ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(matis->ctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (matis->ctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(matis->ctx,pcis->vec1_global,pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (matis->ctx,pcis->vec1_global,pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_delta */
ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecNorm(lambda_global,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"C[coll]: CHECK infty norm of B_delta*w (w continuous): % 1.14e\n",real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
/******************************************************************/
/* TEST D: It should holds E_Dw = w - P_Dw w\in\widetilde{W} */
/* E_D = R_D^TR */
/* P_D = B_{D,delta}^T B_{delta} */
/* eq.44 Mandel Tezaur and Dohrmann 2005 */
/******************************************************************/
/* compute a random vector in \widetilde{W} */
ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr);
scalar_value = 0.0; /* set zero at vertices */
ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr);
for (i=0;i<n_vertices;i++) { array[vertex_indices[i]]=scalar_value; }
ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr);
/* store w for final comparison */
ierr = VecDuplicate(pcis->vec1_B,&test_vec);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,test_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,test_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Jump operator P_D : results stored in pcis->vec1_B */
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_delta */
ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_Ddelta^T */
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr);
/* Average operator E_D : results stored in pcis->vec2_B */
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = PCBDDCScalingExtension(fetidpmat_ctx->pc,pcis->vec2_B,pcis->vec1_global);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_global,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_global,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* test E_D=I-P_D */
scalar_value = 1.0;
ierr = VecAXPY(pcis->vec1_B,scalar_value,pcis->vec2_B);CHKERRQ(ierr);
scalar_value = -1.0;
ierr = VecAXPY(pcis->vec1_B,scalar_value,test_vec);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_B,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = VecDestroy(&test_vec);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"D[%04d] CHECK infty norm of E_D + P_D - I: % 1.14e\n",rank,real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
/******************************************************************/
/* TEST E: It should holds R_D^TP_Dw=0 w\in\widetilde{W} */
/* eq.48 Mandel Tezaur and Dohrmann 2005 */
/******************************************************************/
ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr);
ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr);
scalar_value = 0.0; /* set zero at vertices */
for (i=0;i<n_vertices;i++) { array[vertex_indices[i]]=scalar_value; }
ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr);
/* Jump operator P_D : results stored in pcis->vec1_B */
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_delta */
ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_Ddelta^T */
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr);
/* scaling */
ierr = PCBDDCScalingExtension(fetidpmat_ctx->pc,pcis->vec1_B,pcis->vec1_global);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_global,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"E[coll]: CHECK infty norm of R^T_D P_D: % 1.14e\n",real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (!fully_redundant) {
/******************************************************************/
/* TEST F: It should holds B_{delta}B^T_{D,delta}=I */
/* Corollary thm 14 Mandel Tezaur and Dohrmann 2005 */
/******************************************************************/
ierr = VecDuplicate(lambda_global,&test_vec);CHKERRQ(ierr);
ierr = VecSetRandom(lambda_global,NULL);CHKERRQ(ierr);
/* Action of B_Ddelta^T */
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr);
/* Action of B_delta */
ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSet(test_vec,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,test_vec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,test_vec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
scalar_value = -1.0;
ierr = VecAXPY(lambda_global,scalar_value,test_vec);CHKERRQ(ierr);
ierr = VecNorm(lambda_global,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"E[coll]: CHECK infty norm of P^T_D - I: % 1.14e\n",real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = VecDestroy(&test_vec);CHKERRQ(ierr);
}
}
/* final cleanup */
ierr = PetscFree(vertex_indices);CHKERRQ(ierr);
ierr = VecDestroy(&lambda_global);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
