int Dense3d_MPI::setup()
{
//begin
pA(_srcPos!=NULL && _srcNor!=NULL && _trgPos!=NULL);
//--------------------------------------------------------------------------
/* Create a Scatter context - copies all source position values to each processor
* See http://www-unix.mcs.anl.gov/petsc/petsc-as/snapshots/petsc-current/docs/manualpages/Vec/VecScatterCreateToAll.html */
{
VecScatter ctx;
pC( VecScatterCreateToAll(_srcPos, &ctx, &_srcAllPos) );
// VecScatterBegin(VecScatter inctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
pC( VecScatterBegin(ctx, _srcPos, _srcAllPos, INSERT_VALUES, SCATTER_FORWARD) );
pC( VecScatterEnd(ctx, _srcPos, _srcAllPos, INSERT_VALUES, SCATTER_FORWARD ) );
pC( VecScatterDestroy(ctx) );
}
/* Create a Scatter context - copies all source normal values to each processor */
{
VecScatter ctx;
pC( VecScatterCreateToAll(_srcNor, &ctx, &_srcAllNor) );
pC( VecScatterBegin(ctx, _srcNor, _srcAllNor, INSERT_VALUES, SCATTER_FORWARD) );
pC( VecScatterEnd(ctx, _srcNor, _srcAllNor, INSERT_VALUES, SCATTER_FORWARD) );
pC( VecScatterDestroy(ctx) );
}
return(0);
}
PetscErrorCode MatMult_myMat(Mat A,Vec x,Vec y)
{
PetscFunctionBeginUser;
PetscErrorCode ierr;
model *m;
ierr = MatShellGetContext(A, &m); CHKERRQ(ierr);
PetscInt len_x, len_y;
ierr = VecGetLocalSize(x, &len_x); CHKERRQ(ierr);
ierr = VecGetLocalSize(y, &len_y); CHKERRQ(ierr);
//std::cout << "len_x=" << len_x << "len_y=" << len_y << std::endl;
const PetscScalar *px;
PetscScalar *py, *pgx;
VecScatter ctx;
Vec vout;
VecScatterCreateToAll(x,&ctx,&vout);
// scatter as many times as you need
VecScatterBegin(ctx,x,vout,INSERT_VALUES,SCATTER_FORWARD);
VecScatterEnd(ctx,x,vout,INSERT_VALUES,SCATTER_FORWARD);
ierr = VecGetArray(vout, &pgx); CHKERRQ(ierr);
ierr = VecGetArrayRead(x, &px); CHKERRQ(ierr);
ierr = VecGetArray(y, &py); CHKERRQ(ierr);
PetscInt Istart, Iend;
ierr = MatGetOwnershipRange(A, &Istart, &Iend); CHKERRQ(ierr);
PetscInt A_m, n;
ierr = MatGetSize(A, &A_m, &n); CHKERRQ(ierr);
PetscScalar* pgy = new PetscScalar[n];
rokko::heisenberg_hamiltonian::multiply(m->L, m->lattice, pgx, pgy);
for(int j = 0; j < len_y; ++j) {
py[j] = 0.;
}
int k=0;
for(int i = Istart; i < Iend; ++i) {
py[k] = pgy[i];
++k;
}
// destroy scatter context and local vector when no longer needed
VecScatterDestroy(&ctx);
//VecView(vout, PETSC_VIEWER_STDOUT_WORLD);
VecDestroy(&vout);
ierr = VecRestoreArrayRead(x,&px); CHKERRQ(ierr);
ierr = VecRestoreArray(y,&py); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode PCSVDGetVec(PC pc,PCSide side,AccessMode amode,Vec x,Vec *xred)
{
PC_SVD *jac = (PC_SVD*)pc->data;
PetscErrorCode ierr;
PetscMPIInt size;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)pc),&size);CHKERRQ(ierr);
*xred = NULL;
switch (side) {
case PC_LEFT:
if (size == 1) *xred = x;
else {
if (!jac->left2red) {ierr = VecScatterCreateToAll(x,&jac->left2red,&jac->leftred);CHKERRQ(ierr);}
if (amode & READ) {
ierr = VecScatterBegin(jac->left2red,x,jac->leftred,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(jac->left2red,x,jac->leftred,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
*xred = jac->leftred;
}
break;
case PC_RIGHT:
if (size == 1) *xred = x;
else {
if (!jac->right2red) {ierr = VecScatterCreateToAll(x,&jac->right2red,&jac->rightred);CHKERRQ(ierr);}
if (amode & READ) {
ierr = VecScatterBegin(jac->right2red,x,jac->rightred,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(jac->right2red,x,jac->rightred,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
*xred = jac->rightred;
}
break;
default: SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_PLIB,"Side must be LEFT or RIGHT");
}
PetscFunctionReturn(0);
}
void ReplicatableVector::ReplicatePetscVector(Vec vec)
{
// If the size has changed then we'll need to make a new context
PetscInt isize;
VecGetSize(vec, &isize);
unsigned size = isize;
if (this->GetSize() != size)
{
Resize(size);
}
if (mReplicated == NULL)
{
// This creates mToAll (the scatter context) and mReplicated (to store values)
VecScatterCreateToAll(vec, &mToAll, &mReplicated);
}
// Replicate the data
//PETSc-3.x.x or PETSc-2.3.3
#if ( (PETSC_VERSION_MAJOR == 3) || (PETSC_VERSION_MAJOR == 2 && PETSC_VERSION_MINOR == 3 && PETSC_VERSION_SUBMINOR == 3)) //2.3.3 or 3.x.x
VecScatterBegin(mToAll, vec, mReplicated, INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd (mToAll, vec, mReplicated, INSERT_VALUES, SCATTER_FORWARD);
#else
//PETSc-2.3.2 or previous
VecScatterBegin(vec, mReplicated, INSERT_VALUES, SCATTER_FORWARD, mToAll);
VecScatterEnd (vec, mReplicated, INSERT_VALUES, SCATTER_FORWARD, mToAll);
#endif
// Information is now in mReplicated PETSc vector
// Copy into mData
double* p_replicated;
VecGetArray(mReplicated, &p_replicated);
for (unsigned i=0; i<size; i++)
{
mpData[i] = p_replicated[i];
}
}
int Dense3d_MPI::evaluate(Vec srcDen, Vec trgVal)
{
//begin
// CHECK
pA(_srcPos!=NULL && _srcNor!=NULL && _trgPos!=NULL && srcDen!=NULL && trgVal!=NULL);
//-----------------------------------
int dim = this->dim();
int srcDOF = this->srcDOF();
int trgDOF = this->trgDOF();
/* Get global number of source positions */
PetscInt srcGlbNum = procGlbNum(_srcPos);
/* Get local number of target positions */
PetscInt trgLclNum = procLclNum(_trgPos);
Vec srcAllPos = _srcAllPos;
Vec srcAllNor = _srcAllNor;
Vec srcAllDen;
/* Create scatter context to scatter source densities to all processors */
{
VecScatter ctx;
pC( VecScatterCreateToAll(srcDen, &ctx, &srcAllDen) );
pC( VecScatterBegin(ctx, srcDen, srcAllDen, INSERT_VALUES, SCATTER_FORWARD) );
pC( VecScatterEnd(ctx, srcDen, srcAllDen, INSERT_VALUES, SCATTER_FORWARD) );
pC( VecScatterDestroy(ctx) );
}
Vec trgLclPos = _trgPos;
Vec trgLclVal = trgVal;
/* Create matrices for source positions, normals, densities. See common/nummat.hpp for
* more information on matrices */
double* srcAllPosArr; pC( VecGetArray(srcAllPos, &srcAllPosArr) );
DblNumMat srcAllPosMat(dim, srcGlbNum, false, srcAllPosArr);
double* srcAllNorArr; pC( VecGetArray(srcAllNor, &srcAllNorArr) );
DblNumMat srcAllNorMat(dim, srcGlbNum, false, srcAllNorArr);
double* srcAllDenArr; pC( VecGetArray(srcAllDen, &srcAllDenArr) );
DblNumVec srcAllDenVec(srcDOF*srcGlbNum, false, srcAllDenArr);
/* Create matrices for target positions and values */
double* trgLclPosArr; pC( VecGetArray(trgLclPos, &trgLclPosArr) );
DblNumMat trgLclPosMat(dim, trgLclNum, false, trgLclPosArr);
double* trgLclValArr; pC( VecGetArray(trgLclVal, &trgLclValArr) );
DblNumVec trgLclValVec(trgDOF*trgLclNum, false, trgLclValArr);
/* Create an evaluation context and evaluate based on kernel type */
DblNumMat inter(trgDOF, srcGlbNum*srcDOF);
/* Do multiplication one line of the matrices at a time */
for(int i=0; i<trgLclNum; i++) {
DblNumMat onePosMat(dim, 1, false, trgLclPosMat.clmdata(i));
DblNumVec oneValVec(trgDOF, false, trgLclValVec.data()+trgDOF*i);
/* Create kernel multiplier context based on kernel type */
pC( _knl.buildKnlIntCtx(srcAllPosMat, srcAllNorMat, onePosMat, inter) );
/* Computes 1.0*inter*srcAllDenVec + 0.0*oneValVec = oneValVec */
pC( dgemv(1.0, inter, srcAllDenVec, 0.0, oneValVec) );
}
pC( VecRestoreArray(srcAllPos, &srcAllPosArr) );
pC( VecRestoreArray(srcAllNor, &srcAllNorArr) );
pC( VecRestoreArray(srcAllDen, &srcAllDenArr) );
pC( VecRestoreArray(trgLclPos, &trgLclPosArr) );
pC( VecRestoreArray(trgLclVal, &trgLclValArr) );
pC( VecDestroy(srcAllDen) );
return(0);
}
PETSC_EXTERN void PETSC_STDCALL vecscattercreatetoall_(Vec *vin,VecScatter *ctx,Vec *vout, int *ierr)
{
CHKFORTRANNULLOBJECT(vout);
*ierr = VecScatterCreateToAll(*vin,ctx,vout);
}
PetscErrorCode KSPSolve_CG(KSP ksp)
{
PetscErrorCode ierr;
PetscInt i,stored_max_it,eigs;
PetscScalar dpi = 0.0,a = 1.0,beta,betaold = 1.0,b = 0,*e = 0,*d = 0,delta,dpiold;
PetscReal dp = 0.0;
Vec X,B,Z,R,P,S,W;
KSP_CG *cg;
Mat Amat,Pmat;
PetscBool diagonalscale;
/* Dingwen */
PetscInt itv_d, itv_c;
PetscScalar CKSX1,CKSZ1,CKSR1,CKSP1,CKSS1,CKSW1;
PetscScalar CKSX2,CKSZ2,CKSR2,CKSP2,CKSS2,CKSW2;
Vec CKSAmat1;
Vec CKSAmat2;
Vec C1,C2;
PetscScalar d1,d2;
PetscScalar sumX1,sumR1;
PetscScalar sumX2,sumR2;
Vec CKPX,CKPP;
PetscScalar CKPbetaold;
PetscInt CKPi;
PetscBool flag1 = PETSC_TRUE, flag2 = PETSC_TRUE;
PetscInt pos;
PetscScalar v;
VecScatter ctx;
Vec W_SEQ;
PetscScalar *_W;
/* Dingwen */
PetscFunctionBegin;
ierr = PCGetDiagonalScale(ksp->pc,&diagonalscale);CHKERRQ(ierr);
if (diagonalscale) SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"Krylov method %s does not support diagonal scaling",((PetscObject)ksp)->type_name);
cg = (KSP_CG*)ksp->data;
eigs = ksp->calc_sings;
stored_max_it = ksp->max_it;
X = ksp->vec_sol;
B = ksp->vec_rhs;
R = ksp->work[0];
Z = ksp->work[1];
P = ksp->work[2];
/* Dingwen */
CKPX = ksp->work[3];
CKPP = ksp->work[4];
CKSAmat1 = ksp->work[5];
CKSAmat2 = ksp->work[6];
C1 = ksp->work[7];
C2 = ksp->work[8];
/* Dingwen */
/* Dingwen */
int rank; /* Get MPI variables */
MPI_Comm_rank (MPI_COMM_WORLD,&rank);
/* Dingwen */
#define VecXDot(x,y,a) (((cg->type) == (KSP_CG_HERMITIAN)) ? VecDot(x,y,a) : VecTDot(x,y,a))
if (cg->singlereduction) {
S = ksp->work[9];
W = ksp->work[10];
} else {
S = 0; /* unused */
W = Z;
}
if (eigs) {e = cg->e; d = cg->d; e[0] = 0.0; }
ierr = PCGetOperators(ksp->pc,&Amat,&Pmat);CHKERRQ(ierr);
ksp->its = 0;
if (!ksp->guess_zero) {
ierr = KSP_MatMult(ksp,Amat,X,R);CHKERRQ(ierr); /* r <- b - Ax */
ierr = VecAYPX(R,-1.0,B);CHKERRQ(ierr);
} else {
ierr = VecCopy(B,R);CHKERRQ(ierr); /* r <- b (x is 0) */
}
/* Dingwen */
/* checksum coefficients initialization */
PetscInt size;
ierr = VecGetSize(B,&size);
for (i=0; i<size; i++)
{
v = 1.0;
ierr = VecSetValues(C1,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
v = i;
ierr = VecSetValues(C2,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
}
d1 = 1.0;
d2 = 2.0;
/* Dingwen */
switch (ksp->normtype) {
case KSP_NORM_PRECONDITIONED:
ierr = KSP_PCApply(ksp,R,Z);CHKERRQ(ierr); /* z <- Br */
ierr = VecNorm(Z,NORM_2,&dp);CHKERRQ(ierr); /* dp <- z'*z = e'*A'*B'*B*A'*e' */
break;
case KSP_NORM_UNPRECONDITIONED:
ierr = VecNorm(R,NORM_2,&dp);CHKERRQ(ierr); /* dp <- r'*r = e'*A'*A*e */
break;
case KSP_NORM_NATURAL:
ierr = KSP_PCApply(ksp,R,Z);CHKERRQ(ierr); /* z <- Br */
if (cg->singlereduction) {
ierr = KSP_MatMult(ksp,Amat,Z,S);CHKERRQ(ierr);
ierr = VecXDot(Z,S,&delta);CHKERRQ(ierr);
/* Dingwen */
ierr = VecXDot(C1,S,&CKSS1);CHKERRQ(ierr); /* Compute the initial checksum1(S) */
ierr = VecXDot(C2,S,&CKSS2);CHKERRQ(ierr); /* Compute the initial checksum2(S) */
/* Dingwen */
}
ierr = VecXDot(Z,R,&beta);CHKERRQ(ierr); /* beta <- z'*r */
KSPCheckDot(ksp,beta);
dp = PetscSqrtReal(PetscAbsScalar(beta)); /* dp <- r'*z = r'*B*r = e'*A'*B*A*e */
break;
case KSP_NORM_NONE:
dp = 0.0;
break;
default: SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"%s",KSPNormTypes[ksp->normtype]);
}
ierr = KSPLogResidualHistory(ksp,dp);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,0,dp);CHKERRQ(ierr);
ksp->rnorm = dp;
ierr = (*ksp->converged)(ksp,0,dp,&ksp->reason,ksp->cnvP);CHKERRQ(ierr); /* test for convergence */
if (ksp->reason) PetscFunctionReturn(0);
if (ksp->normtype != KSP_NORM_PRECONDITIONED && (ksp->normtype != KSP_NORM_NATURAL)) {
ierr = KSP_PCApply(ksp,R,Z);CHKERRQ(ierr); /* z <- Br */
}
if (ksp->normtype != KSP_NORM_NATURAL) {
if (cg->singlereduction) {
ierr = KSP_MatMult(ksp,Amat,Z,S);CHKERRQ(ierr);
ierr = VecXDot(Z,S,&delta);CHKERRQ(ierr);
}
ierr = VecXDot(Z,R,&beta);CHKERRQ(ierr); /* beta <- z'*r */
KSPCheckDot(ksp,beta);
}
/* Dingwen */
/* Checksum Initialization */
ierr = VecXDot(C1,X,&CKSX1);CHKERRQ(ierr); /* Compute the initial checksum1(X) */
ierr = VecXDot(C1,W,&CKSW1);CHKERRQ(ierr); /* Compute the initial checksum1(W) */
ierr = VecXDot(C1,R,&CKSR1);CHKERRQ(ierr); /* Compute the initial checksum1(R) */
ierr = VecXDot(C1,Z,&CKSZ1);CHKERRQ(ierr); /* Compute the initial checksum1(Z) */
ierr = VecXDot(C2,X,&CKSX2);CHKERRQ(ierr); /* Compute the initial checksum2(X) */
ierr = VecXDot(C2,W,&CKSW2);CHKERRQ(ierr); /* Compute the initial checksum2(W) */
ierr = VecXDot(C2,R,&CKSR2);CHKERRQ(ierr); /* Compute the initial checksum2(R) */
ierr = VecXDot(C2,Z,&CKSZ2);CHKERRQ(ierr); /* Compute the initial checksum2(Z) */
ierr = KSP_MatMultTranspose(ksp,Amat,C1,CKSAmat1);CHKERRQ(ierr);
ierr = VecAXPY(CKSAmat1,-d1,C1);CHKERRQ(ierr);
ierr = VecAXPY(CKSAmat1,-d2,C2);CHKERRQ(ierr); /* Compute the initial checksum1(A) */
ierr = KSP_MatMultTranspose(ksp,Amat,C2,CKSAmat2);CHKERRQ(ierr);
ierr = VecAXPY(CKSAmat2,-d2,C1);CHKERRQ(ierr);
ierr = VecAXPY(CKSAmat2,-d1,C2);CHKERRQ(ierr); /* Compute the initial checksum2(A) */
itv_c = 2;
itv_d = 10;
/* Dingwen */
i = 0;
do {
/* Dingwen */
if ((i>0) && (i%itv_d == 0))
{
ierr = VecXDot(C1,X,&sumX1);CHKERRQ(ierr);
ierr = VecXDot(C1,R,&sumR1);CHKERRQ(ierr);
if ((PetscAbsScalar(sumX1-CKSX1) > 1.0e-6) || (PetscAbsScalar(sumR1-CKSR1) > 1.0e-6))
{
/* Rollback and Recovery */
if (rank==0) printf ("Recovery start...\n");
if (rank==0) printf ("Rollback from iteration-%d to iteration-%d\n",i,CKPi);
betaold = CKPbetaold; /* Recovery scalar betaold by checkpoint*/
i = CKPi; /* Recovery integer i by checkpoint */
ierr = VecCopy(CKPP,P);CHKERRQ(ierr); /* Recovery vector P from checkpoint */
ierr = VecXDot(C1,P,&CKSP1);CHKERRQ(ierr); /* Recovery checksum1(P) by P */
ierr = VecXDot(C2,P,&CKSP2);CHKERRQ(ierr); /* Recovery checksum2(P) by P */
ierr = KSP_MatMult(ksp,Amat,P,W);CHKERRQ(ierr); /* Recovery vector W by P */
ierr = VecXDot(P,W,&dpi);CHKERRQ(ierr); /* Recovery scalar dpi by P and W */
ierr = VecCopy(CKPX,X);CHKERRQ(ierr); /* Recovery vector X from checkpoint */
ierr = VecXDot(C1,X,&CKSX1);CHKERRQ(ierr); /* Recovery checksum1(X) by X */
ierr = VecXDot(C2,X,&CKSX2);CHKERRQ(ierr); /* Recovery checksum2(X) by X */
ierr = KSP_MatMult(ksp,Amat,X,R);CHKERRQ(ierr); /* Recovery vector R by X */
ierr = VecAYPX(R,-1.0,B);CHKERRQ(ierr);
ierr = VecXDot(C1,R,&CKSR1);CHKERRQ(ierr); /* Recovery checksum1(R) by R */
ierr = VecXDot(C2,R,&CKSR2);CHKERRQ(ierr); /* Recovery checksum2(R) by R */
ierr = KSP_PCApply(ksp,R,Z);CHKERRQ(ierr); /* Recovery vector Z by R */
ierr = VecXDot(C1,Z,&CKSZ1);CHKERRQ(ierr); /* Recovery checksum1(Z) by Z */
ierr = VecXDot(C2,Z,&CKSZ2);CHKERRQ(ierr); /* Recovery checksum2(Z) by Z */
ierr = VecXDot(Z,R,&beta);CHKERRQ(ierr); /* Recovery scalar beta by Z and R */
if (rank==0) printf ("Recovery end.\n");
}
else if (i%(itv_c*itv_d) == 0)
{
if (rank==0) printf ("Checkpoint iteration-%d\n",i);
ierr = VecCopy(X,CKPX);CHKERRQ(ierr);
ierr = VecCopy(P,CKPP);CHKERRQ(ierr);
CKPbetaold = betaold;
CKPi = i;
}
}
ksp->its = i+1;
if (beta == 0.0) {
ksp->reason = KSP_CONVERGED_ATOL;
ierr = PetscInfo(ksp,"converged due to beta = 0\n");CHKERRQ(ierr);
break;
#if !defined(PETSC_USE_COMPLEX)
} else if ((i > 0) && (beta*betaold < 0.0)) {
ksp->reason = KSP_DIVERGED_INDEFINITE_PC;
ierr = PetscInfo(ksp,"diverging due to indefinite preconditioner\n");CHKERRQ(ierr);
break;
#endif
}
if (!i) {
ierr = VecCopy(Z,P);CHKERRQ(ierr); /* p <- z */
b = 0.0;
/* Dingwen */
ierr = VecXDot(C1,P, &CKSP1);CHKERRQ(ierr); /* Compute the initial checksum1(P) */
ierr = VecXDot(C2,P, &CKSP2);CHKERRQ(ierr); /* Compute the initial checksum2(P) */
/* Dingwen */
} else {
b = beta/betaold;
if (eigs) {
if (ksp->max_it != stored_max_it) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"Can not change maxit AND calculate eigenvalues");
e[i] = PetscSqrtReal(PetscAbsScalar(b))/a;
}
ierr = VecAYPX(P,b,Z);CHKERRQ(ierr); /* p <- z + b* p */
/* Dingwen */
CKSP1 = CKSZ1 + b*CKSP1; /* Update checksum1(P) = checksum1(Z) + b*checksum1(P); */
CKSP2 = CKSZ2 + b*CKSP2; /* Update checksum2(P) = checksum2(Z) + b*checksum2(P); */
/* Dingwen */
}
dpiold = dpi;
if (!cg->singlereduction || !i) {
ierr = KSP_MatMult(ksp,Amat,P,W);CHKERRQ(ierr); /* w <- Ap */ /* MVM */
ierr = VecXDot(P,W,&dpi);CHKERRQ(ierr); /* dpi <- p'w */
/* Dingwen */
ierr = VecXDot(CKSAmat1, P, &CKSW1);CHKERRQ(ierr);
CKSW1 = CKSW1 + d1*CKSP1 + d2*CKSP2; /* Update checksum1(W) = checksum1(A)P + d1*checksum1(P) + d2*checksum2(P); */
ierr = VecXDot(CKSAmat2, P, &CKSW2);CHKERRQ(ierr);
CKSW2 = CKSW2 + d2*CKSP1 + d1*CKSP2; /* Update checksum2(W) = checksum2(A)P + d2*checksum1(P) + d1*checksum2(P); */
if((i==41)&&(flag2))
{
pos = 100;
v = 1000;
ierr = VecSetValue(W,pos,v,INSERT_VALUES);CHKERRQ(ierr);
VecAssemblyBegin(W);
VecAssemblyEnd(W);
if (rank==0) printf ("Inject an error in %d-th element of vector W after MVM W=AP at iteration-%d\n", pos,i);
flag2 = PETSC_FALSE;
}
PetscScalar delta1,delta2;
PetscScalar sumW1,sumW2;
ierr = VecXDot(C1,W,&sumW1);CHKERRQ(ierr);
ierr = VecXDot(C2,W,&sumW2);CHKERRQ(ierr);
delta1 = sumW1 - CKSW1;
delta2 = sumW2 - CKSW2;
if (PetscAbsScalar(delta1) > 1.0e-6)
{
VecScatterCreateToAll(W,&ctx,&W_SEQ);
VecScatterBegin(ctx,W,W_SEQ,INSERT_VALUES,SCATTER_FORWARD);
VecScatterEnd(ctx,W,W_SEQ,INSERT_VALUES,SCATTER_FORWARD);
VecGetArray(W_SEQ,&_W);
pos = rint(delta2/delta1);
v = _W[pos];
v = v - delta1;
ierr = VecSetValues(W,1,&pos,&v,INSERT_VALUES);CHKERRQ(ierr);
if (rank==0) printf ("Correct an error of %d-th elements of vector W after MVM W=AP at iteration-%d\n", pos, i);
}
} else {
ierr = VecAYPX(W,beta/betaold,S);CHKERRQ(ierr); /* w <- Ap */
dpi = delta - beta*beta*dpiold/(betaold*betaold); /* dpi <- p'w */
/* Dingwen */
CKSW1 = beta/betaold*CKSW1 + CKSS1; /* Update checksum1(W) = checksum1(S) + beta/betaold*checksum1(W); */
CKSW2 = beta/betaold*CKSW2 + CKSS2; /* Update checksum2(W) = checksum2(S) + beta/betaold*checksum2(W); */
/* Dingwen */
}
betaold = beta;
KSPCheckDot(ksp,beta);
if ((dpi == 0.0) || ((i > 0) && (PetscRealPart(dpi*dpiold) <= 0.0))) {
ksp->reason = KSP_DIVERGED_INDEFINITE_MAT;
ierr = PetscInfo(ksp,"diverging due to indefinite or negative definite matrix\n");CHKERRQ(ierr);
break;
}
a = beta/dpi; /* a = beta/p'w */
if (eigs) d[i] = PetscSqrtReal(PetscAbsScalar(b))*e[i] + 1.0/a;
ierr = VecAXPY(X,a,P);CHKERRQ(ierr); /* x <- x + ap */
/* Dingwen */
CKSX1 = CKSX1 + a*CKSP1; /* Update checksum1(X) = checksum1(X) + a*checksum1(P); */
CKSX2 = CKSX2 + a*CKSP2; /* Update checksum2(X) = checksum2(X) + a*checksum2(P); */
/* Dingwen */
ierr = VecAXPY(R,-a,W);CHKERRQ(ierr); /* r <- r - aw */
/* Dingwen */
CKSR1 = CKSR1 - a*CKSW1; /* Update checksum1(R) = checksum1(R) - a*checksum1(W); */
CKSR2 = CKSR2 - a*CKSW2; /* Update checksum2(R) = checksum2(R) - a*checksum2(W); */
/* Dingwen */
if (ksp->normtype == KSP_NORM_PRECONDITIONED && ksp->chknorm < i+2) {
ierr = KSP_PCApply(ksp,R,Z);CHKERRQ(ierr); /* z <- Br */
/* Dingwen */
ierr = VecXDot(C1,Z, &CKSZ1);CHKERRQ(ierr); /* Update checksum1(Z) */
ierr = VecXDot(C2,Z, &CKSZ2);CHKERRQ(ierr); /* Update checksum2(Z) */
/* Dingwen */
if (cg->singlereduction) {
ierr = KSP_MatMult(ksp,Amat,Z,S);CHKERRQ(ierr); /* MVM */
/* Dingwen */
ierr = VecXDot(CKSAmat1, Z, &CKSS1);CHKERRQ(ierr);
CKSS1 = CKSS1 + d1*CKSZ1 + d2*CKSZ2; /* Update checksum1(S) = checksum1(A)Z + d1*chekcsum1(Z) + d2*checksum2(Z); */
ierr = VecXDot(CKSAmat2, Z, &CKSS2);CHKERRQ(ierr);
CKSS2 = CKSS2 + d2*CKSZ1 + d1*CKSZ2; /* Update checksum2(S) = checksum2(A)Z + d2*chekcsum1(Z) + d1*checksum2(Z); */
/* Dingwen */
}
ierr = VecNorm(Z,NORM_2,&dp);CHKERRQ(ierr); /* dp <- z'*z */
} else if (ksp->normtype == KSP_NORM_UNPRECONDITIONED && ksp->chknorm < i+2) {
ierr = VecNorm(R,NORM_2,&dp);CHKERRQ(ierr); /* dp <- r'*r */
} else if (ksp->normtype == KSP_NORM_NATURAL) {
ierr = KSP_PCApply(ksp,R,Z);CHKERRQ(ierr); /* z <- Br */
/* Dingwen */
ierr = VecXDot(C1,Z, &CKSZ1);CHKERRQ(ierr); /* Update checksum1(Z) */
ierr = VecXDot(C2,Z, &CKSZ2);CHKERRQ(ierr); /* Update checksum2(Z) */
/* Dingwen */
if (cg->singlereduction) {
PetscScalar tmp[2];
Vec vecs[2];
vecs[0] = S; vecs[1] = R;
ierr = KSP_MatMult(ksp,Amat,Z,S);CHKERRQ(ierr);
ierr = VecMDot(Z,2,vecs,tmp);CHKERRQ(ierr);
delta = tmp[0]; beta = tmp[1];
} else {
ierr = VecXDot(Z,R,&beta);CHKERRQ(ierr); /* beta <- r'*z */
}
KSPCheckDot(ksp,beta);
dp = PetscSqrtReal(PetscAbsScalar(beta));
} else {
dp = 0.0;
}
ksp->rnorm = dp;
CHKERRQ(ierr);KSPLogResidualHistory(ksp,dp);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,i+1,dp);CHKERRQ(ierr);
ierr = (*ksp->converged)(ksp,i+1,dp,&ksp->reason,ksp->cnvP);CHKERRQ(ierr);
if (ksp->reason) break;
if ((ksp->normtype != KSP_NORM_PRECONDITIONED && (ksp->normtype != KSP_NORM_NATURAL)) || (ksp->chknorm >= i+2)) {
ierr = KSP_PCApply(ksp,R,Z);CHKERRQ(ierr); /* z <- Br */
/* Dingwen */
ierr = VecXDot(C1,Z, &CKSZ1);CHKERRQ(ierr); /* Update checksum1(Z) */
ierr = VecXDot(C2,Z, &CKSZ2);CHKERRQ(ierr); /* Update checksum2(Z) */
/* Dingwen */
if (cg->singlereduction) {
ierr = KSP_MatMult(ksp,Amat,Z,S);CHKERRQ(ierr);
}
}
if ((ksp->normtype != KSP_NORM_NATURAL) || (ksp->chknorm >= i+2)) {
if (cg->singlereduction) {
PetscScalar tmp[2];
Vec vecs[2];
vecs[0] = S; vecs[1] = R;
ierr = VecMDot(Z,2,vecs,tmp);CHKERRQ(ierr);
delta = tmp[0]; beta = tmp[1];
} else {
ierr = VecXDot(Z,R,&beta);CHKERRQ(ierr); /* beta <- z'*r */
}
KSPCheckDot(ksp,beta);
}
i++;
/* Dingwen */
/* Inject error */
if ((i==50)&&(flag1))
{
pos = 1000;
v = -1;
ierr = VecSetValues(X,1,&pos,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
flag1 = PETSC_FALSE;
if (rank==0)printf ("Inject an error in vector X at the end of iteration-%d\n", i-1);
}
/* Dingwen */
} while (i<ksp->max_it);
/* Dingwen */
ierr = VecXDot(C1,X,&sumX1);CHKERRQ(ierr);
ierr = VecXDot(C1,R,&sumR1);CHKERRQ(ierr);
ierr = VecXDot(C2,X,&sumX2);CHKERRQ(ierr);
ierr = VecXDot(C2,R,&sumR2);CHKERRQ(ierr);
if (rank==0)
{
printf ("sum1 of X = %f\n", sumX1);
printf ("checksum1(X) = %f\n", CKSX1);
printf ("sum2 of X = %f\n", sumX2);
printf ("checksum2(X) = %f\n", CKSX2);
printf ("sum1 of R = %f\n", sumR1);
printf ("checksum1(R) = %f\n", CKSR1);
printf ("sum2 of R = %f\n", sumR2);
printf ("checksum2(R) = %f\n", CKSR2);
}
VecDestroy(&W_SEQ);
VecScatterDestroy(&ctx);
/* Dingwen */
if (i >= ksp->max_it) ksp->reason = KSP_DIVERGED_ITS;
if (eigs) cg->ned = ksp->its;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscInt n = 3,i,len,start,end;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscScalar value,*yy;
Vec x,y,z,y_t;
VecScatter toall,tozero;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* create two vectors */
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE,size*n,&x);CHKERRQ(ierr);
/* each processor inserts its values */
ierr = VecGetOwnershipRange(x,&start,&end);CHKERRQ(ierr);
for (i=start; i<end; i++) {
value = (PetscScalar) i;
ierr = VecSetValues(x,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecScatterCreateToAll(x,&toall,&y);CHKERRQ(ierr);
ierr = VecScatterBegin(toall,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(toall,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(toall);CHKERRQ(ierr);
/* Cannot view the above vector with VecView(), so place it in an MPI Vec
and do a VecView() */
ierr = VecGetArray(y,&yy);CHKERRQ(ierr);
ierr = VecGetLocalSize(y,&len);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,len,PETSC_DECIDE,yy,&y_t);CHKERRQ(ierr);
ierr = VecView(y_t,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(y_t);
ierr = VecRestoreArray(y,&yy);CHKERRQ(ierr);
ierr = VecScatterCreateToAll(x,&tozero,&z);CHKERRQ(ierr);
ierr = VecScatterBegin(tozero,x,z,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(tozero,x,z,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(tozero);CHKERRQ(ierr);
if (!rank) {
ierr = VecView(z,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = VecDestroy(z);CHKERRQ(ierr);
ierr = VecScatterCreateToZero(x,&tozero,&z);CHKERRQ(ierr);
ierr = VecScatterBegin(tozero,x,z,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(tozero,x,z,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(tozero);CHKERRQ(ierr);
ierr = VecDestroy(z);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(y);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
void PETSC_STDCALL vecscattercreatetoall_(Vec vin,VecScatter *ctx,Vec *vout, int *__ierr ){
*__ierr = VecScatterCreateToAll(
(Vec)PetscToPointer((vin) ),ctx,vout);
}
void coulomb_trick(PetscErrorCode ierr, params *params)
{
size_t neval;
PetscReal a,b,abserr,coul_result;
PetscReal mf2,mu2,th2;
PetscInt start,end;
PetscInt rank;
PetscReal *_mu;
PetscReal *_th;
Vec MU_SEQ;
Vec TH_SEQ;
VecScatter ctm;
VecScatter ctt;
int_params int_params;
ierr = PetscPrintf(PETSC_COMM_WORLD,"---Beginning Coulomb Trick---\n");CHKERRV(ierr);
print_progress_header(ierr);
int_params.epsabs=1e-4;
int_params.epsrel=1e-8;
int_params.N_work=4000;
int_params.flag_asy=params->flag_asy;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
for(PetscInt i=0;i<params->nf;i++)
{
int_params.p_bohr.push_back(params->p_bohr[i]);
}
gsl_function F;
F.function = &ct_integrand;
F.params=&int_params;
ierr = VecScatterCreateToAll(params->mu,&ctm,&MU_SEQ);CHKERRV(ierr);
ierr = VecScatterBegin(ctm,params->mu,MU_SEQ,INSERT_VALUES,SCATTER_FORWARD);CHKERRV(ierr);
ierr = VecScatterEnd(ctm,params->mu,MU_SEQ,INSERT_VALUES,SCATTER_FORWARD);CHKERRV(ierr);
ierr = VecGetArray(MU_SEQ,&_mu);CHKERRV(ierr);
ierr = VecScatterCreateToAll(params->theta,&ctt,&TH_SEQ);CHKERRV(ierr);
ierr = VecScatterBegin(ctt,params->theta,TH_SEQ,INSERT_VALUES,SCATTER_FORWARD);CHKERRV(ierr);
ierr = VecScatterEnd(ctt,params->theta,TH_SEQ,INSERT_VALUES,SCATTER_FORWARD);CHKERRV(ierr);
ierr = VecGetArray(TH_SEQ,&_th);CHKERRV(ierr);
ierr = VecGetOwnershipRange(params->CT,&start,&end);CHKERRV(ierr);
PetscInt step;
step=1;
if(params->Jz==0){step=2;start=start+(start%2);}
gsl_integration_cquad_workspace *work = gsl_integration_cquad_workspace_alloc(int_params.N_work);
for (PetscInt i=start; i<end; i+=step)
{
//if((i-start)%((end-start)/10)==0){ierr = PetscPrintf(PETSC_COMM_SELF,"----Cc: r%d -> %d\\%d\n",rank,(i-start),(end-start));CHKERRV(ierr);}
// if(params->Jz !=0 || i%2==0){
get_index(i,&int_params,params);
int_params.mu2 = _mu[params->nm_tot[int_params.index_f]+int_params.index_m];
int_params.th2 = _th[params->nt_tot[int_params.index_f]+int_params.index_t];
// ierr = PetscPrintf(PETSC_COMM_SELF,"Pts: %d %d %f %f\n",int_params.index_m,int_params.index_t,int_params.mu2,int_params.th2);CHKERRV(ierr);
if(int_params.index_s<2){int_params.Jzc = params->Jz;}
else{int_params.Jzc= -params->Jz;}
// if (int_params.Jzc !=0 || int_params.index_s%2==0)
// {
int_params.mf = params->m[int_params.index_f];
mf2=int_params.mf*int_params.mf;
mu2=int_params.mu2;
th2=int_params.th2;
int_params.x2 = (1.0+mu2*th2/sqrt(mf2+mu2*mu2))/2.0;
int_params.k2 = mu2*sqrt(1.0-th2*th2);
a = 0.0;
b = params->lambda[int_params.index_f]/2.0;
// if (int_params.Jzc !=0 || int_params.index_s%2==0)
// {
// gsl_integration_cquad_workspace *work = gsl_integration_cquad_workspace_alloc(int_params.N_work);
gsl_integration_cquad(&F,a,b,int_params.epsabs,int_params.epsrel,work,&coul_result,&abserr,&neval);
// gsl_integration_cquad_workspace_free(work);
ierr = VecSetValues(params->CT,1,&i,&coul_result,ADD_VALUES);CHKERRV(ierr);
if (params->Jz==0)
{
PetscInt ishift;
ishift=i+(3-2*int_params.index_s);
ierr = VecSetValues(params->CT,1,&ishift,&coul_result,ADD_VALUES);CHKERRV(ierr);
}
// }
// ierr = PetscPrintf(PETSC_COMM_SELF,"Pts: %d %d %f %f %f\n",int_params.index_m,int_params.index_t,int_params.mu2,int_params.th2,coul_result);CHKERRV(ierr);
if((i-start)%((end-start)/4)==0){ierr = PetscPrintf(PETSC_COMM_SELF,"|");CHKERRV(ierr);}
}
gsl_integration_cquad_workspace_free(work);
ierr = VecRestoreArray(MU_SEQ,&_mu);CHKERRV(ierr);
ierr = VecScatterDestroy(&ctm);CHKERRV(ierr);
ierr = VecDestroy(&MU_SEQ);CHKERRV(ierr);
ierr = VecRestoreArray(TH_SEQ,&_th);CHKERRV(ierr);
ierr = VecScatterDestroy(&ctt);CHKERRV(ierr);
ierr = VecDestroy(&TH_SEQ);CHKERRV(ierr);
ierr = VecAssemblyBegin(params->CT);CHKERRV(ierr);
ierr = VecAssemblyEnd(params->CT);CHKERRV(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n---Finishing Coulomb Trick---\n");CHKERRV(ierr);
}
