static PetscErrorCode PCApply_Redundant(PC pc,Vec x,Vec y)
{
PC_Redundant *red = (PC_Redundant*)pc->data;
PetscErrorCode ierr;
PetscScalar *array;
PetscFunctionBegin;
/* scatter x to xdup */
ierr = VecScatterBegin(red->scatterin,x,red->xdup,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(red->scatterin,x,red->xdup,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* place xdup's local array into xsub */
ierr = VecGetArray(red->xdup,&array);CHKERRQ(ierr);
ierr = VecPlaceArray(red->xsub,(const PetscScalar*)array);CHKERRQ(ierr);
/* apply preconditioner on each processor */
ierr = PCApply(red->pc,red->xsub,red->ysub);CHKERRQ(ierr);
ierr = VecResetArray(red->xsub);CHKERRQ(ierr);
ierr = VecRestoreArray(red->xdup,&array);CHKERRQ(ierr);
/* place ysub's local array into ydup */
ierr = VecGetArray(red->ysub,&array);CHKERRQ(ierr);
ierr = VecPlaceArray(red->ydup,(const PetscScalar*)array);CHKERRQ(ierr);
/* scatter ydup to y */
ierr = VecScatterBegin(red->scatterout,red->ydup,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(red->scatterout,red->ydup,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecResetArray(red->ydup);CHKERRQ(ierr);
ierr = VecRestoreArray(red->ysub,&array);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode IPMGatherRHS(Tao tao,Vec RHS,Vec X1,Vec X2,Vec X3,Vec X4)
{
TAO_IPM *ipmP = (TAO_IPM *)tao->data;
PetscErrorCode ierr;
PetscFunctionBegin;
/* rhs = [x1 (n)
x2 (me)
x3 (nb)
x4 (nb)] */
if (X1) {
ierr = VecScatterBegin(ipmP->rhs1,X1,RHS,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ipmP->rhs1,X1,RHS,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
if (ipmP->me > 0 && X2) {
ierr = VecScatterBegin(ipmP->rhs2,X2,RHS,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ipmP->rhs2,X2,RHS,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
if (ipmP->nb > 0) {
if (X3) {
ierr = VecScatterBegin(ipmP->rhs3,X3,RHS,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ipmP->rhs3,X3,RHS,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
if (X4) {
ierr = VecScatterBegin(ipmP->rhs4,X4,RHS,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ipmP->rhs4,X4,RHS,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
int Dense3d_MPI::setup()
{
//begin
pA(_srcPos!=NULL && _srcNor!=NULL && _trgPos!=NULL);
//--------------------------------------------------------------------------
/* Create a Scatter context - copies all source position values to each processor
* See http://www-unix.mcs.anl.gov/petsc/petsc-as/snapshots/petsc-current/docs/manualpages/Vec/VecScatterCreateToAll.html */
{
VecScatter ctx;
pC( VecScatterCreateToAll(_srcPos, &ctx, &_srcAllPos) );
// VecScatterBegin(VecScatter inctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
pC( VecScatterBegin(ctx, _srcPos, _srcAllPos, INSERT_VALUES, SCATTER_FORWARD) );
pC( VecScatterEnd(ctx, _srcPos, _srcAllPos, INSERT_VALUES, SCATTER_FORWARD ) );
pC( VecScatterDestroy(ctx) );
}
/* Create a Scatter context - copies all source normal values to each processor */
{
VecScatter ctx;
pC( VecScatterCreateToAll(_srcNor, &ctx, &_srcAllNor) );
pC( VecScatterBegin(ctx, _srcNor, _srcAllNor, INSERT_VALUES, SCATTER_FORWARD) );
pC( VecScatterEnd(ctx, _srcNor, _srcAllNor, INSERT_VALUES, SCATTER_FORWARD) );
pC( VecScatterDestroy(ctx) );
}
return(0);
}
PetscErrorCode IPMScatterStep(Tao tao, Vec STEP, Vec X1, Vec X2, Vec X3, Vec X4)
{
TAO_IPM *ipmP = (TAO_IPM *)tao->data;
PetscErrorCode ierr;
PetscFunctionBegin;
CHKMEMQ;
/* [x1 (n)
x2 (nb) may be 0
x3 (me) may be 0
x4 (nb) may be 0 */
if (X1) {
ierr = VecScatterBegin(ipmP->step1,STEP,X1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ipmP->step1,STEP,X1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
if (X2 && ipmP->nb > 0) {
ierr = VecScatterBegin(ipmP->step2,STEP,X2,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ipmP->step2,STEP,X2,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
if (X3 && ipmP->me > 0) {
ierr = VecScatterBegin(ipmP->step3,STEP,X3,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ipmP->step3,STEP,X3,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
if (X4 && ipmP->nb > 0) {
ierr = VecScatterBegin(ipmP->step4,STEP,X4,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ipmP->step4,STEP,X4,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
CHKMEMQ;
PetscFunctionReturn(0);
}
static PetscErrorCode PCSVDRestoreVec(PC pc,PCSide side,AccessMode amode,Vec x,Vec *xred)
{
PC_SVD *jac = (PC_SVD*)pc->data;
PetscErrorCode ierr;
PetscMPIInt size;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)pc),&size);CHKERRQ(ierr);
switch (side) {
case PC_LEFT:
if (size != 1 && amode & WRITE) {
ierr = VecScatterBegin(jac->left2red,jac->leftred,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(jac->left2red,jac->leftred,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
break;
case PC_RIGHT:
if (size != 1 && amode & WRITE) {
ierr = VecScatterBegin(jac->right2red,jac->rightred,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(jac->right2red,jac->rightred,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
break;
default: SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_PLIB,"Side must be LEFT or RIGHT");
}
*xred = NULL;
PetscFunctionReturn(0);
}
void PetscVector<T>::localize (NumericVector<T>& v_local_in) const
{
this->_restore_array();
// Make sure the NumericVector passed in is really a PetscVector
PetscVector<T>* v_local = libmesh_cast_ptr<PetscVector<T>*>(&v_local_in);
libmesh_assert(v_local);
libmesh_assert_equal_to (v_local->size(), this->size());
PetscErrorCode ierr = 0;
const PetscInt n = this->size();
IS is;
VecScatter scatter;
// Create idx, idx[i] = i;
std::vector<PetscInt> idx(n); Utility::iota (idx.begin(), idx.end(), 0);
// Create the index set & scatter object
ierr = ISCreateLibMesh(libMesh::COMM_WORLD, n, &idx[0], PETSC_USE_POINTER, &is);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterCreate(_vec, is,
v_local->_vec, is,
&scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Perform the scatter
#if PETSC_VERSION_LESS_THAN(2,3,3)
ierr = VecScatterBegin(_vec, v_local->_vec, INSERT_VALUES,
SCATTER_FORWARD, scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd (_vec, v_local->_vec, INSERT_VALUES,
SCATTER_FORWARD, scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
#else
// API argument order change in PETSc 2.3.3
ierr = VecScatterBegin(scatter, _vec, v_local->_vec,
INSERT_VALUES, SCATTER_FORWARD);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = VecScatterEnd (scatter, _vec, v_local->_vec,
INSERT_VALUES, SCATTER_FORWARD);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
#endif
// Clean up
ierr = LibMeshISDestroy (&is);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
ierr = LibMeshVecScatterDestroy(&scatter);
CHKERRABORT(libMesh::COMM_WORLD,ierr);
// Make sure ghost dofs are up to date
if (v_local->type() == GHOSTED)
v_local->close();
}
static PetscErrorCode PCApply_Redistribute(PC pc,Vec b,Vec x)
{
PC_Redistribute *red = (PC_Redistribute*)pc->data;
PetscErrorCode ierr;
PetscInt dcnt = red->dcnt,i;
const PetscInt *drows = red->drows;
PetscScalar *xwork;
const PetscScalar *bwork,*diag = red->diag;
PetscFunctionBegin;
if (!red->work) {
ierr = VecDuplicate(b,&red->work);CHKERRQ(ierr);
}
/* compute the rows of solution that have diagonal entries only */
ierr = VecSet(x,0.0);CHKERRQ(ierr); /* x = diag(A)^{-1} b */
ierr = VecGetArray(x,&xwork);CHKERRQ(ierr);
ierr = VecGetArrayRead(b,&bwork);CHKERRQ(ierr);
for (i=0; i<dcnt; i++) {
xwork[drows[i]] = diag[i]*bwork[drows[i]];
}
ierr = PetscLogFlops(dcnt);CHKERRQ(ierr);
ierr = VecRestoreArray(red->work,&xwork);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(b,&bwork);CHKERRQ(ierr);
/* update the right hand side for the reduced system with diagonal rows (and corresponding columns) removed */
ierr = MatMult(pc->pmat,x,red->work);CHKERRQ(ierr);
ierr = VecAYPX(red->work,-1.0,b);CHKERRQ(ierr); /* red->work = b - A x */
ierr = VecScatterBegin(red->scatter,red->work,red->b,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(red->scatter,red->work,red->b,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = KSPSolve(red->ksp,red->b,red->x);CHKERRQ(ierr);
ierr = VecScatterBegin(red->scatter,red->x,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(red->scatter,red->x,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode MatMultTransposeAdd_SubMatrix(Mat N,Vec v1,Vec v2,Vec v3)
{
Mat_SubMatrix *Na = (Mat_SubMatrix*)N->data;
Vec xx = 0;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PreScaleLeft(N,v1,&xx);CHKERRQ(ierr);
ierr = VecZeroEntries(Na->lwork);CHKERRQ(ierr);
ierr = VecScatterBegin(Na->lrestrict,xx,Na->lwork,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (Na->lrestrict,xx,Na->lwork,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = MatMultTranspose(Na->A,Na->lwork,Na->rwork);CHKERRQ(ierr);
if (v2 == v3) {
if (Na->scale == (PetscScalar)1.0 && !Na->right) {
ierr = VecScatterBegin(Na->rprolong,Na->rwork,v3,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (Na->rprolong,Na->rwork,v3,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
} else {
if (!Na->orwork) {ierr = VecDuplicate(v3,&Na->orwork);CHKERRQ(ierr);}
ierr = VecScatterBegin(Na->rprolong,Na->rwork,Na->orwork,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (Na->rprolong,Na->rwork,Na->orwork,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = PostScaleRight(N,Na->orwork);CHKERRQ(ierr);
ierr = VecAXPY(v3,Na->scale,Na->orwork);CHKERRQ(ierr);
}
} else {
ierr = VecScatterBegin(Na->rprolong,Na->rwork,v3,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (Na->rprolong,Na->rwork,v3,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = PostScaleRight(N,v3);CHKERRQ(ierr);
ierr = VecAYPX(v3,Na->scale,v2);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*
Gather right-hand-side.
Call for Solve step.
Scatter solution.
*/
PetscErrorCode MatSolve_PaStiX(Mat A,Vec b,Vec x)
{
Mat_Pastix *lu=(Mat_Pastix*)A->spptr;
PetscScalar *array;
Vec x_seq;
PetscErrorCode ierr;
PetscFunctionBegin;
lu->rhsnbr = 1;
x_seq = lu->b_seq;
if (lu->commSize > 1) {
/* PaStiX only supports centralized rhs. Scatter b into a seqential rhs vector */
ierr = VecScatterBegin(lu->scat_rhs,b,x_seq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(lu->scat_rhs,b,x_seq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(x_seq,&array);CHKERRQ(ierr);
} else { /* size == 1 */
ierr = VecCopy(b,x);CHKERRQ(ierr);
ierr = VecGetArray(x,&array);CHKERRQ(ierr);
}
lu->rhs = array;
if (lu->commSize == 1) {
ierr = VecRestoreArray(x,&array);CHKERRQ(ierr);
} else {
ierr = VecRestoreArray(x_seq,&array);CHKERRQ(ierr);
}
/* solve phase */
/*-------------*/
lu->iparm[IPARM_START_TASK] = API_TASK_SOLVE;
lu->iparm[IPARM_END_TASK] = API_TASK_REFINE;
lu->iparm[IPARM_RHS_MAKING] = API_RHS_B;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in solve phase: lu->iparm[IPARM_ERROR_NUMBER] = %d\n",lu->iparm[IPARM_ERROR_NUMBER]);
if (lu->commSize == 1) {
ierr = VecRestoreArray(x,&(lu->rhs));CHKERRQ(ierr);
} else {
ierr = VecRestoreArray(x_seq,&(lu->rhs));CHKERRQ(ierr);
}
if (lu->commSize > 1) { /* convert PaStiX centralized solution to petsc mpi x */
ierr = VecScatterBegin(lu->scat_sol,x_seq,x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(lu->scat_sol,x_seq,x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*
FormFunction1 - Evaluates nonlinear function, F(x).
Input Parameters:
. snes - the SNES context
. x - input vector
. ctx - optional user-defined context
Output Parameter:
. f - function vector
*/
PetscErrorCode FormFunction1(SNES snes,Vec x,Vec f,void *ctx)
{
PetscErrorCode ierr;
const PetscScalar *xx;
PetscScalar *ff;
AppCtx *user = (AppCtx*)ctx;
Vec xloc=user->xloc,floc=user->rloc;
VecScatter scatter=user->scatter;
MPI_Comm comm;
PetscMPIInt size,rank;
PetscInt rstart,rend;
ierr = PetscObjectGetComm((PetscObject)snes,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
if (size > 1){
/*
This is a ridiculous case for testing intermidiate steps from sequential
code development to parallel implementation.
(1) scatter x into a sequetial vector;
(2) each process evaluates all values of floc;
(3) scatter floc back to the parallel f.
*/
ierr = VecScatterBegin(scatter,x,xloc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,x,xloc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(f,&rstart,&rend);CHKERRQ(ierr);
ierr = VecGetArrayRead(xloc,&xx);CHKERRQ(ierr);
ierr = VecGetArray(floc,&ff);CHKERRQ(ierr);
ff[0] = xx[0]*xx[0] + xx[0]*xx[1] - 3.0;
ff[1] = xx[0]*xx[1] + xx[1]*xx[1] - 6.0;
ierr = VecRestoreArray(floc,&ff);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(xloc,&xx);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,floc,f,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,floc,f,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
} else {
/*
Get pointers to vector data.
- For default PETSc vectors, VecGetArray() returns a pointer to
the data array. Otherwise, the routine is implementation dependent.
- You MUST call VecRestoreArray() when you no longer need access to
the array.
*/
ierr = VecGetArrayRead(x,&xx);CHKERRQ(ierr);
ierr = VecGetArray(f,&ff);CHKERRQ(ierr);
/* Compute function */
ff[0] = xx[0]*xx[0] + xx[0]*xx[1] - 3.0;
ff[1] = xx[0]*xx[1] + xx[1]*xx[1] - 6.0;
/* Restore vectors */
ierr = VecRestoreArrayRead(x,&xx);CHKERRQ(ierr);
ierr = VecRestoreArray(f,&ff);CHKERRQ(ierr);
}
return 0;
}
PetscErrorCode RHSFunction(TS ts,PetscReal t,Vec globalin,Vec globalout,void *ctx)
{
PetscScalar *inptr,*outptr;
PetscInt i,n,*idx;
PetscErrorCode ierr;
IS from,to;
VecScatter scatter;
Vec tmp_in,tmp_out;
/* Get the length of parallel vector */
ierr = VecGetSize(globalin,&n);CHKERRQ(ierr);
/* Set the index sets */
ierr = PetscMalloc(n*sizeof(PetscInt),&idx);CHKERRQ(ierr);
for(i=0; i<n; i++) idx[i]=i;
/* Create local sequential vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&tmp_in);CHKERRQ(ierr);
ierr = VecDuplicate(tmp_in,&tmp_out);CHKERRQ(ierr);
/* Create scatter context */
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,idx,PETSC_COPY_VALUES,&from);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,idx,PETSC_COPY_VALUES,&to);CHKERRQ(ierr);
ierr = VecScatterCreate(globalin,from,tmp_in,to,&scatter);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,globalin,tmp_in,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,globalin,tmp_in,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter);CHKERRQ(ierr);
/*Extract income array */
ierr = VecGetArray(tmp_in,&inptr);CHKERRQ(ierr);
/* Extract outcome array*/
ierr = VecGetArray(tmp_out,&outptr);CHKERRQ(ierr);
outptr[0] = 2.0*inptr[0]+inptr[1];
outptr[1] = inptr[0]+2.0*inptr[1]+inptr[2];
outptr[2] = inptr[1]+2.0*inptr[2];
ierr = VecRestoreArray(tmp_in,&inptr);CHKERRQ(ierr);
ierr = VecRestoreArray(tmp_out,&outptr);CHKERRQ(ierr);
ierr = VecScatterCreate(tmp_out,from,globalout,to,&scatter);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,tmp_out,globalout,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,tmp_out,globalout,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Destroy idx aand scatter */
ierr = ISDestroy(&from);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter);CHKERRQ(ierr);
ierr = VecDestroy(&tmp_in);CHKERRQ(ierr);
ierr = VecDestroy(&tmp_out);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode PCApply_NN(PC pc,Vec r,Vec z)
{
PC_IS *pcis = (PC_IS*)(pc->data);
PetscErrorCode ierr;
PetscScalar m_one = -1.0;
Vec w = pcis->vec1_global;
PetscFunctionBegin;
/*
Dirichlet solvers.
Solving $ B_I^{(i)}r_I^{(i)} $ at each processor.
Storing the local results at vec2_D
*/
ierr = VecScatterBegin(pcis->global_to_D,r,pcis->vec1_D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_D,r,pcis->vec1_D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = KSPSolve(pcis->ksp_D,pcis->vec1_D,pcis->vec2_D);CHKERRQ(ierr);
/*
Computing $ r_B - \sum_j \tilde R_j^T A_{BI}^{(j)} (B_I^{(j)}r_I^{(j)}) $ .
Storing the result in the interface portion of the global vector w.
*/
ierr = MatMult(pcis->A_BI,pcis->vec2_D,pcis->vec1_B);CHKERRQ(ierr);
ierr = VecScale(pcis->vec1_B,m_one);CHKERRQ(ierr);
ierr = VecCopy(r,w);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,w,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,w,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/*
Apply the interface preconditioner
*/
ierr = PCNNApplyInterfacePreconditioner(pc,w,z,pcis->work_N,pcis->vec1_B,pcis->vec2_B,pcis->vec3_B,pcis->vec1_D,
pcis->vec3_D,pcis->vec1_N,pcis->vec2_N);CHKERRQ(ierr);
/*
Computing $ t_I^{(i)} = A_{IB}^{(i)} \tilde R_i z_B $
The result is stored in vec1_D.
*/
ierr = VecScatterBegin(pcis->global_to_B,z,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,z,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = MatMult(pcis->A_IB,pcis->vec1_B,pcis->vec1_D);CHKERRQ(ierr);
/*
Dirichlet solvers.
Computing $ B_I^{(i)}t_I^{(i)} $ and sticking into the global vector the blocks
$ B_I^{(i)}r_I^{(i)} - B_I^{(i)}t_I^{(i)} $.
*/
ierr = VecScatterBegin(pcis->global_to_D,pcis->vec2_D,z,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_D,pcis->vec2_D,z,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = KSPSolve(pcis->ksp_D,pcis->vec1_D,pcis->vec2_D);CHKERRQ(ierr);
ierr = VecScale(pcis->vec2_D,m_one);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_D,pcis->vec2_D,z,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_D,pcis->vec2_D,z,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
-n <length> : vector length\n\n";
#include <petscvec.h>
#undef __FUNCT__
#define __FUNCT__ "main"
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt n = 5,idx1[2] = {0,3},idx2[2] = {1,4};
PetscScalar one = 1.0,two = 2.0;
Vec x,y;
IS is1,is2;
VecScatter ctx = 0;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
/* create two vector */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
/* create two index sets */
ierr = ISCreateGeneral(PETSC_COMM_SELF,2,idx1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,2,idx2,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr);
ierr = VecSet(x,one);CHKERRQ(ierr);
ierr = VecSet(y,two);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,y,is2,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,y,x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,y,x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"-------\n");CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatMultASPIN(Mat m,Vec X,Vec Y)
{
PetscErrorCode ierr;
void *ctx;
SNES snes;
PetscInt n,i;
VecScatter *oscatter;
SNES *subsnes;
PetscBool match;
MPI_Comm comm;
KSP ksp;
Vec *x,*b;
Vec W;
SNES npc;
Mat subJ,subpJ;
PetscFunctionBegin;
ierr = MatShellGetContext(m,&ctx);CHKERRQ(ierr);
snes = (SNES)ctx;
ierr = SNESGetNPC(snes,&npc);CHKERRQ(ierr);
ierr = SNESGetFunction(npc,&W,NULL,NULL);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)npc,SNESNASM,&match);CHKERRQ(ierr);
if (!match) {
ierr = PetscObjectGetComm((PetscObject)snes,&comm);
SETERRQ(comm,PETSC_ERR_ARG_WRONGSTATE,"MatMultASPIN requires that the nonlinear preconditioner be Nonlinear additive Schwarz");
}
ierr = SNESNASMGetSubdomains(npc,&n,&subsnes,NULL,&oscatter,NULL);CHKERRQ(ierr);
ierr = SNESNASMGetSubdomainVecs(npc,&n,&x,&b,NULL,NULL);CHKERRQ(ierr);
ierr = VecSet(Y,0);CHKERRQ(ierr);
ierr = MatMult(npc->jacobian_pre,X,W);CHKERRQ(ierr);
for (i=0;i<n;i++) {
ierr = VecScatterBegin(oscatter[i],W,b[i],INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
for (i=0;i<n;i++) {
ierr = VecScatterEnd(oscatter[i],W,b[i],INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecSet(x[i],0.);CHKERRQ(ierr);
ierr = SNESGetJacobian(subsnes[i],&subJ,&subpJ,NULL,NULL);CHKERRQ(ierr);
ierr = SNESGetKSP(subsnes[i],&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,subJ,subpJ);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b[i],x[i]);CHKERRQ(ierr);
ierr = VecScatterBegin(oscatter[i],x[i],Y,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
for (i=0;i<n;i++) {
ierr = VecScatterEnd(oscatter[i],x[i],Y,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*
Defines the RHS function that is passed to the time-integrator.
Solves F(U,V) for V and then computes f(U,V)
*/
PetscErrorCode TSFunction(TS ts,PetscReal t,Vec U,Vec F,void *actx)
{
AppCtx *ctx = (AppCtx*)actx;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ctx->t = t;
ierr = VecScatterBegin(ctx->scatterU,U,ctx->UV,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx->scatterU,U,ctx->UV,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = SNESSolve(ctx->snes,NULL,ctx->V);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx->scatterV,ctx->V,ctx->UV,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx->scatterV,ctx->V,ctx->UV,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = (*ctx->f)(t,ctx->UV,F);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Defines the nonlinear function that is passed to the time-integrator
*/
PetscErrorCode TSFunctionI(TS ts,PetscReal t,Vec UV,Vec UVdot,Vec F,void *actx)
{
AppCtx *ctx = (AppCtx*)actx;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = VecCopy(UVdot,F);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx->scatterU,UV,ctx->U,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx->scatterU,UV,ctx->U,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx->scatterV,UV,ctx->V,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx->scatterV,UV,ctx->V,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = (*ctx->F)(t,ctx->U,ctx->V,ctx->VF);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx->scatterV,ctx->VF,F,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx->scatterV,ctx->VF,F,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt n = 5,N,low,high,iglobal,i;
PetscMPIInt size,rank;
PetscScalar value,zero = 0.0;
Vec x,y;
IS is1,is2;
VecScatter ctx;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* create two vectors */
N = size*n;
ierr = VecCreate(PETSC_COMM_WORLD,&y);CHKERRQ(ierr);
ierr = VecSetSizes(y,PETSC_DECIDE,N);CHKERRQ(ierr);
ierr = VecSetFromOptions(y);CHKERRQ(ierr);
if (!rank) {
ierr = VecCreateSeq(PETSC_COMM_SELF,N,&x);CHKERRQ(ierr);
} else {
ierr = VecCreateSeq(PETSC_COMM_SELF,0,&x);CHKERRQ(ierr);
}
/* create two index sets */
if (!rank) {
ierr = ISCreateStride(PETSC_COMM_SELF,N,0,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,N,0,1,&is2);CHKERRQ(ierr);
} else {
ierr = ISCreateStride(PETSC_COMM_SELF,0,0,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,0,0,1,&is2);CHKERRQ(ierr);
}
ierr = VecSet(x,zero);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(y,&low,&high);CHKERRQ(ierr);
for (i=0; i<n; i++) {
iglobal = i + low; value = (PetscScalar) (i + 10*rank);
ierr = VecSetValues(y,1,&iglobal,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(y);CHKERRQ(ierr);
ierr = VecAssemblyEnd(y);CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecScatterCreate(y,is2,x,is1,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,y,x,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,y,x,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
if (!rank)
{printf("----\n"); ierr = VecView(x,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void Save(Vec u, const char* filename) {
int n, rank;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
VecGetSize(u, &n);
Vec uloc;
VecCreateSeq(PETSC_COMM_SELF, n, &uloc);
VecScatter ctx;
VecScatterCreateToZero(u, &ctx, &uloc);
VecScatterBegin(ctx, u, uloc, INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd(ctx, u, uloc, INSERT_VALUES, SCATTER_FORWARD);
const PetscScalar *pu;
VecGetArrayRead(uloc, &pu);
if (rank==0) {
int sq = sqrt(n);
std::ofstream out(filename);
for (size_t i=0; i<n; i++) {
out << pu[i] << std::endl;
if ((i+1)%sq == 0) out << std::endl;
}
}
VecRestoreArrayRead(uloc, &pu);
VecScatterDestroy(&ctx);
VecDestroy(&uloc);
}
PetscErrorCode RetrieveVecPoints(Vec x, int Npt, int *Pos, double *ptValues)
{
PetscErrorCode ierr;
Vec T;
VecScatter scatter;
IS from, to;
ierr = VecCreateSeq(PETSC_COMM_SELF, Npt, &T);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Npt, Pos,PETSC_COPY_VALUES, &from);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,Npt,0,1, &to);CHKERRQ(ierr);
ierr = VecScatterCreate(x,from,T,to,&scatter);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,x,T,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,x,T,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
int ix[Npt];
int i;
for(i=0; i<Npt; i++)
ix[i]=i;
ierr = VecGetValues(T,Npt,ix,ptValues);
ierr = ISDestroy(&from);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter);CHKERRQ(ierr);
ierr = VecDestroy(&T);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PetscVector::copyFromArray( double v[] )
{
int ierr;
Vec sv;
IS is;
VecScatter ctx;
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, n, v, &sv);
assert(ierr == 0);
ierr = ISCreateStride(PETSC_COMM_WORLD, n, 0, 1, &is);
assert( ierr == 0);
ierr = VecScatterCreate( sv, is, pv, is, &ctx);
assert( ierr == 0);
ierr = VecScatterBegin( sv, pv,INSERT_VALUES,SCATTER_FORWARD,
ctx);
assert( ierr == 0);
ierr = VecScatterEnd( sv, pv,INSERT_VALUES,SCATTER_FORWARD,
ctx);
assert( ierr == 0);
ierr = VecScatterDestroy(ctx);
assert( ierr == 0);
ierr = ISDestroy( is );
assert(ierr == 0);
ierr = VecDestroy( sv );
assert(ierr == 0);
}
static PetscErrorCode PetscML_comm(double p[],void *ML_data)
{
PetscErrorCode ierr;
FineGridCtx *ml=(FineGridCtx*)ML_data;
Mat A=ml->A;
Mat_MPIAIJ *a = (Mat_MPIAIJ*)A->data;
PetscMPIInt size;
PetscInt i,in_length=A->rmap->n,out_length=ml->Aloc->cmap->n;
PetscScalar *array;
PetscFunctionBegin;
ierr = MPI_Comm_size(((PetscObject)A)->comm,&size);CHKERRQ(ierr);
if (size == 1) return 0;
ierr = VecPlaceArray(ml->y,p);CHKERRQ(ierr);
ierr = VecScatterBegin(a->Mvctx,ml->y,a->lvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(a->Mvctx,ml->y,a->lvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecResetArray(ml->y);CHKERRQ(ierr);
ierr = VecGetArray(a->lvec,&array);CHKERRQ(ierr);
for (i=in_length; i<out_length; i++){
p[i] = array[i-in_length];
}
ierr = VecRestoreArray(a->lvec,&array);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
Real SAMpatchPETSc::normL2(const Vector& x, char dofType) const
{
#ifdef HAVE_MPI
if (adm.isParallel()) {
if (dofIS.find(dofType) == dofIS.end())
setupIS(dofType);
Vec lx;
VecCreateSeqWithArray(PETSC_COMM_SELF, 1, x.size(), x.data(), &lx);
Vec gx;
VecCreate(*adm.getCommunicator(), &gx);
VecSetSizes(gx, dofIS[dofType].nDofs, PETSC_DETERMINE);
VecSetFromOptions(gx);
PetscInt n;
VecGetSize(gx, &n);
if (!dofIS[dofType].scatterCreated) {
VecScatterCreate(lx, dofIS[dofType].local, gx, dofIS[dofType].global, &dofIS[dofType].ctx);
dofIS[dofType].scatterCreated = true;
}
VecScatterBegin(dofIS[dofType].ctx, lx, gx, INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd(dofIS[dofType].ctx, lx, gx, INSERT_VALUES, SCATTER_FORWARD);
PetscReal d;
VecNorm(gx, NORM_2, &d);
VecDestroy(&lx);
VecDestroy(&gx);
return d / sqrt(double(n));
}
#endif
return this->SAM::normL2(x, dofType);
}
PetscErrorCode DMLocalToGlobalEnd_DA(DM da,Vec l,InsertMode mode,Vec g)
{
PetscErrorCode ierr;
DM_DA *dd = (DM_DA*)da->data;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidHeaderSpecific(l,VEC_CLASSID,2);
PetscValidHeaderSpecific(g,VEC_CLASSID,3);
if (mode == ADD_VALUES) {
ierr = VecScatterEnd(dd->gtol,l,g,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
} else if (mode == INSERT_VALUES) {
ierr = VecScatterEnd(dd->ltog,l,g,mode,SCATTER_FORWARD);CHKERRQ(ierr);
} else SETERRQ(((PetscObject)da)->comm,PETSC_ERR_SUP,"Not yet implemented");
PetscFunctionReturn(0);
}
PetscErrorCode VecReorder(Vec r, Vec order, Vec or){
PetscErrorCode err;
PetscInt n_local, low, high;
PetscInt *to_idx, *from_idx;
PetscScalar *o_array;
IS to, from;
VecScatter scatter;
int i;
err = VecGetLocalSize(r, &n_local); CHKERRQ(err);
err = PetscMalloc1(n_local, &to_idx); CHKERRQ(err);
err = PetscMalloc1(n_local, &from_idx); CHKERRQ(err);
err = VecGetOwnershipRange(r, &low, &high); CHKERRQ(err);
err = VecGetArray(order, &o_array);
for(i = 0; i < n_local; i++){
to_idx[i] = (PetscInt) o_array[low + i];
from_idx[i] = (PetscInt) low + i;
}
err = VecRestoreArray(order, &o_array);
err = ISCreateGeneral(PETSC_COMM_SELF, n_local, from_idx, PETSC_OWN_POINTER, &to); CHKERRQ(err);
err = ISCreateGeneral(PETSC_COMM_SELF, n_local, from_idx, PETSC_OWN_POINTER, &from); CHKERRQ(err);
err = VecScatterCreate(r, from, or, to, &scatter); CHKERRQ(err);
err = VecScatterBegin(scatter, r, or, INSERT_VALUES, SCATTER_FORWARD); CHKERRQ(err);
err = VecScatterEnd(scatter, r, or, INSERT_VALUES, SCATTER_FORWARD); CHKERRQ(err);
err = PetscFree(to_idx); CHKERRQ(err);
err = PetscFree(from_idx); CHKERRQ(err);
return err;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt i,blocks[2],nlocal;
PetscMPIInt size,rank;
PetscScalar value;
Vec x,y;
IS is1,is2;
VecScatter ctx = 0;
PetscViewer subviewer;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (size != 2) SETERRQ(PETSC_COMM_SELF,1,"Must run with 2 processors");
/* create two vectors */
if (!rank) nlocal = 8;
else nlocal = 4;
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,nlocal,12);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&y);CHKERRQ(ierr);
ierr = VecSetSizes(y,8,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(y);CHKERRQ(ierr);
/* create two index sets */
if (!rank) {
blocks[0] = 0; blocks[1] = 2;
} else {
blocks[0] = 1; blocks[1] = 2;
}
ierr = ISCreateBlock(PETSC_COMM_SELF,4,2,blocks,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,8,0,1,&is2);CHKERRQ(ierr);
for (i=0; i<12; i++) {
value = i;
ierr = VecSetValues(x,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecScatterCreateWithData(x,is1,y,is2,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&subviewer);CHKERRQ(ierr);
ierr = VecView(y,subviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&subviewer);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode PCNNApplyInterfacePreconditioner (PC pc, Vec r, Vec z, PetscScalar* work_N, Vec vec1_B, Vec vec2_B, Vec vec3_B, Vec vec1_D,
Vec vec2_D, Vec vec1_N, Vec vec2_N)
{
PetscErrorCode ierr;
PC_IS* pcis = (PC_IS*)(pc->data);
PetscFunctionBegin;
/*
First balancing step.
*/
{
PetscBool flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-pc_nn_turn_off_first_balancing",&flg,PETSC_NULL);CHKERRQ(ierr);
if (!flg) {
ierr = PCNNBalancing(pc,r,(Vec)0,z,vec1_B,vec2_B,(Vec)0,vec1_D,vec2_D,work_N);CHKERRQ(ierr);
} else {
ierr = VecCopy(r,z);CHKERRQ(ierr);
}
}
/*
Extract the local interface part of z and scale it by D
*/
ierr = VecScatterBegin(pcis->global_to_B,z,vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,z,vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPointwiseMult(vec2_B,pcis->D,vec1_B);CHKERRQ(ierr);
/* Neumann Solver */
ierr = PCISApplyInvSchur(pc,vec2_B,vec1_B,vec1_N,vec2_N);CHKERRQ(ierr);
/*
Second balancing step.
*/
{
PetscBool flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-pc_turn_off_second_balancing",&flg,PETSC_NULL);CHKERRQ(ierr);
if (!flg) {
ierr = PCNNBalancing(pc,r,vec1_B,z,vec2_B,vec3_B,(Vec)0,vec1_D,vec2_D,work_N);CHKERRQ(ierr);
} else {
ierr = VecPointwiseMult(vec2_B,pcis->D,vec1_B);CHKERRQ(ierr);
ierr = VecSet(z,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,vec2_B,z,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,vec2_B,z,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
PetscErrorCode PCISApplyInvSchur(PC pc, Vec b, Vec x, Vec vec1_N, Vec vec2_N)
{
PetscErrorCode ierr;
PC_IS *pcis = (PC_IS*)(pc->data);
PetscFunctionBegin;
/*
Neumann solvers.
Applying the inverse of the local Schur complement, i.e, solving a Neumann
Problem with zero at the interior nodes of the RHS and extracting the interface
part of the solution. inverse Schur complement is applied to b and the result
is stored in x.
*/
/* Setting the RHS vec1_N */
ierr = VecSet(vec1_N,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,b,vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,b,vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* Checking for consistency of the RHS */
{
PetscBool flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-pc_is_check_consistency",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscScalar average;
PetscViewer viewer;
ierr = PetscViewerASCIIGetStdout(PetscObjectComm((PetscObject)pc),&viewer);CHKERRQ(ierr);
ierr = VecSum(vec1_N,&average);CHKERRQ(ierr);
average = average / ((PetscReal)pcis->n);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);CHKERRQ(ierr);
if (pcis->pure_neumann) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d is floating. Average = % 1.14e\n",PetscGlobalRank,PetscAbsScalar(average));CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d is fixed. Average = % 1.14e\n",PetscGlobalRank,PetscAbsScalar(average));CHKERRQ(ierr);
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);CHKERRQ(ierr);
}
}
/* Solving the system for vec2_N */
ierr = KSPSolve(pcis->ksp_N,vec1_N,vec2_N);CHKERRQ(ierr);
/* Extracting the local interface vector out of the solution */
ierr = VecScatterBegin(pcis->N_to_B,vec2_N,x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,vec2_N,x,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FAGlobalToLocal(FA fa,Vec g,Vec l)
{
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = VecScatterBegin(fa->vscat,g,l,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(fa->vscat,g,l,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode MatMultTranspose_SubMatrix(Mat N,Vec x,Vec y)
{
Mat_SubMatrix *Na = (Mat_SubMatrix*)N->data;
Vec xx = 0;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PreScaleLeft(N,x,&xx);CHKERRQ(ierr);
ierr = VecZeroEntries(Na->lwork);CHKERRQ(ierr);
ierr = VecScatterBegin(Na->lrestrict,xx,Na->lwork,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (Na->lrestrict,xx,Na->lwork,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = MatMultTranspose(Na->A,Na->lwork,Na->rwork);CHKERRQ(ierr);
ierr = VecScatterBegin(Na->rprolong,Na->rwork,y,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (Na->rprolong,Na->rwork,y,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = PostScaleRight(N,y);CHKERRQ(ierr);
ierr = VecScale(y,Na->scale);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Shared by both sequential and parallel versions of CRL matrix: MATMPIAIJCRL and MATSEQAIJCRL
- the scatter is used only in the parallel version
*/
PetscErrorCode MatMult_AIJCRL(Mat A,Vec xx,Vec yy)
{
Mat_AIJCRL *aijcrl = (Mat_AIJCRL*) A->spptr;
PetscInt m = aijcrl->m; /* Number of rows in the matrix. */
PetscInt rmax = aijcrl->rmax,*icols = aijcrl->icols;
PetscScalar *acols = aijcrl->acols;
PetscErrorCode ierr;
PetscScalar *x,*y;
#if !defined(PETSC_USE_FORTRAN_KERNEL_MULTCRL)
PetscInt i,j,ii;
#endif
#if defined(PETSC_HAVE_PRAGMA_DISJOINT)
#pragma disjoint(*x,*y,*aa)
#endif
PetscFunctionBegin;
if (aijcrl->xscat) {
ierr = VecCopy(xx,aijcrl->xwork);CHKERRQ(ierr);
/* get remote values needed for local part of multiply */
ierr = VecScatterBegin(aijcrl->xscat,xx,aijcrl->fwork,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(aijcrl->xscat,xx,aijcrl->fwork,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
xx = aijcrl->xwork;
}
ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
ierr = VecGetArray(yy,&y);CHKERRQ(ierr);
#if defined(PETSC_USE_FORTRAN_KERNEL_MULTCRL)
fortranmultcrl_(&m,&rmax,x,y,icols,acols);
#else
/* first column */
for (j=0; j<m; j++) y[j] = acols[j]*x[icols[j]];
/* other columns */
#if defined(PETSC_HAVE_CRAY_VECTOR)
#pragma _CRI preferstream
#endif
for (i=1; i<rmax; i++) {
ii = i*m;
#if defined(PETSC_HAVE_CRAY_VECTOR)
#pragma _CRI prefervector
#endif
for (j=0; j<m; j++) y[j] = y[j] + acols[ii+j]*x[icols[ii+j]];
}
#if defined(PETSC_HAVE_CRAY_VECTOR)
#pragma _CRI ivdep
#endif
#endif
ierr = PetscLogFlops(2.0*aijcrl->nz - m);CHKERRQ(ierr);
ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
ierr = VecRestoreArray(yy,&y);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
